diff --git a/library/strategy_base.py b/library/strategy_base.py index 358e42f1..c7f6e39b 100644 --- a/library/strategy_base.py +++ b/library/strategy_base.py @@ -522,6 +522,7 @@ class LatentsCachingStrategy: """ return self._default_load_latents_from_disk(None, npz_path, bucket_reso) + # TODO: Solve Multinode hang def _default_load_latents_from_disk( self, latents_stride: Optional[int], npz_path: str, bucket_reso: Tuple[int, int] ) -> Tuple[Optional[np.ndarray], Optional[List[int]], Optional[List[int]], Optional[np.ndarray], Optional[np.ndarray]]: diff --git a/library/train_util.py b/library/train_util.py index 6a98c4c6..ebe9f706 100644 --- a/library/train_util.py +++ b/library/train_util.py @@ -135,6 +135,7 @@ IMAGE_TRANSFORMS = transforms.Compose( TEXT_ENCODER_OUTPUTS_CACHE_SUFFIX = "_te_outputs.npz" TEXT_ENCODER_OUTPUTS_CACHE_SUFFIX_SD3 = "_sd3_te.npz" +SKIP_NPZ_PATH_CHECK = False def set_skip_npz_path_check(skip: bool): global SKIP_NPZ_PATH_CHECK