mirror of
https://github.com/kohya-ss/sd-scripts.git
synced 2026-04-09 06:45:09 +00:00
format by black
This commit is contained in:
Kohya S
@@ -16,174 +16,192 @@ MIN_DIFF = 1e-6
|
|||||||
|
|
||||||
|
|
||||||
def save_to_file(file_name, model, state_dict, dtype):
|
def save_to_file(file_name, model, state_dict, dtype):
|
||||||
if dtype is not None:
|
if dtype is not None:
|
||||||
for key in list(state_dict.keys()):
|
for key in list(state_dict.keys()):
|
||||||
if type(state_dict[key]) == torch.Tensor:
|
if type(state_dict[key]) == torch.Tensor:
|
||||||
state_dict[key] = state_dict[key].to(dtype)
|
state_dict[key] = state_dict[key].to(dtype)
|
||||||
|
|
||||||
if os.path.splitext(file_name)[1] == '.safetensors':
|
if os.path.splitext(file_name)[1] == ".safetensors":
|
||||||
save_file(model, file_name)
|
save_file(model, file_name)
|
||||||
else:
|
else:
|
||||||
torch.save(model, file_name)
|
torch.save(model, file_name)
|
||||||
|
|
||||||
|
|
||||||
def svd(args):
|
def svd(args):
|
||||||
def str_to_dtype(p):
|
def str_to_dtype(p):
|
||||||
if p == 'float':
|
if p == "float":
|
||||||
return torch.float
|
return torch.float
|
||||||
if p == 'fp16':
|
if p == "fp16":
|
||||||
return torch.float16
|
return torch.float16
|
||||||
if p == 'bf16':
|
if p == "bf16":
|
||||||
return torch.bfloat16
|
return torch.bfloat16
|
||||||
return None
|
return None
|
||||||
|
|
||||||
save_dtype = str_to_dtype(args.save_precision)
|
save_dtype = str_to_dtype(args.save_precision)
|
||||||
|
|
||||||
print(f"loading SD model : {args.model_org}")
|
print(f"loading SD model : {args.model_org}")
|
||||||
text_encoder_o, _, unet_o = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_org)
|
text_encoder_o, _, unet_o = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_org)
|
||||||
print(f"loading SD model : {args.model_tuned}")
|
print(f"loading SD model : {args.model_tuned}")
|
||||||
text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_tuned)
|
text_encoder_t, _, unet_t = model_util.load_models_from_stable_diffusion_checkpoint(args.v2, args.model_tuned)
|
||||||
|
|
||||||
# create LoRA network to extract weights: Use dim (rank) as alpha
|
# create LoRA network to extract weights: Use dim (rank) as alpha
|
||||||
if args.conv_dim is None:
|
if args.conv_dim is None:
|
||||||
kwargs = {}
|
kwargs = {}
|
||||||
else:
|
else:
|
||||||
kwargs = {"conv_dim": args.conv_dim, "conv_alpha": args.conv_dim}
|
kwargs = {"conv_dim": args.conv_dim, "conv_alpha": args.conv_dim}
|
||||||
|
|
||||||
lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o, **kwargs)
|
lora_network_o = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_o, unet_o, **kwargs)
|
||||||
lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t, **kwargs)
|
lora_network_t = lora.create_network(1.0, args.dim, args.dim, None, text_encoder_t, unet_t, **kwargs)
|
||||||
assert len(lora_network_o.text_encoder_loras) == len(
|
assert len(lora_network_o.text_encoder_loras) == len(
|
||||||
lora_network_t.text_encoder_loras), f"model version is different (SD1.x vs SD2.x) / それぞれのモデルのバージョンが違います（SD1.xベースとSD2.xベース） "
|
lora_network_t.text_encoder_loras
|
||||||
|
), f"model version is different (SD1.x vs SD2.x) / それぞれのモデルのバージョンが違います（SD1.xベースとSD2.xベース） "
|
||||||
|
|
||||||
# get diffs
|
# get diffs
|
||||||
diffs = {}
|
|
||||||
text_encoder_different = False
|
|
||||||
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.text_encoder_loras, lora_network_t.text_encoder_loras)):
|
|
||||||
lora_name = lora_o.lora_name
|
|
||||||
module_o = lora_o.org_module
|
|
||||||
module_t = lora_t.org_module
|
|
||||||
diff = module_t.weight - module_o.weight
|
|
||||||
|
|
||||||
# Text Encoder might be same
|
|
||||||
if torch.max(torch.abs(diff)) > MIN_DIFF:
|
|
||||||
text_encoder_different = True
|
|
||||||
|
|
||||||
diff = diff.float()
|
|
||||||
diffs[lora_name] = diff
|
|
||||||
|
|
||||||
if not text_encoder_different:
|
|
||||||
print("Text encoder is same. Extract U-Net only.")
|
|
||||||
lora_network_o.text_encoder_loras = []
|
|
||||||
diffs = {}
|
diffs = {}
|
||||||
|
text_encoder_different = False
|
||||||
|
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.text_encoder_loras, lora_network_t.text_encoder_loras)):
|
||||||
|
lora_name = lora_o.lora_name
|
||||||
|
module_o = lora_o.org_module
|
||||||
|
module_t = lora_t.org_module
|
||||||
|
diff = module_t.weight - module_o.weight
|
||||||
|
|
||||||
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.unet_loras, lora_network_t.unet_loras)):
|
# Text Encoder might be same
|
||||||
lora_name = lora_o.lora_name
|
if torch.max(torch.abs(diff)) > MIN_DIFF:
|
||||||
module_o = lora_o.org_module
|
text_encoder_different = True
|
||||||
module_t = lora_t.org_module
|
|
||||||
diff = module_t.weight - module_o.weight
|
|
||||||
diff = diff.float()
|
|
||||||
|
|
||||||
if args.device:
|
diff = diff.float()
|
||||||
diff = diff.to(args.device)
|
diffs[lora_name] = diff
|
||||||
|
|
||||||
diffs[lora_name] = diff
|
if not text_encoder_different:
|
||||||
|
print("Text encoder is same. Extract U-Net only.")
|
||||||
|
lora_network_o.text_encoder_loras = []
|
||||||
|
diffs = {}
|
||||||
|
|
||||||
# make LoRA with svd
|
for i, (lora_o, lora_t) in enumerate(zip(lora_network_o.unet_loras, lora_network_t.unet_loras)):
|
||||||
print("calculating by svd")
|
lora_name = lora_o.lora_name
|
||||||
lora_weights = {}
|
module_o = lora_o.org_module
|
||||||
with torch.no_grad():
|
module_t = lora_t.org_module
|
||||||
for lora_name, mat in tqdm(list(diffs.items())):
|
diff = module_t.weight - module_o.weight
|
||||||
# if args.conv_dim is None, diffs do not include LoRAs for conv2d-3x3
|
diff = diff.float()
|
||||||
conv2d = (len(mat.size()) == 4)
|
|
||||||
kernel_size = None if not conv2d else mat.size()[2:4]
|
|
||||||
conv2d_3x3 = conv2d and kernel_size != (1, 1)
|
|
||||||
|
|
||||||
rank = args.dim if not conv2d_3x3 or args.conv_dim is None else args.conv_dim
|
if args.device:
|
||||||
out_dim, in_dim = mat.size()[0:2]
|
diff = diff.to(args.device)
|
||||||
|
|
||||||
if args.device:
|
diffs[lora_name] = diff
|
||||||
mat = mat.to(args.device)
|
|
||||||
|
|
||||||
# print(lora_name, mat.size(), mat.device, rank, in_dim, out_dim)
|
# make LoRA with svd
|
||||||
rank = min(rank, in_dim, out_dim) # LoRA rank cannot exceed the original dim
|
print("calculating by svd")
|
||||||
|
lora_weights = {}
|
||||||
|
with torch.no_grad():
|
||||||
|
for lora_name, mat in tqdm(list(diffs.items())):
|
||||||
|
# if args.conv_dim is None, diffs do not include LoRAs for conv2d-3x3
|
||||||
|
conv2d = len(mat.size()) == 4
|
||||||
|
kernel_size = None if not conv2d else mat.size()[2:4]
|
||||||
|
conv2d_3x3 = conv2d and kernel_size != (1, 1)
|
||||||
|
|
||||||
if conv2d:
|
rank = args.dim if not conv2d_3x3 or args.conv_dim is None else args.conv_dim
|
||||||
if conv2d_3x3:
|
out_dim, in_dim = mat.size()[0:2]
|
||||||
mat = mat.flatten(start_dim=1)
|
|
||||||
else:
|
|
||||||
mat = mat.squeeze()
|
|
||||||
|
|
||||||
U, S, Vh = torch.linalg.svd(mat)
|
if args.device:
|
||||||
|
mat = mat.to(args.device)
|
||||||
|
|
||||||
U = U[:, :rank]
|
# print(lora_name, mat.size(), mat.device, rank, in_dim, out_dim)
|
||||||
S = S[:rank]
|
rank = min(rank, in_dim, out_dim) # LoRA rank cannot exceed the original dim
|
||||||
U = U @ torch.diag(S)
|
|
||||||
|
|
||||||
Vh = Vh[:rank, :]
|
if conv2d:
|
||||||
|
if conv2d_3x3:
|
||||||
|
mat = mat.flatten(start_dim=1)
|
||||||
|
else:
|
||||||
|
mat = mat.squeeze()
|
||||||
|
|
||||||
dist = torch.cat([U.flatten(), Vh.flatten()])
|
U, S, Vh = torch.linalg.svd(mat)
|
||||||
hi_val = torch.quantile(dist, CLAMP_QUANTILE)
|
|
||||||
low_val = -hi_val
|
|
||||||
|
|
||||||
U = U.clamp(low_val, hi_val)
|
U = U[:, :rank]
|
||||||
Vh = Vh.clamp(low_val, hi_val)
|
S = S[:rank]
|
||||||
|
U = U @ torch.diag(S)
|
||||||
|
|
||||||
if conv2d:
|
Vh = Vh[:rank, :]
|
||||||
U = U.reshape(out_dim, rank, 1, 1)
|
|
||||||
Vh = Vh.reshape(rank, in_dim, kernel_size[0], kernel_size[1])
|
|
||||||
|
|
||||||
U = U.to("cpu").contiguous()
|
dist = torch.cat([U.flatten(), Vh.flatten()])
|
||||||
Vh = Vh.to("cpu").contiguous()
|
hi_val = torch.quantile(dist, CLAMP_QUANTILE)
|
||||||
|
low_val = -hi_val
|
||||||
|
|
||||||
lora_weights[lora_name] = (U, Vh)
|
U = U.clamp(low_val, hi_val)
|
||||||
|
Vh = Vh.clamp(low_val, hi_val)
|
||||||
|
|
||||||
# make state dict for LoRA
|
if conv2d:
|
||||||
lora_sd = {}
|
U = U.reshape(out_dim, rank, 1, 1)
|
||||||
for lora_name, (up_weight, down_weight) in lora_weights.items():
|
Vh = Vh.reshape(rank, in_dim, kernel_size[0], kernel_size[1])
|
||||||
lora_sd[lora_name + '.lora_up.weight'] = up_weight
|
|
||||||
lora_sd[lora_name + '.lora_down.weight'] = down_weight
|
|
||||||
lora_sd[lora_name + '.alpha'] = torch.tensor(down_weight.size()[0])
|
|
||||||
|
|
||||||
# load state dict to LoRA and save it
|
U = U.to("cpu").contiguous()
|
||||||
lora_network_save, lora_sd = lora.create_network_from_weights(1.0, None, None, text_encoder_o, unet_o, weights_sd=lora_sd)
|
Vh = Vh.to("cpu").contiguous()
|
||||||
lora_network_save.apply_to(text_encoder_o, unet_o) # create internal module references for state_dict
|
|
||||||
|
|
||||||
info = lora_network_save.load_state_dict(lora_sd)
|
lora_weights[lora_name] = (U, Vh)
|
||||||
print(f"Loading extracted LoRA weights: {info}")
|
|
||||||
|
|
||||||
dir_name = os.path.dirname(args.save_to)
|
# make state dict for LoRA
|
||||||
if dir_name and not os.path.exists(dir_name):
|
lora_sd = {}
|
||||||
os.makedirs(dir_name, exist_ok=True)
|
for lora_name, (up_weight, down_weight) in lora_weights.items():
|
||||||
|
lora_sd[lora_name + ".lora_up.weight"] = up_weight
|
||||||
|
lora_sd[lora_name + ".lora_down.weight"] = down_weight
|
||||||
|
lora_sd[lora_name + ".alpha"] = torch.tensor(down_weight.size()[0])
|
||||||
|
|
||||||
# minimum metadata
|
# load state dict to LoRA and save it
|
||||||
metadata = {"ss_network_module": "networks.lora", "ss_network_dim": str(args.dim), "ss_network_alpha": str(args.dim)}
|
lora_network_save, lora_sd = lora.create_network_from_weights(1.0, None, None, text_encoder_o, unet_o, weights_sd=lora_sd)
|
||||||
|
lora_network_save.apply_to(text_encoder_o, unet_o) # create internal module references for state_dict
|
||||||
|
|
||||||
lora_network_save.save_weights(args.save_to, save_dtype, metadata)
|
info = lora_network_save.load_state_dict(lora_sd)
|
||||||
print(f"LoRA weights are saved to: {args.save_to}")
|
print(f"Loading extracted LoRA weights: {info}")
|
||||||
|
|
||||||
|
dir_name = os.path.dirname(args.save_to)
|
||||||
|
if dir_name and not os.path.exists(dir_name):
|
||||||
|
os.makedirs(dir_name, exist_ok=True)
|
||||||
|
|
||||||
|
# minimum metadata
|
||||||
|
metadata = {"ss_network_module": "networks.lora", "ss_network_dim": str(args.dim), "ss_network_alpha": str(args.dim)}
|
||||||
|
|
||||||
|
lora_network_save.save_weights(args.save_to, save_dtype, metadata)
|
||||||
|
print(f"LoRA weights are saved to: {args.save_to}")
|
||||||
|
|
||||||
|
|
||||||
def setup_parser() -> argparse.ArgumentParser:
|
def setup_parser() -> argparse.ArgumentParser:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--v2", action='store_true',
|
parser.add_argument("--v2", action="store_true", help="load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む")
|
||||||
help='load Stable Diffusion v2.x model / Stable Diffusion 2.xのモデルを読み込む')
|
parser.add_argument(
|
||||||
parser.add_argument("--save_precision", type=str, default=None,
|
"--save_precision",
|
||||||
choices=[None, "float", "fp16", "bf16"], help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はfloat")
|
type=str,
|
||||||
parser.add_argument("--model_org", type=str, default=None,
|
default=None,
|
||||||
help="Stable Diffusion original model: ckpt or safetensors file / 元モデル、ckptまたはsafetensors")
|
choices=[None, "float", "fp16", "bf16"],
|
||||||
parser.add_argument("--model_tuned", type=str, default=None,
|
help="precision in saving, same to merging if omitted / 保存時に精度を変更して保存する、省略時はfloat",
|
||||||
help="Stable Diffusion tuned model, LoRA is difference of `original to tuned`: ckpt or safetensors file / 派生モデル（生成されるLoRAは元→派生の差分になります）、ckptまたはsafetensors")
|
)
|
||||||
parser.add_argument("--save_to", type=str, default=None,
|
parser.add_argument(
|
||||||
help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors")
|
"--model_org",
|
||||||
parser.add_argument("--dim", type=int, default=4, help="dimension (rank) of LoRA (default 4) / LoRAの次元数（rank）（デフォルト4）")
|
type=str,
|
||||||
parser.add_argument("--conv_dim", type=int, default=None,
|
default=None,
|
||||||
help="dimension (rank) of LoRA for Conv2d-3x3 (default None, disabled) / LoRAのConv2d-3x3の次元数（rank）（デフォルトNone、適用なし）")
|
help="Stable Diffusion original model: ckpt or safetensors file / 元モデル、ckptまたはsafetensors",
|
||||||
parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--model_tuned",
|
||||||
|
type=str,
|
||||||
|
default=None,
|
||||||
|
help="Stable Diffusion tuned model, LoRA is difference of `original to tuned`: ckpt or safetensors file / 派生モデル（生成されるLoRAは元→派生の差分になります）、ckptまたはsafetensors",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--save_to", type=str, default=None, help="destination file name: ckpt or safetensors file / 保存先のファイル名、ckptまたはsafetensors"
|
||||||
|
)
|
||||||
|
parser.add_argument("--dim", type=int, default=4, help="dimension (rank) of LoRA (default 4) / LoRAの次元数（rank）（デフォルト4）")
|
||||||
|
parser.add_argument(
|
||||||
|
"--conv_dim",
|
||||||
|
type=int,
|
||||||
|
default=None,
|
||||||
|
help="dimension (rank) of LoRA for Conv2d-3x3 (default None, disabled) / LoRAのConv2d-3x3の次元数（rank）（デフォルトNone、適用なし）",
|
||||||
|
)
|
||||||
|
parser.add_argument("--device", type=str, default=None, help="device to use, cuda for GPU / 計算を行うデバイス、cuda でGPUを使う")
|
||||||
|
|
||||||
return parser
|
return parser
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == "__main__":
|
||||||
parser = setup_parser()
|
parser = setup_parser()
|
||||||
|
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
svd(args)
|
svd(args)
|
||||||
|
|||||||
Reference in New Issue
Block a user