From 1cbb3fe2c8fd8b629825025c86297e8b886307b2 Mon Sep 17 00:00:00 2001 From: rockerBOO Date: Sun, 19 Nov 2023 20:46:01 -0500 Subject: [PATCH 1/3] Add LR scheduler metadata for non-standard LR schedulers --- train_network.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train_network.py b/train_network.py index d50916b7..dab17199 100644 --- a/train_network.py +++ b/train_network.py @@ -503,7 +503,8 @@ class NetworkTrainer: "ss_gradient_accumulation_steps": args.gradient_accumulation_steps, "ss_max_train_steps": args.max_train_steps, "ss_lr_warmup_steps": args.lr_warmup_steps, - "ss_lr_scheduler": args.lr_scheduler, + "ss_lr_scheduler": (args.lr_scheduler_type or args.lr_scheduler) + + (f"{args.lr_scheduler_args}" if len(args.lr_scheduler_args) > 0 else ""), "ss_network_module": args.network_module, "ss_network_dim": args.network_dim, # None means default because another network than LoRA may have another default dim "ss_network_alpha": args.network_alpha, # some networks may not have alpha From 4aba1217b03d8c3118a9f618c54ea012aef4c96f Mon Sep 17 00:00:00 2001 From: rockerBOO Date: Sun, 19 Nov 2023 20:58:16 -0500 Subject: [PATCH 2/3] Add parens to match optimizer args --- train_network.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_network.py b/train_network.py index dab17199..15947151 100644 --- a/train_network.py +++ b/train_network.py @@ -504,7 +504,7 @@ class NetworkTrainer: "ss_max_train_steps": args.max_train_steps, "ss_lr_warmup_steps": args.lr_warmup_steps, "ss_lr_scheduler": (args.lr_scheduler_type or args.lr_scheduler) - + (f"{args.lr_scheduler_args}" if len(args.lr_scheduler_args) > 0 else ""), + + (f"({args.lr_scheduler_args})" if len(args.lr_scheduler_args) > 0 else ""), "ss_network_module": args.network_module, "ss_network_dim": args.network_dim, # None means default because another network than LoRA may have another default dim "ss_network_alpha": args.network_alpha, # some networks may not have alpha From aa04692f77f4270fab3f53d34e5646888b9d0882 Mon Sep 17 00:00:00 2001 From: rockerBOO Date: Thu, 23 Nov 2023 10:00:01 -0500 Subject: [PATCH 3/3] Fix handling default state --- train_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/train_network.py b/train_network.py index 15947151..c1593534 100644 --- a/train_network.py +++ b/train_network.py @@ -504,7 +504,10 @@ class NetworkTrainer: "ss_max_train_steps": args.max_train_steps, "ss_lr_warmup_steps": args.lr_warmup_steps, "ss_lr_scheduler": (args.lr_scheduler_type or args.lr_scheduler) - + (f"({args.lr_scheduler_args})" if len(args.lr_scheduler_args) > 0 else ""), + + (f"({args.lr_scheduler_args})" + if args.lr_scheduler_args is not None + and len(args.lr_scheduler_args) > 0 + else ""), "ss_network_module": args.network_module, "ss_network_dim": args.network_dim, # None means default because another network than LoRA may have another default dim "ss_network_alpha": args.network_alpha, # some networks may not have alpha