From 21f5b618c3c583d53036062adb1bf7e90824644f Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 14 Feb 2023 19:46:27 +0900 Subject: [PATCH 1/5] Show the moving average loss --- train_db.py | 11 +++++++---- train_network.py | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/train_db.py b/train_db.py index c210767b..a3154cd1 100644 --- a/train_db.py +++ b/train_db.py @@ -206,6 +206,7 @@ def train(args): if accelerator.is_main_process: accelerator.init_trackers("dreambooth") + loss_list = [] for epoch in range(num_train_epochs): print(f"epoch {epoch+1}/{num_train_epochs}") train_dataset.set_current_epoch(epoch + 1) @@ -216,7 +217,6 @@ def train(args): if args.gradient_checkpointing or global_step < args.stop_text_encoder_training: text_encoder.train() - loss_total = 0 for step, batch in enumerate(train_dataloader): # 指定したステップ数でText Encoderの学習を止める if global_step == args.stop_text_encoder_training: @@ -291,8 +291,11 @@ def train(args): logs = {"loss": current_loss, "lr": lr_scheduler.get_last_lr()[0]} accelerator.log(logs, step=global_step) - loss_total += current_loss - avr_loss = loss_total / (step+1) + if epoch == 0: + loss_list.append(current_loss) + else: + loss_list[step] = current_loss + avr_loss = sum(loss_list) / len(loss_list) logs = {"loss": avr_loss} # , "lr": lr_scheduler.get_last_lr()[0]} progress_bar.set_postfix(**logs) @@ -300,7 +303,7 @@ def train(args): break if args.logging_dir is not None: - logs = {"epoch_loss": loss_total / len(train_dataloader)} + logs = {"epoch_loss": sum(loss_list) / len(loss_list)} accelerator.log(logs, step=epoch+1) accelerator.wait_for_everyone() diff --git a/train_network.py b/train_network.py index bb3159fd..c9c3c468 100644 --- a/train_network.py +++ b/train_network.py @@ -378,6 +378,7 @@ def train(args): if accelerator.is_main_process: accelerator.init_trackers("network_train") + loss_list = [] for epoch in range(num_train_epochs): print(f"epoch {epoch+1}/{num_train_epochs}") train_dataset.set_current_epoch(epoch + 1) @@ -386,7 +387,6 @@ def train(args): network.on_epoch_start(text_encoder, unet) - loss_total = 0 for step, batch in enumerate(train_dataloader): with accelerator.accumulate(network): with torch.no_grad(): @@ -446,8 +446,11 @@ def train(args): global_step += 1 current_loss = loss.detach().item() - loss_total += current_loss - avr_loss = loss_total / (step+1) + if epoch == 0: + loss_list.append(current_loss) + else: + loss_list[step] = current_loss + avr_loss = sum(loss_list) / len(loss_list) logs = {"loss": avr_loss} # , "lr": lr_scheduler.get_last_lr()[0]} progress_bar.set_postfix(**logs) @@ -459,7 +462,7 @@ def train(args): break if args.logging_dir is not None: - logs = {"loss/epoch": loss_total / len(train_dataloader)} + logs = {"loss/epoch": sum(loss_list) / len(loss_list)} accelerator.log(logs, step=epoch+1) accelerator.wait_for_everyone() From 8aed5125deddef0a73fcb9a84c3bacefb0059a11 Mon Sep 17 00:00:00 2001 From: Yuta Hayashibe Date: Tue, 14 Feb 2023 21:11:30 +0900 Subject: [PATCH 2/5] Removed call of sum() --- train_db.py | 7 +++++-- train_network.py | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/train_db.py b/train_db.py index a3154cd1..cbcd8071 100644 --- a/train_db.py +++ b/train_db.py @@ -207,6 +207,7 @@ def train(args): accelerator.init_trackers("dreambooth") loss_list = [] + loss_total = 0.0 for epoch in range(num_train_epochs): print(f"epoch {epoch+1}/{num_train_epochs}") train_dataset.set_current_epoch(epoch + 1) @@ -294,8 +295,10 @@ def train(args): if epoch == 0: loss_list.append(current_loss) else: + loss_total -= loss_list[step] loss_list[step] = current_loss - avr_loss = sum(loss_list) / len(loss_list) + loss_total += current_loss + avr_loss = loss_total / len(loss_list) logs = {"loss": avr_loss} # , "lr": lr_scheduler.get_last_lr()[0]} progress_bar.set_postfix(**logs) @@ -303,7 +306,7 @@ def train(args): break if args.logging_dir is not None: - logs = {"epoch_loss": sum(loss_list) / len(loss_list)} + logs = {"epoch_loss": loss_total / len(loss_list)} accelerator.log(logs, step=epoch+1) accelerator.wait_for_everyone() diff --git a/train_network.py b/train_network.py index c9c3c468..91d2a0bc 100644 --- a/train_network.py +++ b/train_network.py @@ -379,6 +379,7 @@ def train(args): accelerator.init_trackers("network_train") loss_list = [] + loss_total = 0.0 for epoch in range(num_train_epochs): print(f"epoch {epoch+1}/{num_train_epochs}") train_dataset.set_current_epoch(epoch + 1) @@ -449,8 +450,10 @@ def train(args): if epoch == 0: loss_list.append(current_loss) else: + loss_total -= loss_list[step] loss_list[step] = current_loss - avr_loss = sum(loss_list) / len(loss_list) + loss_total += current_loss + avr_loss = loss_total / len(loss_list) logs = {"loss": avr_loss} # , "lr": lr_scheduler.get_last_lr()[0]} progress_bar.set_postfix(**logs) @@ -462,7 +465,7 @@ def train(args): break if args.logging_dir is not None: - logs = {"loss/epoch": sum(loss_list) / len(loss_list)} + logs = {"loss/epoch": loss_total / len(loss_list)} accelerator.log(logs, step=epoch+1) accelerator.wait_for_everyone() From 496c8cdc098262b5e4796167bf15367daa6ca47b Mon Sep 17 00:00:00 2001 From: space-nuko <24979496+space-nuko@users.noreply.github.com> Date: Thu, 16 Feb 2023 02:56:39 -0800 Subject: [PATCH 3/5] Add noise-offset to metadata --- train_network.py | 1 + 1 file changed, 1 insertion(+) diff --git a/train_network.py b/train_network.py index 1b8046d2..d66f9095 100644 --- a/train_network.py +++ b/train_network.py @@ -353,6 +353,7 @@ def train(args): "ss_max_bucket_reso": train_dataset.max_bucket_reso, "ss_seed": args.seed, "ss_keep_tokens": args.keep_tokens, + "ss_noise_offset": args.noise_offset, "ss_dataset_dirs": json.dumps(train_dataset.dataset_dirs_info), "ss_reg_dataset_dirs": json.dumps(train_dataset.reg_dataset_dirs_info), "ss_tag_frequency": json.dumps(train_dataset.tag_frequency), From ffdfd5f6153280afff5929a75c1fc4321bfdfc91 Mon Sep 17 00:00:00 2001 From: Kohya S Date: Thu, 16 Feb 2023 22:21:36 +0900 Subject: [PATCH 4/5] fix name of loss for epoch --- train_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train_db.py b/train_db.py index d36bd8d0..e4f1e54c 100644 --- a/train_db.py +++ b/train_db.py @@ -309,7 +309,7 @@ def train(args): break if args.logging_dir is not None: - logs = {"epoch_loss": loss_total / len(loss_list)} + logs = {"loss/epoch": loss_total / len(loss_list)} accelerator.log(logs, step=epoch+1) accelerator.wait_for_everyone() From 3bc0d83769ca7fcf322be3c1f9a025f0dc375880 Mon Sep 17 00:00:00 2001 From: Kohya S Date: Thu, 16 Feb 2023 22:21:51 +0900 Subject: [PATCH 5/5] update readme --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 62551f27..03ee5d01 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,11 @@ The majority of scripts is licensed under ASL 2.0 (including codes from Diffuser ## Change History +- 16 Feb. 2023, 2023/2/16: + - Noise offset is recorded to the metadata. Thanks to space-nuko! + - Show the moving average loss to prevent loss jumping in ``train_network.py`` and ``train_db.py``. Thanks to shirayu! + - Noise offsetがメタデータに記録されるようになりました。space-nuko氏に感謝します。 + - ``train_network.py``と``train_db.py``で学習中に表示されるlossの値が移動平均になりました。epochの先頭で表示されるlossが大きく変動する事象を解決します。shirayu氏に感謝します。 - 14 Feb. 2023, 2023/2/14: - Add support with multi-gpu trainining for ``train_network.py``. Thanks to Isotr0py! - Add ``--verbose`` option for ``resize_lora.py``. For details, see [this PR](https://github.com/kohya-ss/sd-scripts/pull/179). Thanks to mgz-dev!