From 41d195715d00d8db42335ab4ffef2a2e07fb97ff Mon Sep 17 00:00:00 2001 From: Kohya S Date: Sun, 16 Jul 2023 15:56:29 +0900 Subject: [PATCH] fix scheduler steps with gradient accumulation --- library/train_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library/train_util.py b/library/train_util.py index c36651e2..a4021c4f 100644 --- a/library/train_util.py +++ b/library/train_util.py @@ -3348,7 +3348,7 @@ def get_scheduler_fix(args, optimizer: Optimizer, num_processes: int): """ name = args.lr_scheduler num_warmup_steps: Optional[int] = args.lr_warmup_steps - num_training_steps = args.max_train_steps * num_processes * args.gradient_accumulation_steps + num_training_steps = args.max_train_steps * num_processes # * args.gradient_accumulation_steps num_cycles = args.lr_scheduler_num_cycles power = args.lr_scheduler_power