added setting to save checkpoint only after X number of steps

if save_every_n_steps is set, and save_every_n_steps_after_x is set, then it will save only after the number of steps defined by save_every_n_steps_after_x
This commit is contained in:
yushan777
2023-09-24 12:22:36 +01:00
parent 1e395ed285
commit 96f06d917e
9 changed files with 27 additions and 8 deletions

View File

@@ -482,7 +482,9 @@ def train(args):
)
# 指定ステップごとにモデルを保存
if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0:
if args.save_every_n_steps is not None and global_step % args.save_every_n_steps == 0 and \
args.save_every_n_steps_after_x is not None and global_step >= args.save_every_n_steps_after_x:
accelerator.wait_for_everyone()
if accelerator.is_main_process:
ckpt_name = train_util.get_step_ckpt_name(args, "." + args.save_model_as, global_step)