From 6d06b215bfe27d751d1864166c54d28487f0566f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 1 Oct 2023 22:51:32 +0000 Subject: [PATCH 1/6] Bump actions/checkout from 3 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/typos.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml index 90132c33..b6865dbf 100644 --- a/.github/workflows/typos.yml +++ b/.github/workflows/typos.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: typos-action uses: crate-ci/typos@v1.16.15 From 592014923fdc2905588c37d7b417549f608d49c5 Mon Sep 17 00:00:00 2001 From: Isotr0py <41363108+Isotr0py@users.noreply.github.com> Date: Wed, 4 Oct 2023 21:48:25 +0800 Subject: [PATCH 2/6] Support JPEG-XL on windows --- library/train_util.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/library/train_util.py b/library/train_util.py index 5433357a..51610e70 100644 --- a/library/train_util.py +++ b/library/train_util.py @@ -96,6 +96,7 @@ try: except: pass +# JPEG-XL on Linux try: from jxlpy import JXLImagePlugin @@ -103,6 +104,14 @@ try: except: pass +# JPEG-XL on Windows +try: + import pillow_jxl + + IMAGE_EXTENSIONS.extend([".jxl", ".JXL"]) +except: + pass + IMAGE_TRANSFORMS = transforms.Compose( [ transforms.ToTensor(), From a4857fa764effdbdb099fbb6bd54c6d1b46b8238 Mon Sep 17 00:00:00 2001 From: alexds9 Date: Thu, 5 Oct 2023 21:26:09 +0300 Subject: [PATCH 3/6] Add append_captions feature to wd14 tagger This feature allows for appending new tags to the existing content of caption files. If the caption file for an image already exists, the tags generated from the current run are appended to the existing ones. Duplicate tags are checked and avoided. --- finetune/tag_images_by_wd14_tagger.py | 31 ++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/finetune/tag_images_by_wd14_tagger.py b/finetune/tag_images_by_wd14_tagger.py index 91e4f573..dde586c7 100644 --- a/finetune/tag_images_by_wd14_tagger.py +++ b/finetune/tag_images_by_wd14_tagger.py @@ -165,12 +165,35 @@ def main(args): if len(character_tag_text) > 0: character_tag_text = character_tag_text[2:] + caption_file = os.path.splitext(image_path)[0] + args.caption_extension + tag_text = ", ".join(combined_tags) - with open(os.path.splitext(image_path)[0] + args.caption_extension, "wt", encoding="utf-8") as f: + if args.append_captions: + # Check if file exists + if os.path.exists(caption_file): + + with open(caption_file, "rt", encoding="utf-8") as f: + + # Read file and remove new lines + existing_content = f.read().strip("\n") # Remove trailing comma, whitespace, and newlines + + # Split the content into tags and store them in a list + existing_tags = [tag.strip() for tag in existing_content.split(",") if tag.strip()] + + # Check and remove repeating tags in tag_text + tag_text = ", ".join([tag for tag in combined_tags if tag not in existing_tags]) + + # If the file has content, prepend a comma to tag_text + if existing_content.strip() and tag_text: + tag_text = ", ".join(existing_tags) + ", " + tag_text + + + with open(caption_file, "wt", encoding="utf-8") as f: f.write(tag_text + "\n") if args.debug: - print(f"\n{image_path}:\n Character tags: {character_tag_text}\n General tags: {general_tag_text}") + print( + f"\n{image_path}:\n Character tags: {character_tag_text}\n General tags: {general_tag_text}") # 読み込みの高速化のためにDataLoaderを使うオプション if args.max_data_loader_n_workers is not None: @@ -282,7 +305,9 @@ def setup_parser() -> argparse.ArgumentParser: default="", help="comma-separated list of undesired tags to remove from the output / 出力から除外したいタグのカンマ区切りのリスト", ) - parser.add_argument("--frequency_tags", action="store_true", help="Show frequency of tags for images / 画像ごとのタグの出現頻度を表示する") + parser.add_argument("--frequency_tags", action="store_true", + help="Show frequency of tags for images / 画像ごとのタグの出現頻度を表示する") + parser.add_argument("--append_captions", action="store_true", help="Append captions instead of overwriting") return parser From 9378da3c8266c0a87d893a2145196ec6efeb76a0 Mon Sep 17 00:00:00 2001 From: alexds9 Date: Thu, 5 Oct 2023 21:29:46 +0300 Subject: [PATCH 4/6] Fix comment --- finetune/tag_images_by_wd14_tagger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/finetune/tag_images_by_wd14_tagger.py b/finetune/tag_images_by_wd14_tagger.py index dde586c7..e2ac5c1d 100644 --- a/finetune/tag_images_by_wd14_tagger.py +++ b/finetune/tag_images_by_wd14_tagger.py @@ -176,7 +176,7 @@ def main(args): with open(caption_file, "rt", encoding="utf-8") as f: # Read file and remove new lines - existing_content = f.read().strip("\n") # Remove trailing comma, whitespace, and newlines + existing_content = f.read().strip("\n") # Remove newlines # Split the content into tags and store them in a list existing_tags = [tag.strip() for tag in existing_content.split(",") if tag.strip()] From 025368f51c31050544934a972ad77b149276bcf1 Mon Sep 17 00:00:00 2001 From: Kohya S Date: Mon, 9 Oct 2023 14:06:58 +0900 Subject: [PATCH 5/6] may work dropout in LyCORIS #859 --- train_network.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/train_network.py b/train_network.py index 99179814..2232a384 100644 --- a/train_network.py +++ b/train_network.py @@ -283,7 +283,10 @@ class NetworkTrainer: if args.dim_from_weights: network, _ = network_module.create_network_from_weights(1, args.network_weights, vae, text_encoder, unet, **net_kwargs) else: - # LyCORIS will work with this... + if "dropout" not in net_kwargs: + # workaround for LyCORIS (;^ω^) + net_kwargs["dropout"] = args.network_dropout + network = network_module.create_network( 1.0, args.network_dim, From 0d4e8b50d0ce23437a16d4735f785190a4457af3 Mon Sep 17 00:00:00 2001 From: Kohya S Date: Mon, 9 Oct 2023 15:09:54 +0900 Subject: [PATCH 6/6] change option to append_tags, minor update --- finetune/tag_images_by_wd14_tagger.py | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/finetune/tag_images_by_wd14_tagger.py b/finetune/tag_images_by_wd14_tagger.py index e2ac5c1d..31ee93bc 100644 --- a/finetune/tag_images_by_wd14_tagger.py +++ b/finetune/tag_images_by_wd14_tagger.py @@ -169,31 +169,26 @@ def main(args): tag_text = ", ".join(combined_tags) - if args.append_captions: + if args.append_tags: # Check if file exists if os.path.exists(caption_file): - with open(caption_file, "rt", encoding="utf-8") as f: - # Read file and remove new lines existing_content = f.read().strip("\n") # Remove newlines - # Split the content into tags and store them in a list - existing_tags = [tag.strip() for tag in existing_content.split(",") if tag.strip()] + # Split the content into tags and store them in a list + existing_tags = [tag.strip() for tag in existing_content.split(",") if tag.strip()] # Check and remove repeating tags in tag_text - tag_text = ", ".join([tag for tag in combined_tags if tag not in existing_tags]) - - # If the file has content, prepend a comma to tag_text - if existing_content.strip() and tag_text: - tag_text = ", ".join(existing_tags) + ", " + tag_text + new_tags = [tag for tag in combined_tags if tag not in existing_tags] + # Create new tag_text + tag_text = ", ".join(existing_tags + new_tags) with open(caption_file, "wt", encoding="utf-8") as f: f.write(tag_text + "\n") if args.debug: - print( - f"\n{image_path}:\n Character tags: {character_tag_text}\n General tags: {general_tag_text}") + print(f"\n{image_path}:\n Character tags: {character_tag_text}\n General tags: {general_tag_text}") # 読み込みの高速化のためにDataLoaderを使うオプション if args.max_data_loader_n_workers is not None: @@ -305,15 +300,15 @@ def setup_parser() -> argparse.ArgumentParser: default="", help="comma-separated list of undesired tags to remove from the output / 出力から除外したいタグのカンマ区切りのリスト", ) - parser.add_argument("--frequency_tags", action="store_true", - help="Show frequency of tags for images / 画像ごとのタグの出現頻度を表示する") - parser.add_argument("--append_captions", action="store_true", help="Append captions instead of overwriting") + parser.add_argument("--frequency_tags", action="store_true", help="Show frequency of tags for images / 画像ごとのタグの出現頻度を表示する") + parser.add_argument("--append_tags", action="store_true", help="Append captions instead of overwriting / 上書きではなくキャプションを追記する") return parser + if __name__ == "__main__": parser = setup_parser() - + args = parser.parse_args() # スペルミスしていたオプションを復元する