VictorMylle/Kohya-ss-sd-scripts
1
0
Fork 1
mirror of https://github.com/kohya-ss/sd-scripts.git synced 2026-04-09 06:45:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

warn and continue if huggingface uploading failed

Browse Source
This commit is contained in:
Kohya S
2023-05-31 20:48:33 +09:00
parent 6fbd526931
commit 3a06968332
2 changed files with 31 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

52
library/huggingface_util.py
View File

@@ -6,9 +6,7 @@ import os
from library.utils import fire_in_thread from library.utils import fire_in_thread
def exists_repo( def exists_repo(repo_id: str, repo_type: str, revision: str = "main", token: str = None):
repo_id: str, repo_type: str, revision: str = "main", token: str = None
):
api = HfApi( api = HfApi(
token=token, token=token,
) )
@@ -32,27 +30,35 @@ def upload(
private = args.huggingface_repo_visibility is None or args.huggingface_repo_visibility != "public" private = args.huggingface_repo_visibility is None or args.huggingface_repo_visibility != "public"
api = HfApi(token=token) api = HfApi(token=token)
if not exists_repo(repo_id=repo_id, repo_type=repo_type, token=token): if not exists_repo(repo_id=repo_id, repo_type=repo_type, token=token):
api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private) try:
api.create_repo(repo_id=repo_id, repo_type=repo_type, private=private)
except Exception as e: # とりあえずRepositoryNotFoundErrorは確認したが他にあると困るので
print("===========================================")
print(f"failed to create HuggingFace repo / HuggingFaceのリポジトリの作成に失敗しました : {e}")
print("===========================================")
is_folder = (type(src) == str and os.path.isdir(src)) or ( is_folder = (type(src) == str and os.path.isdir(src)) or (isinstance(src, Path) and src.is_dir())
isinstance(src, Path) and src.is_dir()
)
def uploader(): def uploader():
if is_folder: try:
api.upload_folder( if is_folder:
repo_id=repo_id, api.upload_folder(
repo_type=repo_type, repo_id=repo_id,
folder_path=src, repo_type=repo_type,
path_in_repo=path_in_repo, folder_path=src,
) path_in_repo=path_in_repo,
else: )
api.upload_file( else:
repo_id=repo_id, api.upload_file(
repo_type=repo_type, repo_id=repo_id,
path_or_fileobj=src, repo_type=repo_type,
path_in_repo=path_in_repo, path_or_fileobj=src,
) path_in_repo=path_in_repo,
)
except Exception as e: # RuntimeErrorを確認済みだが他にあると困るので
print("===========================================")
print(f"failed to upload to HuggingFace / HuggingFaceへのアップロードに失敗しました : {e}")
print("===========================================")
if args.async_upload and not force_sync_upload: if args.async_upload and not force_sync_upload:
fire_in_thread(uploader) fire_in_thread(uploader)
@@ -71,7 +77,5 @@ def list_dir(
token=token, token=token,
) )
repo_info = api.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type) repo_info = api.repo_info(repo_id=repo_id, revision=revision, repo_type=repo_type)
file_list = [ file_list = [file for file in repo_info.siblings if file.rfilename.startswith(subfolder)]
file for file in repo_info.siblings if file.rfilename.startswith(subfolder)
]
return file_list return file_list

4
library/train_util.py
View File

@@ -1087,7 +1087,9 @@ class DreamBoothDataset(BaseDataset):
for img_path in img_paths: for img_path in img_paths:
cap_for_img = read_caption(img_path, subset.caption_extension) cap_for_img = read_caption(img_path, subset.caption_extension)
if cap_for_img is None and subset.class_tokens is None: if cap_for_img is None and subset.class_tokens is None:
print(f"neither caption file nor class tokens are found. use empty caption for {img_path} / キャプションファイルもclass tokenも見つかりませんでした。空のキャプションを使用します: {img_path}") print(
f"neither caption file nor class tokens are found. use empty caption for {img_path} / キャプションファイルもclass tokenも見つかりませんでした。空のキャプションを使用します: {img_path}"
)
captions.append("") captions.append("")
missing_captions.append(img_path) missing_captions.append(img_path)
else: else: