Merge 31e339c6a3 into 51435f1718

README-ja.md

@@ -50,9 +50,6 @@ Stable Diffusion等の画像生成モデルの学習、モデルによる画像
 
 ### 更新履歴
 
-- 次のリリースに含まれる予定の主な変更点は以下の通りです。リリース前の変更点は予告なく変更される可能性があります。
-    - Intel GPUの互換性を向上しました。[PR #2307](https://github.com/kohya-ss/sd-scripts/pull/2307) WhitePr氏に感謝します。
-
 - **Version 0.10.3 (2026-04-02):**
     - Animaでfp16で学習する際の安定性をさらに改善しました。[PR #2302](https://github.com/kohya-ss/sd-scripts/pull/2302) 問題をご報告いただいた方々に深く感謝します。
 

README.md

@@ -47,9 +47,6 @@ If you find this project helpful, please consider supporting its development via
 
 ### Change History
 
-- The following are the main changes planned for the next release. Please note that these changes may be subject to change without notice before the release.
-    - Improved compatibility with Intel GPUs. Thanks to WhitePr for [PR #2307](https://github.com/kohya-ss/sd-scripts/pull/2307).
-
 - **Version 0.10.3 (2026-04-02):**
     - Stability when training with fp16 on Anima has been further improved. See [PR #2302](https://github.com/kohya-ss/sd-scripts/pull/2302) for details. We deeply appreciate those who reported the issue.
 

gen_img.py

@@ -542,10 +542,20 @@ class PipelineLike:
                 uncond_embeddings = torch.cat([uncond_embeddings, tes_uncond_embs[i]], dim=2)  # n,77,2048
 
         if do_classifier_free_guidance:
+            lcm = uncond_embeddings.shape[1] * text_embeddings.shape[1] // math.gcd(uncond_embeddings.shape[1], text_embeddings.shape[1])
+            
             if negative_scale is None:
-                text_embeddings = torch.cat([uncond_embeddings, text_embeddings])
+                text_embeddings = torch.cat([
+                    uncond_embeddings.repeat(1, lcm // uncond_embeddings.shape[1], 1), 
+                    text_embeddings.repeat(1, lcm // text_embeddings.shape[1], 1),
+                ])
             else:
-                text_embeddings = torch.cat([uncond_embeddings, text_embeddings, real_uncond_embeddings])
+                lcm = real_uncond_embeddings.shape[1] * text_embeddings.shape[1] // math.gcd(real_uncond_embeddings.shape[1], text_embeddings.shape[1])
+                text_embeddings = torch.cat([
+                    uncond_embeddings.repeat(1, lcm // uncond_embeddings.shape[1], 1), 
+                    text_embeddings.repeat(1, lcm // text_embeddings.shape[1], 1), 
+                    real_uncond_embeddings.repeat(1, lcm // real_uncond_embeddings.shape[1], 1)
+                ])
 
         if self.control_net_lllites or (self.control_nets and self.is_sdxl):
             # ControlNetのhintにguide imageを流用する。ControlNetの場合はControlNet側で行う
@@ -1105,22 +1115,17 @@ def pad_tokens_and_weights(tokens, weights, max_length, bos, eos, pad, no_boseos
     """
     max_embeddings_multiples = (max_length - 2) // (chunk_length - 2)
     weights_length = max_length if no_boseos_middle else max_embeddings_multiples * chunk_length
+    lcm = chunk_length
     for i in range(len(tokens)):
-        tokens[i] = [bos] + tokens[i] + [eos] + [pad] * (max_length - 2 - len(tokens[i]))
-        if no_boseos_middle:
-            weights[i] = [1.0] + weights[i] + [1.0] * (max_length - 1 - len(weights[i]))
-        else:
-            w = []
-            if len(weights[i]) == 0:
-                w = [1.0] * weights_length
-            else:
-                for j in range(max_embeddings_multiples):
-                    w.append(1.0)  # weight for starting token in this chunk
-                    w += weights[i][j * (chunk_length - 2) : min(len(weights[i]), (j + 1) * (chunk_length - 2))]
-                    w.append(1.0)  # weight for ending token in this chunk
-                w += [1.0] * (weights_length - len(w))
-            weights[i] = w[:]
-
+        target_length = ((len(tokens[i]) + 2) // chunk_length + 1) * chunk_length
+        lcm = target_length * lcm // math.gcd(target_length, lcm)
+        tokens[i] = [bos] + tokens[i] + [eos] + [pad] * (target_length - 2 - len(tokens[i]))
+        weights[i] = [1.0] + weights[i] + [1.0] * (target_length - 1 - len(weights[i]))
+            
+    for i in range(len(tokens)):
+        tokens[i] = tokens[i] * (lcm // len(tokens[i]))
+        weights[i] = weights[i] * (lcm // len(weights[i]))
+    
     return tokens, weights
 
 
@@ -1138,56 +1143,21 @@ def get_unweighted_text_embeddings(
     When the length of tokens is a multiple of the capacity of the text encoder,
     it should be split into chunks and sent to the text encoder individually.
     """
-    max_embeddings_multiples = (text_input.shape[1] - 2) // (chunk_length - 2)
-    if max_embeddings_multiples > 1:
-        text_embeddings = []
-        pool = None
-        for i in range(max_embeddings_multiples):
-            # extract the i-th chunk
-            text_input_chunk = text_input[:, i * (chunk_length - 2) : (i + 1) * (chunk_length - 2) + 2].clone()
-
-            # cover the head and the tail by the starting and the ending tokens
-            text_input_chunk[:, 0] = text_input[0, 0]
-            if pad == eos:  # v1
-                text_input_chunk[:, -1] = text_input[0, -1]
-            else:  # v2
-                for j in range(len(text_input_chunk)):
-                    if text_input_chunk[j, -1] != eos and text_input_chunk[j, -1] != pad:  # 最後に普通の文字がある
-                        text_input_chunk[j, -1] = eos
-                    if text_input_chunk[j, 1] == pad:  # BOSだけであとはPAD
-                        text_input_chunk[j, 1] = eos
-
-            # in sdxl, value of clip_skip is same for Text Encoder 1 and 2
-            enc_out = text_encoder(text_input_chunk, output_hidden_states=True, return_dict=True)
-            text_embedding = enc_out["hidden_states"][-clip_skip]
-            if not is_sdxl:  # SD 1.5 requires final_layer_norm
-                text_embedding = text_encoder.text_model.final_layer_norm(text_embedding)
-            if pool is None:
-                pool = enc_out.get("text_embeds", None)  # use 1st chunk, if provided
-                if pool is not None:
-                    pool = train_util.pool_workaround(text_encoder, enc_out["last_hidden_state"], text_input_chunk, eos)
-
-            if no_boseos_middle:
-                if i == 0:
-                    # discard the ending token
-                    text_embedding = text_embedding[:, :-1]
-                elif i == max_embeddings_multiples - 1:
-                    # discard the starting token
-                    text_embedding = text_embedding[:, 1:]
-                else:
-                    # discard both starting and ending tokens
-                    text_embedding = text_embedding[:, 1:-1]
-
-            text_embeddings.append(text_embedding)
-        text_embeddings = torch.concat(text_embeddings, axis=1)
-    else:
-        enc_out = text_encoder(text_input, output_hidden_states=True, return_dict=True)
-        text_embeddings = enc_out["hidden_states"][-clip_skip]
+    pool = None
+    text_embeddings = []
+    
+    for chunk in text_input.chunk(text_input.shape[1] // chunk_length, dim=1):
+        enc_out = text_encoder(chunk, output_hidden_states=True, return_dict=True)
+        text_embedding = enc_out["hidden_states"][-clip_skip]
         if not is_sdxl:  # SD 1.5 requires final_layer_norm
-            text_embeddings = text_encoder.text_model.final_layer_norm(text_embeddings)
-        pool = enc_out.get("text_embeds", None)  # text encoder 1 doesn't return this
-        if pool is not None:
-            pool = train_util.pool_workaround(text_encoder, enc_out["last_hidden_state"], text_input, eos)
+            text_embedding = text_encoder.text_model.final_layer_norm(text_embedding)
+        text_embeddings.append(text_embedding)
+        if pool is None:
+            pool = enc_out.get("text_embeds", None)  # text encoder 1 doesn't return this
+            if pool is not None:
+                pool = train_util.pool_workaround(text_encoder, enc_out["last_hidden_state"], text_input, eos)
+        
+    text_embeddings = torch.cat(text_embeddings, dim=1)
     return text_embeddings, pool
 
 

library/ipex/__init__.py

@@ -1,7 +1,6 @@
 import os
 import sys
 import torch
-from packaging import version
 try:
     import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
     has_ipex = True
@@ -9,7 +8,7 @@ except Exception:
     has_ipex = False
 from .hijacks import ipex_hijacks
 
-torch_version = version.parse(torch.__version__)
+torch_version = float(torch.__version__[:3])
 
 # pylint: disable=protected-access, missing-function-docstring, line-too-long
 
@@ -57,6 +56,7 @@ def ipex_init(): # pylint: disable=too-many-statements
             torch.cuda.__path__ = torch.xpu.__path__
             torch.cuda.set_stream = torch.xpu.set_stream
             torch.cuda.torch = torch.xpu.torch
+            torch.cuda.Union = torch.xpu.Union
             torch.cuda.__annotations__ = torch.xpu.__annotations__
             torch.cuda.__package__ = torch.xpu.__package__
             torch.cuda.__builtins__ = torch.xpu.__builtins__
@@ -64,12 +64,14 @@ def ipex_init(): # pylint: disable=too-many-statements
             torch.cuda.StreamContext = torch.xpu.StreamContext
             torch.cuda._lazy_call = torch.xpu._lazy_call
             torch.cuda.random = torch.xpu.random
+            torch.cuda._device = torch.xpu._device
             torch.cuda.__name__ = torch.xpu.__name__
+            torch.cuda._device_t = torch.xpu._device_t
             torch.cuda.__spec__ = torch.xpu.__spec__
             torch.cuda.__file__ = torch.xpu.__file__
             # torch.cuda.is_current_stream_capturing = torch.xpu.is_current_stream_capturing
 
-            if torch_version < version.parse("2.3"):
+            if torch_version < 2.3:
                 torch.cuda._initialization_lock = torch.xpu.lazy_init._initialization_lock
                 torch.cuda._initialized = torch.xpu.lazy_init._initialized
                 torch.cuda._is_in_bad_fork = torch.xpu.lazy_init._is_in_bad_fork
@@ -112,22 +114,17 @@ def ipex_init(): # pylint: disable=too-many-statements
                 torch.cuda.threading = torch.xpu.threading
                 torch.cuda.traceback = torch.xpu.traceback
 
-            if torch_version < version.parse("2.5"):
+            if torch_version < 2.5:
                 torch.cuda.os = torch.xpu.os
                 torch.cuda.Device = torch.xpu.Device
                 torch.cuda.warnings = torch.xpu.warnings
                 torch.cuda.classproperty = torch.xpu.classproperty
                 torch.UntypedStorage.cuda = torch.UntypedStorage.xpu
 
-            if torch_version < version.parse("2.7"):
+            if torch_version < 2.7:
                 torch.cuda.Tuple = torch.xpu.Tuple
                 torch.cuda.List = torch.xpu.List
 
-            if torch_version < version.parse("2.11"):
-                torch.cuda._device_t = torch.xpu._device_t
-                torch.cuda._device = torch.xpu._device
-                torch.cuda.Union = torch.xpu.Union
-
 
             # Memory:
             if 'linux' in sys.platform and "WSL2" in os.popen("uname -a").read():
@@ -163,7 +160,7 @@ def ipex_init(): # pylint: disable=too-many-statements
             torch.cuda.initial_seed = torch.xpu.initial_seed
 
             # C
-            if torch_version < version.parse("2.3"):
+            if torch_version < 2.3:
                 torch._C._cuda_getCurrentRawStream = ipex._C._getCurrentRawStream
                 ipex._C._DeviceProperties.multi_processor_count = ipex._C._DeviceProperties.gpu_subslice_count
                 ipex._C._DeviceProperties.major = 12