VictorMylle/Kohya-ss-sd-scripts
1
0
Fork 1
mirror of https://github.com/kohya-ss/sd-scripts.git synced 2026-04-18 01:30:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

IPEX attention optimizations

Browse Source
This commit is contained in:
Disty0
2023-09-28 14:02:25 +03:00
parent 1e395ed285
commit 209eafb631
3 changed files with 25 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
library/ipex/__init__.py
View File

@@ -144,7 +144,7 @@ def ipex_init(): # pylint: disable=too-many-statements
ipex._C._DeviceProperties.minor = 2 ipex._C._DeviceProperties.minor = 2
#Fix functions with ipex: #Fix functions with ipex:
torch.cuda.mem_get_info = lambda device=None: [(torch.xpu.get_device_properties(device).total_memory - torch.xpu.memory_allocated(device)), torch.xpu.get_device_properties(device).total_memory] torch.cuda.mem_get_info = lambda device=None: [(torch.xpu.get_device_properties(device).total_memory - torch.xpu.memory_reserved(device)), torch.xpu.get_device_properties(device).total_memory]
torch._utils._get_available_device_type = lambda: "xpu" torch._utils._get_available_device_type = lambda: "xpu"
torch.has_cuda = True torch.has_cuda = True
torch.cuda.has_half = True torch.cuda.has_half = True
@@ -156,7 +156,6 @@ def ipex_init(): # pylint: disable=too-many-statements
torch.cuda.get_device_properties.minor = 7 torch.cuda.get_device_properties.minor = 7
torch.cuda.ipc_collect = lambda *args, **kwargs: None torch.cuda.ipc_collect = lambda *args, **kwargs: None
torch.cuda.utilization = lambda *args, **kwargs: 0 torch.cuda.utilization = lambda *args, **kwargs: 0
# getDeviceIdListForCard is renamed since https://github.com/intel/intel-extension-for-pytorch/commit/835b41fd5c8b6facf9efee8312f20699850ee592
if hasattr(torch.xpu, 'getDeviceIdListForCard'): if hasattr(torch.xpu, 'getDeviceIdListForCard'):
torch.cuda.getDeviceIdListForCard = torch.xpu.getDeviceIdListForCard torch.cuda.getDeviceIdListForCard = torch.xpu.getDeviceIdListForCard
torch.cuda.get_device_id_list_per_card = torch.xpu.getDeviceIdListForCard torch.cuda.get_device_id_list_per_card = torch.xpu.getDeviceIdListForCard

33
library/ipex/attention.py
View File

@@ -10,13 +10,15 @@ def torch_bmm(input, mat2, *, out=None):
#ARC GPUs can't allocate more than 4GB to a single block, Slice it: #ARC GPUs can't allocate more than 4GB to a single block, Slice it:
batch_size_attention, input_tokens, mat2_shape = input.shape[0], input.shape[1], mat2.shape[2] batch_size_attention, input_tokens, mat2_shape = input.shape[0], input.shape[1], mat2.shape[2]
block_multiply = 2.4 if input.dtype == torch.float32 else 1.2 block_multiply = input.element_size()
block_size = (batch_size_attention * input_tokens * mat2_shape) / 1024 * block_multiply #MB slice_block_size = input_tokens * mat2_shape / 1024 / 1024 * block_multiply
block_size = batch_size_attention * slice_block_size
split_slice_size = batch_size_attention split_slice_size = batch_size_attention
if block_size >= 4000: if block_size > 4:
do_split = True do_split = True
#Find something divisible with the input_tokens #Find something divisible with the input_tokens
while ((split_slice_size * input_tokens * mat2_shape) / 1024 * block_multiply) > 4000: while (split_slice_size * slice_block_size) > 4:
split_slice_size = split_slice_size // 2 split_slice_size = split_slice_size // 2
if split_slice_size <= 1: if split_slice_size <= 1:
split_slice_size = 1 split_slice_size = 1
@@ -24,12 +26,12 @@ def torch_bmm(input, mat2, *, out=None):
else: else:
do_split = False do_split = False
split_block_size = (split_slice_size * input_tokens * mat2_shape) / 1024 * block_multiply #MB
split_2_slice_size = input_tokens split_2_slice_size = input_tokens
if split_block_size >= 4000: if split_slice_size * slice_block_size > 4:
slice_block_size2 = split_slice_size * mat2_shape / 1024 / 1024 * block_multiply
do_split_2 = True do_split_2 = True
#Find something divisible with the input_tokens #Find something divisible with the input_tokens
while ((split_slice_size * split_2_slice_size * mat2_shape) / 1024 * block_multiply) > 4000: while (split_2_slice_size * slice_block_size2) > 4:
split_2_slice_size = split_2_slice_size // 2 split_2_slice_size = split_2_slice_size // 2
if split_2_slice_size <= 1: if split_2_slice_size <= 1:
split_2_slice_size = 1 split_2_slice_size = 1
@@ -71,13 +73,16 @@ def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.
else: else:
shape_one, batch_size_attention, query_tokens, shape_four = query.shape shape_one, batch_size_attention, query_tokens, shape_four = query.shape
no_shape_one = False no_shape_one = False
block_multiply = 3.6 if query.dtype == torch.float32 else 1.8
block_size = (shape_one * batch_size_attention * query_tokens * shape_four) / 1024 * block_multiply #MB block_multiply = query.element_size()
slice_block_size = shape_one * query_tokens * shape_four / 1024 / 1024 * block_multiply
block_size = batch_size_attention * slice_block_size
split_slice_size = batch_size_attention split_slice_size = batch_size_attention
if block_size >= 4000: if block_size > 4:
do_split = True do_split = True
#Find something divisible with the shape_one #Find something divisible with the shape_one
while ((shape_one * split_slice_size * query_tokens * shape_four) / 1024 * block_multiply) > 4000: while (split_slice_size * slice_block_size) > 4:
split_slice_size = split_slice_size // 2 split_slice_size = split_slice_size // 2
if split_slice_size <= 1: if split_slice_size <= 1:
split_slice_size = 1 split_slice_size = 1
@@ -85,12 +90,12 @@ def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.
else: else:
do_split = False do_split = False
split_block_size = (shape_one * split_slice_size * query_tokens * shape_four) / 1024 * block_multiply #MB
split_2_slice_size = query_tokens split_2_slice_size = query_tokens
if split_block_size >= 4000: if split_slice_size * slice_block_size > 4:
slice_block_size2 = shape_one * split_slice_size * shape_four / 1024 / 1024 * block_multiply
do_split_2 = True do_split_2 = True
#Find something divisible with the batch_size_attention #Find something divisible with the batch_size_attention
while ((shape_one * split_slice_size * split_2_slice_size * shape_four) / 1024 * block_multiply) > 4000: while (split_2_slice_size * slice_block_size2) > 4:
split_2_slice_size = split_2_slice_size // 2 split_2_slice_size = split_2_slice_size // 2
if split_2_slice_size <= 1: if split_2_slice_size <= 1:
split_2_slice_size = 1 split_2_slice_size = 1

9
library/ipex/diffusers.py
View File

@@ -55,13 +55,14 @@ class SlicedAttnProcessor: # pylint: disable=too-few-public-methods
) )
#ARC GPUs can't allocate more than 4GB to a single block, Slice it: #ARC GPUs can't allocate more than 4GB to a single block, Slice it:
block_multiply = 2.4 if query.dtype == torch.float32 else 1.2 block_multiply = query.element_size()
block_size = (batch_size_attention * query_tokens * shape_three) / 1024 * block_multiply #MB slice_block_size = self.slice_size * shape_three / 1024 / 1024 * block_multiply
block_size = query_tokens * slice_block_size
split_2_slice_size = query_tokens split_2_slice_size = query_tokens
if block_size >= 4000: if block_size > 4:
do_split_2 = True do_split_2 = True
#Find something divisible with the query_tokens #Find something divisible with the query_tokens
while ((self.slice_size * split_2_slice_size * shape_three) / 1024 * block_multiply) > 4000: while (split_2_slice_size * slice_block_size) > 4:
split_2_slice_size = split_2_slice_size // 2 split_2_slice_size = split_2_slice_size // 2
if split_2_slice_size <= 1: if split_2_slice_size <= 1:
split_2_slice_size = 1 split_2_slice_size = 1