diff --git a/library/ipex/attention.py b/library/ipex/attention.py index ced59637..2e61f2c9 100644 --- a/library/ipex/attention.py +++ b/library/ipex/attention.py @@ -20,22 +20,21 @@ def torch_bmm_32_bit(input, mat2, *, out=None): if split_slice_size <= 1: split_slice_size = 1 break + split_2_slice_size = input_tokens + if split_slice_size * slice_block_size > 4: + slice_block_size_2 = split_slice_size * mat2_shape / 1024 / 1024 * block_multiply + do_split_2 = True + # Find something divisible with the input_tokens + while (split_2_slice_size * slice_block_size_2) > 4: + split_2_slice_size = split_2_slice_size // 2 + if split_2_slice_size <= 1: + split_2_slice_size = 1 + break + else: + do_split_2 = False else: do_split = False - split_2_slice_size = input_tokens - if split_slice_size * slice_block_size > 4: - slice_block_size2 = split_slice_size * mat2_shape / 1024 / 1024 * block_multiply - do_split_2 = True - # Find something divisible with the input_tokens - while (split_2_slice_size * slice_block_size2) > 4: - split_2_slice_size = split_2_slice_size // 2 - if split_2_slice_size <= 1: - split_2_slice_size = 1 - break - else: - do_split_2 = False - if do_split: hidden_states = torch.zeros(input.shape[0], input.shape[1], mat2.shape[2], device=input.device, dtype=input.dtype) for i in range(batch_size_attention // split_slice_size): @@ -64,42 +63,51 @@ original_scaled_dot_product_attention = torch.nn.functional.scaled_dot_product_a def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False): # ARC GPUs can't allocate more than 4GB to a single block, Slice it: if len(query.shape) == 3: - batch_size_attention, query_tokens, shape_four = query.shape - shape_one = 1 - no_shape_one = True + batch_size_attention, query_tokens, shape_three = query.shape + shape_four = 1 else: - shape_one, batch_size_attention, query_tokens, shape_four = query.shape - no_shape_one = False + batch_size_attention, query_tokens, shape_three, shape_four = query.shape block_multiply = query.element_size() - slice_block_size = shape_one * query_tokens * shape_four / 1024 / 1024 * block_multiply + slice_block_size = query_tokens * shape_three * shape_four / 1024 / 1024 * block_multiply block_size = batch_size_attention * slice_block_size split_slice_size = batch_size_attention if block_size > 4: do_split = True - # Find something divisible with the shape_one + # Find something divisible with the batch_size_attention while (split_slice_size * slice_block_size) > 4: split_slice_size = split_slice_size // 2 if split_slice_size <= 1: split_slice_size = 1 break + split_2_slice_size = query_tokens + if split_slice_size * slice_block_size > 4: + slice_block_size_2 = split_slice_size * shape_three * shape_four / 1024 / 1024 * block_multiply + do_split_2 = True + # Find something divisible with the query_tokens + while (split_2_slice_size * slice_block_size_2) > 4: + split_2_slice_size = split_2_slice_size // 2 + if split_2_slice_size <= 1: + split_2_slice_size = 1 + break + split_3_slice_size = shape_three + if split_2_slice_size * slice_block_size_2 > 4: + slice_block_size_3 = split_slice_size * split_2_slice_size * shape_four / 1024 / 1024 * block_multiply + do_split_3 = True + # Find something divisible with the shape_three + while (split_3_slice_size * slice_block_size_3) > 4: + split_3_slice_size = split_3_slice_size // 2 + if split_3_slice_size <= 1: + split_3_slice_size = 1 + break + else: + do_split_3 = False + else: + do_split_2 = False else: do_split = False - split_2_slice_size = query_tokens - if split_slice_size * slice_block_size > 4: - slice_block_size2 = shape_one * split_slice_size * shape_four / 1024 / 1024 * block_multiply - do_split_2 = True - # Find something divisible with the batch_size_attention - while (split_2_slice_size * slice_block_size2) > 4: - split_2_slice_size = split_2_slice_size // 2 - if split_2_slice_size <= 1: - split_2_slice_size = 1 - break - else: - do_split_2 = False - if do_split: hidden_states = torch.zeros(query.shape, device=query.device, dtype=query.dtype) for i in range(batch_size_attention // split_slice_size): @@ -109,7 +117,18 @@ def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropo for i2 in range(query_tokens // split_2_slice_size): # pylint: disable=invalid-name start_idx_2 = i2 * split_2_slice_size end_idx_2 = (i2 + 1) * split_2_slice_size - if no_shape_one: + if do_split_3: + for i3 in range(shape_three // split_3_slice_size): # pylint: disable=invalid-name + start_idx_3 = i3 * split_3_slice_size + end_idx_3 = (i3 + 1) * split_3_slice_size + hidden_states[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3] = original_scaled_dot_product_attention( + query[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], + key[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], + value[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3], + attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2, start_idx_3:end_idx_3] if attn_mask is not None else attn_mask, + dropout_p=dropout_p, is_causal=is_causal + ) + else: hidden_states[start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention( query[start_idx:end_idx, start_idx_2:end_idx_2], key[start_idx:end_idx, start_idx_2:end_idx_2], @@ -117,31 +136,14 @@ def scaled_dot_product_attention_32_bit(query, key, value, attn_mask=None, dropo attn_mask=attn_mask[start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask, dropout_p=dropout_p, is_causal=is_causal ) - else: - hidden_states[:, start_idx:end_idx, start_idx_2:end_idx_2] = original_scaled_dot_product_attention( - query[:, start_idx:end_idx, start_idx_2:end_idx_2], - key[:, start_idx:end_idx, start_idx_2:end_idx_2], - value[:, start_idx:end_idx, start_idx_2:end_idx_2], - attn_mask=attn_mask[:, start_idx:end_idx, start_idx_2:end_idx_2] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal - ) else: - if no_shape_one: - hidden_states[start_idx:end_idx] = original_scaled_dot_product_attention( - query[start_idx:end_idx], - key[start_idx:end_idx], - value[start_idx:end_idx], - attn_mask=attn_mask[start_idx:end_idx] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal - ) - else: - hidden_states[:, start_idx:end_idx] = original_scaled_dot_product_attention( - query[:, start_idx:end_idx], - key[:, start_idx:end_idx], - value[:, start_idx:end_idx], - attn_mask=attn_mask[:, start_idx:end_idx] if attn_mask is not None else attn_mask, - dropout_p=dropout_p, is_causal=is_causal - ) + hidden_states[start_idx:end_idx] = original_scaled_dot_product_attention( + query[start_idx:end_idx], + key[start_idx:end_idx], + value[start_idx:end_idx], + attn_mask=attn_mask[start_idx:end_idx] if attn_mask is not None else attn_mask, + dropout_p=dropout_p, is_causal=is_causal + ) else: return original_scaled_dot_product_attention( query, key, value, attn_mask=attn_mask, dropout_p=dropout_p, is_causal=is_causal diff --git a/library/ipex/hijacks.py b/library/ipex/hijacks.py index a699e1e4..eb5f779f 100644 --- a/library/ipex/hijacks.py +++ b/library/ipex/hijacks.py @@ -185,6 +185,10 @@ def ipex_hijacks(): CondFunc('torch.Generator', lambda orig_func, device=None: torch.xpu.Generator(return_xpu(device)), lambda orig_func, device=None: device is not None and device != torch.device("cpu") and device != "cpu") + else: + CondFunc('torch.Generator', + lambda orig_func, device=None: orig_func(return_xpu(device)), + lambda orig_func, device=None: check_device(device)) # TiledVAE and ControlNet: CondFunc('torch.batch_norm',