Cleanup IPEX libs

library/ipex/__init__.py

@@ -4,7 +4,6 @@ import contextlib
 import torch
 import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
 from .hijacks import ipex_hijacks
-from .attention import attention_init
 
 # pylint: disable=protected-access, missing-function-docstring, line-too-long
 
@@ -166,7 +165,11 @@ def ipex_init(): # pylint: disable=too-many-statements
 
         ipex_hijacks()
         if not torch.xpu.has_fp64_dtype():
+            try:
+                from .attention import attention_init
                 attention_init()
+            except Exception: # pylint: disable=broad-exception-caught
+                pass
             try:
                 from .diffusers import ipex_diffusers
                 ipex_diffusers()