@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 78f4590b60
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix][XPU] fix silu_and_mul (#11823)

Signed-off-by: yan ma <yan.ma@intel.com>

Yan Ma committed Jan 9, 2025 at 00:11 UTC 78f4590b60161dee1a444870ae682ba45f633502
2 files changed +4 -5
vllm/model_executor/layers/activation.py
+2 -2
@@ -64,8 +64,8 @@ class SiluAndMul(CustomOp):
64 if current_platform.is_cuda_alike() or current_platform.is_cpu():
65 self.op = torch.ops._C.silu_and_mul
66 elif current_platform.is_xpu():
67 - import intel_extension_for_pytorch as ipex
68 - self.op = ipex.llm.functional.silu_and_mul
67 + from vllm._ipex_ops import ipex_ops
68 + self.op = ipex_ops.silu_and_mul
69
70 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
71 """PyTorch-native implementation equivalent to forward()."""
vllm/plugins/__init__.py
+2 -3
@@ -63,8 +63,8 @@ def load_general_plugins():
63 from vllm.platforms import current_platform
64
65 if current_platform.is_xpu():
66 - # see https://github.com/pytorch/pytorch/blob/8cada5cbe5450e17c26fb8b358116785324537b2/torch/_dynamo/config.py#L158 # noqa
67 - os.environ['TORCH_COMPILE_DISABLE'] = 'True'
66 + # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
67 + torch._dynamo.config.disable = True
68 if current_platform.is_hpu():
69 # NOTE(kzawora): PT HPU lazy backend (PT_HPU_LAZY_MODE = 1)
70 # does not support torch.compile
@@ -72,7 +72,6 @@ def load_general_plugins():
72 # torch.compile support
73 is_lazy = os.environ.get('PT_HPU_LAZY_MODE', '1') == '1'
74 if is_lazy:
75 - # see https://github.com/pytorch/pytorch/blob/43c5f59/torch/_dynamo/config.py#L158
75 torch._dynamo.config.disable = True
76 # NOTE(kzawora) multi-HPU inference with HPUGraphs (lazy-only)
77 # requires enabling lazy collectives