@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d4e6194570
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[CI/Build][CPU][Bugfix] Fix CPU CI (#12150)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Jan 17, 2025 at 19:39 UTC d4e619457075c0dd917b84644f467f7f8aae10f0
2 files changed +8 -4
.buildkite/run-cpu-test.sh
+2 -2
@@ -83,6 +83,6 @@ function cpu_tests() {
83 tests/lora/test_qwen2vl.py"
84 }
85
86 -# All of CPU tests are expected to be finished less than 25 mins.
86 +# All of CPU tests are expected to be finished less than 40 mins.
87 export -f cpu_tests
88 -timeout 30m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
88 +timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
vllm/model_executor/layers/activation.py
+6 -2
@@ -30,8 +30,10 @@ class FatreluAndMul(CustomOp):
30 def __init__(self, threshold: float = 0.):
31 super().__init__()
32 self.threshold = threshold
33 - if current_platform.is_cuda_alike() or current_platform.is_cpu():
33 + if current_platform.is_cuda_alike():
34 self.op = torch.ops._C.fatrelu_and_mul
35 + elif current_platform.is_cpu():
36 + self._forward_method = self.forward_native
37
38 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
39 d = x.shape[-1] // 2
@@ -100,11 +102,13 @@ class MulAndSilu(CustomOp):
102
103 def __init__(self):
104 super().__init__()
103 - if current_platform.is_cuda_alike() or current_platform.is_cpu():
105 + if current_platform.is_cuda_alike():
106 self.op = torch.ops._C.mul_and_silu
107 elif current_platform.is_xpu():
108 from vllm._ipex_ops import ipex_ops
109 self.op = ipex_ops.silu_and_mul
110 + elif current_platform.is_cpu():
111 + self._forward_method = self.forward_native
112
113 def forward_native(self, x: torch.Tensor) -> torch.Tensor:
114 """PyTorch-native implementation equivalent to forward()."""