A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[CI/Build][CPU][Bugfix] Fix CPU CI (#12150)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jan 17, 2025 at 19:39 UTC
d4e619457075c0dd917b84644f467f7f8aae10f0
2 files changed
+8
-4
.buildkite/run-cpu-test.sh
+2
-2
@@ -83,6 +83,6 @@ function cpu_tests() {
83
tests/lora/test_qwen2vl.py"
84
}
85
86
-# All of CPU tests are expected to be finished less than 25 mins.
86
+# All of CPU tests are expected to be finished less than 40 mins.
87
export -f cpu_tests
88
-timeout 30m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
88
+timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
vllm/model_executor/layers/activation.py
+6
-2
@@ -30,8 +30,10 @@ class FatreluAndMul(CustomOp):
30
def __init__(self, threshold: float = 0.):
31
super().__init__()
32
self.threshold = threshold
33
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
33
+ if current_platform.is_cuda_alike():
34
self.op = torch.ops._C.fatrelu_and_mul
35
+ elif current_platform.is_cpu():
36
+ self._forward_method = self.forward_native
37
38
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
39
d = x.shape[-1] // 2
@@ -100,11 +102,13 @@ class MulAndSilu(CustomOp):
102
103
def __init__(self):
104
super().__init__()
103
- if current_platform.is_cuda_alike() or current_platform.is_cpu():
105
+ if current_platform.is_cuda_alike():
106
self.op = torch.ops._C.mul_and_silu
107
elif current_platform.is_xpu():
108
from vllm._ipex_ops import ipex_ops
109
self.op = ipex_ops.silu_and_mul
110
+ elif current_platform.is_cpu():
111
+ self._forward_method = self.forward_native
112
113
def forward_native(self, x: torch.Tensor) -> torch.Tensor:
114
"""PyTorch-native implementation equivalent to forward()."""