A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 19:45 UTC
[CPU] Upgrade CPU backend to torch-2.6 (#13381)
Signed-off-by: jiang1.li <jiang1.li@intel.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Li, Jiang committed
Mar 12, 2025 at 18:41 UTC
ff47aab05640394a513a5b2ac772a115ddc2e05a
9 files changed
+23
-13
.buildkite/run-cpu-test.sh
+5
-3
@@ -19,13 +19,14 @@ remove_docker_container
19
20
# Run the image, setting --shm-size=4g for tensor parallel.
21
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \
22
- --cpuset-mems="$NUMA_NODE" --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"
22
+ --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"
23
docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --cpuset-cpus="$CORE_RANGE" \
24
- --cpuset-mems="$NUMA_NODE" --privileged=true --network host -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2
24
+ --cpuset-mems="$NUMA_NODE" --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --shm-size=4g --name cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2
25
26
function cpu_tests() {
27
set -e
28
export NUMA_NODE=$2
29
+ export BUILDKITE_BUILD_NUMBER=$3
30
31
# offline inference
32
docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-avx2-"$NUMA_NODE" bash -c "
@@ -36,6 +37,7 @@ function cpu_tests() {
37
docker exec cpu-test-"$BUILDKITE_BUILD_NUMBER"-"$NUMA_NODE" bash -c "
38
set -e
39
pip install -r vllm/requirements/test.txt
40
+ pip install -r vllm/requirements/cpu.txt
41
pytest -v -s tests/models/decoder_only/language -m cpu_model
42
pytest -v -s tests/models/embedding/language -m cpu_model
43
pytest -v -s tests/models/encoder_decoder/language -m cpu_model
@@ -85,4 +87,4 @@ function cpu_tests() {
87
88
# All of CPU tests are expected to be finished less than 40 mins.
89
export -f cpu_tests
88
-timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
90
+timeout 40m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE $BUILDKITE_BUILD_NUMBER"
Dockerfile.cpu
+1
-1
@@ -22,7 +22,7 @@ ENV LD_PRELOAD="/usr/lib/x86_64-linux-gnu/libtcmalloc_minimal.so.4:/usr/local/li
22
23
RUN echo 'ulimit -c 0' >> ~/.bashrc
24
25
-RUN pip install intel_extension_for_pytorch==2.5.0
25
+RUN pip install intel_extension_for_pytorch==2.6.0
26
27
WORKDIR /workspace
28
cmake/cpu_extension.cmake
+1
-1
@@ -149,7 +149,7 @@ if (AVX512_FOUND AND NOT AVX512_DISABLED)
149
FetchContent_Declare(
150
oneDNN
151
GIT_REPOSITORY https://github.com/oneapi-src/oneDNN.git
152
- GIT_TAG v3.6
152
+ GIT_TAG v3.7.1
153
GIT_PROGRESS TRUE
154
GIT_SHALLOW TRUE
155
)
requirements/cpu.txt
+1
-1
@@ -2,7 +2,7 @@
2
-r common.txt
3
4
# Dependencies for CPUs
5
-torch==2.5.1+cpu; platform_machine != "ppc64le" and platform_machine != "aarch64" and platform_system != "Darwin" and platform_machine != "s390x"
5
+torch==2.6.0+cpu; platform_machine == "x86_64"
6
torch==2.5.1; platform_machine == "ppc64le" or platform_machine == "aarch64" or platform_system == "Darwin"
7
torch==2.7.0.dev20250304; platform_machine == "s390x"
8
tests/lora/test_qwen2vl.py
+1
-1
@@ -12,7 +12,7 @@ from vllm.lora.request import LoRARequest
12
from vllm.platforms import current_platform
13
14
15
-@pytest.fixture(autouse=True)
15
+@pytest.fixture(autouse=not current_platform.is_cpu())
16
def v1(run_with_both_engines_lora):
17
# Simple autouse wrapper to run both engines for each test
18
# This can be promoted up to conftest.py to run for every
vllm/attention/ops/ipex_attn.py
+1
-1
@@ -17,7 +17,7 @@ class _PagedAttention:
17
18
@staticmethod
19
def get_supported_head_sizes() -> List[int]:
20
- return [32, 64, 80, 96, 112, 128, 256]
20
+ return [32, 64, 80, 96, 112, 128, 192, 256]
21
22
@staticmethod
23
def get_kv_cache_shape(
vllm/executor/multiproc_worker_utils.py
+5
-4
@@ -254,10 +254,11 @@ def _run_worker_process(
254
# online (in situ) tuning is enabled.
255
# Offline tuning API (record_untuned_is_enabled()) only
256
# available in PyTorch 2.6 or later.
257
- import torch.cuda.tunable as tunable
258
- if (tunable.is_enabled() and tunable.tuning_is_enabled()
259
- and not tunable.record_untuned_is_enabled()):
260
- tunable.write_file()
257
+ if torch.cuda.is_available():
258
+ import torch.cuda.tunable as tunable
259
+ if (tunable.is_enabled() and tunable.tuning_is_enabled()
260
+ and not tunable.record_untuned_is_enabled()):
261
+ tunable.write_file()
262
263
logger.info("Worker exiting")
264
vllm/model_executor/layers/fused_moe/layer.py
+5
-1
@@ -193,10 +193,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
193
global_num_experts: int = -1,
194
expert_map: Optional[torch.Tensor] = None,
195
custom_routing_function: Optional[Callable] = None,
196
+ scoring_func: str = "softmax",
197
+ e_score_correction_bias: Optional[torch.Tensor] = None,
198
activation: str = "silu",
199
**kwargs,
200
):
199
- assert custom_routing_function is None
201
assert activation == "silu", f"{activation} is not supported."
202
return layer.ipex_fusion(
203
x,
@@ -206,6 +207,9 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, CustomOp):
207
renormalize,
208
topk_group,
209
num_expert_group,
210
+ custom_routing_function,
211
+ scoring_func,
212
+ e_score_correction_bias,
213
)
214
215
def forward_tpu(
vllm/platforms/cpu.py
+3
@@ -121,6 +121,9 @@ class CpuPlatform(Platform):
121
# Disable torch async compiling which won't work with daemonic processes
122
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
123
124
+ # MLA attention is not supported
125
+ os.environ["VLLM_MLA_DISABLE"] = "1"
126
+
127
# Intel OpenMP setting
128
ld_prealod_str = os.getenv("LD_PRELOAD", "")
129
if "libiomp5.so" in ld_prealod_str: