A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[CPU] Fix torch version in x86 CPU backend (#19258)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jun 26, 2025 at 18:34 UTC
0567c8249fdbff59a05f000cb326aed7cf5c8567
8 files changed
+52
-20
csrc/cpu/torch_bindings.cpp
+9
-4
@@ -131,16 +131,19 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
131
132
// Quantization
133
#ifdef __AVX512F__
134
+ at::Tag stride_tag = at::Tag::needs_fixed_stride_order;
135
// Compute int8 quantized tensor for given scaling factor.
136
ops.def(
137
"static_scaled_int8_quant(Tensor! out, Tensor input, Tensor scale,"
137
- "Tensor? azp) -> ()");
138
+ "Tensor? azp) -> ()",
139
+ {stride_tag});
140
ops.impl("static_scaled_int8_quant", torch::kCPU, &static_scaled_int8_quant);
141
142
// Compute int8 quantized tensor and scaling factor
143
ops.def(
144
"dynamic_scaled_int8_quant(Tensor! out, Tensor input, Tensor! scale, "
143
- "Tensor!? azp) -> ()");
145
+ "Tensor!? azp) -> ()",
146
+ {stride_tag});
147
ops.impl("dynamic_scaled_int8_quant", torch::kCPU,
148
&dynamic_scaled_int8_quant);
149
// W8A8 GEMM, supporting symmetric per-tensor or per-row/column
@@ -148,7 +151,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
151
ops.def(
152
"cutlass_scaled_mm(Tensor! out, Tensor a,"
153
" Tensor b, Tensor a_scales,"
151
- " Tensor b_scales, Tensor? bias) -> ()");
154
+ " Tensor b_scales, Tensor? bias) -> ()",
155
+ {stride_tag});
156
ops.impl("cutlass_scaled_mm", torch::kCPU, &int8_scaled_mm);
157
// w8a8 GEMM, supporting asymmetric per-tensor or per-row/column
158
// quantization.
@@ -156,7 +160,8 @@ TORCH_LIBRARY_EXPAND(TORCH_EXTENSION_NAME, ops) {
160
"cutlass_scaled_mm_azp(Tensor! out, Tensor a,"
161
" Tensor b, Tensor a_scales,"
162
" Tensor b_scales, Tensor azp_adj,"
159
- " Tensor? azp, Tensor? bias) -> ()");
163
+ " Tensor? azp, Tensor? bias) -> ()",
164
+ {stride_tag});
165
ops.impl("cutlass_scaled_mm_azp", torch::kCPU, &int8_scaled_mm_azp);
166
#elif defined(__powerpc64__)
167
// Compute int8 quantized tensor for given scaling factor.
docker/Dockerfile.cpu
+20
-13
@@ -66,7 +66,7 @@ ENV VLLM_CPU_DISABLE_AVX512=${VLLM_CPU_DISABLE_AVX512}
66
WORKDIR /workspace/vllm
67
68
RUN --mount=type=cache,target=/root/.cache/uv \
69
- --mount=type=bind,src=requirements/build.txt,target=requirements/build.txt \
69
+ --mount=type=bind,src=requirements/cpu-build.txt,target=requirements/build.txt \
70
uv pip install -r requirements/build.txt
71
72
COPY . .
@@ -79,6 +79,22 @@ RUN --mount=type=cache,target=/root/.cache/uv \
79
--mount=type=bind,source=.git,target=.git \
80
VLLM_TARGET_DEVICE=cpu python3 setup.py bdist_wheel
81
82
+######################### TEST DEPS #########################
83
+FROM base AS vllm-test-deps
84
+
85
+WORKDIR /workspace/vllm
86
+
87
+RUN --mount=type=bind,src=requirements/test.in,target=requirements/test.in \
88
+ cp requirements/test.in requirements/cpu-test.in && \
89
+ sed -i '/mamba_ssm/d' requirements/cpu-test.in && \
90
+ sed -i 's/torch==.*/torch==2.6.0/g' requirements/cpu-test.in && \
91
+ sed -i 's/torchaudio.*/torchaudio/g' requirements/cpu-test.in && \
92
+ sed -i 's/torchvision.*/torchvision/g' requirements/cpu-test.in && \
93
+ uv pip compile requirements/cpu-test.in -o requirements/cpu-test.txt --index-strategy unsafe-best-match --torch-backend cpu
94
+
95
+RUN --mount=type=cache,target=/root/.cache/uv \
96
+ uv pip install -r requirements/cpu-test.txt
97
+
98
######################### DEV IMAGE #########################
99
FROM vllm-build AS vllm-dev
100
@@ -97,28 +113,19 @@ RUN --mount=type=cache,target=/root/.cache/uv \
113
--mount=type=bind,source=.git,target=.git \
114
VLLM_TARGET_DEVICE=cpu python3 setup.py develop
115
116
+COPY --from=vllm-test-deps /workspace/vllm/requirements/cpu-test.txt requirements/test.txt
117
+
118
RUN --mount=type=cache,target=/root/.cache/uv \
101
- --mount=type=bind,src=requirements/test.in,target=requirements/test.in \
102
- cp requirements/test.in requirements/test-cpu.in && \
103
- sed -i '/mamba_ssm/d' requirements/test-cpu.in && \
104
- uv pip compile requirements/test-cpu.in -o requirements/test.txt && \
119
uv pip install -r requirements/dev.txt && \
120
pre-commit install --hook-type pre-commit --hook-type commit-msg
121
122
ENTRYPOINT ["bash"]
123
124
######################### TEST IMAGE #########################
111
-FROM base AS vllm-test
125
+FROM vllm-test-deps AS vllm-test
126
127
WORKDIR /workspace/
128
115
-RUN --mount=type=cache,target=/root/.cache/uv \
116
- --mount=type=bind,src=requirements/test.in,target=requirements/test.in \
117
- cp requirements/test.in requirements/test-cpu.in && \
118
- sed -i '/mamba_ssm/d' requirements/test-cpu.in && \
119
- uv pip compile requirements/test-cpu.in -o requirements/cpu-test.txt && \
120
- uv pip install -r requirements/cpu-test.txt
121
-
129
RUN --mount=type=cache,target=/root/.cache/uv \
130
--mount=type=bind,from=vllm-build,src=/workspace/vllm/dist,target=dist \
131
uv pip install dist/*.whl
requirements/cpu-build.txt
new
+12
@@ -0,0 +1,12 @@
1
+# Temporarily used for x86 CPU backend to avoid performance regression of torch>2.6.0+cpu,
2
+# see https://github.com/pytorch/pytorch/pull/151218
3
+cmake>=3.26.1
4
+ninja
5
+packaging>=24.2
6
+setuptools>=77.0.3,<80.0.0
7
+setuptools-scm>=8
8
+--extra-index-url https://download.pytorch.org/whl/cpu
9
+torch==2.6.0+cpu
10
+wheel
11
+jinja2>=3.1.6
12
+regex
requirements/cpu.txt
+3
-2
@@ -8,7 +8,7 @@ numba == 0.61.2; python_version > '3.9'
8
packaging>=24.2
9
setuptools>=77.0.3,<80.0.0
10
--extra-index-url https://download.pytorch.org/whl/cpu
11
-torch==2.7.0+cpu; platform_machine == "x86_64"
11
+torch==2.6.0+cpu; platform_machine == "x86_64" # torch>2.6.0+cpu has performance regression on x86 platform, see https://github.com/pytorch/pytorch/pull/151218
12
torch==2.7.0; platform_system == "Darwin"
13
torch==2.7.0; platform_machine == "ppc64le" or platform_machine == "aarch64"
14
@@ -23,6 +23,7 @@ datasets # for benchmark scripts
23
24
# Intel Extension for PyTorch, only for x86_64 CPUs
25
intel-openmp==2024.2.1; platform_machine == "x86_64"
26
-intel_extension_for_pytorch==2.7.0; platform_machine == "x86_64"
26
+intel_extension_for_pytorch==2.6.0; platform_machine == "x86_64" # torch>2.6.0+cpu has performance regression on x86 platform, see https://github.com/pytorch/pytorch/pull/151218
27
py-libnuma; platform_system != "Darwin"
28
psutil; platform_system != "Darwin"
29
+triton==3.2.0; platform_machine == "x86_64" # Triton is required for torch 2.6+cpu, as it is imported in torch.compile.
tests/models/multimodal/generation/test_common.py
+2
@@ -107,6 +107,8 @@ VLM_TEST_SETTINGS = {
107
),
108
limit_mm_per_prompt={"image": 4},
109
)],
110
+ # TODO: Revert to "auto" when CPU backend can use torch > 2.6
111
+ dtype="bfloat16" if current_platform.is_cpu() else "auto",
112
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
113
),
114
"paligemma": VLMTestInfo(
tests/models/multimodal/generation/vlm_utils/builders.py
+3
@@ -203,6 +203,9 @@ def build_embedding_inputs_from_test_info(
203
204
images = [asset.pil_image for asset in image_assets]
205
embeds = test_info.convert_assets_to_embeddings(image_assets)
206
+ if test_info.dtype != "auto":
207
+ dtype = getattr(torch, test_info.dtype) # type: ignore
208
+ embeds = [e.to(dtype=dtype) for e in embeds]
209
assert len(images) == len(model_prompts)
210
211
inputs = build_single_image_inputs(images, model_prompts, size_wrapper)
vllm/model_executor/layers/fused_moe/layer.py
+2
@@ -54,6 +54,8 @@ else:
54
if is_rocm_aiter_moe_enabled():
55
from vllm.model_executor.layers.fused_moe.rocm_aiter_fused_moe import ( # noqa: E501
56
rocm_aiter_grouped_topk as grouped_topk)
57
+elif current_platform.is_cpu():
58
+ pass
59
else:
60
from vllm.model_executor.layers.fused_moe.fused_moe import grouped_topk
61
if current_platform.is_tpu():
vllm/model_executor/layers/quantization/ipex_quant.py
+1
-1
@@ -15,7 +15,7 @@ from vllm.model_executor.layers.quantization.base_config import (
15
from vllm.model_executor.layers.quantization.gptq import GPTQLinearMethod
16
from vllm.platforms import current_platform
17
18
-MIN_IPEX_VERSION = "2.7.0"
18
+MIN_IPEX_VERSION = "2.6.0"
19
20
21
class IPEXConfig(QuantizationConfig):