A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[CPU] Refine default config for the CPU backend (#19539)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Jun 13, 2025 at 13:27 UTC
6458721108be64708023ee397c4fe08bf7189022
4 files changed
+60
-25
.buildkite/scripts/hardware_ci/run-cpu-test.sh
+12
-3
@@ -24,13 +24,22 @@ numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --tag cpu-test-"$NUMA_NODE
24
numactl -C "$CORE_RANGE" -N "$NUMA_NODE" docker build --build-arg VLLM_CPU_DISABLE_AVX512="true" --tag cpu-test-"$NUMA_NODE"-avx2 --target vllm-test -f docker/Dockerfile.cpu .
25
26
# Run the image, setting --shm-size=4g for tensor parallel.
27
-docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --env VLLM_CPU_OMP_THREADS_BIND="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"
28
-docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --env VLLM_CPU_OMP_THREADS_BIND="$OMP_CORE_RANGE" --shm-size=4g --name cpu-test-"$NUMA_NODE"-avx2 cpu-test-"$NUMA_NODE"-avx2
27
+docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --env VLLM_CPU_OMP_THREADS_BIND="$OMP_CORE_RANGE" --env VLLM_CPU_CI_ENV=1 --shm-size=4g --name cpu-test-"$NUMA_NODE" cpu-test-"$NUMA_NODE"
28
+docker run -itd --cpuset-cpus="$CORE_RANGE" --cpuset-mems="$NUMA_NODE" --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/huggingface --privileged=true -e HF_TOKEN --env VLLM_CPU_KVCACHE_SPACE=4 --env VLLM_CPU_OMP_THREADS_BIND="$OMP_CORE_RANGE" --env VLLM_CPU_CI_ENV=1 --shm-size=4g --name cpu-test-"$NUMA_NODE"-avx2 cpu-test-"$NUMA_NODE"-avx2
29
30
function cpu_tests() {
31
set -e
32
export NUMA_NODE=$2
33
34
+ # list packages
35
+ docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
36
+ set -e
37
+ pip list"
38
+
39
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
40
+ set -e
41
+ pip list"
42
+
43
# offline inference
44
docker exec cpu-test-"$NUMA_NODE"-avx2 bash -c "
45
set -e
@@ -72,7 +81,7 @@ function cpu_tests() {
81
set -e
82
python3 -m vllm.entrypoints.openai.api_server --model facebook/opt-125m --dtype half &
83
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
75
- python3 benchmarks/benchmark_serving.py \
84
+ VLLM_CPU_CI_ENV=0 python3 benchmarks/benchmark_serving.py \
85
--backend vllm \
86
--dataset-name random \
87
--model facebook/opt-125m \
vllm/engine/arg_utils.py
+22
-4
@@ -1562,14 +1562,20 @@ class EngineArgs:
1562
UsageContext.LLM_CLASS: 16384,
1563
UsageContext.OPENAI_API_SERVER: 8192,
1564
}
1565
- default_max_num_seqs = 1024
1565
+ default_max_num_seqs = {
1566
+ UsageContext.LLM_CLASS: 1024,
1567
+ UsageContext.OPENAI_API_SERVER: 1024,
1568
+ }
1569
else:
1570
# TODO(woosuk): Tune the default values for other hardware.
1571
default_max_num_batched_tokens = {
1572
UsageContext.LLM_CLASS: 8192,
1573
UsageContext.OPENAI_API_SERVER: 2048,
1574
}
1572
- default_max_num_seqs = 256
1575
+ default_max_num_seqs = {
1576
+ UsageContext.LLM_CLASS: 256,
1577
+ UsageContext.OPENAI_API_SERVER: 256,
1578
+ }
1579
1580
# tpu specific default values.
1581
if current_platform.is_tpu():
@@ -1586,6 +1592,17 @@ class EngineArgs:
1592
}
1593
}
1594
1595
+ # cpu specific default values.
1596
+ if current_platform.is_cpu():
1597
+ default_max_num_batched_tokens = {
1598
+ UsageContext.LLM_CLASS: 4096,
1599
+ UsageContext.OPENAI_API_SERVER: 2048,
1600
+ }
1601
+ default_max_num_seqs = {
1602
+ UsageContext.LLM_CLASS: 128,
1603
+ UsageContext.OPENAI_API_SERVER: 32,
1604
+ }
1605
+
1606
use_context_value = usage_context.value if usage_context else None
1607
if (self.max_num_batched_tokens is None
1608
and usage_context in default_max_num_batched_tokens):
@@ -1606,8 +1623,9 @@ class EngineArgs:
1623
"Setting max_num_batched_tokens to %d for %s usage context.",
1624
self.max_num_batched_tokens, use_context_value)
1625
1609
- if self.max_num_seqs is None:
1610
- self.max_num_seqs = default_max_num_seqs
1626
+ if (self.max_num_seqs is None
1627
+ and usage_context in default_max_num_seqs):
1628
+ self.max_num_seqs = default_max_num_seqs[usage_context]
1629
1630
logger.debug("Setting max_num_seqs to %d for %s usage context.",
1631
self.max_num_seqs, use_context_value)
vllm/platforms/cpu.py
+14
-6
@@ -89,10 +89,6 @@ class CpuPlatform(Platform):
89
import vllm.envs as envs
90
from vllm.utils import GiB_bytes
91
model_config = vllm_config.model_config
92
- # Reminder: Please update docs/features/compatibility_matrix.md
93
- # If the feature combo become valid
94
- if not model_config.enforce_eager:
95
- model_config.enforce_eager = True
92
93
model_config.disable_cascade_attn = True
94
@@ -171,9 +167,21 @@ class CpuPlatform(Platform):
167
compilation_config = vllm_config.compilation_config
168
if (envs.VLLM_USE_V1 and vllm_config.compilation_config.level
169
== CompilationLevel.PIECEWISE):
170
+
171
+ # Note: vLLM V1 is using PIECEWISE level compilation, which will
172
+ # take time to compile kernels just-in-time with the inductor
173
+ # backend. For CPU CI tests, most of them are executed fast and
174
+ # compilations consume too much time, even with torch compile
175
+ # cache. So use VLLM_CPU_CI_ENV to indicate the CI environment,
176
+ # and just execute model with dynamo + eager mode to save time.
177
+ # VLLM_CPU_CI_ENV is only used as an internal variable.
178
+ if os.environ.get("VLLM_CPU_CI_ENV", "0") != "0":
179
+ backend = "eager"
180
+ else:
181
+ backend = "inductor"
182
+
183
compilation_config.level = CompilationLevel.DYNAMO_ONCE
175
- compilation_config.backend = "eager"
176
- compilation_config.custom_ops += ["none"]
184
+ compilation_config.backend = backend
185
compilation_config.inductor_compile_config.update({
186
"dce":
187
True,
vllm/v1/worker/cpu_model_runner.py
+12
-12
@@ -60,7 +60,8 @@ class CPUModelRunner(GPUModelRunner):
60
def warming_up_model(self) -> None:
61
logger.info("Warming up model for the compilation...")
62
# Only generate graph for the generic shape
63
- self._dummy_run(max(16, self.max_num_reqs))
63
+ with _set_global_compilation_settings(self.vllm_config):
64
+ self._dummy_run(max(16, self.max_num_reqs))
65
logger.info("Warming up done.")
66
67
def _init_device_properties(self) -> None:
@@ -71,16 +72,15 @@ class CPUModelRunner(GPUModelRunner):
72
73
74
@contextmanager
74
-def _set_global_compilation_settings():
75
+def _set_global_compilation_settings(config: VllmConfig):
76
import torch._inductor.config
77
77
- # Note: The CPPGEMM backend requires freezing parameters.
78
- freezing_value = torch._inductor.config.freezing
79
- torch._inductor.config.freezing = True
80
- # Note: workaround for "ValueError: fast mode: can't pickle cyclic objects
81
- # including object type dict"
82
- force_disable_caches = torch._inductor.config.force_disable_caches
83
- torch._inductor.config.force_disable_caches = True
84
- yield
85
- torch._inductor.config.freezing = freezing_value
86
- torch._inductor.config.force_disable_caches = force_disable_caches
78
+ inductor_config = config.compilation_config.inductor_compile_config
79
+ try:
80
+ # Note: The MKLDNN and CPPGEMM backend requires freezing parameters.
81
+ freezing_value = torch._inductor.config.freezing
82
+ if inductor_config.get("max_autotune", False):
83
+ torch._inductor.config.freezing = True
84
+ yield
85
+ finally:
86
+ torch._inductor.config.freezing = freezing_value