A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 19:45 UTC
[Bugfix][CPU] Skip set_num_threads after thread binding (#38535)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Mar 30, 2026 at 20:13 UTC
6557f4937fd2937ae4824beb492ff67625895d89
2 files changed
+12
-2
.buildkite/scripts/hardware_ci/run-cpu-distributed-smoke-test.sh
+3
-2
@@ -1,9 +1,10 @@
1
#!/bin/bash
2
set -euox pipefail
3
export VLLM_CPU_CI_ENV=0
4
+export VLLM_CPU_KVCACHE_SPACE=1 # avoid OOM
5
6
echo "--- PP+TP"
6
-vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 &
7
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -pp=2 --max-model-len=4096 &
8
server_pid=$!
9
timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
10
vllm bench serve \
@@ -23,7 +24,7 @@ if [ "$failed_req" -ne 0 ]; then
24
fi
25
26
echo "--- DP+TP"
26
-vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 &
27
+vllm serve meta-llama/Llama-3.2-3B-Instruct -tp=2 -dp=2 --max-model-len=4096 &
28
server_pid=$!
29
timeout 600 bash -c "until curl localhost:8000/v1/models > /dev/null 2>&1; do sleep 1; done" || exit 1
30
vllm bench serve \
vllm/v1/worker/cpu_worker.py
+9
@@ -108,6 +108,15 @@ class CPUWorker(Worker):
108
if ret:
109
logger.info(ret)
110
111
+ # After the thread binding, changing thread num is not allowed
112
+ def skip_set_num_threads(x: int):
113
+ logger.warning(
114
+ "CPU backend doesn't allow to use "
115
+ "`torch.set_num_threads` after the thread binding, skip it."
116
+ )
117
+
118
+ torch.set_num_threads = skip_set_num_threads
119
+
120
# Note: unique identifier for creating allreduce shared memory
121
os.environ["VLLM_DIST_IDENT"] = self.distributed_init_method.split(":")[-1]
122
# Initialize the distributed environment.