A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 17:00 UTC
[HARDWARE][CPU] Add Option for Disabling Binding to Specific CPU Cores (#27953)
Signed-off-by: Stan Hatko <stan_hatko@live.com> Co-authored-by: Li, Jiang <jiang1.li@intel.com>
StanHatko committed
Nov 6, 2025 at 10:47 UTC
e52e4da9714962b8db623359992ac3a5853879f7
3 files changed
+15
-8
docs/getting_started/installation/cpu.md
+1
-1
@@ -94,7 +94,7 @@ Currently, there are no pre-built CPU wheels.
94
## Related runtime environment variables
95
96
- `VLLM_CPU_KVCACHE_SPACE`: specify the KV Cache size (e.g, `VLLM_CPU_KVCACHE_SPACE=40` means 40 GiB space for KV cache), larger setting will allow vLLM running more requests in parallel. This parameter should be set based on the hardware configuration and memory management pattern of users. Default value is `0`.
97
-- `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists or `auto` (by default). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively.
97
+- `VLLM_CPU_OMP_THREADS_BIND`: specify the CPU cores dedicated to the OpenMP threads, can be set as CPU id lists, `auto` (by default), or `nobind` (to disable binding to individual CPU cores and to inherit user-defined OpenMP variables). For example, `VLLM_CPU_OMP_THREADS_BIND=0-31` means there will be 32 OpenMP threads bound on 0-31 CPU cores. `VLLM_CPU_OMP_THREADS_BIND=0-31|32-63` means there will be 2 tensor parallel processes, 32 OpenMP threads of rank0 are bound on 0-31 CPU cores, and the OpenMP threads of rank1 are bound on 32-63 CPU cores. By setting to `auto`, the OpenMP threads of each rank are bound to the CPU cores in each NUMA node respectively. If set to `nobind`, the number of OpenMP threads is determined by the standard `OMP_NUM_THREADS` environment variable.
98
- `VLLM_CPU_NUM_OF_RESERVED_CPU`: specify the number of CPU cores which are not dedicated to the OpenMP threads for each rank. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. Default value is `None`. If the value is not set and use `auto` thread binding, no CPU will be reserved for `world_size == 1`, 1 CPU per rank will be reserved for `world_size > 1`.
99
- `CPU_VISIBLE_MEMORY_NODES`: specify visible NUMA memory nodes for vLLM CPU workers, similar to ```CUDA_VISIBLE_DEVICES```. The variable only takes effect when VLLM_CPU_OMP_THREADS_BIND is set to `auto`. The variable provides more control for the auto thread-binding feature, such as masking nodes and changing nodes binding sequence.
100
- `VLLM_CPU_MOE_PREPACK` (x86 only): whether to use prepack for MoE layer. This will be passed to `ipex.llm.modules.GatedMLPMOE`. Default is `1` (True). On unsupported CPUs, you might need to set this to `0` (False).
vllm/platforms/cpu.py
+9
-4
@@ -14,6 +14,7 @@ from typing import TYPE_CHECKING
14
import regex as re
15
import torch
16
17
+from vllm import envs
18
from vllm.logger import init_logger
19
from vllm.utils import DEFAULT_MAX_NUM_BATCHED_TOKENS
20
@@ -151,7 +152,6 @@ class CpuPlatform(Platform):
152
153
@classmethod
154
def get_device_total_memory(cls, device_id: int = 0) -> int:
154
- import vllm.envs as envs
155
from vllm.utils.mem_constants import GiB_bytes
156
157
kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
@@ -289,11 +289,16 @@ class CpuPlatform(Platform):
289
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
290
291
# Note: to avoid the error 'nthreads cannot be larger than environment
292
- # variable "NUMEXPR_MAX_THREADS" (64)'.
292
+ # variable "NUMEXPR_MAX_THREADS" (64)'.
293
os.environ["NUMEXPR_MAX_THREADS"] = str(get_max_threads())
294
295
- # Set default threads num for OpenMP parallel
296
- os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())
295
+ if envs.VLLM_CPU_OMP_THREADS_BIND != "nobind":
296
+ # Set default threads num for OpenMP parallel
297
+ os.environ["OMP_NUM_THREADS"] = str(torch.get_num_threads())
298
+ else:
299
+ # In this case, setting the OpenMP configuration via
300
+ # OMP_NUM_THREADS is up to the user.
301
+ logger.info("Disabling binding processes to CPU cores...")
302
303
# Disable torch async compiling which won't work with daemonic processes
304
os.environ["TORCHINDUCTOR_COMPILE_THREADS"] = "1"
vllm/v1/worker/cpu_worker.py
+5
-3
@@ -69,13 +69,15 @@ class CPUWorker(Worker):
69
self.local_omp_cpuid = self._get_autobind_cpu_ids(
70
lambda cpus: [cpu for cpu in cpus if cpu.id % 8 < 4]
71
)
72
- elif current_platform.get_cpu_architecture() == CpuArchEnum.X86:
72
+ elif cpu_arch == CpuArchEnum.X86:
73
# For x86 SMT-2, use 1 CPU per core
74
self.local_omp_cpuid = self._get_autobind_cpu_ids(
75
lambda cpus: cpus[-1:]
76
)
77
else:
78
- self.local_omp_cpuid = "all"
78
+ self.local_omp_cpuid = "nobind"
79
+ elif omp_cpuids == "nobind":
80
+ self.local_omp_cpuid = "nobind"
81
else:
82
local_dp_rank = self.parallel_config.data_parallel_rank_local
83
omp_cpuids = omp_cpuids.split("|")
@@ -86,7 +88,7 @@ class CPUWorker(Worker):
88
]
89
self.local_omp_cpuid = omp_cpuids[self.rank]
90
89
- if self.local_omp_cpuid != "all":
91
+ if self.local_omp_cpuid != "nobind":
92
ret = torch.ops._C_utils.init_cpu_threads_env(self.local_omp_cpuid)
93
if ret:
94
logger.info(ret)