A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 16:30 UTC
Add NUMA Core binding in nixl_connector for CPU xPyD (#32365)
Signed-off-by: Hongming Zheng <hongming.zheng@intel.com> Signed-off-by: ZhengHongming888 <hongming.zheng@intel.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
ZhengHongming888 committed
Feb 9, 2026 at 07:39 UTC
cb62e86f83bf859fb25936a0c39709a31515fddc
3 files changed
+73
vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py
+11
@@ -926,6 +926,17 @@ class NixlConnectorWorker:
926
else:
927
self.use_host_buffer = self.kv_buffer_device == "cpu"
928
929
+ # reserve different cores for start_load_kv() from model_forward()
930
+ if self.device_type == "cpu":
931
+ numa_core_list = current_platform.discover_numa_topology()
932
+ # setup one last core in each numa for kv transfer.
933
+ rsv_cores_for_kv = [
934
+ max(each_numa_core_list) for each_numa_core_list in numa_core_list
935
+ ]
936
+
937
+ if rsv_cores_for_kv:
938
+ os.sched_setaffinity(0, rsv_cores_for_kv)
939
+
940
# support for oot platform which can't register nixl memory
941
# type based on kv_buffer_device
942
nixl_memory_type = current_platform.get_nixl_memory_type()
vllm/platforms/cpu.py
+61
@@ -213,6 +213,13 @@ class CpuPlatform(Platform):
213
214
cache_config.cpu_kvcache_space_bytes = CpuPlatform.get_device_total_memory()
215
216
+ # reserve at least one core for nixl_connector under p/d case
217
+ if vllm_config.kv_transfer_config and (
218
+ envs.VLLM_CPU_NUM_OF_RESERVED_CPU == 0
219
+ or envs.VLLM_CPU_NUM_OF_RESERVED_CPU is None
220
+ ):
221
+ os.environ["VLLM_CPU_NUM_OF_RESERVED_CPU"] = "1"
222
+
223
parallel_config = vllm_config.parallel_config
224
if (
225
parallel_config.world_size > 1
@@ -396,6 +403,60 @@ class CpuPlatform(Platform):
403
404
return allowed_numa_nodes_list, logical_cpu_list
405
406
+ @classmethod
407
+ def discover_numa_topology(cls) -> list[list[int]]:
408
+ """
409
+ Discover NUMA topology and keep the last physical core of each numa
410
+ into one core group list for nixl start_kv_load()
411
+ """
412
+ SYS_NODE = "/sys/devices/system/node"
413
+ SYS_CPU = "/sys/devices/system/cpu"
414
+
415
+ if not (os.path.exists(SYS_NODE) and os.path.exists(SYS_CPU)):
416
+ return []
417
+
418
+ core_rsv_for_kv = []
419
+ for node in os.listdir(SYS_NODE):
420
+ if not node.startswith("node") or not node[4:].isdigit():
421
+ continue
422
+ node_path = f"{SYS_NODE}/{node}"
423
+
424
+ seen_phys = set()
425
+ for cpu in os.listdir(node_path):
426
+ if not cpu.startswith("cpu") or not cpu[3:].isdigit():
427
+ continue
428
+
429
+ cpu_id = int(cpu[3:])
430
+ # thread_siblings based on cpu_id
431
+ path = f"{SYS_CPU}/cpu{cpu_id}/topology/thread_siblings_list"
432
+
433
+ if os.path.exists(path):
434
+ try:
435
+ with open(path) as f:
436
+ s = f.read()
437
+ cpus: list[int] = []
438
+ for part in s.strip().split(","):
439
+ if "-" in part:
440
+ a, b = map(int, part.split("-"))
441
+ cpus.extend(range(a, b + 1))
442
+ else:
443
+ cpus.append(int(part))
444
+ siblings = cpus if cpus else [cpu_id]
445
+ except (OSError, ValueError):
446
+ siblings = [cpu_id]
447
+ else:
448
+ siblings = [cpu_id]
449
+
450
+ phys = min(siblings)
451
+
452
+ if phys not in seen_phys:
453
+ seen_phys.add(phys)
454
+
455
+ if len(seen_phys) > 0:
456
+ core_rsv_for_kv.append(list(seen_phys))
457
+
458
+ return core_rsv_for_kv
459
+
460
@classmethod
461
def is_pin_memory_available(cls) -> bool:
462
return False
vllm/v1/worker/cpu_worker.py
+1
@@ -54,6 +54,7 @@ class CPUWorker(Worker):
54
def init_device(self):
55
# Setup OpenMP threads affinity.
56
omp_cpuids = envs.VLLM_CPU_OMP_THREADS_BIND
57
+ # Under numa binding some cores reserved for kv transfer in nixl_connector.py
58
if omp_cpuids == "auto" and platform.system() == "Linux":
59
cpu_arch = current_platform.get_cpu_architecture()
60
if cpu_arch in (CpuArchEnum.POWERPC, CpuArchEnum.S390X):