A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 12:15 UTC
[Misc][NUMA] Auto-bind to PCT priority cores on DGX B300 + widen EngineCore across shard NUMA nodes (#43270)
Signed-off-by: Vadim Gimpelson <vadim.gimpelson@gmail.com> Co-authored-by: Cursor <noreply@cursor.com>
Vadim Gimpelson committed
May 29, 2026 at 06:07 UTC
bf18d7e0b45353cef43f3455e9a8b4262377636e
3 files changed
+622
-33
tests/utils_/test_numa_utils.py
+338
-4
@@ -10,6 +10,33 @@ from vllm.config import ParallelConfig
10
from vllm.utils import numa_utils
11
12
13
+@pytest.fixture(autouse=True)
14
+def _disable_pct_by_default(monkeypatch):
15
+ """Force PCT detection OFF unless a test opts in via ``_patch_pct_gates``.
16
+
17
+ The CI / dev machines themselves can be Xeon 6776P with PCT enabled, so a
18
+ plain ``cache_clear`` would let the gate auto-detect ``True`` from the
19
+ live filesystem and silently re-route "baseline" tests through the PCT
20
+ path. Stub ``/proc/cpuinfo`` and ``acpi_cppc/highest_perf`` to a state
21
+ that fails the gate; ``_patch_pct_gates`` re-stubs on top when needed.
22
+ """
23
+ from io import StringIO
24
+
25
+ real_open = open
26
+
27
+ def _no_pct_open(path, *args, **kwargs):
28
+ if path == numa_utils._PROC_CPUINFO_PATH:
29
+ return StringIO("processor\t: 0\nmodel name\t: Generic Test CPU\n")
30
+ if path == numa_utils._PCT_HIGHEST_PERF_PATH:
31
+ raise OSError("PCT disabled by autouse fixture")
32
+ return real_open(path, *args, **kwargs)
33
+
34
+ monkeypatch.setattr("builtins.open", _no_pct_open)
35
+ numa_utils._pct_sku_config.cache_clear()
36
+ yield
37
+ numa_utils._pct_sku_config.cache_clear()
38
+
39
+
40
def _make_config(**parallel_kwargs):
41
parallel_defaults = dict(
42
numa_bind=False,
@@ -31,7 +58,7 @@ def _make_config(**parallel_kwargs):
58
def test_get_numactl_args_with_node_binding():
59
vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
60
assert (
34
- numa_utils._get_numactl_args(vllm_config, local_rank=1)
61
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
62
== "--cpunodebind=1 --membind=1"
63
)
64
@@ -43,7 +70,300 @@ def test_get_numactl_args_with_cpu_binding():
70
numa_bind_cpus=["0-3", "4-7"],
71
)
72
assert (
46
- numa_utils._get_numactl_args(vllm_config, local_rank=1)
73
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
74
+ == "--physcpubind=4-7 --membind=1"
75
+ )
76
+
77
+
78
+def _patch_pct_gates(
79
+ monkeypatch,
80
+ *,
81
+ model_match: bool,
82
+ highest_perf: int | None,
83
+ cpulist: str | None = "0-31,64-95",
84
+ cpulist_by_node: dict[int, str | None] | None = None,
85
+ sku: str = "6776P",
86
+):
87
+ """Force `_pct_sku_config` and node cpulist read to deterministic state.
88
+
89
+ ``cpulist`` is the default returned for any node not present in
90
+ ``cpulist_by_node``. ``cpulist_by_node`` lets a test return different
91
+ cpulists for different NUMA nodes (e.g. node 0 vs node 1). ``sku`` lets
92
+ the test pick which Granite Rapids SKU appears in the fake
93
+ ``/proc/cpuinfo`` ``model name`` (only used when ``model_match=True``).
94
+ """
95
+ import pathlib
96
+ from io import StringIO
97
+
98
+ import regex as re
99
+
100
+ cpuinfo = (
101
+ f"processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum {sku} CPU @ 2.40GHz\n"
102
+ if model_match
103
+ else "processor\t: 0\nmodel name\t: Intel(R) Xeon(R) Platinum 8480+\n"
104
+ )
105
+
106
+ real_open = open
107
+
108
+ def fake_open(path, *args, **kwargs):
109
+ if path == numa_utils._PROC_CPUINFO_PATH:
110
+ return StringIO(cpuinfo)
111
+ if path == numa_utils._PCT_HIGHEST_PERF_PATH:
112
+ if highest_perf is None:
113
+ raise OSError("missing")
114
+ return StringIO(f"{highest_perf}\n")
115
+ return real_open(path, *args, **kwargs)
116
+
117
+ real_read_text = pathlib.Path.read_text
118
+ cpulist_by_node = cpulist_by_node or {}
119
+
120
+ def fake_read_text(self, *args, **kwargs):
121
+ path_str = str(self)
122
+ if path_str.endswith("/cpulist") and "/sys/devices/system/node" in path_str:
123
+ match = re.search(r"/node(\d+)/cpulist$", path_str)
124
+ if match:
125
+ node_id = int(match.group(1))
126
+ if node_id in cpulist_by_node:
127
+ val = cpulist_by_node[node_id]
128
+ if val is None:
129
+ raise OSError(f"missing cpulist for node{node_id}")
130
+ return val
131
+ if cpulist is None:
132
+ raise OSError("missing cpulist")
133
+ return cpulist
134
+ return real_read_text(self, *args, **kwargs)
135
+
136
+ monkeypatch.setattr("builtins.open", fake_open)
137
+ monkeypatch.setattr("pathlib.Path.read_text", fake_read_text)
138
+ numa_utils._pct_sku_config.cache_clear()
139
+
140
+
141
+def test_pct_binding_filters_cpus(monkeypatch):
142
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
143
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) == [0, 1, 16, 17, 64, 65, 80, 81]
144
+
145
+
146
+@pytest.mark.parametrize(
147
+ "sku,expected_cpus",
148
+ [
149
+ # 64-core SKUs (stride 16): cpus from "0-31,64-95" with cpu_id % 16
150
+ # in (0, 1) -> 0, 1, 16, 17, 64, 65, 80, 81.
151
+ ("6776P", [0, 1, 16, 17, 64, 65, 80, 81]),
152
+ ("6774P", [0, 1, 16, 17, 64, 65, 80, 81]),
153
+ # 72-core SKU (stride 18): cpus from "0-31,64-95" with cpu_id % 18
154
+ # in (0, 1) -> 0, 1, 18, 19, 72, 73, 90, 91.
155
+ ("6962P", [0, 1, 18, 19, 72, 73, 90, 91]),
156
+ ],
157
+)
158
+def test_pct_binding_fires_on_every_capable_sku(monkeypatch, sku, expected_cpus):
159
+ """Each SKU in ``_PCT_CAPABLE_SKUS`` engages the gate at its own
160
+ expected ``highest_perf`` and uses its own priority-core stride."""
161
+ sku_config = numa_utils._PCT_CAPABLE_SKUS[sku]
162
+ _patch_pct_gates(
163
+ monkeypatch,
164
+ model_match=True,
165
+ highest_perf=sku_config.highest_perf,
166
+ sku=sku,
167
+ )
168
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) == expected_cpus
169
+
170
+
171
+def test_pct_binding_fails_closed_when_sku_perf_mismatch(monkeypatch):
172
+ """6962P with 6776P's highest_perf (46 vs expected 44) must fail closed."""
173
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, sku="6962P")
174
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
175
+
176
+
177
+def test_pct_binding_disabled_when_cpu_model_mismatch(monkeypatch):
178
+ _patch_pct_gates(monkeypatch, model_match=False, highest_perf=46)
179
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
180
+
181
+
182
+def test_pct_binding_disabled_when_highest_perf_does_not_match(monkeypatch):
183
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=42)
184
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
185
+
186
+
187
+def test_pct_binding_disabled_when_files_missing(monkeypatch):
188
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=None)
189
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
190
+
191
+
192
+def test_pct_binding_returns_none_when_node_cpulist_filter_empty(monkeypatch):
193
+ _patch_pct_gates(
194
+ monkeypatch,
195
+ model_match=True,
196
+ highest_perf=46,
197
+ cpulist="2-15,18-31",
198
+ )
199
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
200
+
201
+
202
+def test_pct_binding_returns_none_when_node_cpulist_missing(monkeypatch):
203
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46, cpulist=None)
204
+ assert numa_utils._maybe_get_pct_cpu_binding([0]) is None
205
+
206
+
207
+def test_get_numactl_args_uses_pct_when_user_did_not_specify_cpus(monkeypatch):
208
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
209
+ vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
210
+ assert (
211
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
212
+ == "--physcpubind=0,1,16,17,64,65,80,81 --membind=1"
213
+ )
214
+
215
+
216
+def test_get_numactl_args_engine_core_baseline_single_node_shard():
217
+ """Baseline (no PCT): single-NUMA shard -> single-node bind."""
218
+ vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0, 1])
219
+ assert (
220
+ numa_utils._get_numactl_enginecore_args(
221
+ vllm_config.parallel_config, local_rank=0
222
+ )
223
+ == "--cpunodebind=0 --membind=0"
224
+ )
225
+
226
+
227
+def test_get_numactl_args_engine_core_baseline_spans_shard_numa_nodes():
228
+ """Baseline (no PCT): a TP=4 shard spanning both NUMA nodes -> bind to both."""
229
+ vllm_config = _make_config(
230
+ numa_bind=True,
231
+ numa_bind_nodes=[0, 0, 1, 1],
232
+ tensor_parallel_size=4,
233
+ )
234
+ assert (
235
+ numa_utils._get_numactl_enginecore_args(
236
+ vllm_config.parallel_config, local_rank=0
237
+ )
238
+ == "--cpunodebind=0,1 --membind=0,1"
239
+ )
240
+
241
+
242
+def test_get_numactl_args_engine_core_pct_spans_shard_numa_nodes(monkeypatch):
243
+ """PCT: EngineCore for a multi-NUMA shard binds to the union of priority
244
+ cores across all shard nodes, so worker `--physcpubind` is always a
245
+ subset of EngineCore's `cpus_allowed`."""
246
+ _patch_pct_gates(
247
+ monkeypatch,
248
+ model_match=True,
249
+ highest_perf=46,
250
+ cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
251
+ )
252
+ vllm_config = _make_config(
253
+ numa_bind=True,
254
+ numa_bind_nodes=[0, 0, 1, 1],
255
+ tensor_parallel_size=4,
256
+ )
257
+ assert numa_utils._get_numactl_enginecore_args(
258
+ vllm_config.parallel_config, local_rank=0
259
+ ) == (
260
+ "--physcpubind="
261
+ "0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
262
+ " --membind=0,1"
263
+ )
264
+
265
+
266
+def test_get_numactl_args_engine_core_pct_dp_shard_picks_local_nodes(monkeypatch):
267
+ """With DP=2, each shard's EngineCore binds only to its own NUMA nodes."""
268
+ _patch_pct_gates(
269
+ monkeypatch,
270
+ model_match=True,
271
+ highest_perf=46,
272
+ cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
273
+ )
274
+ vllm_config = _make_config(
275
+ numa_bind=True,
276
+ numa_bind_nodes=[0, 0, 1, 1],
277
+ tensor_parallel_size=2,
278
+ data_parallel_rank_local=1,
279
+ )
280
+ # Shard 1 owns gpu_indices 2 and 3 -> nodes [1, 1] -> {1}.
281
+ assert (
282
+ numa_utils._get_numactl_enginecore_args(
283
+ vllm_config.parallel_config, local_rank=0
284
+ )
285
+ == "--physcpubind=64,65,80,81,192,193,208,209 --membind=1"
286
+ )
287
+
288
+
289
+def test_get_numactl_args_engine_core_pct_external_launcher_spans_local_nodes(
290
+ monkeypatch,
291
+):
292
+ """external_launcher (or multi-node-within-DP, or Ray) hits the
293
+ fallback branch. EngineCore must still span every local NUMA node
294
+ so it can mp-spawn its local workers without ``--physcpubind``
295
+ strict-validation failures."""
296
+ _patch_pct_gates(
297
+ monkeypatch,
298
+ model_match=True,
299
+ highest_perf=46,
300
+ cpulist_by_node={0: "0-31,128-159", 1: "64-95,192-223"},
301
+ )
302
+ vllm_config = _make_config(
303
+ numa_bind=True,
304
+ numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
305
+ distributed_executor_backend="external_launcher",
306
+ tensor_parallel_size=8,
307
+ )
308
+ assert numa_utils._get_numactl_enginecore_args(
309
+ vllm_config.parallel_config, local_rank=0
310
+ ) == (
311
+ "--physcpubind="
312
+ "0,1,16,17,64,65,80,81,128,129,144,145,192,193,208,209"
313
+ " --membind=0,1"
314
+ )
315
+
316
+
317
+def test_get_numactl_args_engine_core_baseline_multi_node_within_dp_spans_locals():
318
+ """Multi-node-within-DP fallback: bind EngineCore to all local NUMA
319
+ nodes that the visible ``numa_bind_nodes`` reference."""
320
+ vllm_config = _make_config(
321
+ numa_bind=True,
322
+ numa_bind_nodes=[0, 0, 0, 0, 1, 1, 1, 1],
323
+ nnodes_within_dp=2,
324
+ tensor_parallel_size=8,
325
+ )
326
+ assert (
327
+ numa_utils._get_numactl_enginecore_args(
328
+ vllm_config.parallel_config, local_rank=0
329
+ )
330
+ == "--cpunodebind=0,1 --membind=0,1"
331
+ )
332
+
333
+
334
+def test_get_numactl_args_engine_core_skips_user_cpu_list(monkeypatch):
335
+ """EngineCore ignores ``--numa-bind-cpus``.
336
+
337
+ Those are per-worker lists; binding EngineCore to any of them would
338
+ shrink its ``cpus_allowed`` below the strict-superset workers'
339
+ ``--physcpubind`` spawns need. We fall back to ``--cpunodebind`` over
340
+ the shard's NUMA nodes instead. PCT auto-detect is also bypassed when
341
+ the user is explicit (its priority-core union may not be a superset
342
+ of the user's per-worker cores)."""
343
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
344
+ vllm_config = _make_config(
345
+ numa_bind=True,
346
+ numa_bind_nodes=[0, 0, 1, 1],
347
+ numa_bind_cpus=["0-3", "4-7", "64-67", "68-71"],
348
+ tensor_parallel_size=4,
349
+ )
350
+ assert (
351
+ numa_utils._get_numactl_enginecore_args(
352
+ vllm_config.parallel_config, local_rank=0
353
+ )
354
+ == "--cpunodebind=0,1 --membind=0,1"
355
+ )
356
+
357
+
358
+def test_get_numactl_args_user_cpus_override_pct(monkeypatch):
359
+ _patch_pct_gates(monkeypatch, model_match=True, highest_perf=46)
360
+ vllm_config = _make_config(
361
+ numa_bind=True,
362
+ numa_bind_nodes=[0, 1],
363
+ numa_bind_cpus=["0-3", "4-7"],
364
+ )
365
+ assert (
366
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
367
== "--physcpubind=4-7 --membind=1"
368
)
369
@@ -57,7 +377,7 @@ def test_get_numactl_args_uses_dp_offset():
377
tensor_parallel_size=2,
378
)
379
assert (
60
- numa_utils._get_numactl_args(vllm_config, local_rank=1)
380
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=1)
381
== "--cpunodebind=1 --membind=1"
382
)
383
@@ -66,7 +386,21 @@ def test_get_numactl_args_requires_detectable_nodes(monkeypatch):
386
vllm_config = _make_config(numa_bind=True)
387
monkeypatch.setattr(numa_utils, "get_auto_numa_nodes", lambda: None)
388
with pytest.raises(RuntimeError):
69
- numa_utils._get_numactl_args(vllm_config, local_rank=0)
389
+ numa_utils._get_numactl_worker_args(vllm_config.parallel_config, local_rank=0)
390
+
391
+
392
+def test_configure_subprocess_rejects_unknown_process_kind():
393
+ """configure_subprocess only knows 'worker' and 'EngineCore'; anything
394
+ else must raise ValueError instead of silently routing to the worker
395
+ path."""
396
+ vllm_config = _make_config(numa_bind=True, numa_bind_nodes=[0])
397
+ with (
398
+ pytest.raises(ValueError, match="process_kind"),
399
+ numa_utils.configure_subprocess(
400
+ vllm_config, local_rank=0, process_kind="bogus"
401
+ ),
402
+ ):
403
+ pass
404
405
406
def test_log_numactl_show(monkeypatch):
vllm/config/parallel.py
+6
-1
@@ -267,7 +267,12 @@ class ParallelConfig:
267
"""num of nodes for multi-node distributed
268
inference when distributed_executor_backend is mp."""
269
numa_bind: bool = False
270
- """Enable NUMA binding for GPU worker subprocesses."""
270
+ """Enable NUMA binding for GPU worker subprocesses.
271
+
272
+ By default, workers are pinned to their GPU's NUMA-local CPUs and
273
+ memory; on PCT-capable Xeons they also auto-bind to the SKU's
274
+ PCT priority cores.
275
+ """
276
numa_bind_nodes: list[int] | None = None
277
"""NUMA node to bind each GPU worker to.
278
vllm/utils/numa_utils.py
+278
-28
@@ -14,7 +14,7 @@ import subprocess
14
from contextlib import contextmanager
15
from functools import cache
16
from pathlib import Path
17
-from typing import TYPE_CHECKING
17
+from typing import TYPE_CHECKING, NamedTuple
18
19
import psutil
20
@@ -112,6 +112,117 @@ def get_auto_numa_nodes() -> list[int] | None:
112
return numa_nodes
113
114
115
+# PCT (Priority Core Turbo) auto-detection workaround for Granite Rapids
116
+# Xeon SKUs.
117
+#
118
+# Background:
119
+# * The Linux kernel does not expose PCT priority-core membership via any
120
+# unprivileged sysfs path. The official interface
121
+# (/dev/isst_interface, used by `intel-speed-select`) is root-only,
122
+# which is a non-starter in most production deployments (shared
123
+# clusters, prebuilt containers, managed cloud).
124
+# * Even recent stable kernels (e.g. 6.14, March 2025) do not yet
125
+# preferentially schedule work on PCT priority cores, so vLLM cannot
126
+# just "let the scheduler handle it".
127
+#
128
+# Empirical heuristic (DGX B300 / Xeon 6776P, the SKU we measured):
129
+# * /proc/cpuinfo `model name` contains the SKU number.
130
+# * cpu0 is a PCT priority core on these SKUs, so it reports the
131
+# priority-cohort CPPC `highest_perf` (the value matches the SKU's
132
+# "Max PCT core frequency" in 100 MHz units, e.g. 4.6 GHz -> 46).
133
+# * Priority cores within each NUMA node satisfy `cpu_id % S in (0, 1)`
134
+# intersected with the node's cpulist, where `S` is the SKU's logical
135
+# CPUs per priority "group" (= total threads / 8 priority cores; 16 on
136
+# 64-core SKUs, 18 on 72-core SKUs).
137
+#
138
+# SKU table:
139
+# ``_PCT_CAPABLE_SKUS`` maps each known PCT-capable Granite Rapids part
140
+# to a ``_PctSku(highest_perf, priority_stride)`` config:
141
+# * highest_perf is the expected ``acpi_cppc/highest_perf`` on cpu0,
142
+# derived from Intel ARK's "Max PCT core frequency" * 10 (CPPC max
143
+# ratio reports in 100 MHz units).
144
+# * priority_stride is the SKU's "Total Cores" / 4 (= total HT threads
145
+# / 8 priority cores), used in the ``cpu_id % stride`` filter above.
146
+# Values:
147
+# * 6776P - 4.6 GHz, 64C/128T -> (46, 16) measured on DGX B300
148
+# * 6774P - 4.6 GHz, 64C/128T -> (46, 16) per Intel ARK, not measured
149
+# * 6962P - 4.4 GHz, 72C/144T -> (44, 18) per Intel ARK, not measured
150
+# The non-measured SKUs are listed best-effort: the gate fails closed
151
+# (no PCT engagement) if a host's actual highest_perf doesn't match the
152
+# table value, so adding entries is safe. If you have access to a 6962P
153
+# or 6774P box and find a different value or cpu-id pattern, update the
154
+# table below.
155
+#
156
+# This whole block is a stop-gap until the kernel exposes PCT membership
157
+# in an unprivileged way; see the tracking issue linked from the PR.
158
+
159
+
160
+class _PctSku(NamedTuple):
161
+ """Per-SKU config used by the PCT auto-detection gate."""
162
+
163
+ highest_perf: int
164
+ priority_stride: int
165
+
166
+
167
+_PCT_CAPABLE_SKUS: dict[str, _PctSku] = {
168
+ "6776P": _PctSku(highest_perf=46, priority_stride=16),
169
+ "6774P": _PctSku(highest_perf=46, priority_stride=16),
170
+ "6962P": _PctSku(highest_perf=44, priority_stride=18),
171
+}
172
+_PCT_HIGHEST_PERF_PATH = "/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf"
173
+_PROC_CPUINFO_PATH = "/proc/cpuinfo"
174
+
175
+
176
+def _pct_sku_from_cpuinfo() -> _PctSku | None:
177
+ """Return the ``_PctSku`` config for this host's SKU, or None.
178
+
179
+ Reads ``/proc/cpuinfo``'s ``model name`` and looks the SKU up in
180
+ ``_PCT_CAPABLE_SKUS``. Returns ``None`` when the host is not a known
181
+ PCT-capable Granite Rapids Xeon (or when ``/proc/cpuinfo`` is
182
+ unreadable).
183
+ """
184
+ try:
185
+ with open(_PROC_CPUINFO_PATH) as f:
186
+ for line in f:
187
+ if not line.lstrip().lower().startswith("model name"):
188
+ continue
189
+ for sku, config in _PCT_CAPABLE_SKUS.items():
190
+ if sku in line:
191
+ return config
192
+ except OSError:
193
+ return None
194
+ return None
195
+
196
+
197
+@cache
198
+def _pct_sku_config() -> _PctSku | None:
199
+ """Detect a PCT-capable Granite Rapids Xeon with PCT enabled.
200
+
201
+ See the comment block above ``_PCT_CAPABLE_SKUS`` for the full context
202
+ (why we hard-code SKUs, why we read CPPC ``highest_perf``, etc.).
203
+
204
+ Returns the matching ``_PctSku`` config when both gates hold:
205
+ * ``/proc/cpuinfo`` ``model name`` contains an SKU listed in
206
+ ``_PCT_CAPABLE_SKUS``.
207
+ * ``/sys/devices/system/cpu/cpu0/acpi_cppc/highest_perf`` matches
208
+ that SKU's expected ``highest_perf``.
209
+ Otherwise returns ``None`` and the caller falls back to the default
210
+ NUMA-node bind.
211
+ """
212
+ sku = _pct_sku_from_cpuinfo()
213
+ if sku is None:
214
+ return None
215
+
216
+ try:
217
+ with open(_PCT_HIGHEST_PERF_PATH) as f:
218
+ actual = int(f.read().strip())
219
+ except (OSError, ValueError):
220
+ return None
221
+ if actual != sku.highest_perf:
222
+ return None
223
+ return sku
224
+
225
+
226
def _get_gpu_index(
227
parallel_config, local_rank: int, dp_local_rank: int | None = None
228
) -> int:
@@ -156,55 +267,183 @@ def _get_numa_node(parallel_config, gpu_index: int) -> int:
267
return numa_nodes[gpu_index]
268
269
159
-def _get_cpu_binding(parallel_config, gpu_index: int) -> str | None:
270
+def _maybe_get_pct_cpu_binding(numa_nodes: list[int]) -> list[int] | None:
271
+ """Return the union of PCT priority cores across ``numa_nodes`` (or None).
272
+
273
+ PCT (Priority Core Turbo) lets a subset of cores boost above the rest;
274
+ we want workers and the EngineCore on those cores. The Linux kernel does
275
+ not expose PCT membership without root, so we use the empirical heuristic
276
+ documented above ``_PCT_CAPABLE_SKUS``: priority cores within each NUMA
277
+ node satisfy ``cpu_id % stride in (0, 1)`` intersected with the node's
278
+ ``cpulist``, where ``stride`` is the SKU's logical CPUs per priority
279
+ group (16 on 64-core SKUs, 18 on 72-core SKUs). Only triggers on the
280
+ SKUs in ``_PCT_CAPABLE_SKUS`` with the expected CPPC ``highest_perf``
281
+ signal; on any other host it returns None and the caller falls back to
282
+ the default NUMA-node bind.
283
+
284
+ Returns the sorted CPU ids as a ``list[int]``; the caller is expected
285
+ to format them for the chosen tool (e.g. comma-joined for
286
+ ``numactl --physcpubind``).
287
+ """
288
+ sku = _pct_sku_config()
289
+ if sku is None:
290
+ return None
291
+
292
+ from vllm.utils.cpu_resource_utils import parse_id_list
293
+
294
+ stride = sku.priority_stride
295
+ union_cpus: set[int] = set()
296
+ for numa_node in numa_nodes:
297
+ cpulist_path = Path(f"/sys/devices/system/node/node{numa_node}/cpulist")
298
+ try:
299
+ cpulist_raw = cpulist_path.read_text().strip()
300
+ except OSError:
301
+ continue
302
+ if not cpulist_raw:
303
+ continue
304
+ try:
305
+ node_cpus = parse_id_list(cpulist_raw)
306
+ except ValueError:
307
+ continue
308
+
309
+ priority = [cpu for cpu in node_cpus if cpu % stride in (0, 1)]
310
+ if not priority:
311
+ continue
312
+ union_cpus.update(priority)
313
+ logger.info(
314
+ "Detected PCT-capable Granite Rapids Xeon (stride=%d); "
315
+ "NUMA node %d priority cores: %s",
316
+ stride,
317
+ numa_node,
318
+ ",".join(str(c) for c in priority),
319
+ )
320
+
321
+ if not union_cpus:
322
+ return None
323
+ return sorted(union_cpus)
324
+
325
+
326
+def _get_cpu_binding(
327
+ parallel_config, gpu_index: int, numa_nodes: list[int]
328
+) -> str | None:
329
+ """Return the CPU list a process should be pinned to (or None)."""
330
cpu_bindings = parallel_config.numa_bind_cpus
331
if cpu_bindings is None:
162
- return None
332
+ pct_cpus = _maybe_get_pct_cpu_binding(numa_nodes)
333
+ if pct_cpus is None:
334
+ return None
335
+ return ",".join(str(c) for c in pct_cpus)
336
337
if gpu_index >= len(cpu_bindings):
338
raise ValueError(
339
f"GPU index {gpu_index} exceeds numa_bind_cpus size "
340
f"{len(cpu_bindings)}. Ensure the binding lists cover every visible GPU."
341
)
169
-
342
return cpu_bindings[gpu_index]
343
344
173
-def _get_numactl_args(
174
- vllm_config: "VllmConfig",
175
- local_rank: int,
176
- dp_local_rank: int | None = None,
177
- process_kind: str = "worker",
178
-) -> str | None:
179
- parallel_config = vllm_config.parallel_config
180
- if not parallel_config.numa_bind:
181
- return None
182
-
345
+def _get_numactl_worker_args(
346
+ parallel_config, local_rank: int, dp_local_rank: int | None = None
347
+) -> str:
348
+ """Compute the numactl args for a single TP/PP worker subprocess."""
349
gpu_index = _get_gpu_index(parallel_config, local_rank, dp_local_rank)
350
numa_node = _get_numa_node(parallel_config, gpu_index)
185
- cpu_binding = _get_cpu_binding(parallel_config, gpu_index)
351
+ cpu_binding = _get_cpu_binding(parallel_config, gpu_index, [numa_node])
352
353
if cpu_binding is not None:
188
- bind_arg = f"--physcpubind={cpu_binding}"
354
logger.info(
190
- "Binding %s subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
191
- process_kind,
355
+ "Binding worker subprocess (local_rank=%s, gpu_index=%s) to CPUs %s and NUMA node %s", # noqa: E501
356
local_rank,
357
gpu_index,
358
cpu_binding,
359
numa_node,
360
)
361
+ return f"--physcpubind={cpu_binding} --membind={numa_node}"
362
+
363
+ logger.info(
364
+ "Binding worker subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
365
+ local_rank,
366
+ gpu_index,
367
+ numa_node,
368
+ )
369
+ return f"--cpunodebind={numa_node} --membind={numa_node}"
370
+
371
+
372
+def _get_enginecore_numa_nodes(
373
+ parallel_config, dp_local_rank: int | None = None
374
+) -> list[int]:
375
+ """Return the sorted, unique NUMA nodes of the EngineCore's DP shard."""
376
+ numa_nodes = parallel_config.numa_bind_nodes
377
+ if numa_nodes is None:
378
+ # Trigger auto-detection (it caches into parallel_config).
379
+ _get_numa_node(parallel_config, 0)
380
+ numa_nodes = parallel_config.numa_bind_nodes
381
+
382
+ if (
383
+ parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
384
+ and parallel_config.data_parallel_backend != "ray"
385
+ and parallel_config.nnodes_within_dp == 1
386
+ ):
387
+ if dp_local_rank is None:
388
+ dp_local_rank = parallel_config.data_parallel_rank_local
389
+ if dp_local_rank is None:
390
+ dp_local_rank = parallel_config.data_parallel_index
391
+
392
+ tp_pp_world_size = (
393
+ parallel_config.pipeline_parallel_size
394
+ * parallel_config.tensor_parallel_size
395
+ )
396
+ shard_start = dp_local_rank * tp_pp_world_size
397
+ shard_end = min(shard_start + tp_pp_world_size, len(numa_nodes))
398
+ shard_indices: range | tuple[int, ...] = range(shard_start, shard_end)
399
else:
198
- bind_arg = f"--cpunodebind={numa_node}"
400
+ shard_indices = range(len(numa_nodes))
401
+
402
+ if not shard_indices:
403
+ return [numa_nodes[0]]
404
+ return sorted({numa_nodes[i] for i in shard_indices})
405
+
406
+
407
+def _get_numactl_enginecore_args(
408
+ parallel_config, local_rank: int, dp_local_rank: int | None = None
409
+) -> str:
410
+ """Compute the numactl args for an EngineCore subprocess.
411
+
412
+ ``--numa-bind-cpus`` is deliberately ignored here: the user provides a
413
+ per-worker CPU list, and binding EngineCore to any of those entries
414
+ would shrink its ``cpus_allowed`` below the strict-superset that the
415
+ workers' ``--physcpubind`` spawns require. We fall back to
416
+ ``--cpunodebind=<shard nodes>`` instead, which is always a safe
417
+ superset. PCT auto-detection still applies when the user did not pass
418
+ ``--numa-bind-cpus`` (its priority-core union across the shard nodes
419
+ is also a safe superset by construction).
420
+ """
421
+ shard_nodes = _get_enginecore_numa_nodes(parallel_config, dp_local_rank)
422
+ membind_arg = ",".join(str(n) for n in shard_nodes)
423
+
424
+ pct_cpus = (
425
+ None
426
+ if parallel_config.numa_bind_cpus is not None
427
+ else _maybe_get_pct_cpu_binding(shard_nodes)
428
+ )
429
+
430
+ if pct_cpus is not None:
431
+ cpu_binding = ",".join(str(c) for c in pct_cpus)
432
logger.info(
200
- "Binding %s subprocess (local_rank=%s, gpu_index=%s) to NUMA node %s",
201
- process_kind,
433
+ "Binding EngineCore subprocess (local_rank=%s) to CPUs %s "
434
+ "and NUMA nodes %s",
435
local_rank,
203
- gpu_index,
204
- numa_node,
436
+ cpu_binding,
437
+ membind_arg,
438
)
439
+ return f"--physcpubind={cpu_binding} --membind={membind_arg}"
440
207
- return f"{bind_arg} --membind={numa_node}"
441
+ logger.info(
442
+ "Binding EngineCore subprocess (local_rank=%s) to NUMA nodes %s",
443
+ local_rank,
444
+ membind_arg,
445
+ )
446
+ return f"--cpunodebind={membind_arg} --membind={membind_arg}"
447
448
449
def _log_numactl_show(label: str) -> bool:
@@ -242,13 +481,24 @@ def configure_subprocess(
481
process_kind: str = "worker",
482
):
483
"""Temporarily replace the multiprocessing executable with a numactl wrapper."""
245
- numactl_args = _get_numactl_args(
246
- vllm_config, local_rank, dp_local_rank, process_kind
247
- )
248
- if numactl_args is None:
484
+ parallel_config = vllm_config.parallel_config
485
+ if not parallel_config.numa_bind:
486
yield
487
return
488
489
+ if process_kind == "EngineCore":
490
+ numactl_args = _get_numactl_enginecore_args(
491
+ parallel_config, local_rank, dp_local_rank
492
+ )
493
+ elif process_kind == "worker":
494
+ numactl_args = _get_numactl_worker_args(
495
+ parallel_config, local_rank, dp_local_rank
496
+ )
497
+ else:
498
+ raise ValueError(
499
+ f"Unknown process_kind {process_kind!r}; expected 'worker' or 'EngineCore'."
500
+ )
501
+
502
executable, debug_str = _get_numactl_executable()
503
python_executable = os.fsdecode(multiprocessing.spawn.get_executable())
504
with (