A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 17:30 UTC
[ROCm] Implement GPU-to-NUMA-node detection (#40015)
Signed-off-by: Patrick Schlangen <pschlan@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
pschlan-amd committed
Apr 23, 2026 at 17:08 UTC
0098db9ec1b19138843fee3147b61bbdbec0cd05
2 files changed
+31
-3
docs/configuration/optimization.md
+3
-3
@@ -155,9 +155,9 @@ switch to `--physcpubind=<cpu-list> --membind=<node>`.
155
156
These `--numa-bind*` options only apply to GPU execution processes. They do not
157
configure the CPU backend's separate thread-affinity controls. Automatic
158
-GPU-to-NUMA detection is currently implemented for CUDA/NVML-based platforms;
159
-other GPU backends must provide explicit binding lists if they use these
160
-options.
158
+GPU-to-NUMA detection is currently implemented for CUDA/NVML-based as well as
159
+ROCM-based platforms; other GPU backends must provide explicit binding lists if
160
+they use these options.
161
162
`--numa-bind-nodes` takes one non-negative NUMA node index per visible GPU, in
163
the same order as the GPU indices.
vllm/platforms/rocm.py
+28
@@ -33,6 +33,7 @@ try:
33
amdsmi_init,
34
amdsmi_shut_down,
35
amdsmi_topo_get_link_type,
36
+ amdsmi_topo_get_numa_node_number,
37
)
38
except ImportError as e:
39
logger.warning("Failed to import from amdsmi with %r", e)
@@ -955,3 +956,30 @@ class RocmPlatform(Platform):
956
rms_norm = default
957
958
return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm)
959
+
960
+ @classmethod
961
+ @with_amdsmi_context
962
+ def get_all_device_numa_nodes(cls) -> list[int] | None:
963
+ """Get NUMA nodes for all visible GPU devices."""
964
+ try:
965
+ handles = amdsmi_get_processor_handles()
966
+ numa_nodes = []
967
+ for device_id in range(cls.device_count()):
968
+ physical_device_id = cls.device_id_to_physical_device_id(device_id)
969
+ try:
970
+ numa_node = amdsmi_topo_get_numa_node_number(
971
+ handles[physical_device_id]
972
+ )
973
+ except AmdSmiException as e:
974
+ logger.warning(
975
+ "Could not detect NUMA node for GPU %d, "
976
+ "disabling automatic NUMA binding: %s",
977
+ device_id,
978
+ e,
979
+ )
980
+ return None
981
+ numa_nodes.append(numa_node)
982
+ return numa_nodes
983
+ except Exception as e:
984
+ logger.warning("Failed to get NUMA nodes for GPUs: %s", e)
985
+ return None