A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 23:45 UTC
[Misc] Improve comment for DPEngineCoreActor._set_cuda_visible_devices() (#21501)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Jul 24, 2025 at 03:13 UTC
73e3949d07cb11f0f9ebfb85cf0ee39d29db6be4
1 file changed
+7
-2
vllm/v1/engine/core.py
+7
-2
@@ -1082,8 +1082,13 @@ class DPEngineCoreActor(DPEngineCoreProc):
1082
# RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES, but vLLM workers created
1083
# thereafter would have CUDA_VISIBLE_DEVICES set, which is sticky:
1084
# https://github.com/ray-project/ray/blob/e752fc319ddedd9779a0989b6d3613909bad75c9/python/ray/_private/worker.py#L456 # noqa: E501
1085
- # But vLLM worker assumes visibility into all local GPUs, therefore
1086
- # this results in incorrect indexing into the GPU ID list.
1085
+ # This is problematic because when the vLLM worker (a Ray actor)
1086
+ # executes a task, it indexes into the sticky CUDA_VISIBLE_DEVICES
1087
+ # rather than directly using the GPU ID, potentially resulting in
1088
+ # index out of bounds error. See:
1089
+ # https://github.com/ray-project/ray/pull/40461/files#diff-31e8159767361e4bc259b6d9883d9c0d5e5db780fcea4a52ead4ee3ee4a59a78R1860 # noqa: E501
1090
+ # and get_accelerator_ids_for_accelerator_resource() in worker.py
1091
+ # of ray.
1092
self._set_cuda_visible_devices(vllm_config, local_dp_rank)
1093
1094
super().__init__(vllm_config, local_client, "", executor_class,