@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1b44aaf4e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[bugfix][distributed] fix 16 gpus local rank arrangement (#5604)

youkaichao committed Jun 17, 2024 at 14:35 UTC 1b44aaf4e3559e4e321f32715b08f1aa7e4f3d50
1 file changed +6
vllm/executor/ray_gpu_executor.py
+6
@@ -137,6 +137,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
137
138 for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
139 node_workers[node_id].append(i)
140 + # `gpu_ids` can be a list of strings or integers.
141 + # convert them to integers for consistency.
142 + # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
143 + # string sorting is not sufficient.
144 + # see https://github.com/vllm-project/vllm/issues/5590
145 + gpu_ids = [int(x) for x in gpu_ids]
146 node_gpus[node_id].extend(gpu_ids)
147 for node_id, gpu_ids in node_gpus.items():
148 node_gpus[node_id] = sorted(gpu_ids)