A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[bugfix][distributed] fix 16 gpus local rank arrangement (#5604)
youkaichao committed
Jun 17, 2024 at 14:35 UTC
1b44aaf4e3559e4e321f32715b08f1aa7e4f3d50
1 file changed
+6
vllm/executor/ray_gpu_executor.py
+6
@@ -137,6 +137,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
137
138
for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
139
node_workers[node_id].append(i)
140
+ # `gpu_ids` can be a list of strings or integers.
141
+ # convert them to integers for consistency.
142
+ # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
143
+ # string sorting is not sufficient.
144
+ # see https://github.com/vllm-project/vllm/issues/5590
145
+ gpu_ids = [int(x) for x in gpu_ids]
146
node_gpus[node_id].extend(gpu_ids)
147
for node_id, gpu_ids in node_gpus.items():
148
node_gpus[node_id] = sorted(gpu_ids)