A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[core][distributed] fix ray worker rank assignment (#6235)
youkaichao committed
Jul 8, 2024 at 21:31 UTC
70c232f85a9e83421a4d9ca95e6384364271f2bc
1 file changed
+28
-6
vllm/executor/ray_gpu_executor.py
+28
-6
@@ -134,11 +134,32 @@ class RayGPUExecutor(DistributedGPUExecutor):
134
worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
135
use_dummy_driver=True)
136
137
- node_workers = defaultdict(list)
138
- node_gpus = defaultdict(list)
139
-
140
- for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
141
- node_workers[node_id].append(i)
137
+ # the order in `worker_node_and_gpu_ids` does not necessarily match
138
+ # the machine boundaries. We need to make sure that workers in the
139
+ # same node are assigned consecutive ranks.
140
+ # examples:
141
+ # [('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [1]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [2]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [3]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [1]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [2]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [3])] # noqa
142
+
143
+ # initialize worker ranks with -1 (unassigned)
144
+ worker_ranks = [-1 for x in worker_node_and_gpu_ids]
145
+ current_rank = 0
146
+ while -1 in worker_ranks:
147
+ # whenever we find an unassigned worker, find the node
148
+ index = worker_ranks.index(-1)
149
+ current_node_id = worker_node_and_gpu_ids[index][0]
150
+ # assign ranks to all workers in the same node
151
+ for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
152
+ if node_id == current_node_id:
153
+ worker_ranks[i] = current_rank
154
+ current_rank += 1
155
+ # with the above example, worker_ranks will be [0, 4, 5, 6, 7, 1, 2, 3]
156
+
157
+ node_workers = defaultdict(list) # node id -> list of worker ranks
158
+ node_gpus = defaultdict(list) # node id -> list of gpu ids
159
+
160
+ for worker_rank, (node_id, gpu_ids) in zip(worker_ranks,
161
+ worker_node_and_gpu_ids):
162
+ node_workers[node_id].append(worker_rank)
163
# `gpu_ids` can be a list of strings or integers.
164
# convert them to integers for consistency.
165
# NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
@@ -184,7 +205,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
205
local_rank=node_workers[node_id].index(rank),
206
rank=rank,
207
distributed_init_method=distributed_init_method,
187
- ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
208
+ ) for rank, (node_id,
209
+ _) in zip(worker_ranks, worker_node_and_gpu_ids)
210
]
211
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
212