@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70c232f85a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[core][distributed] fix ray worker rank assignment (#6235)

youkaichao committed Jul 8, 2024 at 21:31 UTC 70c232f85a9e83421a4d9ca95e6384364271f2bc
1 file changed +28 -6
vllm/executor/ray_gpu_executor.py
+28 -6
@@ -134,11 +134,32 @@ class RayGPUExecutor(DistributedGPUExecutor):
134 worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
135 use_dummy_driver=True)
136
137 - node_workers = defaultdict(list)
138 - node_gpus = defaultdict(list)
139 -
140 - for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
141 - node_workers[node_id].append(i)
137 + # the order in `worker_node_and_gpu_ids` does not necessarily match
138 + # the machine boundaries. We need to make sure that workers in the
139 + # same node are assigned consecutive ranks.
140 + # examples:
141 + # [('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [1]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [2]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [3]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [1]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [2]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [3])] # noqa
142 +
143 + # initialize worker ranks with -1 (unassigned)
144 + worker_ranks = [-1 for x in worker_node_and_gpu_ids]
145 + current_rank = 0
146 + while -1 in worker_ranks:
147 + # whenever we find an unassigned worker, find the node
148 + index = worker_ranks.index(-1)
149 + current_node_id = worker_node_and_gpu_ids[index][0]
150 + # assign ranks to all workers in the same node
151 + for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
152 + if node_id == current_node_id:
153 + worker_ranks[i] = current_rank
154 + current_rank += 1
155 + # with the above example, worker_ranks will be [0, 4, 5, 6, 7, 1, 2, 3]
156 +
157 + node_workers = defaultdict(list) # node id -> list of worker ranks
158 + node_gpus = defaultdict(list) # node id -> list of gpu ids
159 +
160 + for worker_rank, (node_id, gpu_ids) in zip(worker_ranks,
161 + worker_node_and_gpu_ids):
162 + node_workers[node_id].append(worker_rank)
163 # `gpu_ids` can be a list of strings or integers.
164 # convert them to integers for consistency.
165 # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
@@ -184,7 +205,8 @@ class RayGPUExecutor(DistributedGPUExecutor):
205 local_rank=node_workers[node_id].index(rank),
206 rank=rank,
207 distributed_init_method=distributed_init_method,
187 - ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
208 + ) for rank, (node_id,
209 + _) in zip(worker_ranks, worker_node_and_gpu_ids)
210 ]
211 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
212