@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f26c4aeecb
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Misc] Optimize ray worker initialization time (#11275)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>

Rui Qiao committed Dec 18, 2024 at 23:38 UTC f26c4aeecba481ce1445be7a998b0b97460a13bb
1 file changed +22 -13
vllm/executor/ray_gpu_executor.py
+22 -13
@@ -123,6 +123,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
123
124 # Create the workers.
125 driver_ip = get_ip()
126 + workers = []
127 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128 if not bundle.get("GPU", 0):
129 continue
@@ -138,20 +139,30 @@ class RayGPUExecutor(DistributedGPUExecutor):
139 scheduling_strategy=scheduling_strategy,
140 **ray_remote_kwargs,
141 )(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142 + workers.append(worker)
143
142 - if self.use_ray_spmd_worker:
143 - self.workers.append(worker)
144 - else:
145 - worker_ip = ray.get(worker.get_node_ip.remote())
146 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
144 + worker_ip_refs = [
145 + worker.get_node_ip.remote() # type: ignore[attr-defined]
146 + for worker in workers
147 + ]
148 + worker_ips = ray.get(worker_ip_refs)
149 +
150 + if not self.use_ray_spmd_worker:
151 + for i in range(len(workers)):
152 + worker = workers[i]
153 + worker_ip = worker_ips[i]
154 + if self.driver_dummy_worker is None and worker_ip == driver_ip:
155 # If the worker is on the same node as the driver, we use it
156 # as the resource holder for the driver process.
157 self.driver_dummy_worker = worker
158 self.driver_worker = RayWorkerWrapper(
159 vllm_config=self.vllm_config)
152 - else:
153 - # Else, added to the list of workers.
154 - self.workers.append(worker)
160 + workers.pop(i)
161 + worker_ips.pop(i)
162 + self.workers = workers
163 + break
164 + else:
165 + self.workers = workers
166
167 logger.debug("workers: %s", self.workers)
168 logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
@@ -161,14 +172,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
172 "adjusting the Ray placement group or running the driver on a "
173 "GPU node.")
174
164 - worker_ips = [
165 - ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
166 - for worker in self.workers
167 - ]
175 ip_counts: Dict[str, int] = {}
176 for ip in worker_ips:
177 ip_counts[ip] = ip_counts.get(ip, 0) + 1
178
179 + worker_to_ip = dict(zip(self.workers, worker_ips))
180 +
181 def sort_by_driver_then_worker_ip(worker):
182 """
183 Sort the workers based on 3 properties:
@@ -179,7 +188,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
188 3. Finally, if the work is on a node with smaller IP address, it
189 should be placed first.
190 """
182 - ip = ray.get(worker.get_node_ip.remote())
191 + ip = worker_to_ip[worker]
192 return (ip != driver_ip, ip_counts[ip], ip)
193
194 # After sorting, the workers on the same node will be