A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc] Optimize ray worker initialization time (#11275)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Cody Yu <hao.yu.cody@gmail.com>
Rui Qiao committed
Dec 18, 2024 at 23:38 UTC
f26c4aeecba481ce1445be7a998b0b97460a13bb
1 file changed
+22
-13
vllm/executor/ray_gpu_executor.py
+22
-13
@@ -123,6 +123,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
123
124
# Create the workers.
125
driver_ip = get_ip()
126
+ workers = []
127
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
128
if not bundle.get("GPU", 0):
129
continue
@@ -138,20 +139,30 @@ class RayGPUExecutor(DistributedGPUExecutor):
139
scheduling_strategy=scheduling_strategy,
140
**ray_remote_kwargs,
141
)(RayWorkerWrapper).remote(vllm_config=self.vllm_config)
142
+ workers.append(worker)
143
142
- if self.use_ray_spmd_worker:
143
- self.workers.append(worker)
144
- else:
145
- worker_ip = ray.get(worker.get_node_ip.remote())
146
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
144
+ worker_ip_refs = [
145
+ worker.get_node_ip.remote() # type: ignore[attr-defined]
146
+ for worker in workers
147
+ ]
148
+ worker_ips = ray.get(worker_ip_refs)
149
+
150
+ if not self.use_ray_spmd_worker:
151
+ for i in range(len(workers)):
152
+ worker = workers[i]
153
+ worker_ip = worker_ips[i]
154
+ if self.driver_dummy_worker is None and worker_ip == driver_ip:
155
# If the worker is on the same node as the driver, we use it
156
# as the resource holder for the driver process.
157
self.driver_dummy_worker = worker
158
self.driver_worker = RayWorkerWrapper(
159
vllm_config=self.vllm_config)
152
- else:
153
- # Else, added to the list of workers.
154
- self.workers.append(worker)
160
+ workers.pop(i)
161
+ worker_ips.pop(i)
162
+ self.workers = workers
163
+ break
164
+ else:
165
+ self.workers = workers
166
167
logger.debug("workers: %s", self.workers)
168
logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
@@ -161,14 +172,12 @@ class RayGPUExecutor(DistributedGPUExecutor):
172
"adjusting the Ray placement group or running the driver on a "
173
"GPU node.")
174
164
- worker_ips = [
165
- ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
166
- for worker in self.workers
167
- ]
175
ip_counts: Dict[str, int] = {}
176
for ip in worker_ips:
177
ip_counts[ip] = ip_counts.get(ip, 0) + 1
178
179
+ worker_to_ip = dict(zip(self.workers, worker_ips))
180
+
181
def sort_by_driver_then_worker_ip(worker):
182
"""
183
Sort the workers based on 3 properties:
@@ -179,7 +188,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
188
3. Finally, if the work is on a node with smaller IP address, it
189
should be placed first.
190
"""
182
- ip = ray.get(worker.get_node_ip.remote())
191
+ ip = worker_to_ip[worker]
192
return (ip != driver_ip, ip_counts[ip], ip)
193
194
# After sorting, the workers on the same node will be