@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 61e592747c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Core] Introduce SPMD worker execution using Ray accelerated DAG (#6032)

Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Stephanie Wang <swang@cs.berkeley.edu>

Rui Qiao committed Jul 17, 2024 at 22:27 UTC 61e592747c28c9fbd6861e48b825c796e09da02f
8 files changed +218 -121
.buildkite/test-pipeline.yaml
+3
@@ -84,6 +84,8 @@ steps:
84 - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
87 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
88 + - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
89 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90 - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91 - TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
@@ -108,6 +110,7 @@ steps:
110 # We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
111 # See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
112 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
113 + - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
114 - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
115 - pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
116
vllm/engine/llm_engine.py
+5
@@ -6,6 +6,7 @@ from typing import Set, Type, TypeVar, Union
6
7 from transformers import PreTrainedTokenizer
8
9 +import vllm.envs as envs
10 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11 LoRAConfig, ModelConfig, MultiModalConfig,
12 ObservabilityConfig, ParallelConfig,
@@ -414,6 +415,9 @@ class LLMEngine:
415 elif distributed_executor_backend == "mp":
416 from vllm.executor.multiproc_gpu_executor import (
417 MultiprocessingGPUExecutor)
418 + assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
419 + "multiprocessing distributed executor backend does not "
420 + "support VLLM_USE_RAY_SPMD_WORKER=1")
421 executor_class = MultiprocessingGPUExecutor
422 else:
423 from vllm.executor.gpu_executor import GPUExecutor
@@ -426,6 +430,7 @@ class LLMEngine:
430 usage_context=usage_context,
431 stat_loggers=stat_loggers,
432 )
433 +
434 return engine
435
436 def __reduce__(self):
vllm/envs.py
+8
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
34 VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
35 VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
36 VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
37 + VLLM_USE_RAY_SPMD_WORKER: bool = False
38 VLLM_USE_RAY_COMPILED_DAG: bool = False
39 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
40 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
@@ -261,6 +262,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
262 "VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":
263 lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)),
264
265 + # If the env var is set, then all workers will execute as separate
266 + # processes from the engine, and we use the same mechanism to trigger
267 + # execution on all workers.
268 + # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
269 + "VLLM_USE_RAY_SPMD_WORKER":
270 + lambda: bool(os.getenv("VLLM_USE_RAY_SPMD_WORKER", 0)),
271 +
272 # If the env var is set, it uses the Ray's compiled DAG API
273 # which optimizes the control plane overhead.
274 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
vllm/executor/distributed_gpu_executor.py
+5 -3
@@ -64,8 +64,8 @@ class DistributedGPUExecutor(GPUExecutor):
64 num_cpu_blocks=num_cpu_blocks)
65
66 def execute_model(
67 - self, execute_model_req: ExecuteModelRequest
68 - ) -> Optional[List[SamplerOutput]]:
67 + self,
68 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
69 if self.parallel_worker_tasks is None:
70 self.parallel_worker_tasks = self._run_workers(
71 "start_worker_execution_loop",
@@ -73,7 +73,9 @@ class DistributedGPUExecutor(GPUExecutor):
73 **self.extra_execute_model_run_workers_kwargs)
74
75 # Only the driver worker returns the sampling results.
76 - return self._driver_execute_model(execute_model_req)
76 + driver_outputs = self._driver_execute_model(execute_model_req)
77 + assert driver_outputs is not None
78 + return driver_outputs
79
80 def stop_remote_worker_execution_loop(self) -> None:
81 if self.parallel_worker_tasks is None:
vllm/executor/ray_gpu_executor.py
+138 -74
@@ -1,6 +1,5 @@
1 import asyncio
2 import os
3 -import pickle
3 from collections import defaultdict
4 from itertools import islice, repeat
5 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
@@ -23,12 +22,30 @@ if TYPE_CHECKING:
22
23 logger = init_logger(__name__)
24
26 -USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
27 -
25
26 class RayGPUExecutor(DistributedGPUExecutor):
27
28 def _init_executor(self) -> None:
29 + # If the env var is set, it uses the Ray's compiled DAG API
30 + # which optimizes the control plane overhead.
31 + # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
32 + # Currently, this requires USE_RAY_SPMD_WORKER=True.
33 + self.use_ray_compiled_dag = envs.VLLM_USE_RAY_COMPILED_DAG
34 + # If the env var is set, then we do not distinguish between the
35 + # "driver worker" vs other workers. Also, the rank 0 worker will
36 + # be executed in a remote Ray worker. Currently this requires
37 + # USE_RAY_COMPILED_DAG=True.
38 + self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
39 + if self.use_ray_compiled_dag:
40 + assert self.use_ray_spmd_worker, (
41 + "VLLM_USE_RAY_COMPILED_DAG=1 requires "
42 + "VLLM_USE_RAY_SPMD_WORKER=1")
43 + if self.use_ray_spmd_worker:
44 + # TODO: Support SPMD worker for non-DAG Ray executor.
45 + assert self.use_ray_compiled_dag, (
46 + "VLLM_USE_RAY_SPMD_WORKER=1 requires "
47 + "VLLM_USE_RAY_COMPILED_DAG=1")
48 +
49 assert self.parallel_config.distributed_executor_backend == "ray"
50 placement_group = self.parallel_config.placement_group
51
@@ -40,11 +57,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
57 # Create the parallel GPU workers.
58 self._init_workers_ray(placement_group)
59
43 - self.forward_dag = None
44 - if USE_RAY_COMPILED_DAG:
45 - self.forward_dag = self._compiled_ray_dag()
46 - self.extra_execute_model_run_workers_kwargs[
47 - "use_ray_compiled_dag"] = True
60 + self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
61
62 def _configure_ray_workers_use_nsight(self,
63 ray_remote_kwargs) -> Dict[str, Any]:
@@ -110,21 +123,24 @@ class RayGPUExecutor(DistributedGPUExecutor):
123 trust_remote_code=self.model_config.trust_remote_code,
124 )
125
113 - worker_ip = ray.get(worker.get_node_ip.remote())
114 - if worker_ip == driver_ip and self.driver_dummy_worker is None:
115 - # If the worker is on the same node as the driver, we use it
116 - # as the resource holder for the driver process.
117 - self.driver_dummy_worker = worker
118 - self.driver_worker = RayWorkerWrapper(
119 - worker_module_name=worker_module_name,
120 - worker_class_name=worker_class_name,
121 - trust_remote_code=self.model_config.trust_remote_code,
122 - )
123 - else:
124 - # Else, added to the list of workers.
126 + if self.use_ray_spmd_worker:
127 self.workers.append(worker)
126 -
127 - if self.driver_dummy_worker is None:
128 + else:
129 + worker_ip = ray.get(worker.get_node_ip.remote())
130 + if worker_ip == driver_ip and self.driver_dummy_worker is None:
131 + # If the worker is on the same node as the driver, we use it
132 + # as the resource holder for the driver process.
133 + self.driver_dummy_worker = worker
134 + self.driver_worker = RayWorkerWrapper(
135 + worker_module_name=worker_module_name,
136 + worker_class_name=worker_class_name,
137 + trust_remote_code=self.model_config.trust_remote_code,
138 + )
139 + else:
140 + # Else, added to the list of workers.
141 + self.workers.append(worker)
142 +
143 + if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
144 raise ValueError(
145 "Ray does not allocate any GPUs on the driver node. Consider "
146 "adjusting the Ray placement group or running the driver on a "
@@ -254,9 +270,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
270 Passing None will cause the driver to stop the model execution
271 loop running in each of the remote workers.
272 """
273 + assert not self.use_ray_spmd_worker, (
274 + "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
275 return self.driver_worker.execute_method("execute_model",
276 execute_model_req)
277
278 + def execute_model(
279 + self,
280 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
281 + if not self.use_ray_spmd_worker:
282 + return super().execute_model(execute_model_req)
283 +
284 + if self.forward_dag is None:
285 + self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
286 +
287 + outputs = ray.get(self.forward_dag.execute(execute_model_req))
288 + return outputs[0]
289 +
290 def _run_workers(
291 self,
292 method: str,
@@ -266,7 +296,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
296 all_kwargs: Optional[List[Dict[str, Any]]] = None,
297 use_dummy_driver: bool = False,
298 max_concurrent_workers: Optional[int] = None,
269 - use_ray_compiled_dag: bool = False,
299 **kwargs,
300 ) -> Any:
301 """Runs the given method on all workers. Can be used in the following
@@ -281,6 +310,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
310 - all_args/all_kwargs: args/kwargs for each worker are specified
311 individually
312 """
313 + if self.use_ray_spmd_worker:
314 + assert not async_run_tensor_parallel_workers_only, (
315 + "async_run_tensor_parallel_workers_only is not supported for "
316 + "spmd mode.")
317
318 if max_concurrent_workers:
319 raise NotImplementedError(
@@ -289,71 +322,69 @@ class RayGPUExecutor(DistributedGPUExecutor):
322 count = len(self.workers) if not \
323 async_run_tensor_parallel_workers_only \
324 else len(self.non_driver_workers)
325 + # If using SPMD worker, all workers are the same, so we should execute
326 + # the args on all workers. Otherwise, we skip the first worker's args
327 + # because those args will go to the driver worker.
328 + first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
329 all_worker_args = repeat(args, count) if all_args is None \
293 - else islice(all_args, 1, None)
330 + else islice(all_args, first_worker_args_index, None)
331 all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
295 - else islice(all_kwargs, 1, None)
296 -
297 - if use_ray_compiled_dag:
298 - # Right now, compiled DAG can only accept a single
299 - # input. TODO(sang): Fix it.
300 - assert self.forward_dag is not None
301 - output_channels = self.forward_dag.execute(1)
302 - ray_worker_outputs = []
303 - else:
304 - # Start the ray workers first.
305 - ray_workers = self.workers
306 - if async_run_tensor_parallel_workers_only:
307 - ray_workers = self.non_driver_workers
308 - ray_worker_outputs = [
309 - worker.execute_method.remote(method, *worker_args,
310 - **worker_kwargs)
311 - for (worker, worker_args, worker_kwargs
312 - ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
313 - ]
332 + else islice(all_kwargs, first_worker_args_index, None)
333 +
334 + # Start the ray workers first.
335 + ray_workers = self.workers
336 + if async_run_tensor_parallel_workers_only:
337 + ray_workers = self.non_driver_workers
338 + ray_worker_outputs = [
339 + worker.execute_method.remote(method, *worker_args, **worker_kwargs)
340 + for (worker, worker_args, worker_kwargs
341 + ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
342 + ]
343
344 if async_run_tensor_parallel_workers_only:
345 # Just return futures
346 return ray_worker_outputs
347
319 - driver_args = args if all_args is None else all_args[0]
320 - driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
348 + driver_worker_output = []
349 + # In SPMD mode, the driver worker is the same as any other worker,
350 + # so we only explicitly execute on the driver worker if using a
351 + # non-SPMD worker class.
352 + if not self.use_ray_spmd_worker:
353 + driver_args = args if all_args is None else all_args[0]
354 + driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
355 +
356 + # Start the driver worker after all the ray workers.
357 + if not use_dummy_driver:
358 + driver_worker_output = [
359 + self.driver_worker.execute_method(method, *driver_args,
360 + **driver_kwargs)
361 + ]
362 + else:
363 + assert self.driver_dummy_worker is not None
364 + driver_worker_output = [
365 + ray.get(
366 + self.driver_dummy_worker.execute_method.remote(
367 + method, *driver_args, **driver_kwargs))
368 + ]
369
322 - # Start the driver worker after all the ray workers.
323 - if not use_dummy_driver:
324 - driver_worker_output = self.driver_worker.execute_method(
325 - method, *driver_args, **driver_kwargs)
326 - else:
327 - assert self.driver_dummy_worker is not None
328 - driver_worker_output = ray.get(
329 - self.driver_dummy_worker.execute_method.remote(
330 - method, *driver_args, **driver_kwargs))
370 # Get the results of the ray workers.
371 if self.workers:
333 - if use_ray_compiled_dag:
334 - try:
335 - ray_worker_outputs = [
336 - pickle.loads(chan.begin_read())
337 - for chan in output_channels
338 - ]
339 - finally:
340 - # Has to call end_read in order to reuse the DAG.
341 - for chan in output_channels:
342 - chan.end_read()
343 - else:
344 - ray_worker_outputs = ray.get(ray_worker_outputs)
372 + ray_worker_outputs = ray.get(ray_worker_outputs)
373
346 - return [driver_worker_output] + ray_worker_outputs
374 + return driver_worker_output + ray_worker_outputs
375
376 def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
377 """Wait for futures returned from _run_workers() with
378 async_run_remote_workers_only to complete."""
379 ray.get(parallel_worker_tasks)
380
353 - def _compiled_ray_dag(self):
381 + def _compiled_ray_dag(self, enable_asyncio: bool):
382 import pkg_resources
355 - required_version = "2.9"
356 - current_version = pkg_resources.get_distribution("ray").version
383 + from packaging import version
384 +
385 + required_version = version.parse("2.32")
386 + current_version = version.parse(
387 + pkg_resources.get_distribution("ray").version)
388 if current_version < required_version:
389 raise ValueError(f"Ray version {required_version} or greater is "
390 f"required, but found {current_version}")
@@ -365,23 +396,47 @@ class RayGPUExecutor(DistributedGPUExecutor):
396 # a dummy value for now. It will be fixed soon.
397 with InputNode() as input_data:
398 forward_dag = MultiOutputNode([
368 - worker.execute_model_compiled_dag_remote.
369 - bind( # type: ignore[attr-defined]
399 + worker.execute_model_spmd.bind( # type: ignore[attr-defined]
400 input_data) for worker in self.workers
401 ])
372 - return forward_dag.experimental_compile()
402 + return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
403 +
404 + def __del__(self):
405 + if self.forward_dag is not None:
406 + self.forward_dag.teardown()
407 + import ray
408 + for worker in self.workers:
409 + ray.kill(worker)
410
411
412 class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
413
414 def __init__(self, *args, **kwargs):
415 super().__init__(*args, **kwargs)
379 - self.driver_exec_method = make_async(self.driver_worker.execute_method)
416 + self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
417 + if not self.use_ray_compiled_dag:
418 + self.driver_exec_method = make_async(
419 + self.driver_worker.execute_method)
420 +
421 + async def execute_model_async(
422 + self,
423 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
424 + if not self.use_ray_spmd_worker:
425 + return await super().execute_model_async(execute_model_req)
426 +
427 + if self.forward_dag is None:
428 + self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
429 +
430 + dag_future = await self.forward_dag.execute_async(execute_model_req)
431 + outputs = await dag_future
432 + return outputs[0]
433
434 async def _driver_execute_model_async(
435 self,
436 execute_model_req: Optional[ExecuteModelRequest] = None
437 ) -> List[SamplerOutput]:
438 + assert not self.use_ray_spmd_worker, (
439 + "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
440 if self.pp_locks is None:
441 # This locks each pipeline parallel stage so multiple virtual
442 # engines can't execute on the same stage at the same time
@@ -415,8 +470,17 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
470 return results[-1]
471
472 async def _start_worker_execution_loop(self):
473 + assert not self.use_ray_spmd_worker, (
474 + "worker loop is disabled for VLLM_USE_RAY_SPMD_WORKER=1")
475 coros = [
476 worker.execute_method.remote("start_worker_execution_loop")
477 for worker in self.non_driver_workers
478 ]
479 return await asyncio.gather(*coros)
480 +
481 + def __del__(self):
482 + if self.forward_dag is not None:
483 + self.forward_dag.teardown()
484 + import ray
485 + for worker in self.workers:
486 + ray.kill(worker)
vllm/executor/ray_utils.py
+8 -6
@@ -1,8 +1,8 @@
1 -import pickle
1 from typing import List, Optional, Tuple
2
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
5 +from vllm.sequence import ExecuteModelRequest
6 from vllm.utils import get_ip, is_hip, is_xpu
7 from vllm.worker.worker_base import WorkerWrapperBase
8
@@ -31,16 +31,18 @@ try:
31 gpu_ids = ray.get_gpu_ids()
32 return node_id, gpu_ids
33
34 - def execute_model_compiled_dag_remote(self, ignored):
35 - """Used only when compiled DAG is enabled."""
34 + def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
35 + """Used only when SPMD worker and compiled DAG are both
36 + enabled."""
37 + # TODO(swang): This is needed right now because Ray aDAG executes
38 + # on a background thread, so we need to reset torch's current
39 + # device.
40 import torch
41 if not self.compiled_dag_cuda_device_set:
42 torch.cuda.set_device(self.worker.device)
43 self.compiled_dag_cuda_device_set = True
44
41 - output = self.worker.execute_model()
42 - output = pickle.dumps(output)
43 - return output
45 + return self.worker._execute_model_spmd(execute_model_req)
46
47 ray_import_err = None
48
vllm/executor/ray_xpu_executor.py
+21 -37
@@ -1,11 +1,11 @@
1 import asyncio
2 import os
3 -import pickle
3 from collections import defaultdict
4 from itertools import islice, repeat
5 from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
6 Tuple, Union)
7
8 +import vllm.envs as envs
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, MultiModalConfig, ParallelConfig,
11 PromptAdapterConfig, SchedulerConfig,
@@ -30,7 +30,7 @@ logger = init_logger(__name__)
30 # If the env var is set, it uses the Ray's compiled DAG API
31 # which optimizes the control plane overhead.
32 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
33 -USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
33 +USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
34
35
36 class RayXPUExecutor(DistributedGPUExecutor):
@@ -72,10 +72,9 @@ class RayXPUExecutor(DistributedGPUExecutor):
72 # Create the parallel GPU workers.
73 self._init_workers_ray(placement_group)
74
75 - # Profile the memory usage and initialize the cache.
75 self.forward_dag = None
76 if USE_RAY_COMPILED_DAG:
78 - self.forward_dag = self._compiled_ray_dag()
77 + self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
78
79 # This is non-None when the execute model loop is running
80 # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
@@ -270,7 +269,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
269 all_kwargs: Optional[List[Dict[str, Any]]] = None,
270 use_dummy_driver: bool = False,
271 max_concurrent_workers: Optional[int] = None,
273 - use_ray_compiled_dag: bool = False,
272 **kwargs,
273 ) -> Any:
274 """Runs the given method on all workers. Can be used in the following
@@ -293,26 +291,20 @@ class RayXPUExecutor(DistributedGPUExecutor):
291 all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
292 else islice(all_kwargs, 1, None)
293
296 - if use_ray_compiled_dag:
297 - # Right now, compiled DAG can only accept a single
298 - # input. TODO(sang): Fix it.
299 - assert self.forward_dag is not None
300 - output_channels = self.forward_dag.execute(1)
301 - else:
302 - # Start the ray workers first.
303 - ray_worker_outputs = [
304 - worker.execute_method.remote(method, *worker_args,
305 - **worker_kwargs)
306 - for (worker, worker_args, worker_kwargs
307 - ) in zip(self.workers, all_worker_args, all_worker_kwargs)
308 - ]
294 + # Start the ray workers first.
295 + ray_worker_outputs = [
296 + worker.execute_method.remote(method, *worker_args, **worker_kwargs)
297 + for (worker, worker_args, worker_kwargs
298 + ) in zip(self.workers, all_worker_args, all_worker_kwargs)
299 + ]
300 +
301 if async_run_remote_workers_only:
302 # Just return futures
303 return ray_worker_outputs
304
305 + driver_worker_output = []
306 driver_args = args if all_args is None else all_args[0]
307 driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
315 -
308 # Start the driver worker after all the ray workers.
309 if not use_dummy_driver:
310 driver_worker_output = self.driver_worker.execute_method(
@@ -324,36 +316,28 @@ class RayXPUExecutor(DistributedGPUExecutor):
316 method, *driver_args, **driver_kwargs))
317 # Get the results of the ray workers.
318 if self.workers:
327 - if use_ray_compiled_dag:
328 - try:
329 - ray_worker_outputs = [
330 - pickle.loads(chan.begin_read())
331 - for chan in output_channels
332 - ]
333 - finally:
334 - # Has to call end_read in order to reuse the DAG.
335 - for chan in output_channels:
336 - chan.end_read()
337 - else:
338 - ray_worker_outputs = ray.get(ray_worker_outputs)
319 + ray_worker_outputs = ray.get(ray_worker_outputs)
320
340 - return [driver_worker_output] + ray_worker_outputs
321 + return driver_worker_output + ray_worker_outputs
322
323 def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
324 """Wait for futures returned from _run_workers() with
325 async_run_remote_workers_only to complete."""
326 ray.get(parallel_worker_tasks)
327
347 - def _compiled_ray_dag(self):
328 + def _compiled_ray_dag(self, enable_asyncio: bool):
329 import pkg_resources
349 - required_version = "2.9"
350 - current_version = pkg_resources.get_distribution("ray").version
330 + from packaging import version
331 +
332 + required_version = version.parse("2.32")
333 + current_version = version.parse(
334 + pkg_resources.get_distribution("ray").version)
335 if current_version < required_version:
336 raise ValueError(f"Ray version {required_version} or greater is "
337 f"required, but found {current_version}")
338
339 from ray.dag import InputNode, MultiOutputNode
356 - assert self.parallel_config.worker_use_ray
340 + assert self.parallel_config.distributed_executor_backend == "ray"
341
342 # Right now, compiled DAG requires at least 1 arg. We send
343 # a dummy value for now. It will be fixed soon.
@@ -363,7 +347,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
347 bind( # type: ignore[attr-defined]
348 input_data) for worker in self.workers
349 ])
366 - return forward_dag.experimental_compile()
350 + return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
351
352 def check_health(self) -> None:
353 """Raises an error if engine is unhealthy."""
vllm/worker/worker_base.py
+30 -1
@@ -281,6 +281,33 @@ class LocalOrDistributedWorkerBase(WorkerBase):
281 # list to conform to interface.
282 return output
283
284 + def _execute_model_spmd(
285 + self, execute_model_req: ExecuteModelRequest
286 + ) -> Optional[List[SamplerOutput]]:
287 + """
288 + Execute model in Single Program Multiple Data (SPMD) fashion.
289 + All workers take the same request, prepare the input and
290 + execute the model.
291 + """
292 + assert execute_model_req is not None, (
293 + "_execute_model_spmd() requires each worker to take in an "
294 + "ExecuteModelRequest")
295 + worker_input: WorkerInput = self.prepare_worker_input(
296 + execute_model_req=execute_model_req)
297 + model_input: ModelRunnerInputBase = (
298 + self.model_runner.prepare_model_input(
299 + execute_model_req.seq_group_metadata_list))
300 +
301 + self.execute_worker(worker_input)
302 +
303 + # If there is no input, we don't need to execute the model.
304 + if worker_input.num_seq_groups == 0:
305 + return []
306 +
307 + return self.model_runner.execute_model(
308 + model_input, self.kv_cache[worker_input.virtual_engine]
309 + if self.kv_cache is not None else None)
310 +
311
312 class WorkerWrapperBase:
313 """
@@ -296,7 +323,7 @@ class WorkerWrapperBase:
323 trust_remote_code: bool = False) -> None:
324 self.worker_module_name = worker_module_name
325 self.worker_class_name = worker_class_name
299 - self.worker = None
326 + self.worker: Optional[WorkerBase] = None
327 if trust_remote_code:
328 # note: lazy import to avoid importing torch before initializing
329 from vllm.utils import init_cached_hf_modules
@@ -323,7 +350,9 @@ class WorkerWrapperBase:
350
351 mod = importlib.import_module(self.worker_module_name)
352 worker_class = getattr(mod, self.worker_class_name)
353 +
354 self.worker = worker_class(*args, **kwargs)
355 + assert self.worker is not None
356
357 def execute_method(self, method, *args, **kwargs):
358 try: