A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Core] Introduce SPMD worker execution using Ray accelerated DAG (#6032)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Stephanie Wang <swang@cs.berkeley.edu>
Rui Qiao committed
Jul 17, 2024 at 22:27 UTC
61e592747c28c9fbd6861e48b825c796e09da02f
8 files changed
+218
-121
.buildkite/test-pipeline.yaml
+3
@@ -84,6 +84,8 @@ steps:
84
- VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py
85
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
86
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
87
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
88
+ - TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
89
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
90
- TEST_DIST_MODEL=meta-llama/Llama-2-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_chunked_prefill_distributed.py
91
- TEST_DIST_MODEL=llava-hf/llava-1.5-7b-hf DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_multimodal_broadcast.py
@@ -108,6 +110,7 @@ steps:
110
# We want to test that models which use 2 GPUs work with 4 GPUs, which is why we duplicate them here.
111
# See https://github.com/vllm-project/vllm/pull/5473#issuecomment-2166601837 for context.
112
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
113
+ - TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray VLLM_USE_RAY_SPMD_WORKER=1 VLLM_USE_RAY_COMPILED_DAG=1 pytest -v -s distributed/test_basic_distributed_correctness.py
114
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
115
- pytest -v -s spec_decode/e2e/test_integration_dist_tp4.py
116
vllm/engine/llm_engine.py
+5
@@ -6,6 +6,7 @@ from typing import Set, Type, TypeVar, Union
6
7
from transformers import PreTrainedTokenizer
8
9
+import vllm.envs as envs
10
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
11
LoRAConfig, ModelConfig, MultiModalConfig,
12
ObservabilityConfig, ParallelConfig,
@@ -414,6 +415,9 @@ class LLMEngine:
415
elif distributed_executor_backend == "mp":
416
from vllm.executor.multiproc_gpu_executor import (
417
MultiprocessingGPUExecutor)
418
+ assert not envs.VLLM_USE_RAY_SPMD_WORKER, (
419
+ "multiprocessing distributed executor backend does not "
420
+ "support VLLM_USE_RAY_SPMD_WORKER=1")
421
executor_class = MultiprocessingGPUExecutor
422
else:
423
from vllm.executor.gpu_executor import GPUExecutor
@@ -426,6 +430,7 @@ class LLMEngine:
430
usage_context=usage_context,
431
stat_loggers=stat_loggers,
432
)
433
+
434
return engine
435
436
def __reduce__(self):
vllm/envs.py
+8
@@ -34,6 +34,7 @@ if TYPE_CHECKING:
34
VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS: bool = False
35
VLLM_XLA_CACHE_PATH: str = os.path.join(VLLM_CACHE_ROOT, "xla_cache")
36
VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
37
+ VLLM_USE_RAY_SPMD_WORKER: bool = False
38
VLLM_USE_RAY_COMPILED_DAG: bool = False
39
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
40
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
@@ -261,6 +262,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
262
"VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS":
263
lambda: bool(os.getenv("VLLM_OPENVINO_ENABLE_QUANTIZED_WEIGHTS", False)),
264
265
+ # If the env var is set, then all workers will execute as separate
266
+ # processes from the engine, and we use the same mechanism to trigger
267
+ # execution on all workers.
268
+ # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
269
+ "VLLM_USE_RAY_SPMD_WORKER":
270
+ lambda: bool(os.getenv("VLLM_USE_RAY_SPMD_WORKER", 0)),
271
+
272
# If the env var is set, it uses the Ray's compiled DAG API
273
# which optimizes the control plane overhead.
274
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
vllm/executor/distributed_gpu_executor.py
+5
-3
@@ -64,8 +64,8 @@ class DistributedGPUExecutor(GPUExecutor):
64
num_cpu_blocks=num_cpu_blocks)
65
66
def execute_model(
67
- self, execute_model_req: ExecuteModelRequest
68
- ) -> Optional[List[SamplerOutput]]:
67
+ self,
68
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
69
if self.parallel_worker_tasks is None:
70
self.parallel_worker_tasks = self._run_workers(
71
"start_worker_execution_loop",
@@ -73,7 +73,9 @@ class DistributedGPUExecutor(GPUExecutor):
73
**self.extra_execute_model_run_workers_kwargs)
74
75
# Only the driver worker returns the sampling results.
76
- return self._driver_execute_model(execute_model_req)
76
+ driver_outputs = self._driver_execute_model(execute_model_req)
77
+ assert driver_outputs is not None
78
+ return driver_outputs
79
80
def stop_remote_worker_execution_loop(self) -> None:
81
if self.parallel_worker_tasks is None:
vllm/executor/ray_gpu_executor.py
+138
-74
@@ -1,6 +1,5 @@
1
import asyncio
2
import os
3
-import pickle
3
from collections import defaultdict
4
from itertools import islice, repeat
5
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
@@ -23,12 +22,30 @@ if TYPE_CHECKING:
22
23
logger = init_logger(__name__)
24
26
-USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
27
-
25
26
class RayGPUExecutor(DistributedGPUExecutor):
27
28
def _init_executor(self) -> None:
29
+ # If the env var is set, it uses the Ray's compiled DAG API
30
+ # which optimizes the control plane overhead.
31
+ # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
32
+ # Currently, this requires USE_RAY_SPMD_WORKER=True.
33
+ self.use_ray_compiled_dag = envs.VLLM_USE_RAY_COMPILED_DAG
34
+ # If the env var is set, then we do not distinguish between the
35
+ # "driver worker" vs other workers. Also, the rank 0 worker will
36
+ # be executed in a remote Ray worker. Currently this requires
37
+ # USE_RAY_COMPILED_DAG=True.
38
+ self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
39
+ if self.use_ray_compiled_dag:
40
+ assert self.use_ray_spmd_worker, (
41
+ "VLLM_USE_RAY_COMPILED_DAG=1 requires "
42
+ "VLLM_USE_RAY_SPMD_WORKER=1")
43
+ if self.use_ray_spmd_worker:
44
+ # TODO: Support SPMD worker for non-DAG Ray executor.
45
+ assert self.use_ray_compiled_dag, (
46
+ "VLLM_USE_RAY_SPMD_WORKER=1 requires "
47
+ "VLLM_USE_RAY_COMPILED_DAG=1")
48
+
49
assert self.parallel_config.distributed_executor_backend == "ray"
50
placement_group = self.parallel_config.placement_group
51
@@ -40,11 +57,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
57
# Create the parallel GPU workers.
58
self._init_workers_ray(placement_group)
59
43
- self.forward_dag = None
44
- if USE_RAY_COMPILED_DAG:
45
- self.forward_dag = self._compiled_ray_dag()
46
- self.extra_execute_model_run_workers_kwargs[
47
- "use_ray_compiled_dag"] = True
60
+ self.forward_dag: Optional["ray.dag.CompiledDAG"] = None
61
62
def _configure_ray_workers_use_nsight(self,
63
ray_remote_kwargs) -> Dict[str, Any]:
@@ -110,21 +123,24 @@ class RayGPUExecutor(DistributedGPUExecutor):
123
trust_remote_code=self.model_config.trust_remote_code,
124
)
125
113
- worker_ip = ray.get(worker.get_node_ip.remote())
114
- if worker_ip == driver_ip and self.driver_dummy_worker is None:
115
- # If the worker is on the same node as the driver, we use it
116
- # as the resource holder for the driver process.
117
- self.driver_dummy_worker = worker
118
- self.driver_worker = RayWorkerWrapper(
119
- worker_module_name=worker_module_name,
120
- worker_class_name=worker_class_name,
121
- trust_remote_code=self.model_config.trust_remote_code,
122
- )
123
- else:
124
- # Else, added to the list of workers.
126
+ if self.use_ray_spmd_worker:
127
self.workers.append(worker)
126
-
127
- if self.driver_dummy_worker is None:
128
+ else:
129
+ worker_ip = ray.get(worker.get_node_ip.remote())
130
+ if worker_ip == driver_ip and self.driver_dummy_worker is None:
131
+ # If the worker is on the same node as the driver, we use it
132
+ # as the resource holder for the driver process.
133
+ self.driver_dummy_worker = worker
134
+ self.driver_worker = RayWorkerWrapper(
135
+ worker_module_name=worker_module_name,
136
+ worker_class_name=worker_class_name,
137
+ trust_remote_code=self.model_config.trust_remote_code,
138
+ )
139
+ else:
140
+ # Else, added to the list of workers.
141
+ self.workers.append(worker)
142
+
143
+ if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
144
raise ValueError(
145
"Ray does not allocate any GPUs on the driver node. Consider "
146
"adjusting the Ray placement group or running the driver on a "
@@ -254,9 +270,23 @@ class RayGPUExecutor(DistributedGPUExecutor):
270
Passing None will cause the driver to stop the model execution
271
loop running in each of the remote workers.
272
"""
273
+ assert not self.use_ray_spmd_worker, (
274
+ "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
275
return self.driver_worker.execute_method("execute_model",
276
execute_model_req)
277
278
+ def execute_model(
279
+ self,
280
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
281
+ if not self.use_ray_spmd_worker:
282
+ return super().execute_model(execute_model_req)
283
+
284
+ if self.forward_dag is None:
285
+ self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
286
+
287
+ outputs = ray.get(self.forward_dag.execute(execute_model_req))
288
+ return outputs[0]
289
+
290
def _run_workers(
291
self,
292
method: str,
@@ -266,7 +296,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
296
all_kwargs: Optional[List[Dict[str, Any]]] = None,
297
use_dummy_driver: bool = False,
298
max_concurrent_workers: Optional[int] = None,
269
- use_ray_compiled_dag: bool = False,
299
**kwargs,
300
) -> Any:
301
"""Runs the given method on all workers. Can be used in the following
@@ -281,6 +310,10 @@ class RayGPUExecutor(DistributedGPUExecutor):
310
- all_args/all_kwargs: args/kwargs for each worker are specified
311
individually
312
"""
313
+ if self.use_ray_spmd_worker:
314
+ assert not async_run_tensor_parallel_workers_only, (
315
+ "async_run_tensor_parallel_workers_only is not supported for "
316
+ "spmd mode.")
317
318
if max_concurrent_workers:
319
raise NotImplementedError(
@@ -289,71 +322,69 @@ class RayGPUExecutor(DistributedGPUExecutor):
322
count = len(self.workers) if not \
323
async_run_tensor_parallel_workers_only \
324
else len(self.non_driver_workers)
325
+ # If using SPMD worker, all workers are the same, so we should execute
326
+ # the args on all workers. Otherwise, we skip the first worker's args
327
+ # because those args will go to the driver worker.
328
+ first_worker_args_index: int = 0 if self.use_ray_spmd_worker else 1
329
all_worker_args = repeat(args, count) if all_args is None \
293
- else islice(all_args, 1, None)
330
+ else islice(all_args, first_worker_args_index, None)
331
all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
295
- else islice(all_kwargs, 1, None)
296
-
297
- if use_ray_compiled_dag:
298
- # Right now, compiled DAG can only accept a single
299
- # input. TODO(sang): Fix it.
300
- assert self.forward_dag is not None
301
- output_channels = self.forward_dag.execute(1)
302
- ray_worker_outputs = []
303
- else:
304
- # Start the ray workers first.
305
- ray_workers = self.workers
306
- if async_run_tensor_parallel_workers_only:
307
- ray_workers = self.non_driver_workers
308
- ray_worker_outputs = [
309
- worker.execute_method.remote(method, *worker_args,
310
- **worker_kwargs)
311
- for (worker, worker_args, worker_kwargs
312
- ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
313
- ]
332
+ else islice(all_kwargs, first_worker_args_index, None)
333
+
334
+ # Start the ray workers first.
335
+ ray_workers = self.workers
336
+ if async_run_tensor_parallel_workers_only:
337
+ ray_workers = self.non_driver_workers
338
+ ray_worker_outputs = [
339
+ worker.execute_method.remote(method, *worker_args, **worker_kwargs)
340
+ for (worker, worker_args, worker_kwargs
341
+ ) in zip(ray_workers, all_worker_args, all_worker_kwargs)
342
+ ]
343
344
if async_run_tensor_parallel_workers_only:
345
# Just return futures
346
return ray_worker_outputs
347
319
- driver_args = args if all_args is None else all_args[0]
320
- driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
348
+ driver_worker_output = []
349
+ # In SPMD mode, the driver worker is the same as any other worker,
350
+ # so we only explicitly execute on the driver worker if using a
351
+ # non-SPMD worker class.
352
+ if not self.use_ray_spmd_worker:
353
+ driver_args = args if all_args is None else all_args[0]
354
+ driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
355
+
356
+ # Start the driver worker after all the ray workers.
357
+ if not use_dummy_driver:
358
+ driver_worker_output = [
359
+ self.driver_worker.execute_method(method, *driver_args,
360
+ **driver_kwargs)
361
+ ]
362
+ else:
363
+ assert self.driver_dummy_worker is not None
364
+ driver_worker_output = [
365
+ ray.get(
366
+ self.driver_dummy_worker.execute_method.remote(
367
+ method, *driver_args, **driver_kwargs))
368
+ ]
369
322
- # Start the driver worker after all the ray workers.
323
- if not use_dummy_driver:
324
- driver_worker_output = self.driver_worker.execute_method(
325
- method, *driver_args, **driver_kwargs)
326
- else:
327
- assert self.driver_dummy_worker is not None
328
- driver_worker_output = ray.get(
329
- self.driver_dummy_worker.execute_method.remote(
330
- method, *driver_args, **driver_kwargs))
370
# Get the results of the ray workers.
371
if self.workers:
333
- if use_ray_compiled_dag:
334
- try:
335
- ray_worker_outputs = [
336
- pickle.loads(chan.begin_read())
337
- for chan in output_channels
338
- ]
339
- finally:
340
- # Has to call end_read in order to reuse the DAG.
341
- for chan in output_channels:
342
- chan.end_read()
343
- else:
344
- ray_worker_outputs = ray.get(ray_worker_outputs)
372
+ ray_worker_outputs = ray.get(ray_worker_outputs)
373
346
- return [driver_worker_output] + ray_worker_outputs
374
+ return driver_worker_output + ray_worker_outputs
375
376
def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
377
"""Wait for futures returned from _run_workers() with
378
async_run_remote_workers_only to complete."""
379
ray.get(parallel_worker_tasks)
380
353
- def _compiled_ray_dag(self):
381
+ def _compiled_ray_dag(self, enable_asyncio: bool):
382
import pkg_resources
355
- required_version = "2.9"
356
- current_version = pkg_resources.get_distribution("ray").version
383
+ from packaging import version
384
+
385
+ required_version = version.parse("2.32")
386
+ current_version = version.parse(
387
+ pkg_resources.get_distribution("ray").version)
388
if current_version < required_version:
389
raise ValueError(f"Ray version {required_version} or greater is "
390
f"required, but found {current_version}")
@@ -365,23 +396,47 @@ class RayGPUExecutor(DistributedGPUExecutor):
396
# a dummy value for now. It will be fixed soon.
397
with InputNode() as input_data:
398
forward_dag = MultiOutputNode([
368
- worker.execute_model_compiled_dag_remote.
369
- bind( # type: ignore[attr-defined]
399
+ worker.execute_model_spmd.bind( # type: ignore[attr-defined]
400
input_data) for worker in self.workers
401
])
372
- return forward_dag.experimental_compile()
402
+ return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
403
+
404
+ def __del__(self):
405
+ if self.forward_dag is not None:
406
+ self.forward_dag.teardown()
407
+ import ray
408
+ for worker in self.workers:
409
+ ray.kill(worker)
410
411
412
class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
413
414
def __init__(self, *args, **kwargs):
415
super().__init__(*args, **kwargs)
379
- self.driver_exec_method = make_async(self.driver_worker.execute_method)
416
+ self.use_ray_spmd_worker = envs.VLLM_USE_RAY_SPMD_WORKER
417
+ if not self.use_ray_compiled_dag:
418
+ self.driver_exec_method = make_async(
419
+ self.driver_worker.execute_method)
420
+
421
+ async def execute_model_async(
422
+ self,
423
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
424
+ if not self.use_ray_spmd_worker:
425
+ return await super().execute_model_async(execute_model_req)
426
+
427
+ if self.forward_dag is None:
428
+ self.forward_dag = self._compiled_ray_dag(enable_asyncio=True)
429
+
430
+ dag_future = await self.forward_dag.execute_async(execute_model_req)
431
+ outputs = await dag_future
432
+ return outputs[0]
433
434
async def _driver_execute_model_async(
435
self,
436
execute_model_req: Optional[ExecuteModelRequest] = None
437
) -> List[SamplerOutput]:
438
+ assert not self.use_ray_spmd_worker, (
439
+ "driver_worker does not exist for VLLM_USE_RAY_SPMD_WORKER=1")
440
if self.pp_locks is None:
441
# This locks each pipeline parallel stage so multiple virtual
442
# engines can't execute on the same stage at the same time
@@ -415,8 +470,17 @@ class RayGPUExecutorAsync(RayGPUExecutor, DistributedGPUExecutorAsync):
470
return results[-1]
471
472
async def _start_worker_execution_loop(self):
473
+ assert not self.use_ray_spmd_worker, (
474
+ "worker loop is disabled for VLLM_USE_RAY_SPMD_WORKER=1")
475
coros = [
476
worker.execute_method.remote("start_worker_execution_loop")
477
for worker in self.non_driver_workers
478
]
479
return await asyncio.gather(*coros)
480
+
481
+ def __del__(self):
482
+ if self.forward_dag is not None:
483
+ self.forward_dag.teardown()
484
+ import ray
485
+ for worker in self.workers:
486
+ ray.kill(worker)
vllm/executor/ray_utils.py
+8
-6
@@ -1,8 +1,8 @@
1
-import pickle
1
from typing import List, Optional, Tuple
2
3
from vllm.config import ParallelConfig
4
from vllm.logger import init_logger
5
+from vllm.sequence import ExecuteModelRequest
6
from vllm.utils import get_ip, is_hip, is_xpu
7
from vllm.worker.worker_base import WorkerWrapperBase
8
@@ -31,16 +31,18 @@ try:
31
gpu_ids = ray.get_gpu_ids()
32
return node_id, gpu_ids
33
34
- def execute_model_compiled_dag_remote(self, ignored):
35
- """Used only when compiled DAG is enabled."""
34
+ def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
35
+ """Used only when SPMD worker and compiled DAG are both
36
+ enabled."""
37
+ # TODO(swang): This is needed right now because Ray aDAG executes
38
+ # on a background thread, so we need to reset torch's current
39
+ # device.
40
import torch
41
if not self.compiled_dag_cuda_device_set:
42
torch.cuda.set_device(self.worker.device)
43
self.compiled_dag_cuda_device_set = True
44
41
- output = self.worker.execute_model()
42
- output = pickle.dumps(output)
43
- return output
45
+ return self.worker._execute_model_spmd(execute_model_req)
46
47
ray_import_err = None
48
vllm/executor/ray_xpu_executor.py
+21
-37
@@ -1,11 +1,11 @@
1
import asyncio
2
import os
3
-import pickle
3
from collections import defaultdict
4
from itertools import islice, repeat
5
from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
6
Tuple, Union)
7
8
+import vllm.envs as envs
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, MultiModalConfig, ParallelConfig,
11
PromptAdapterConfig, SchedulerConfig,
@@ -30,7 +30,7 @@ logger = init_logger(__name__)
30
# If the env var is set, it uses the Ray's compiled DAG API
31
# which optimizes the control plane overhead.
32
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
33
-USE_RAY_COMPILED_DAG = bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0))
33
+USE_RAY_COMPILED_DAG = envs.VLLM_USE_RAY_COMPILED_DAG
34
35
36
class RayXPUExecutor(DistributedGPUExecutor):
@@ -72,10 +72,9 @@ class RayXPUExecutor(DistributedGPUExecutor):
72
# Create the parallel GPU workers.
73
self._init_workers_ray(placement_group)
74
75
- # Profile the memory usage and initialize the cache.
75
self.forward_dag = None
76
if USE_RAY_COMPILED_DAG:
78
- self.forward_dag = self._compiled_ray_dag()
77
+ self.forward_dag = self._compiled_ray_dag(enable_asyncio=False)
78
79
# This is non-None when the execute model loop is running
80
# in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
@@ -270,7 +269,6 @@ class RayXPUExecutor(DistributedGPUExecutor):
269
all_kwargs: Optional[List[Dict[str, Any]]] = None,
270
use_dummy_driver: bool = False,
271
max_concurrent_workers: Optional[int] = None,
273
- use_ray_compiled_dag: bool = False,
272
**kwargs,
273
) -> Any:
274
"""Runs the given method on all workers. Can be used in the following
@@ -293,26 +291,20 @@ class RayXPUExecutor(DistributedGPUExecutor):
291
all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
292
else islice(all_kwargs, 1, None)
293
296
- if use_ray_compiled_dag:
297
- # Right now, compiled DAG can only accept a single
298
- # input. TODO(sang): Fix it.
299
- assert self.forward_dag is not None
300
- output_channels = self.forward_dag.execute(1)
301
- else:
302
- # Start the ray workers first.
303
- ray_worker_outputs = [
304
- worker.execute_method.remote(method, *worker_args,
305
- **worker_kwargs)
306
- for (worker, worker_args, worker_kwargs
307
- ) in zip(self.workers, all_worker_args, all_worker_kwargs)
308
- ]
294
+ # Start the ray workers first.
295
+ ray_worker_outputs = [
296
+ worker.execute_method.remote(method, *worker_args, **worker_kwargs)
297
+ for (worker, worker_args, worker_kwargs
298
+ ) in zip(self.workers, all_worker_args, all_worker_kwargs)
299
+ ]
300
+
301
if async_run_remote_workers_only:
302
# Just return futures
303
return ray_worker_outputs
304
305
+ driver_worker_output = []
306
driver_args = args if all_args is None else all_args[0]
307
driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
315
-
308
# Start the driver worker after all the ray workers.
309
if not use_dummy_driver:
310
driver_worker_output = self.driver_worker.execute_method(
@@ -324,36 +316,28 @@ class RayXPUExecutor(DistributedGPUExecutor):
316
method, *driver_args, **driver_kwargs))
317
# Get the results of the ray workers.
318
if self.workers:
327
- if use_ray_compiled_dag:
328
- try:
329
- ray_worker_outputs = [
330
- pickle.loads(chan.begin_read())
331
- for chan in output_channels
332
- ]
333
- finally:
334
- # Has to call end_read in order to reuse the DAG.
335
- for chan in output_channels:
336
- chan.end_read()
337
- else:
338
- ray_worker_outputs = ray.get(ray_worker_outputs)
319
+ ray_worker_outputs = ray.get(ray_worker_outputs)
320
340
- return [driver_worker_output] + ray_worker_outputs
321
+ return driver_worker_output + ray_worker_outputs
322
323
def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
324
"""Wait for futures returned from _run_workers() with
325
async_run_remote_workers_only to complete."""
326
ray.get(parallel_worker_tasks)
327
347
- def _compiled_ray_dag(self):
328
+ def _compiled_ray_dag(self, enable_asyncio: bool):
329
import pkg_resources
349
- required_version = "2.9"
350
- current_version = pkg_resources.get_distribution("ray").version
330
+ from packaging import version
331
+
332
+ required_version = version.parse("2.32")
333
+ current_version = version.parse(
334
+ pkg_resources.get_distribution("ray").version)
335
if current_version < required_version:
336
raise ValueError(f"Ray version {required_version} or greater is "
337
f"required, but found {current_version}")
338
339
from ray.dag import InputNode, MultiOutputNode
356
- assert self.parallel_config.worker_use_ray
340
+ assert self.parallel_config.distributed_executor_backend == "ray"
341
342
# Right now, compiled DAG requires at least 1 arg. We send
343
# a dummy value for now. It will be fixed soon.
@@ -363,7 +347,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
347
bind( # type: ignore[attr-defined]
348
input_data) for worker in self.workers
349
])
366
- return forward_dag.experimental_compile()
350
+ return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
351
352
def check_health(self) -> None:
353
"""Raises an error if engine is unhealthy."""
vllm/worker/worker_base.py
+30
-1
@@ -281,6 +281,33 @@ class LocalOrDistributedWorkerBase(WorkerBase):
281
# list to conform to interface.
282
return output
283
284
+ def _execute_model_spmd(
285
+ self, execute_model_req: ExecuteModelRequest
286
+ ) -> Optional[List[SamplerOutput]]:
287
+ """
288
+ Execute model in Single Program Multiple Data (SPMD) fashion.
289
+ All workers take the same request, prepare the input and
290
+ execute the model.
291
+ """
292
+ assert execute_model_req is not None, (
293
+ "_execute_model_spmd() requires each worker to take in an "
294
+ "ExecuteModelRequest")
295
+ worker_input: WorkerInput = self.prepare_worker_input(
296
+ execute_model_req=execute_model_req)
297
+ model_input: ModelRunnerInputBase = (
298
+ self.model_runner.prepare_model_input(
299
+ execute_model_req.seq_group_metadata_list))
300
+
301
+ self.execute_worker(worker_input)
302
+
303
+ # If there is no input, we don't need to execute the model.
304
+ if worker_input.num_seq_groups == 0:
305
+ return []
306
+
307
+ return self.model_runner.execute_model(
308
+ model_input, self.kv_cache[worker_input.virtual_engine]
309
+ if self.kv_cache is not None else None)
310
+
311
312
class WorkerWrapperBase:
313
"""
@@ -296,7 +323,7 @@ class WorkerWrapperBase:
323
trust_remote_code: bool = False) -> None:
324
self.worker_module_name = worker_module_name
325
self.worker_class_name = worker_class_name
299
- self.worker = None
326
+ self.worker: Optional[WorkerBase] = None
327
if trust_remote_code:
328
# note: lazy import to avoid importing torch before initializing
329
from vllm.utils import init_cached_hf_modules
@@ -323,7 +350,9 @@ class WorkerWrapperBase:
350
351
mod = importlib.import_module(self.worker_module_name)
352
worker_class = getattr(mod, self.worker_class_name)
353
+
354
self.worker = worker_class(*args, **kwargs)
355
+ assert self.worker is not None
356
357
def execute_method(self, method, *args, **kwargs):
358
try: