A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Core] Pipeline parallel with Ray ADAG (#6837)
Support pipeline-parallelism with Ray accelerated DAG. Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Aug 2, 2024 at 13:55 UTC
05308891e203329a733bcf29a3452b15b75b5eb4
10 files changed
+199
-77
Dockerfile
+2
@@ -42,6 +42,7 @@ WORKDIR /workspace
42
43
# install build and runtime dependencies
44
COPY requirements-common.txt requirements-common.txt
45
+COPY requirements-adag.txt requirements-adag.txt
46
COPY requirements-cuda.txt requirements-cuda.txt
47
RUN --mount=type=cache,target=/root/.cache/pip \
48
python3 -m pip install -r requirements-cuda.txt
@@ -78,6 +79,7 @@ COPY setup.py setup.py
79
COPY cmake cmake
80
COPY CMakeLists.txt CMakeLists.txt
81
COPY requirements-common.txt requirements-common.txt
82
+COPY requirements-adag.txt requirements-adag.txt
83
COPY requirements-cuda.txt requirements-cuda.txt
84
COPY pyproject.toml pyproject.toml
85
COPY vllm vllm
MANIFEST.in
+1
@@ -1,4 +1,5 @@
1
include LICENSE
2
+include requirements-adag.txt
3
include requirements-common.txt
4
include requirements-cuda.txt
5
include requirements-rocm.txt
requirements-adag.txt
new
+3
@@ -0,0 +1,3 @@
1
+# Dependencies for Ray accelerated DAG
2
+cupy-cuda12x
3
+ray >= 2.32
\ No newline at end of file
requirements-test.txt
+3
@@ -1,3 +1,6 @@
1
+# Needed for Ray accelerated DAG tests
2
+-r requirements-adag.txt
3
+
4
# testing
5
pytest
6
tensorizer>=2.9.0
tests/distributed/test_pipeline_parallel.py
+35
-16
@@ -15,22 +15,31 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
15
16
17
@pytest.mark.parametrize(
18
- "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME, DIST_BACKEND",
19
- [
20
- (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
21
- (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
22
- (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
23
- (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
24
- (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
25
- (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
26
- (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27
- (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
28
- (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
29
- (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30
- ])
31
-@fork_new_process_for_each_test
18
+ ("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, "
19
+ "MODEL_NAME, DIST_BACKEND, USE_RAY_ADAG, USE_RAY_ADAG_NCCL"), [
20
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
21
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
22
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
23
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
24
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
25
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
26
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
27
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
28
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
29
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
30
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
31
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
32
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
33
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
34
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
35
+ (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
36
+ (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
37
+ (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
38
+ (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
39
+ (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
40
+ ])
41
def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
33
- DIST_BACKEND):
42
+ DIST_BACKEND, USE_RAY_ADAG, USE_RAY_ADAG_NCCL):
43
if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
44
pytest.skip("Skipping multi-node pipeline parallel test for "
45
"multiprocessing distributed backend")
@@ -67,8 +76,18 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
76
if EAGER_MODE:
77
pp_args.append("--enforce-eager")
78
tp_args.append("--enforce-eager")
79
+ pp_env = None
80
+ if USE_RAY_ADAG:
81
+ assert DIST_BACKEND == "ray", (
82
+ "Ray ADAG is only supported with Ray distributed backend")
83
+ pp_env = {
84
+ "VLLM_USE_RAY_COMPILED_DAG": "1",
85
+ "VLLM_USE_RAY_SPMD_WORKER": "1",
86
+ "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
87
+ str(int(USE_RAY_ADAG_NCCL)),
88
+ }
89
71
- compare_two_settings(MODEL_NAME, pp_args, tp_args)
90
+ compare_two_settings(MODEL_NAME, pp_args, tp_args, pp_env)
91
92
93
@pytest.mark.parametrize("PP_SIZE, MODEL_NAME", [
tests/utils.py
+25
-6
@@ -7,7 +7,7 @@ import time
7
import warnings
8
from contextlib import contextmanager
9
from pathlib import Path
10
-from typing import Any, Dict, List
10
+from typing import Any, Dict, List, Optional
11
12
import openai
13
import ray
@@ -57,6 +57,7 @@ class RemoteOpenAIServer:
57
model: str,
58
cli_args: List[str],
59
*,
60
+ env_dict: Optional[Dict[str, str]] = None,
61
auto_port: bool = True,
62
) -> None:
63
if auto_port:
@@ -77,6 +78,8 @@ class RemoteOpenAIServer:
78
# the current process might initialize cuda,
79
# to be safe, we should use spawn method
80
env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
81
+ if env_dict is not None:
82
+ env.update(env_dict)
83
self.proc = subprocess.Popen(["vllm", "serve"] + [model] + cli_args,
84
env=env,
85
stdout=sys.stdout,
@@ -89,6 +92,11 @@ class RemoteOpenAIServer:
92
93
def __exit__(self, exc_type, exc_value, traceback):
94
self.proc.terminate()
95
+ try:
96
+ self.proc.wait(3)
97
+ except subprocess.TimeoutExpired:
98
+ # force kill if needed
99
+ self.proc.kill()
100
101
def _wait_for_server(self, *, url: str, timeout: float):
102
# run health check
@@ -127,10 +135,21 @@ class RemoteOpenAIServer:
135
)
136
137
130
-def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
138
+def compare_two_settings(model: str,
139
+ arg1: List[str],
140
+ arg2: List[str],
141
+ env1: Optional[Dict[str, str]] = None,
142
+ env2: Optional[Dict[str, str]] = None):
143
"""
132
- Launch API server with two different sets of arguments and compare the
133
- results of the API calls. The arguments are after the model name.
144
+ Launch API server with two different sets of arguments/environments
145
+ and compare the results of the API calls.
146
+
147
+ Args:
148
+ model: The model to test.
149
+ arg1: The first set of arguments to pass to the API server.
150
+ arg2: The second set of arguments to pass to the API server.
151
+ env1: The first set of environment variables to pass to the API server.
152
+ env2: The second set of environment variables to pass to the API server.
153
"""
154
155
tokenizer = AutoTokenizer.from_pretrained(model)
@@ -138,8 +157,8 @@ def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
157
prompt = "Hello, my name is"
158
token_ids = tokenizer(prompt)["input_ids"]
159
results = []
141
- for args in (arg1, arg2):
142
- with RemoteOpenAIServer(model, args) as server:
160
+ for args, env in ((arg1, env1), (arg2, env2)):
161
+ with RemoteOpenAIServer(model, args, env_dict=env) as server:
162
client = server.get_client()
163
164
# test models list
vllm/envs.py
+10
-2
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
38
VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
39
VLLM_USE_RAY_SPMD_WORKER: bool = False
40
VLLM_USE_RAY_COMPILED_DAG: bool = False
41
+ VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
42
VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
43
VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
44
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -273,13 +274,20 @@ environment_variables: Dict[str, Callable[[], Any]] = {
274
# execution on all workers.
275
# Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
276
"VLLM_USE_RAY_SPMD_WORKER":
276
- lambda: bool(os.getenv("VLLM_USE_RAY_SPMD_WORKER", 0)),
277
+ lambda: bool(int(os.getenv("VLLM_USE_RAY_SPMD_WORKER", "0"))),
278
279
# If the env var is set, it uses the Ray's compiled DAG API
280
# which optimizes the control plane overhead.
281
# Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
282
"VLLM_USE_RAY_COMPILED_DAG":
282
- lambda: bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0)),
283
+ lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG", "0"))),
284
+
285
+ # If the env var is set, it uses NCCL for communication in
286
+ # Ray's compiled DAG. This flag is ignored if
287
+ # VLLM_USE_RAY_COMPILED_DAG is not set.
288
+ "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
289
+ lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
290
+ ),
291
292
# Use dedicated multiprocess context for workers.
293
# Both spawn and fork work
vllm/executor/ray_gpu_executor.py
+92
-45
@@ -105,12 +105,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
105
# The remaining workers are the actual ray actors.
106
self.workers: List[RayWorkerWrapper] = []
107
108
+ # Used in ray compiled DAG: indexed first by PP rank,
109
+ # and then TP rank. In other words, the inner list is
110
+ # the TP group of workers for a PP rank.
111
+ self.pp_tp_workers: List[List[RayWorkerWrapper]] = []
112
+
113
if self.parallel_config.ray_workers_use_nsight:
114
ray_remote_kwargs = self._configure_ray_workers_use_nsight(
115
ray_remote_kwargs)
116
117
+ logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
118
# Create the workers.
119
driver_ip = get_ip()
120
+ logger.info("driver_ip: %s", driver_ip)
121
worker_wrapper_kwargs = self._get_worker_wrapper_args()
122
for bundle_id, bundle in enumerate(placement_group.bundle_specs):
123
if not bundle.get("GPU", 0):
@@ -142,42 +149,49 @@ class RayGPUExecutor(DistributedGPUExecutor):
149
# Else, added to the list of workers.
150
self.workers.append(worker)
151
152
+ logger.debug("workers: %s", self.workers)
153
+ logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
154
if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
155
raise ValueError(
156
"Ray does not allocate any GPUs on the driver node. Consider "
157
"adjusting the Ray placement group or running the driver on a "
158
"GPU node.")
159
160
+ worker_ips = [
161
+ ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
162
+ for worker in self.workers
163
+ ]
164
+ ip_counts: Dict[str, int] = {}
165
+ for ip in worker_ips:
166
+ ip_counts[ip] = ip_counts.get(ip, 0) + 1
167
+
168
+ def sort_by_driver_then_worker_ip(worker):
169
+ """
170
+ Sort the workers based on 3 properties:
171
+ 1. If the worker is on the same node as the driver (vllm engine),
172
+ it should be placed first.
173
+ 2. Then, if the worker is on a node with fewer workers, it should
174
+ be placed first.
175
+ 3. Finally, if the work is on a node with smaller IP address, it
176
+ should be placed first.
177
+ """
178
+ ip = ray.get(worker.get_node_ip.remote())
179
+ return (ip != driver_ip, ip_counts[ip], ip)
180
+
181
+ # After sorting, the workers on the same node will be
182
+ # close to each other, and the workers on the driver
183
+ # node will be placed first.
184
+ self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
185
+
186
# Get the set of GPU IDs used on each node.
187
worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
188
use_dummy_driver=True)
189
155
- # the order in `worker_node_and_gpu_ids` does not necessarily match
156
- # the machine boundaries. We need to make sure that workers in the
157
- # same node are assigned consecutive ranks.
158
- # examples:
159
- # [('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [1]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [2]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [3]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [1]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [2]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [3])] # noqa
160
-
161
- # initialize worker ranks with -1 (unassigned)
162
- worker_ranks = [-1 for x in worker_node_and_gpu_ids]
163
- current_rank = 0
164
- while -1 in worker_ranks:
165
- # whenever we find an unassigned worker, find the node
166
- index = worker_ranks.index(-1)
167
- current_node_id = worker_node_and_gpu_ids[index][0]
168
- # assign ranks to all workers in the same node
169
- for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
170
- if node_id == current_node_id:
171
- worker_ranks[i] = current_rank
172
- current_rank += 1
173
- # with the above example, worker_ranks will be [0, 4, 5, 6, 7, 1, 2, 3]
174
-
190
node_workers = defaultdict(list) # node id -> list of worker ranks
191
node_gpus = defaultdict(list) # node id -> list of gpu ids
192
178
- for worker_rank, (node_id, gpu_ids) in zip(worker_ranks,
179
- worker_node_and_gpu_ids):
180
- node_workers[node_id].append(worker_rank)
193
+ for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
194
+ node_workers[node_id].append(i)
195
# `gpu_ids` can be a list of strings or integers.
196
# convert them to integers for consistency.
197
# NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
@@ -202,16 +216,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
216
self._run_workers("update_environment_variables",
217
all_args=all_args_to_update_environment_variables)
218
205
- if len(node_gpus) == 1:
206
- # in single node case, we don't need to get the IP address.
207
- # the loopback address is sufficient
208
- # NOTE: a node may have several IP addresses, one for each
209
- # network interface. `get_ip()` might return any of them,
210
- # while they might not work for communication inside the node
211
- # if the network setup is complicated. Using the loopback address
212
- # solves this issue, as it always works for communication inside
213
- # the node.
214
- driver_ip = "127.0.0.1"
219
distributed_init_method = get_distributed_init_method(
220
driver_ip, get_open_port())
221
@@ -221,8 +225,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
225
local_rank=node_workers[node_id].index(rank),
226
rank=rank,
227
distributed_init_method=distributed_init_method,
224
- ) for rank, (node_id,
225
- _) in zip(worker_ranks, worker_node_and_gpu_ids)
228
+ ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
229
]
230
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
231
@@ -231,6 +234,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
234
max_concurrent_workers=self.parallel_config.
235
max_parallel_loading_workers)
236
237
+ if self.use_ray_spmd_worker:
238
+ for pp_rank in range(self.parallel_config.pipeline_parallel_size):
239
+ self.pp_tp_workers.append([])
240
+ for tp_rank in range(
241
+ self.parallel_config.tensor_parallel_size):
242
+ # PP=2, TP=4
243
+ # pp_tp_workers = [[0, 1, 2, 3], [4, 5, 6, 7]]
244
+ rank = (pp_rank * self.parallel_config.tensor_parallel_size
245
+ ) + tp_rank
246
+ assert len(self.pp_tp_workers[pp_rank]) == tp_rank
247
+ assert pp_rank < len(self.pp_tp_workers)
248
+ self.pp_tp_workers[pp_rank].append(self.workers[rank])
249
+
250
# This is the list of workers that are rank 0 of each TP group EXCEPT
251
# global rank 0. These are the workers that will broadcast to the
252
# rest of the workers.
@@ -241,9 +257,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
257
self.non_driver_workers: List[RayWorkerWrapper] = []
258
259
# Enforce rank order for correct rank to return final output.
244
- for rank, worker in sorted(zip(worker_ranks[1:], self.workers)):
245
- # We need to skip the driver worker, which we
246
- # do by skipping worker_ranks[0] which is always 0.
260
+ for index, worker in enumerate(self.workers):
261
+ # The driver worker is rank 0 and not in self.workers.
262
+ rank = index + 1
263
if rank % self.parallel_config.tensor_parallel_size == 0:
264
self.tp_driver_workers.append(worker)
265
else:
@@ -376,16 +392,47 @@ class RayGPUExecutor(DistributedGPUExecutor):
392
raise ValueError(f"Ray version {required_version} or greater is "
393
f"required, but found {current_version}")
394
379
- from ray.dag import InputNode, MultiOutputNode
395
assert self.parallel_config.use_ray
396
+ from ray.dag import InputNode, MultiOutputNode
397
+ from ray.experimental.channel.torch_tensor_type import TorchTensorType
398
382
- # Right now, compiled DAG requires at least 1 arg. We send
383
- # a dummy value for now. It will be fixed soon.
399
+ logger.info("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL = %s",
400
+ envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL)
401
with InputNode() as input_data:
385
- forward_dag = MultiOutputNode([
386
- worker.execute_model_spmd.bind( # type: ignore[attr-defined]
387
- input_data) for worker in self.workers
388
- ])
402
+ # Example DAG: PP=2, TP=4
403
+ # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
404
+ # -> 1 -> (ExecuteModelReq, IntermediateOutput) -> 5 -> SamplerOutput # noqa: E501
405
+ # -> 2 -> (ExecuteModelReq, IntermediateOutput) -> 6 -> SamplerOutput # noqa: E501
406
+ # -> 3 -> (ExecuteModelReq, IntermediateOutput) -> 7 -> SamplerOutput # noqa: E501
407
+
408
+ # All workers in the first TP group will take in the
409
+ # ExecuteModelRequest as input.
410
+ outputs = [input_data for _ in self.pp_tp_workers[0]]
411
+ for pp_rank, tp_group in enumerate(self.pp_tp_workers):
412
+ # Each PP worker takes in the output of the previous PP worker,
413
+ # and the TP group executes in SPMD fashion.
414
+ outputs = [
415
+ worker.execute_model_spmd.
416
+ bind( # type: ignore[attr-defined]
417
+ outputs[i]) for i, worker in enumerate(tp_group)
418
+ ]
419
+
420
+ last_pp_rank = len(self.pp_tp_workers) - 1
421
+ if pp_rank < last_pp_rank:
422
+ # Specify how intermediate tensors should be passed
423
+ # between pp stages, no need to specify for the last
424
+ # pp stage.
425
+ transport = "nccl" \
426
+ if envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL \
427
+ else "auto"
428
+ outputs = [
429
+ output.with_type_hint(
430
+ TorchTensorType(transport=transport))
431
+ for output in outputs
432
+ ]
433
+
434
+ forward_dag = MultiOutputNode(outputs)
435
+
436
return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
437
438
def __del__(self):
vllm/executor/ray_utils.py
+24
-6
@@ -1,8 +1,8 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Tuple, Union
2
3
from vllm.config import ParallelConfig
4
from vllm.logger import init_logger
5
-from vllm.sequence import ExecuteModelRequest
5
+from vllm.sequence import ExecuteModelRequest, IntermediateTensors
6
from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
7
from vllm.worker.worker_base import WorkerWrapperBase
8
@@ -31,9 +31,17 @@ try:
31
gpu_ids = ray.get_gpu_ids()
32
return node_id, gpu_ids
33
34
- def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
35
- """Used only when SPMD worker and compiled DAG are both
36
- enabled."""
34
+ def execute_model_spmd(
35
+ self, req_or_tuple: Union[ExecuteModelRequest,
36
+ Tuple[ExecuteModelRequest,
37
+ IntermediateTensors]]):
38
+ """Execute model in SPMD fashion: used only when SPMD worker and
39
+ compiled DAG are both enabled.
40
+
41
+ Args:
42
+ req_or_tuple: The request to execute the model, or a tuple
43
+ containing the request and intermediate tensors.
44
+ """
45
# TODO(swang): This is needed right now because Ray aDAG executes
46
# on a background thread, so we need to reset torch's current
47
# device.
@@ -42,7 +50,17 @@ try:
50
torch.cuda.set_device(self.worker.device)
51
self.compiled_dag_cuda_device_set = True
52
45
- return self.worker._execute_model_spmd(execute_model_req)
53
+ if isinstance(req_or_tuple, tuple):
54
+ execute_model_req, intermediate_tensors = req_or_tuple
55
+ else:
56
+ execute_model_req = req_or_tuple
57
+ intermediate_tensors = None
58
+
59
+ output = self.worker._execute_model_spmd(execute_model_req,
60
+ intermediate_tensors)
61
+ if isinstance(output, IntermediateTensors):
62
+ return execute_model_req, output
63
+ return output
64
65
ray_import_err = None
66
vllm/worker/worker_base.py
+4
-2
@@ -285,7 +285,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
285
return output
286
287
def _execute_model_spmd(
288
- self, execute_model_req: ExecuteModelRequest
288
+ self,
289
+ execute_model_req: ExecuteModelRequest,
290
+ intermediate_tensors: Optional[IntermediateTensors] = None
291
) -> Optional[List[SamplerOutput]]:
292
"""
293
Execute model in Single Program Multiple Data (SPMD) fashion.
@@ -309,7 +311,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
311
312
return self.model_runner.execute_model(
313
model_input, self.kv_cache[worker_input.virtual_engine]
312
- if self.kv_cache is not None else None)
314
+ if self.kv_cache is not None else None, intermediate_tensors)
315
316
317
class WorkerWrapperBase: