@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 05308891e2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Core] Pipeline parallel with Ray ADAG (#6837)

Support pipeline-parallelism with Ray accelerated DAG. Signed-off-by: Rui Qiao <ruisearch42@gmail.com>

Rui Qiao committed Aug 2, 2024 at 13:55 UTC 05308891e203329a733bcf29a3452b15b75b5eb4
10 files changed +199 -77
Dockerfile
+2
@@ -42,6 +42,7 @@ WORKDIR /workspace
42
43 # install build and runtime dependencies
44 COPY requirements-common.txt requirements-common.txt
45 +COPY requirements-adag.txt requirements-adag.txt
46 COPY requirements-cuda.txt requirements-cuda.txt
47 RUN --mount=type=cache,target=/root/.cache/pip \
48 python3 -m pip install -r requirements-cuda.txt
@@ -78,6 +79,7 @@ COPY setup.py setup.py
79 COPY cmake cmake
80 COPY CMakeLists.txt CMakeLists.txt
81 COPY requirements-common.txt requirements-common.txt
82 +COPY requirements-adag.txt requirements-adag.txt
83 COPY requirements-cuda.txt requirements-cuda.txt
84 COPY pyproject.toml pyproject.toml
85 COPY vllm vllm
MANIFEST.in
+1
@@ -1,4 +1,5 @@
1 include LICENSE
2 +include requirements-adag.txt
3 include requirements-common.txt
4 include requirements-cuda.txt
5 include requirements-rocm.txt
requirements-adag.txt new
+3
@@ -0,0 +1,3 @@
1 +# Dependencies for Ray accelerated DAG
2 +cupy-cuda12x
3 +ray >= 2.32
\ No newline at end of file
requirements-test.txt
+3
@@ -1,3 +1,6 @@
1 +# Needed for Ray accelerated DAG tests
2 +-r requirements-adag.txt
3 +
4 # testing
5 pytest
6 tensorizer>=2.9.0
tests/distributed/test_pipeline_parallel.py
+35 -16
@@ -15,22 +15,31 @@ VLLM_MULTI_NODE = os.getenv("VLLM_MULTI_NODE", "0") == "1"
15
16
17 @pytest.mark.parametrize(
18 - "TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME, DIST_BACKEND",
19 - [
20 - (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
21 - (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
22 - (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
23 - (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray"),
24 - (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray"),
25 - (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
26 - (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
27 - (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
28 - (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp"),
29 - (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp"),
30 - ])
31 -@fork_new_process_for_each_test
18 + ("TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, "
19 + "MODEL_NAME, DIST_BACKEND, USE_RAY_ADAG, USE_RAY_ADAG_NCCL"), [
20 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
21 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
22 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
23 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
24 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", False, False),
25 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
26 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
27 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
28 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
29 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, False),
30 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
31 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
32 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
33 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
34 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "ray", True, True),
35 + (2, 2, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
36 + (2, 2, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
37 + (1, 3, 0, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
38 + (1, 4, 0, 1, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
39 + (1, 4, 1, 0, "meta-llama/Meta-Llama-3-8B", "mp", False, False),
40 + ])
41 def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
33 - DIST_BACKEND):
42 + DIST_BACKEND, USE_RAY_ADAG, USE_RAY_ADAG_NCCL):
43 if VLLM_MULTI_NODE and DIST_BACKEND == "mp":
44 pytest.skip("Skipping multi-node pipeline parallel test for "
45 "multiprocessing distributed backend")
@@ -67,8 +76,18 @@ def test_compare_tp(TP_SIZE, PP_SIZE, EAGER_MODE, CHUNKED_PREFILL, MODEL_NAME,
76 if EAGER_MODE:
77 pp_args.append("--enforce-eager")
78 tp_args.append("--enforce-eager")
79 + pp_env = None
80 + if USE_RAY_ADAG:
81 + assert DIST_BACKEND == "ray", (
82 + "Ray ADAG is only supported with Ray distributed backend")
83 + pp_env = {
84 + "VLLM_USE_RAY_COMPILED_DAG": "1",
85 + "VLLM_USE_RAY_SPMD_WORKER": "1",
86 + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
87 + str(int(USE_RAY_ADAG_NCCL)),
88 + }
89
71 - compare_two_settings(MODEL_NAME, pp_args, tp_args)
90 + compare_two_settings(MODEL_NAME, pp_args, tp_args, pp_env)
91
92
93 @pytest.mark.parametrize("PP_SIZE, MODEL_NAME", [
tests/utils.py
+25 -6
@@ -7,7 +7,7 @@ import time
7 import warnings
8 from contextlib import contextmanager
9 from pathlib import Path
10 -from typing import Any, Dict, List
10 +from typing import Any, Dict, List, Optional
11
12 import openai
13 import ray
@@ -57,6 +57,7 @@ class RemoteOpenAIServer:
57 model: str,
58 cli_args: List[str],
59 *,
60 + env_dict: Optional[Dict[str, str]] = None,
61 auto_port: bool = True,
62 ) -> None:
63 if auto_port:
@@ -77,6 +78,8 @@ class RemoteOpenAIServer:
78 # the current process might initialize cuda,
79 # to be safe, we should use spawn method
80 env['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
81 + if env_dict is not None:
82 + env.update(env_dict)
83 self.proc = subprocess.Popen(["vllm", "serve"] + [model] + cli_args,
84 env=env,
85 stdout=sys.stdout,
@@ -89,6 +92,11 @@ class RemoteOpenAIServer:
92
93 def __exit__(self, exc_type, exc_value, traceback):
94 self.proc.terminate()
95 + try:
96 + self.proc.wait(3)
97 + except subprocess.TimeoutExpired:
98 + # force kill if needed
99 + self.proc.kill()
100
101 def _wait_for_server(self, *, url: str, timeout: float):
102 # run health check
@@ -127,10 +135,21 @@ class RemoteOpenAIServer:
135 )
136
137
130 -def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
138 +def compare_two_settings(model: str,
139 + arg1: List[str],
140 + arg2: List[str],
141 + env1: Optional[Dict[str, str]] = None,
142 + env2: Optional[Dict[str, str]] = None):
143 """
132 - Launch API server with two different sets of arguments and compare the
133 - results of the API calls. The arguments are after the model name.
144 + Launch API server with two different sets of arguments/environments
145 + and compare the results of the API calls.
146 +
147 + Args:
148 + model: The model to test.
149 + arg1: The first set of arguments to pass to the API server.
150 + arg2: The second set of arguments to pass to the API server.
151 + env1: The first set of environment variables to pass to the API server.
152 + env2: The second set of environment variables to pass to the API server.
153 """
154
155 tokenizer = AutoTokenizer.from_pretrained(model)
@@ -138,8 +157,8 @@ def compare_two_settings(model: str, arg1: List[str], arg2: List[str]):
157 prompt = "Hello, my name is"
158 token_ids = tokenizer(prompt)["input_ids"]
159 results = []
141 - for args in (arg1, arg2):
142 - with RemoteOpenAIServer(model, args) as server:
160 + for args, env in ((arg1, env1), (arg2, env2)):
161 + with RemoteOpenAIServer(model, args, env_dict=env) as server:
162 client = server.get_client()
163
164 # test models list
vllm/envs.py
+10 -2
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
38 VLLM_FUSED_MOE_CHUNK_SIZE: int = 64 * 1024
39 VLLM_USE_RAY_SPMD_WORKER: bool = False
40 VLLM_USE_RAY_COMPILED_DAG: bool = False
41 + VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL: bool = True
42 VLLM_WORKER_MULTIPROC_METHOD: str = "fork"
43 VLLM_ASSETS_CACHE: str = os.path.join(VLLM_CACHE_ROOT, "assets")
44 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -273,13 +274,20 @@ environment_variables: Dict[str, Callable[[], Any]] = {
274 # execution on all workers.
275 # Run vLLM with VLLM_USE_RAY_SPMD_WORKER=1 to enable it.
276 "VLLM_USE_RAY_SPMD_WORKER":
276 - lambda: bool(os.getenv("VLLM_USE_RAY_SPMD_WORKER", 0)),
277 + lambda: bool(int(os.getenv("VLLM_USE_RAY_SPMD_WORKER", "0"))),
278
279 # If the env var is set, it uses the Ray's compiled DAG API
280 # which optimizes the control plane overhead.
281 # Run vLLM with VLLM_USE_RAY_COMPILED_DAG=1 to enable it.
282 "VLLM_USE_RAY_COMPILED_DAG":
282 - lambda: bool(os.getenv("VLLM_USE_RAY_COMPILED_DAG", 0)),
283 + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG", "0"))),
284 +
285 + # If the env var is set, it uses NCCL for communication in
286 + # Ray's compiled DAG. This flag is ignored if
287 + # VLLM_USE_RAY_COMPILED_DAG is not set.
288 + "VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL":
289 + lambda: bool(int(os.getenv("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL", "1"))
290 + ),
291
292 # Use dedicated multiprocess context for workers.
293 # Both spawn and fork work
vllm/executor/ray_gpu_executor.py
+92 -45
@@ -105,12 +105,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
105 # The remaining workers are the actual ray actors.
106 self.workers: List[RayWorkerWrapper] = []
107
108 + # Used in ray compiled DAG: indexed first by PP rank,
109 + # and then TP rank. In other words, the inner list is
110 + # the TP group of workers for a PP rank.
111 + self.pp_tp_workers: List[List[RayWorkerWrapper]] = []
112 +
113 if self.parallel_config.ray_workers_use_nsight:
114 ray_remote_kwargs = self._configure_ray_workers_use_nsight(
115 ray_remote_kwargs)
116
117 + logger.info("use_ray_spmd_worker: %s", self.use_ray_spmd_worker)
118 # Create the workers.
119 driver_ip = get_ip()
120 + logger.info("driver_ip: %s", driver_ip)
121 worker_wrapper_kwargs = self._get_worker_wrapper_args()
122 for bundle_id, bundle in enumerate(placement_group.bundle_specs):
123 if not bundle.get("GPU", 0):
@@ -142,42 +149,49 @@ class RayGPUExecutor(DistributedGPUExecutor):
149 # Else, added to the list of workers.
150 self.workers.append(worker)
151
152 + logger.debug("workers: %s", self.workers)
153 + logger.debug("driver_dummy_worker: %s", self.driver_dummy_worker)
154 if not self.use_ray_spmd_worker and self.driver_dummy_worker is None:
155 raise ValueError(
156 "Ray does not allocate any GPUs on the driver node. Consider "
157 "adjusting the Ray placement group or running the driver on a "
158 "GPU node.")
159
160 + worker_ips = [
161 + ray.get(worker.get_node_ip.remote()) # type: ignore[attr-defined]
162 + for worker in self.workers
163 + ]
164 + ip_counts: Dict[str, int] = {}
165 + for ip in worker_ips:
166 + ip_counts[ip] = ip_counts.get(ip, 0) + 1
167 +
168 + def sort_by_driver_then_worker_ip(worker):
169 + """
170 + Sort the workers based on 3 properties:
171 + 1. If the worker is on the same node as the driver (vllm engine),
172 + it should be placed first.
173 + 2. Then, if the worker is on a node with fewer workers, it should
174 + be placed first.
175 + 3. Finally, if the work is on a node with smaller IP address, it
176 + should be placed first.
177 + """
178 + ip = ray.get(worker.get_node_ip.remote())
179 + return (ip != driver_ip, ip_counts[ip], ip)
180 +
181 + # After sorting, the workers on the same node will be
182 + # close to each other, and the workers on the driver
183 + # node will be placed first.
184 + self.workers = sorted(self.workers, key=sort_by_driver_then_worker_ip)
185 +
186 # Get the set of GPU IDs used on each node.
187 worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
188 use_dummy_driver=True)
189
155 - # the order in `worker_node_and_gpu_ids` does not necessarily match
156 - # the machine boundaries. We need to make sure that workers in the
157 - # same node are assigned consecutive ranks.
158 - # examples:
159 - # [('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [0]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [1]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [2]), ('dfaad7adfdae57a694cc74490db45bd112c9f31243523e43ddc2e7f0', [3]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [1]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [2]), ('852a09a13c7503ef126d7c828454c741494b1be33a8627a5206604d9', [3])] # noqa
160 -
161 - # initialize worker ranks with -1 (unassigned)
162 - worker_ranks = [-1 for x in worker_node_and_gpu_ids]
163 - current_rank = 0
164 - while -1 in worker_ranks:
165 - # whenever we find an unassigned worker, find the node
166 - index = worker_ranks.index(-1)
167 - current_node_id = worker_node_and_gpu_ids[index][0]
168 - # assign ranks to all workers in the same node
169 - for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
170 - if node_id == current_node_id:
171 - worker_ranks[i] = current_rank
172 - current_rank += 1
173 - # with the above example, worker_ranks will be [0, 4, 5, 6, 7, 1, 2, 3]
174 -
190 node_workers = defaultdict(list) # node id -> list of worker ranks
191 node_gpus = defaultdict(list) # node id -> list of gpu ids
192
178 - for worker_rank, (node_id, gpu_ids) in zip(worker_ranks,
179 - worker_node_and_gpu_ids):
180 - node_workers[node_id].append(worker_rank)
193 + for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids):
194 + node_workers[node_id].append(i)
195 # `gpu_ids` can be a list of strings or integers.
196 # convert them to integers for consistency.
197 # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs),
@@ -202,16 +216,6 @@ class RayGPUExecutor(DistributedGPUExecutor):
216 self._run_workers("update_environment_variables",
217 all_args=all_args_to_update_environment_variables)
218
205 - if len(node_gpus) == 1:
206 - # in single node case, we don't need to get the IP address.
207 - # the loopback address is sufficient
208 - # NOTE: a node may have several IP addresses, one for each
209 - # network interface. `get_ip()` might return any of them,
210 - # while they might not work for communication inside the node
211 - # if the network setup is complicated. Using the loopback address
212 - # solves this issue, as it always works for communication inside
213 - # the node.
214 - driver_ip = "127.0.0.1"
219 distributed_init_method = get_distributed_init_method(
220 driver_ip, get_open_port())
221
@@ -221,8 +225,7 @@ class RayGPUExecutor(DistributedGPUExecutor):
225 local_rank=node_workers[node_id].index(rank),
226 rank=rank,
227 distributed_init_method=distributed_init_method,
224 - ) for rank, (node_id,
225 - _) in zip(worker_ranks, worker_node_and_gpu_ids)
228 + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
229 ]
230 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
231
@@ -231,6 +234,19 @@ class RayGPUExecutor(DistributedGPUExecutor):
234 max_concurrent_workers=self.parallel_config.
235 max_parallel_loading_workers)
236
237 + if self.use_ray_spmd_worker:
238 + for pp_rank in range(self.parallel_config.pipeline_parallel_size):
239 + self.pp_tp_workers.append([])
240 + for tp_rank in range(
241 + self.parallel_config.tensor_parallel_size):
242 + # PP=2, TP=4
243 + # pp_tp_workers = [[0, 1, 2, 3], [4, 5, 6, 7]]
244 + rank = (pp_rank * self.parallel_config.tensor_parallel_size
245 + ) + tp_rank
246 + assert len(self.pp_tp_workers[pp_rank]) == tp_rank
247 + assert pp_rank < len(self.pp_tp_workers)
248 + self.pp_tp_workers[pp_rank].append(self.workers[rank])
249 +
250 # This is the list of workers that are rank 0 of each TP group EXCEPT
251 # global rank 0. These are the workers that will broadcast to the
252 # rest of the workers.
@@ -241,9 +257,9 @@ class RayGPUExecutor(DistributedGPUExecutor):
257 self.non_driver_workers: List[RayWorkerWrapper] = []
258
259 # Enforce rank order for correct rank to return final output.
244 - for rank, worker in sorted(zip(worker_ranks[1:], self.workers)):
245 - # We need to skip the driver worker, which we
246 - # do by skipping worker_ranks[0] which is always 0.
260 + for index, worker in enumerate(self.workers):
261 + # The driver worker is rank 0 and not in self.workers.
262 + rank = index + 1
263 if rank % self.parallel_config.tensor_parallel_size == 0:
264 self.tp_driver_workers.append(worker)
265 else:
@@ -376,16 +392,47 @@ class RayGPUExecutor(DistributedGPUExecutor):
392 raise ValueError(f"Ray version {required_version} or greater is "
393 f"required, but found {current_version}")
394
379 - from ray.dag import InputNode, MultiOutputNode
395 assert self.parallel_config.use_ray
396 + from ray.dag import InputNode, MultiOutputNode
397 + from ray.experimental.channel.torch_tensor_type import TorchTensorType
398
382 - # Right now, compiled DAG requires at least 1 arg. We send
383 - # a dummy value for now. It will be fixed soon.
399 + logger.info("VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL = %s",
400 + envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL)
401 with InputNode() as input_data:
385 - forward_dag = MultiOutputNode([
386 - worker.execute_model_spmd.bind( # type: ignore[attr-defined]
387 - input_data) for worker in self.workers
388 - ])
402 + # Example DAG: PP=2, TP=4
403 + # (ExecuteModelReq, None) -> 0 -> (ExecuteModelReq, IntermediateOutput) -> 4 -> SamplerOutput # noqa: E501
404 + # -> 1 -> (ExecuteModelReq, IntermediateOutput) -> 5 -> SamplerOutput # noqa: E501
405 + # -> 2 -> (ExecuteModelReq, IntermediateOutput) -> 6 -> SamplerOutput # noqa: E501
406 + # -> 3 -> (ExecuteModelReq, IntermediateOutput) -> 7 -> SamplerOutput # noqa: E501
407 +
408 + # All workers in the first TP group will take in the
409 + # ExecuteModelRequest as input.
410 + outputs = [input_data for _ in self.pp_tp_workers[0]]
411 + for pp_rank, tp_group in enumerate(self.pp_tp_workers):
412 + # Each PP worker takes in the output of the previous PP worker,
413 + # and the TP group executes in SPMD fashion.
414 + outputs = [
415 + worker.execute_model_spmd.
416 + bind( # type: ignore[attr-defined]
417 + outputs[i]) for i, worker in enumerate(tp_group)
418 + ]
419 +
420 + last_pp_rank = len(self.pp_tp_workers) - 1
421 + if pp_rank < last_pp_rank:
422 + # Specify how intermediate tensors should be passed
423 + # between pp stages, no need to specify for the last
424 + # pp stage.
425 + transport = "nccl" \
426 + if envs.VLLM_USE_RAY_COMPILED_DAG_NCCL_CHANNEL \
427 + else "auto"
428 + outputs = [
429 + output.with_type_hint(
430 + TorchTensorType(transport=transport))
431 + for output in outputs
432 + ]
433 +
434 + forward_dag = MultiOutputNode(outputs)
435 +
436 return forward_dag.experimental_compile(enable_asyncio=enable_asyncio)
437
438 def __del__(self):
vllm/executor/ray_utils.py
+24 -6
@@ -1,8 +1,8 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Tuple, Union
2
3 from vllm.config import ParallelConfig
4 from vllm.logger import init_logger
5 -from vllm.sequence import ExecuteModelRequest
5 +from vllm.sequence import ExecuteModelRequest, IntermediateTensors
6 from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
7 from vllm.worker.worker_base import WorkerWrapperBase
8
@@ -31,9 +31,17 @@ try:
31 gpu_ids = ray.get_gpu_ids()
32 return node_id, gpu_ids
33
34 - def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
35 - """Used only when SPMD worker and compiled DAG are both
36 - enabled."""
34 + def execute_model_spmd(
35 + self, req_or_tuple: Union[ExecuteModelRequest,
36 + Tuple[ExecuteModelRequest,
37 + IntermediateTensors]]):
38 + """Execute model in SPMD fashion: used only when SPMD worker and
39 + compiled DAG are both enabled.
40 +
41 + Args:
42 + req_or_tuple: The request to execute the model, or a tuple
43 + containing the request and intermediate tensors.
44 + """
45 # TODO(swang): This is needed right now because Ray aDAG executes
46 # on a background thread, so we need to reset torch's current
47 # device.
@@ -42,7 +50,17 @@ try:
50 torch.cuda.set_device(self.worker.device)
51 self.compiled_dag_cuda_device_set = True
52
45 - return self.worker._execute_model_spmd(execute_model_req)
53 + if isinstance(req_or_tuple, tuple):
54 + execute_model_req, intermediate_tensors = req_or_tuple
55 + else:
56 + execute_model_req = req_or_tuple
57 + intermediate_tensors = None
58 +
59 + output = self.worker._execute_model_spmd(execute_model_req,
60 + intermediate_tensors)
61 + if isinstance(output, IntermediateTensors):
62 + return execute_model_req, output
63 + return output
64
65 ray_import_err = None
66
vllm/worker/worker_base.py
+4 -2
@@ -285,7 +285,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
285 return output
286
287 def _execute_model_spmd(
288 - self, execute_model_req: ExecuteModelRequest
288 + self,
289 + execute_model_req: ExecuteModelRequest,
290 + intermediate_tensors: Optional[IntermediateTensors] = None
291 ) -> Optional[List[SamplerOutput]]:
292 """
293 Execute model in Single Program Multiple Data (SPMD) fashion.
@@ -309,7 +311,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
311
312 return self.model_runner.execute_model(
313 model_input, self.kv_cache[worker_input.virtual_engine]
312 - if self.kv_cache is not None else None)
314 + if self.kv_cache is not None else None, intermediate_tensors)
315
316
317 class WorkerWrapperBase: