@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 52f07e3dec
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Hardware][TPU] Implement tensor parallelism with Ray (#5871)

Woosuk Kwon committed Jul 26, 2024 at 20:54 UTC 52f07e3dec2b76045208f5cfea5670b85a719cc6
6 files changed +365 -21
requirements-tpu.txt
+1
@@ -4,4 +4,5 @@
4 # Dependencies for TPU
5 # Currently, the TPU backend uses a nightly version of PyTorch XLA.
6 # You can install the dependencies in Dockerfile.tpu.
7 +ray
8 triton # To avoid import errors
vllm/attention/backends/pallas.py
+2 -2
@@ -55,8 +55,8 @@ class PallasMetadata(AttentionMetadata):
55
56 # Currently, input sequences can only contain all prefills
57 # or all decoding.
58 - block_tables: Optional[torch.Tensor]
59 - context_lens: Optional[torch.Tensor]
58 + block_tables: Optional[torch.Tensor] = None
59 + context_lens: Optional[torch.Tensor] = None
60
61 @property
62 def prefill_metadata(self) -> Optional["PallasMetadata"]:
vllm/engine/llm_engine.py
+8 -2
@@ -394,8 +394,14 @@ class LLMEngine:
394 from vllm.executor.neuron_executor import NeuronExecutor
395 executor_class = NeuronExecutor
396 elif engine_config.device_config.device_type == "tpu":
397 - from vllm.executor.tpu_executor import TPUExecutor
398 - executor_class = TPUExecutor
397 + if distributed_executor_backend == "ray":
398 + initialize_ray_cluster(engine_config.parallel_config)
399 + from vllm.executor.ray_tpu_executor import RayTPUExecutor
400 + executor_class = RayTPUExecutor
401 + else:
402 + assert distributed_executor_backend is None
403 + from vllm.executor.tpu_executor import TPUExecutor
404 + executor_class = TPUExecutor
405 elif engine_config.device_config.device_type == "cpu":
406 from vllm.executor.cpu_executor import CPUExecutor
407 executor_class = CPUExecutor
vllm/executor/ray_tpu_executor.py new
+313
@@ -0,0 +1,313 @@
1 +import asyncio
2 +import os
3 +from collections import defaultdict
4 +from itertools import islice, repeat
5 +from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple,
6 + Union)
7 +
8 +import vllm.envs as envs
9 +from vllm.executor.executor_base import ExecutorAsyncBase
10 +from vllm.executor.ray_utils import RayWorkerWrapper, ray
11 +from vllm.executor.tpu_executor import TPUExecutor
12 +from vllm.logger import init_logger
13 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
14 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
15 + get_vllm_instance_id, make_async)
16 +
17 +if ray is not None:
18 + from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
19 +
20 +if TYPE_CHECKING:
21 + from ray.util.placement_group import PlacementGroup
22 +
23 +logger = init_logger(__name__)
24 +
25 +
26 +class RayTPUExecutor(TPUExecutor):
27 +
28 + def __init__(self, *args, **kwargs):
29 + # This is non-None when the execute model loop is running
30 + # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
31 + self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
32 + # Updated by implementations that require additional args to be passed
33 + # to the _run_workers execute_model call
34 + self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
35 +
36 + super().__init__(*args, **kwargs)
37 +
38 + def _init_executor(self) -> None:
39 + assert self.parallel_config.distributed_executor_backend == "ray"
40 + placement_group = self.parallel_config.placement_group
41 +
42 + # Disable Ray usage stats collection.
43 + ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
44 + if ray_usage != "1":
45 + os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
46 +
47 + # Create the parallel TPU workers.
48 + self._init_workers_ray(placement_group)
49 +
50 + def _init_workers_ray(self, placement_group: "PlacementGroup",
51 + **ray_remote_kwargs):
52 + # The driver dummy worker does not actually use any resources.
53 + # It holds the resource for the driver worker.
54 + self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
55 + # The remaining workers are the actual ray actors.
56 + self.workers: List[RayWorkerWrapper] = []
57 +
58 + # Create the workers.
59 + driver_ip = get_ip()
60 + for bundle_id, bundle in enumerate(placement_group.bundle_specs):
61 + if not bundle.get("TPU", 0):
62 + continue
63 + scheduling_strategy = PlacementGroupSchedulingStrategy(
64 + placement_group=placement_group,
65 + placement_group_capture_child_tasks=True,
66 + placement_group_bundle_index=bundle_id,
67 + )
68 +
69 + assert self.speculative_config is None
70 + worker_module_name = "vllm.worker.tpu_worker"
71 + worker_class_name = "TPUWorker"
72 +
73 + worker = ray.remote(
74 + num_cpus=0,
75 + resources={"TPU": 1},
76 + scheduling_strategy=scheduling_strategy,
77 + **ray_remote_kwargs,
78 + )(RayWorkerWrapper).remote(
79 + worker_module_name=worker_module_name,
80 + worker_class_name=worker_class_name,
81 + trust_remote_code=self.model_config.trust_remote_code,
82 + )
83 +
84 + worker_ip = ray.get(worker.get_node_ip.remote())
85 + if worker_ip == driver_ip and self.driver_dummy_worker is None:
86 + # If the worker is on the same node as the driver, we use it
87 + # as the resource holder for the driver process.
88 + self.driver_dummy_worker = worker
89 + self.driver_worker = RayWorkerWrapper(
90 + worker_module_name=worker_module_name,
91 + worker_class_name=worker_class_name,
92 + trust_remote_code=self.model_config.trust_remote_code,
93 + )
94 + else:
95 + # Else, added to the list of workers.
96 + self.workers.append(worker)
97 +
98 + if self.driver_dummy_worker is None:
99 + raise ValueError(
100 + "Ray does not allocate any TPUs on the driver node. Consider "
101 + "adjusting the Ray placement group or running the driver on a "
102 + "TPU node.")
103 +
104 + # Get the set of TPU IDs used on each node.
105 + worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
106 + use_dummy_driver=True)
107 +
108 + node_workers = defaultdict(list)
109 + for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
110 + node_workers[node_id].append(i)
111 +
112 + VLLM_INSTANCE_ID = get_vllm_instance_id()
113 +
114 + # Set environment variables for the driver and workers.
115 + all_args_to_update_environment_variables = [({
116 + "VLLM_INSTANCE_ID":
117 + VLLM_INSTANCE_ID,
118 + "VLLM_TRACE_FUNCTION":
119 + str(envs.VLLM_TRACE_FUNCTION),
120 + }, ) for _ in worker_node_and_gpu_ids]
121 + self._run_workers("update_environment_variables",
122 + all_args=all_args_to_update_environment_variables)
123 +
124 + if len(node_workers) == 1:
125 + # in single node case, we don't need to get the IP address.
126 + # the loopback address is sufficient
127 + # NOTE: a node may have several IP addresses, one for each
128 + # network interface. `get_ip()` might return any of them,
129 + # while they might not work for communication inside the node
130 + # if the network setup is complicated. Using the loopback address
131 + # solves this issue, as it always works for communication inside
132 + # the node.
133 + driver_ip = "127.0.0.1"
134 + distributed_init_method = get_distributed_init_method(
135 + driver_ip, get_open_port())
136 +
137 + # Initialize the actual workers inside worker wrapper.
138 + init_worker_all_kwargs = [
139 + self._get_worker_kwargs(
140 + local_rank=node_workers[node_id].index(rank),
141 + rank=rank,
142 + distributed_init_method=distributed_init_method,
143 + ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
144 + ]
145 + self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
146 +
147 + self._run_workers("init_device")
148 + self._run_workers("load_model",
149 + max_concurrent_workers=self.parallel_config.
150 + max_parallel_loading_workers)
151 +
152 + def _driver_execute_model(
153 + self,
154 + execute_model_req: Optional[ExecuteModelRequest] = None
155 + ) -> List[SamplerOutput]:
156 + """Run execute_model in the driver worker.
157 +
158 + Passing None will cause the driver to stop the model execution
159 + loop running in each of the remote workers.
160 + """
161 + return self.driver_worker.execute_method("execute_model",
162 + execute_model_req)
163 +
164 + def _run_workers(
165 + self,
166 + method: str,
167 + *args,
168 + async_run_remote_workers_only: bool = False,
169 + all_args: Optional[List[Tuple[Any, ...]]] = None,
170 + all_kwargs: Optional[List[Dict[str, Any]]] = None,
171 + use_dummy_driver: bool = False,
172 + max_concurrent_workers: Optional[int] = None,
173 + use_ray_compiled_dag: bool = False,
174 + **kwargs,
175 + ) -> Any:
176 + """Runs the given method on all workers. Can be used in the following
177 + ways:
178 +
179 + - async_run_remote_workers_only: If True the method will be run only
180 + in the remote workers, not the driver worker. It will also be
181 + run asynchronously and return a list of futures rather than blocking
182 + on the results.
183 + - args/kwargs: All workers share the same args/kwargs
184 + - all_args/all_kwargs: args/kwargs for each worker are specified
185 + individually
186 + """
187 +
188 + if max_concurrent_workers:
189 + raise NotImplementedError(
190 + "max_concurrent_workers is not supported yet.")
191 +
192 + count = len(self.workers)
193 + all_worker_args = repeat(args, count) if all_args is None \
194 + else islice(all_args, 1, None)
195 + all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
196 + else islice(all_kwargs, 1, None)
197 +
198 + # Start the ray workers first.
199 + ray_worker_outputs = [
200 + worker.execute_method.remote(method, *worker_args, **worker_kwargs)
201 + for (worker, worker_args, worker_kwargs
202 + ) in zip(self.workers, all_worker_args, all_worker_kwargs)
203 + ]
204 +
205 + if async_run_remote_workers_only:
206 + # Just return futures
207 + return ray_worker_outputs
208 +
209 + driver_args = args if all_args is None else all_args[0]
210 + driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
211 +
212 + # Start the driver worker after all the ray workers.
213 + if not use_dummy_driver:
214 + driver_worker_output = self.driver_worker.execute_method(
215 + method, *driver_args, **driver_kwargs)
216 + else:
217 + assert self.driver_dummy_worker is not None
218 + driver_worker_output = ray.get(
219 + self.driver_dummy_worker.execute_method.remote(
220 + method, *driver_args, **driver_kwargs))
221 + # Get the results of the ray workers.
222 + if self.workers:
223 + ray_worker_outputs = ray.get(ray_worker_outputs)
224 +
225 + return [driver_worker_output] + ray_worker_outputs
226 +
227 + def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
228 + """Wait for futures returned from _run_workers() with
229 + async_run_remote_workers_only to complete."""
230 + ray.get(parallel_worker_tasks)
231 +
232 + def determine_num_available_blocks(self) -> Tuple[int, int]:
233 + num_blocks = self._run_workers("determine_num_available_blocks", )
234 + num_tpu_blocks = min(b[0] for b in num_blocks)
235 + num_cpu_blocks = min(b[1] for b in num_blocks)
236 + return num_tpu_blocks, num_cpu_blocks
237 +
238 + def initialize_cache(self, num_gpu_blocks: int,
239 + num_cpu_blocks: int) -> None:
240 + logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
241 + num_cpu_blocks)
242 + self.cache_config.num_gpu_blocks = num_gpu_blocks
243 + self.cache_config.num_cpu_blocks = num_cpu_blocks
244 + self._run_workers("initialize_cache",
245 + num_gpu_blocks=num_gpu_blocks,
246 + num_cpu_blocks=num_cpu_blocks)
247 +
248 + def execute_model(
249 + self,
250 + execute_model_req: ExecuteModelRequest,
251 + ) -> List[SamplerOutput]:
252 + if self.parallel_worker_tasks is None:
253 + self.parallel_worker_tasks = self._run_workers(
254 + "start_worker_execution_loop",
255 + async_run_remote_workers_only=True,
256 + **self.extra_execute_model_run_workers_kwargs)
257 +
258 + # Only the driver worker returns the sampling results.
259 + return self._driver_execute_model(execute_model_req)
260 +
261 + def stop_remote_worker_execution_loop(self) -> None:
262 + if self.parallel_worker_tasks is None:
263 + return
264 +
265 + self._driver_execute_model()
266 + parallel_worker_tasks = self.parallel_worker_tasks
267 + self.parallel_worker_tasks = None
268 + # Ensure that workers exit model loop cleanly
269 + # (this will raise otherwise)
270 + self._wait_for_tasks_completion(parallel_worker_tasks)
271 +
272 +
273 +class RayTPUExecutorAsync(RayTPUExecutor, ExecutorAsyncBase):
274 +
275 + def __init__(self, *args, **kwargs):
276 + super().__init__(*args, **kwargs)
277 + self.driver_exec_method = make_async(self.driver_worker.execute_method)
278 +
279 + async def execute_model_async(
280 + self,
281 + execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
282 + if self.parallel_worker_tasks is None:
283 + # Start model execution loop running in the parallel workers
284 + self.parallel_worker_tasks = asyncio.create_task(
285 + self._start_worker_execution_loop())
286 +
287 + # Only the driver worker returns the sampling results.
288 + return await self._driver_execute_model_async(execute_model_req)
289 +
290 + async def stop_remote_worker_execution_loop_async(self) -> None:
291 + if self.parallel_worker_tasks is None:
292 + return
293 +
294 + await self._driver_execute_model_async()
295 + parallel_worker_tasks = self.parallel_worker_tasks
296 + self.parallel_worker_tasks = None
297 + # Ensure that workers exit model loop cleanly
298 + # (this will raise otherwise)
299 + await parallel_worker_tasks
300 +
301 + async def _driver_execute_model_async(
302 + self,
303 + execute_model_req: Optional[ExecuteModelRequest] = None
304 + ) -> List[SamplerOutput]:
305 + return await self.driver_exec_method("execute_model",
306 + execute_model_req)
307 +
308 + async def _start_worker_execution_loop(self):
309 + coros = [
310 + worker.execute_method.remote("start_worker_execution_loop")
311 + for worker in self.workers
312 + ]
313 + return await asyncio.gather(*coros)
vllm/worker/tpu_model_runner.py
+31 -11
@@ -1,6 +1,7 @@
1 import time
2 from dataclasses import dataclass
3 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4 +from unittest.mock import patch
5
6 import numpy as np
7 import torch
@@ -45,6 +46,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
46 num_samples: int
47 best_of: List[int]
48 seq_groups: List[List[int]]
49 + virtual_engine: int = 0
50
51 def as_broadcastable_tensor_dict(
52 self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -55,6 +57,9 @@ class ModelInputForTPU(ModelRunnerInputBase):
57 "t": self.t,
58 "p": self.p,
59 "num_samples": self.num_samples,
60 + "best_of": self.best_of,
61 + "seq_groups": self.seq_groups,
62 + "virtual_engine": self.virtual_engine,
63 }
64 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
65 return tensor_dict
@@ -113,16 +118,30 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
118 def load_model(self) -> None:
119 self.device = self.device_config.device
120
116 - model = get_model(
117 - model_config=self.model_config,
118 - load_config=self.load_config,
119 - device_config=self.device_config,
120 - parallel_config=self.parallel_config,
121 - cache_config=self.cache_config,
122 - scheduler_config=self.scheduler_config,
123 - multimodal_config=self.multimodal_config,
124 - lora_config=None,
125 - )
121 + # NOTE(woosuk): While the executor assigns the TP ranks to the worker
122 + # process, the ranks can be different from the ranks internally assigned
123 + # by the xm runtime. Therefore, there is a mismatch in the rank
124 + # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
125 + # This is not a problem in linear layers because all-reduce is
126 + # rank-agnostic. However, it matters for all-gather as the ranks
127 + # determine the order of concatenating the output tensors.
128 + # As a workaround, we use the xm's rank assignment only when loading
129 + # the embedding weights.
130 + xm_tp_rank = xm.get_ordinal()
131 + with patch(
132 + "vllm.model_executor.layers.vocab_parallel_embedding."
133 + "get_tensor_model_parallel_rank",
134 + return_value=xm_tp_rank):
135 + model = get_model(
136 + model_config=self.model_config,
137 + load_config=self.load_config,
138 + device_config=self.device_config,
139 + parallel_config=self.parallel_config,
140 + cache_config=self.cache_config,
141 + scheduler_config=self.scheduler_config,
142 + multimodal_config=self.multimodal_config,
143 + lora_config=None,
144 + )
145 model = model.eval()
146 xm.wait_device_ops()
147
@@ -463,10 +482,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
482 tensor_dict, attn_backend=self.attn_backend)
483 return model_input
484
485 + @torch.no_grad()
486 def execute_model(
487 self,
488 model_input: ModelInputForTPU,
469 - kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
489 + kv_caches: Optional[List[Any]],
490 intermediate_tensors: Optional[IntermediateTensors] = None,
491 num_steps: int = 1,
492 ) -> List[SamplerOutput]:
vllm/worker/tpu_worker.py
+10 -6
@@ -70,13 +70,13 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
70
71 def init_device(self) -> None:
72 os.environ["PJRT_DEVICE"] = "TPU"
73 - self.device = xm.xla_device()
74 - self.device_config.device = self.device
73 torch.set_grad_enabled(False)
74 torch.set_default_dtype(self.model_config.dtype)
75
78 - # NOTE(woosuk): This is just a hack to initialize the TP group.
79 - # This cannot perform the actual communication ops.
76 + # NOTE(woosuk): This is just to initialize the TP group and broadcast
77 + # the input objects on CPU. The all-reduce and all-gather ops on TPU
78 + # are invoked by `xm.all_reduce` and `xm.all_gather` which use their
79 + # own context.
80 init_distributed_environment(
81 world_size=self.parallel_config.world_size,
82 rank=self.rank,
@@ -88,6 +88,11 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
88 self.parallel_config.tensor_parallel_size,
89 self.parallel_config.pipeline_parallel_size)
90
91 + # Device initialization should happen after initializing the distributed
92 + # runtime.
93 + self.device = xm.xla_device()
94 + self.device_config.device = self.device
95 +
96 # Set random seed.
97 set_random_seed(self.model_config.seed)
98 xm.set_rng_state(self.model_config.seed, self.device)
@@ -200,8 +205,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
205
206 @property
207 def do_metadata_broadcast(self) -> bool:
203 - # TODO(woosuk): Support TP.
204 - return False
208 + return self.parallel_config.tensor_parallel_size > 1
209
210 @property
211 def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: