A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Hardware][TPU] Implement tensor parallelism with Ray (#5871)
Woosuk Kwon committed
Jul 26, 2024 at 20:54 UTC
52f07e3dec2b76045208f5cfea5670b85a719cc6
6 files changed
+365
-21
requirements-tpu.txt
+1
@@ -4,4 +4,5 @@
4
# Dependencies for TPU
5
# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6
# You can install the dependencies in Dockerfile.tpu.
7
+ray
8
triton # To avoid import errors
vllm/attention/backends/pallas.py
+2
-2
@@ -55,8 +55,8 @@ class PallasMetadata(AttentionMetadata):
55
56
# Currently, input sequences can only contain all prefills
57
# or all decoding.
58
- block_tables: Optional[torch.Tensor]
59
- context_lens: Optional[torch.Tensor]
58
+ block_tables: Optional[torch.Tensor] = None
59
+ context_lens: Optional[torch.Tensor] = None
60
61
@property
62
def prefill_metadata(self) -> Optional["PallasMetadata"]:
vllm/engine/llm_engine.py
+8
-2
@@ -394,8 +394,14 @@ class LLMEngine:
394
from vllm.executor.neuron_executor import NeuronExecutor
395
executor_class = NeuronExecutor
396
elif engine_config.device_config.device_type == "tpu":
397
- from vllm.executor.tpu_executor import TPUExecutor
398
- executor_class = TPUExecutor
397
+ if distributed_executor_backend == "ray":
398
+ initialize_ray_cluster(engine_config.parallel_config)
399
+ from vllm.executor.ray_tpu_executor import RayTPUExecutor
400
+ executor_class = RayTPUExecutor
401
+ else:
402
+ assert distributed_executor_backend is None
403
+ from vllm.executor.tpu_executor import TPUExecutor
404
+ executor_class = TPUExecutor
405
elif engine_config.device_config.device_type == "cpu":
406
from vllm.executor.cpu_executor import CPUExecutor
407
executor_class = CPUExecutor
vllm/executor/ray_tpu_executor.py
new
+313
@@ -0,0 +1,313 @@
1
+import asyncio
2
+import os
3
+from collections import defaultdict
4
+from itertools import islice, repeat
5
+from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Tuple,
6
+ Union)
7
+
8
+import vllm.envs as envs
9
+from vllm.executor.executor_base import ExecutorAsyncBase
10
+from vllm.executor.ray_utils import RayWorkerWrapper, ray
11
+from vllm.executor.tpu_executor import TPUExecutor
12
+from vllm.logger import init_logger
13
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
14
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
15
+ get_vllm_instance_id, make_async)
16
+
17
+if ray is not None:
18
+ from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
19
+
20
+if TYPE_CHECKING:
21
+ from ray.util.placement_group import PlacementGroup
22
+
23
+logger = init_logger(__name__)
24
+
25
+
26
+class RayTPUExecutor(TPUExecutor):
27
+
28
+ def __init__(self, *args, **kwargs):
29
+ # This is non-None when the execute model loop is running
30
+ # in the parallel workers. It's a coroutine in the AsyncLLMEngine case.
31
+ self.parallel_worker_tasks: Optional[Union[Any, Awaitable[Any]]] = None
32
+ # Updated by implementations that require additional args to be passed
33
+ # to the _run_workers execute_model call
34
+ self.extra_execute_model_run_workers_kwargs: Dict[str, Any] = {}
35
+
36
+ super().__init__(*args, **kwargs)
37
+
38
+ def _init_executor(self) -> None:
39
+ assert self.parallel_config.distributed_executor_backend == "ray"
40
+ placement_group = self.parallel_config.placement_group
41
+
42
+ # Disable Ray usage stats collection.
43
+ ray_usage = os.environ.get("RAY_USAGE_STATS_ENABLED", "0")
44
+ if ray_usage != "1":
45
+ os.environ["RAY_USAGE_STATS_ENABLED"] = "0"
46
+
47
+ # Create the parallel TPU workers.
48
+ self._init_workers_ray(placement_group)
49
+
50
+ def _init_workers_ray(self, placement_group: "PlacementGroup",
51
+ **ray_remote_kwargs):
52
+ # The driver dummy worker does not actually use any resources.
53
+ # It holds the resource for the driver worker.
54
+ self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
55
+ # The remaining workers are the actual ray actors.
56
+ self.workers: List[RayWorkerWrapper] = []
57
+
58
+ # Create the workers.
59
+ driver_ip = get_ip()
60
+ for bundle_id, bundle in enumerate(placement_group.bundle_specs):
61
+ if not bundle.get("TPU", 0):
62
+ continue
63
+ scheduling_strategy = PlacementGroupSchedulingStrategy(
64
+ placement_group=placement_group,
65
+ placement_group_capture_child_tasks=True,
66
+ placement_group_bundle_index=bundle_id,
67
+ )
68
+
69
+ assert self.speculative_config is None
70
+ worker_module_name = "vllm.worker.tpu_worker"
71
+ worker_class_name = "TPUWorker"
72
+
73
+ worker = ray.remote(
74
+ num_cpus=0,
75
+ resources={"TPU": 1},
76
+ scheduling_strategy=scheduling_strategy,
77
+ **ray_remote_kwargs,
78
+ )(RayWorkerWrapper).remote(
79
+ worker_module_name=worker_module_name,
80
+ worker_class_name=worker_class_name,
81
+ trust_remote_code=self.model_config.trust_remote_code,
82
+ )
83
+
84
+ worker_ip = ray.get(worker.get_node_ip.remote())
85
+ if worker_ip == driver_ip and self.driver_dummy_worker is None:
86
+ # If the worker is on the same node as the driver, we use it
87
+ # as the resource holder for the driver process.
88
+ self.driver_dummy_worker = worker
89
+ self.driver_worker = RayWorkerWrapper(
90
+ worker_module_name=worker_module_name,
91
+ worker_class_name=worker_class_name,
92
+ trust_remote_code=self.model_config.trust_remote_code,
93
+ )
94
+ else:
95
+ # Else, added to the list of workers.
96
+ self.workers.append(worker)
97
+
98
+ if self.driver_dummy_worker is None:
99
+ raise ValueError(
100
+ "Ray does not allocate any TPUs on the driver node. Consider "
101
+ "adjusting the Ray placement group or running the driver on a "
102
+ "TPU node.")
103
+
104
+ # Get the set of TPU IDs used on each node.
105
+ worker_node_and_gpu_ids = self._run_workers("get_node_and_gpu_ids",
106
+ use_dummy_driver=True)
107
+
108
+ node_workers = defaultdict(list)
109
+ for i, (node_id, _) in enumerate(worker_node_and_gpu_ids):
110
+ node_workers[node_id].append(i)
111
+
112
+ VLLM_INSTANCE_ID = get_vllm_instance_id()
113
+
114
+ # Set environment variables for the driver and workers.
115
+ all_args_to_update_environment_variables = [({
116
+ "VLLM_INSTANCE_ID":
117
+ VLLM_INSTANCE_ID,
118
+ "VLLM_TRACE_FUNCTION":
119
+ str(envs.VLLM_TRACE_FUNCTION),
120
+ }, ) for _ in worker_node_and_gpu_ids]
121
+ self._run_workers("update_environment_variables",
122
+ all_args=all_args_to_update_environment_variables)
123
+
124
+ if len(node_workers) == 1:
125
+ # in single node case, we don't need to get the IP address.
126
+ # the loopback address is sufficient
127
+ # NOTE: a node may have several IP addresses, one for each
128
+ # network interface. `get_ip()` might return any of them,
129
+ # while they might not work for communication inside the node
130
+ # if the network setup is complicated. Using the loopback address
131
+ # solves this issue, as it always works for communication inside
132
+ # the node.
133
+ driver_ip = "127.0.0.1"
134
+ distributed_init_method = get_distributed_init_method(
135
+ driver_ip, get_open_port())
136
+
137
+ # Initialize the actual workers inside worker wrapper.
138
+ init_worker_all_kwargs = [
139
+ self._get_worker_kwargs(
140
+ local_rank=node_workers[node_id].index(rank),
141
+ rank=rank,
142
+ distributed_init_method=distributed_init_method,
143
+ ) for rank, (node_id, _) in enumerate(worker_node_and_gpu_ids)
144
+ ]
145
+ self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
146
+
147
+ self._run_workers("init_device")
148
+ self._run_workers("load_model",
149
+ max_concurrent_workers=self.parallel_config.
150
+ max_parallel_loading_workers)
151
+
152
+ def _driver_execute_model(
153
+ self,
154
+ execute_model_req: Optional[ExecuteModelRequest] = None
155
+ ) -> List[SamplerOutput]:
156
+ """Run execute_model in the driver worker.
157
+
158
+ Passing None will cause the driver to stop the model execution
159
+ loop running in each of the remote workers.
160
+ """
161
+ return self.driver_worker.execute_method("execute_model",
162
+ execute_model_req)
163
+
164
+ def _run_workers(
165
+ self,
166
+ method: str,
167
+ *args,
168
+ async_run_remote_workers_only: bool = False,
169
+ all_args: Optional[List[Tuple[Any, ...]]] = None,
170
+ all_kwargs: Optional[List[Dict[str, Any]]] = None,
171
+ use_dummy_driver: bool = False,
172
+ max_concurrent_workers: Optional[int] = None,
173
+ use_ray_compiled_dag: bool = False,
174
+ **kwargs,
175
+ ) -> Any:
176
+ """Runs the given method on all workers. Can be used in the following
177
+ ways:
178
+
179
+ - async_run_remote_workers_only: If True the method will be run only
180
+ in the remote workers, not the driver worker. It will also be
181
+ run asynchronously and return a list of futures rather than blocking
182
+ on the results.
183
+ - args/kwargs: All workers share the same args/kwargs
184
+ - all_args/all_kwargs: args/kwargs for each worker are specified
185
+ individually
186
+ """
187
+
188
+ if max_concurrent_workers:
189
+ raise NotImplementedError(
190
+ "max_concurrent_workers is not supported yet.")
191
+
192
+ count = len(self.workers)
193
+ all_worker_args = repeat(args, count) if all_args is None \
194
+ else islice(all_args, 1, None)
195
+ all_worker_kwargs = repeat(kwargs, count) if all_kwargs is None \
196
+ else islice(all_kwargs, 1, None)
197
+
198
+ # Start the ray workers first.
199
+ ray_worker_outputs = [
200
+ worker.execute_method.remote(method, *worker_args, **worker_kwargs)
201
+ for (worker, worker_args, worker_kwargs
202
+ ) in zip(self.workers, all_worker_args, all_worker_kwargs)
203
+ ]
204
+
205
+ if async_run_remote_workers_only:
206
+ # Just return futures
207
+ return ray_worker_outputs
208
+
209
+ driver_args = args if all_args is None else all_args[0]
210
+ driver_kwargs = kwargs if all_kwargs is None else all_kwargs[0]
211
+
212
+ # Start the driver worker after all the ray workers.
213
+ if not use_dummy_driver:
214
+ driver_worker_output = self.driver_worker.execute_method(
215
+ method, *driver_args, **driver_kwargs)
216
+ else:
217
+ assert self.driver_dummy_worker is not None
218
+ driver_worker_output = ray.get(
219
+ self.driver_dummy_worker.execute_method.remote(
220
+ method, *driver_args, **driver_kwargs))
221
+ # Get the results of the ray workers.
222
+ if self.workers:
223
+ ray_worker_outputs = ray.get(ray_worker_outputs)
224
+
225
+ return [driver_worker_output] + ray_worker_outputs
226
+
227
+ def _wait_for_tasks_completion(self, parallel_worker_tasks: Any) -> None:
228
+ """Wait for futures returned from _run_workers() with
229
+ async_run_remote_workers_only to complete."""
230
+ ray.get(parallel_worker_tasks)
231
+
232
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
233
+ num_blocks = self._run_workers("determine_num_available_blocks", )
234
+ num_tpu_blocks = min(b[0] for b in num_blocks)
235
+ num_cpu_blocks = min(b[1] for b in num_blocks)
236
+ return num_tpu_blocks, num_cpu_blocks
237
+
238
+ def initialize_cache(self, num_gpu_blocks: int,
239
+ num_cpu_blocks: int) -> None:
240
+ logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
241
+ num_cpu_blocks)
242
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
243
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
244
+ self._run_workers("initialize_cache",
245
+ num_gpu_blocks=num_gpu_blocks,
246
+ num_cpu_blocks=num_cpu_blocks)
247
+
248
+ def execute_model(
249
+ self,
250
+ execute_model_req: ExecuteModelRequest,
251
+ ) -> List[SamplerOutput]:
252
+ if self.parallel_worker_tasks is None:
253
+ self.parallel_worker_tasks = self._run_workers(
254
+ "start_worker_execution_loop",
255
+ async_run_remote_workers_only=True,
256
+ **self.extra_execute_model_run_workers_kwargs)
257
+
258
+ # Only the driver worker returns the sampling results.
259
+ return self._driver_execute_model(execute_model_req)
260
+
261
+ def stop_remote_worker_execution_loop(self) -> None:
262
+ if self.parallel_worker_tasks is None:
263
+ return
264
+
265
+ self._driver_execute_model()
266
+ parallel_worker_tasks = self.parallel_worker_tasks
267
+ self.parallel_worker_tasks = None
268
+ # Ensure that workers exit model loop cleanly
269
+ # (this will raise otherwise)
270
+ self._wait_for_tasks_completion(parallel_worker_tasks)
271
+
272
+
273
+class RayTPUExecutorAsync(RayTPUExecutor, ExecutorAsyncBase):
274
+
275
+ def __init__(self, *args, **kwargs):
276
+ super().__init__(*args, **kwargs)
277
+ self.driver_exec_method = make_async(self.driver_worker.execute_method)
278
+
279
+ async def execute_model_async(
280
+ self,
281
+ execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
282
+ if self.parallel_worker_tasks is None:
283
+ # Start model execution loop running in the parallel workers
284
+ self.parallel_worker_tasks = asyncio.create_task(
285
+ self._start_worker_execution_loop())
286
+
287
+ # Only the driver worker returns the sampling results.
288
+ return await self._driver_execute_model_async(execute_model_req)
289
+
290
+ async def stop_remote_worker_execution_loop_async(self) -> None:
291
+ if self.parallel_worker_tasks is None:
292
+ return
293
+
294
+ await self._driver_execute_model_async()
295
+ parallel_worker_tasks = self.parallel_worker_tasks
296
+ self.parallel_worker_tasks = None
297
+ # Ensure that workers exit model loop cleanly
298
+ # (this will raise otherwise)
299
+ await parallel_worker_tasks
300
+
301
+ async def _driver_execute_model_async(
302
+ self,
303
+ execute_model_req: Optional[ExecuteModelRequest] = None
304
+ ) -> List[SamplerOutput]:
305
+ return await self.driver_exec_method("execute_model",
306
+ execute_model_req)
307
+
308
+ async def _start_worker_execution_loop(self):
309
+ coros = [
310
+ worker.execute_method.remote("start_worker_execution_loop")
311
+ for worker in self.workers
312
+ ]
313
+ return await asyncio.gather(*coros)
vllm/worker/tpu_model_runner.py
+31
-11
@@ -1,6 +1,7 @@
1
import time
2
from dataclasses import dataclass
3
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
4
+from unittest.mock import patch
5
6
import numpy as np
7
import torch
@@ -45,6 +46,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
46
num_samples: int
47
best_of: List[int]
48
seq_groups: List[List[int]]
49
+ virtual_engine: int = 0
50
51
def as_broadcastable_tensor_dict(
52
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -55,6 +57,9 @@ class ModelInputForTPU(ModelRunnerInputBase):
57
"t": self.t,
58
"p": self.p,
59
"num_samples": self.num_samples,
60
+ "best_of": self.best_of,
61
+ "seq_groups": self.seq_groups,
62
+ "virtual_engine": self.virtual_engine,
63
}
64
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
65
return tensor_dict
@@ -113,16 +118,30 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
118
def load_model(self) -> None:
119
self.device = self.device_config.device
120
116
- model = get_model(
117
- model_config=self.model_config,
118
- load_config=self.load_config,
119
- device_config=self.device_config,
120
- parallel_config=self.parallel_config,
121
- cache_config=self.cache_config,
122
- scheduler_config=self.scheduler_config,
123
- multimodal_config=self.multimodal_config,
124
- lora_config=None,
125
- )
121
+ # NOTE(woosuk): While the executor assigns the TP ranks to the worker
122
+ # process, the ranks can be different from the ranks internally assigned
123
+ # by the xm runtime. Therefore, there is a mismatch in the rank
124
+ # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
125
+ # This is not a problem in linear layers because all-reduce is
126
+ # rank-agnostic. However, it matters for all-gather as the ranks
127
+ # determine the order of concatenating the output tensors.
128
+ # As a workaround, we use the xm's rank assignment only when loading
129
+ # the embedding weights.
130
+ xm_tp_rank = xm.get_ordinal()
131
+ with patch(
132
+ "vllm.model_executor.layers.vocab_parallel_embedding."
133
+ "get_tensor_model_parallel_rank",
134
+ return_value=xm_tp_rank):
135
+ model = get_model(
136
+ model_config=self.model_config,
137
+ load_config=self.load_config,
138
+ device_config=self.device_config,
139
+ parallel_config=self.parallel_config,
140
+ cache_config=self.cache_config,
141
+ scheduler_config=self.scheduler_config,
142
+ multimodal_config=self.multimodal_config,
143
+ lora_config=None,
144
+ )
145
model = model.eval()
146
xm.wait_device_ops()
147
@@ -463,10 +482,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
482
tensor_dict, attn_backend=self.attn_backend)
483
return model_input
484
485
+ @torch.no_grad()
486
def execute_model(
487
self,
488
model_input: ModelInputForTPU,
469
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
489
+ kv_caches: Optional[List[Any]],
490
intermediate_tensors: Optional[IntermediateTensors] = None,
491
num_steps: int = 1,
492
) -> List[SamplerOutput]:
vllm/worker/tpu_worker.py
+10
-6
@@ -70,13 +70,13 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
70
71
def init_device(self) -> None:
72
os.environ["PJRT_DEVICE"] = "TPU"
73
- self.device = xm.xla_device()
74
- self.device_config.device = self.device
73
torch.set_grad_enabled(False)
74
torch.set_default_dtype(self.model_config.dtype)
75
78
- # NOTE(woosuk): This is just a hack to initialize the TP group.
79
- # This cannot perform the actual communication ops.
76
+ # NOTE(woosuk): This is just to initialize the TP group and broadcast
77
+ # the input objects on CPU. The all-reduce and all-gather ops on TPU
78
+ # are invoked by `xm.all_reduce` and `xm.all_gather` which use their
79
+ # own context.
80
init_distributed_environment(
81
world_size=self.parallel_config.world_size,
82
rank=self.rank,
@@ -88,6 +88,11 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
88
self.parallel_config.tensor_parallel_size,
89
self.parallel_config.pipeline_parallel_size)
90
91
+ # Device initialization should happen after initializing the distributed
92
+ # runtime.
93
+ self.device = xm.xla_device()
94
+ self.device_config.device = self.device
95
+
96
# Set random seed.
97
set_random_seed(self.model_config.seed)
98
xm.set_rng_state(self.model_config.seed, self.device)
@@ -200,8 +205,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
205
206
@property
207
def do_metadata_broadcast(self) -> bool:
203
- # TODO(woosuk): Support TP.
204
- return False
208
+ return self.parallel_config.tensor_parallel_size > 1
209
210
@property
211
def kv_cache(self) -> Optional[List[List[torch.Tensor]]]: