A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware][TPU] Refactor TPU backend (#5831)
Woosuk Kwon committed
Jun 25, 2024 at 15:25 UTC
bc34937d68e9715d8416457539fb528301cf6269
3 files changed
+65
-32
vllm/executor/tpu_executor.py
+37
-21
@@ -1,4 +1,4 @@
1
-from typing import List, Set, Tuple
1
+from typing import Any, Dict, List, Optional, Set, Tuple
2
3
import torch
4
@@ -26,29 +26,45 @@ class TPUExecutor(ExecutorBase):
26
self.model_config.dtype = torch.bfloat16
27
28
# Instantiate the worker and load the model to the device.
29
- self._init_worker()
30
-
31
- def _init_worker(self):
32
- from vllm.worker.tpu_worker import TPUWorker
29
+ self.driver_worker = self._create_worker()
30
+ self.driver_worker.init_device()
31
+ self.driver_worker.load_model()
32
34
- assert self.parallel_config.world_size == 1, (
35
- "TPUExecutor currently only supports a single TPU chip.")
36
- distributed_init_method = get_distributed_init_method(
37
- get_ip(), get_open_port())
38
- self.driver_worker = TPUWorker(
39
- self.model_config,
40
- self.parallel_config,
41
- self.scheduler_config,
42
- self.device_config,
43
- self.cache_config,
44
- self.load_config,
45
- self.vision_language_config,
46
- local_rank=0,
47
- rank=0,
33
+ def _get_worker_kwargs(
34
+ self,
35
+ local_rank: int = 0,
36
+ rank: int = 0,
37
+ distributed_init_method: Optional[str] = None,
38
+ ) -> Dict[str, Any]:
39
+ """Return worker init args for a given rank."""
40
+ if distributed_init_method is None:
41
+ distributed_init_method = get_distributed_init_method(
42
+ get_ip(), get_open_port())
43
+ return dict(
44
+ model_config=self.model_config,
45
+ parallel_config=self.parallel_config,
46
+ scheduler_config=self.scheduler_config,
47
+ device_config=self.device_config,
48
+ cache_config=self.cache_config,
49
+ load_config=self.load_config,
50
+ local_rank=local_rank,
51
+ rank=rank,
52
distributed_init_method=distributed_init_method,
53
+ vision_language_config=self.vision_language_config,
54
+ is_driver_worker=rank == 0,
55
)
50
- self.driver_worker.init_device()
51
- self.driver_worker.load_model()
56
+
57
+ def _create_worker(
58
+ self,
59
+ local_rank: int = 0,
60
+ rank: int = 0,
61
+ distributed_init_method: Optional[str] = None,
62
+ ):
63
+ from vllm.worker.tpu_worker import TPUWorker
64
+
65
+ worker = TPUWorker(**self._get_worker_kwargs(local_rank, rank,
66
+ distributed_init_method))
67
+ return worker
68
69
def initialize_cache(
70
self,
vllm/worker/tpu_model_runner.py
+4
@@ -33,6 +33,7 @@ class TPUModelRunner:
33
cache_config: CacheConfig,
34
load_config: LoadConfig,
35
vision_language_config: Optional[VisionLanguageConfig] = None,
36
+ is_driver_worker: bool = False,
37
):
38
self.model_config = model_config
39
self.parallel_config = parallel_config
@@ -41,6 +42,7 @@ class TPUModelRunner:
42
self.cache_config = cache_config
43
self.load_config = load_config
44
self.vision_language_config = vision_language_config
45
+ self.is_driver_worker = is_driver_worker
46
47
self.block_size = self.cache_config.block_size
48
self.max_num_blocks_per_seq = (self.model_config.max_model_len //
@@ -373,6 +375,8 @@ class TPUModelRunner:
375
inputs = self.prepare_inputs(seq_group_metadata_list)
376
next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
377
*inputs[2:])
378
+ if not self.is_driver_worker:
379
+ return []
380
next_token_ids = next_token_ids.cpu().tolist()
381
382
i = 0
vllm/worker/tpu_worker.py
+24
-11
@@ -34,6 +34,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
34
local_rank: int,
35
rank: int,
36
distributed_init_method: str,
37
+ is_driver_worker: bool,
38
) -> None:
39
self.model_config = model_config
40
self.parallel_config = parallel_config
@@ -45,6 +46,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
46
self.local_rank = local_rank
47
self.rank = rank
48
self.distributed_init_method = distributed_init_method
49
+ self.is_driver_worker = is_driver_worker
50
51
assert self.device_config.device_type == "tpu"
52
if self.cache_config.cache_dtype == "auto":
@@ -53,10 +55,14 @@ class TPUWorker(LoraNotSupportedWorkerBase):
55
self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
56
self.cache_config.cache_dtype]
57
56
- self.model_runner = TPUModelRunner(model_config, parallel_config,
57
- scheduler_config, device_config,
58
- cache_config, load_config,
59
- vision_language_config)
58
+ self.model_runner = TPUModelRunner(model_config,
59
+ parallel_config,
60
+ scheduler_config,
61
+ device_config,
62
+ cache_config,
63
+ load_config,
64
+ vision_language_config,
65
+ is_driver_worker=is_driver_worker)
66
67
def init_device(self) -> None:
68
os.environ["PJRT_DEVICE"] = "TPU"
@@ -175,16 +181,13 @@ class TPUWorker(LoraNotSupportedWorkerBase):
181
182
def execute_model(
183
self,
178
- execute_model_req: Optional[ExecuteModelRequest] = None
184
+ execute_model_req: Optional[ExecuteModelRequest] = None,
185
) -> List[SamplerOutput]:
180
- if execute_model_req is None:
181
- return []
182
-
183
- seq_group_metadata_list = execute_model_req.seq_group_metadata_list
184
- num_seq_groups = len(seq_group_metadata_list)
185
- if num_seq_groups == 0:
186
+ if not self.is_driver_worker:
187
+ self._execute_model_non_driver()
188
return []
189
190
+ assert execute_model_req is not None
191
# Currently, TPUWorker does not support swapping.
192
# TODO(woosuk): Support block copying.
193
assert len(execute_model_req.blocks_to_swap_in) == 0, (
@@ -193,6 +196,16 @@ class TPUWorker(LoraNotSupportedWorkerBase):
196
"Swapping is not supported for the TPU backend.")
197
assert len(execute_model_req.blocks_to_copy) == 0
198
199
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
200
+ assert len(seq_group_metadata_list) > 0
201
output = self.model_runner.execute_model(seq_group_metadata_list,
202
self.tpu_cache)
203
return [output]
204
+
205
+ def start_worker_execution_loop(self) -> None:
206
+ while self._execute_model_non_driver():
207
+ pass
208
+
209
+ def _execute_model_non_driver(self) -> bool:
210
+ self.model_runner.execute_model(None, self.tpu_cache)
211
+ return True