@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bc34937d68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware][TPU] Refactor TPU backend (#5831)

Woosuk Kwon committed Jun 25, 2024 at 15:25 UTC bc34937d68e9715d8416457539fb528301cf6269
3 files changed +65 -32
vllm/executor/tpu_executor.py
+37 -21
@@ -1,4 +1,4 @@
1 -from typing import List, Set, Tuple
1 +from typing import Any, Dict, List, Optional, Set, Tuple
2
3 import torch
4
@@ -26,29 +26,45 @@ class TPUExecutor(ExecutorBase):
26 self.model_config.dtype = torch.bfloat16
27
28 # Instantiate the worker and load the model to the device.
29 - self._init_worker()
30 -
31 - def _init_worker(self):
32 - from vllm.worker.tpu_worker import TPUWorker
29 + self.driver_worker = self._create_worker()
30 + self.driver_worker.init_device()
31 + self.driver_worker.load_model()
32
34 - assert self.parallel_config.world_size == 1, (
35 - "TPUExecutor currently only supports a single TPU chip.")
36 - distributed_init_method = get_distributed_init_method(
37 - get_ip(), get_open_port())
38 - self.driver_worker = TPUWorker(
39 - self.model_config,
40 - self.parallel_config,
41 - self.scheduler_config,
42 - self.device_config,
43 - self.cache_config,
44 - self.load_config,
45 - self.vision_language_config,
46 - local_rank=0,
47 - rank=0,
33 + def _get_worker_kwargs(
34 + self,
35 + local_rank: int = 0,
36 + rank: int = 0,
37 + distributed_init_method: Optional[str] = None,
38 + ) -> Dict[str, Any]:
39 + """Return worker init args for a given rank."""
40 + if distributed_init_method is None:
41 + distributed_init_method = get_distributed_init_method(
42 + get_ip(), get_open_port())
43 + return dict(
44 + model_config=self.model_config,
45 + parallel_config=self.parallel_config,
46 + scheduler_config=self.scheduler_config,
47 + device_config=self.device_config,
48 + cache_config=self.cache_config,
49 + load_config=self.load_config,
50 + local_rank=local_rank,
51 + rank=rank,
52 distributed_init_method=distributed_init_method,
53 + vision_language_config=self.vision_language_config,
54 + is_driver_worker=rank == 0,
55 )
50 - self.driver_worker.init_device()
51 - self.driver_worker.load_model()
56 +
57 + def _create_worker(
58 + self,
59 + local_rank: int = 0,
60 + rank: int = 0,
61 + distributed_init_method: Optional[str] = None,
62 + ):
63 + from vllm.worker.tpu_worker import TPUWorker
64 +
65 + worker = TPUWorker(**self._get_worker_kwargs(local_rank, rank,
66 + distributed_init_method))
67 + return worker
68
69 def initialize_cache(
70 self,
vllm/worker/tpu_model_runner.py
+4
@@ -33,6 +33,7 @@ class TPUModelRunner:
33 cache_config: CacheConfig,
34 load_config: LoadConfig,
35 vision_language_config: Optional[VisionLanguageConfig] = None,
36 + is_driver_worker: bool = False,
37 ):
38 self.model_config = model_config
39 self.parallel_config = parallel_config
@@ -41,6 +42,7 @@ class TPUModelRunner:
42 self.cache_config = cache_config
43 self.load_config = load_config
44 self.vision_language_config = vision_language_config
45 + self.is_driver_worker = is_driver_worker
46
47 self.block_size = self.cache_config.block_size
48 self.max_num_blocks_per_seq = (self.model_config.max_model_len //
@@ -373,6 +375,8 @@ class TPUModelRunner:
375 inputs = self.prepare_inputs(seq_group_metadata_list)
376 next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
377 *inputs[2:])
378 + if not self.is_driver_worker:
379 + return []
380 next_token_ids = next_token_ids.cpu().tolist()
381
382 i = 0
vllm/worker/tpu_worker.py
+24 -11
@@ -34,6 +34,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
34 local_rank: int,
35 rank: int,
36 distributed_init_method: str,
37 + is_driver_worker: bool,
38 ) -> None:
39 self.model_config = model_config
40 self.parallel_config = parallel_config
@@ -45,6 +46,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
46 self.local_rank = local_rank
47 self.rank = rank
48 self.distributed_init_method = distributed_init_method
49 + self.is_driver_worker = is_driver_worker
50
51 assert self.device_config.device_type == "tpu"
52 if self.cache_config.cache_dtype == "auto":
@@ -53,10 +55,14 @@ class TPUWorker(LoraNotSupportedWorkerBase):
55 self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
56 self.cache_config.cache_dtype]
57
56 - self.model_runner = TPUModelRunner(model_config, parallel_config,
57 - scheduler_config, device_config,
58 - cache_config, load_config,
59 - vision_language_config)
58 + self.model_runner = TPUModelRunner(model_config,
59 + parallel_config,
60 + scheduler_config,
61 + device_config,
62 + cache_config,
63 + load_config,
64 + vision_language_config,
65 + is_driver_worker=is_driver_worker)
66
67 def init_device(self) -> None:
68 os.environ["PJRT_DEVICE"] = "TPU"
@@ -175,16 +181,13 @@ class TPUWorker(LoraNotSupportedWorkerBase):
181
182 def execute_model(
183 self,
178 - execute_model_req: Optional[ExecuteModelRequest] = None
184 + execute_model_req: Optional[ExecuteModelRequest] = None,
185 ) -> List[SamplerOutput]:
180 - if execute_model_req is None:
181 - return []
182 -
183 - seq_group_metadata_list = execute_model_req.seq_group_metadata_list
184 - num_seq_groups = len(seq_group_metadata_list)
185 - if num_seq_groups == 0:
186 + if not self.is_driver_worker:
187 + self._execute_model_non_driver()
188 return []
189
190 + assert execute_model_req is not None
191 # Currently, TPUWorker does not support swapping.
192 # TODO(woosuk): Support block copying.
193 assert len(execute_model_req.blocks_to_swap_in) == 0, (
@@ -193,6 +196,16 @@ class TPUWorker(LoraNotSupportedWorkerBase):
196 "Swapping is not supported for the TPU backend.")
197 assert len(execute_model_req.blocks_to_copy) == 0
198
199 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
200 + assert len(seq_group_metadata_list) > 0
201 output = self.model_runner.execute_model(seq_group_metadata_list,
202 self.tpu_cache)
203 return [output]
204 +
205 + def start_worker_execution_loop(self) -> None:
206 + while self._execute_model_non_driver():
207 + pass
208 +
209 + def _execute_model_non_driver(self) -> bool:
210 + self.model_runner.execute_model(None, self.tpu_cache)
211 + return True