A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[TPU] Use mark_dynamic only for dummy run (#7634)
Woosuk Kwon committed
Aug 18, 2024 at 00:18 UTC
0c2fa50b84dddd4866313ac37074255d29a94055
1 file changed
+28
-48
vllm/worker/tpu_model_runner.py
+28
-48
@@ -144,7 +144,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
144
)
145
model = model.eval()
146
xm.wait_device_ops()
147
- self.model = CompiledModelWrapper(model)
147
+ model = ModelWrapper(model)
148
+ self.model = torch.compile(model,
149
+ backend="openxla",
150
+ fullgraph=True,
151
+ dynamic=False)
152
153
def _dummy_run(
154
self,
@@ -206,9 +210,31 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
210
)
211
t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
212
p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
213
+ num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
214
215
+ # NOTE(woosuk): There are two stages of compilation: torch.compile and
216
+ # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
217
+ # overhead by reusing the FX graph for different shapes.
218
+ # However, the XLA graph will still require static shapes and needs to
219
+ # be re-compiled for every different shapes. This overhead is inevitable
220
+ # in the first run, but can be skipped afterwards as we cache the XLA
221
+ # graphs in the disk (VLLM_XLA_CACHE_PATH).
222
+ if is_prompt:
223
+ # Prefll
224
+ torch._dynamo.mark_dynamic(token_ids, 1)
225
+ torch._dynamo.mark_dynamic(position_ids, 1)
226
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
227
+ else:
228
+ # Decode
229
+ torch._dynamo.mark_dynamic(token_ids, 0)
230
+ torch._dynamo.mark_dynamic(position_ids, 0)
231
+ torch._dynamo.mark_dynamic(input_lens, 0)
232
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
233
+ torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
234
+ torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
235
+ torch._dynamo.mark_dynamic(t, 0)
236
+ torch._dynamo.mark_dynamic(p, 0)
237
# Dummy run.
211
- num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
238
self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
239
num_samples, kv_caches)
240
@@ -682,52 +708,6 @@ class ModelWrapper(nn.Module):
708
return next_token_ids
709
710
685
-class CompiledModelWrapper:
686
-
687
- def __init__(self, model: nn.Module):
688
- model = ModelWrapper(model)
689
- self.model = torch.compile(model,
690
- backend="openxla",
691
- fullgraph=True,
692
- dynamic=False)
693
-
694
- def __call__(
695
- self,
696
- token_ids: torch.Tensor,
697
- position_ids: torch.Tensor,
698
- attn_metadata: AttentionMetadata,
699
- input_lens: torch.Tensor,
700
- t: torch.Tensor,
701
- p: torch.Tensor,
702
- num_samples: int,
703
- kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
704
- ) -> torch.Tensor:
705
- # NOTE(woosuk): There are two stages of compilation: torch.compile and
706
- # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
707
- # overhead by reusing the FX graph for different shapes.
708
- # However, the XLA graph will still require static shapes and needs to
709
- # be re-compiled for every different shapes. This overhead is inevitable
710
- # in the first run, but can be skipped afterwards as we cache the XLA
711
- # graphs in the disk (VLLM_XLA_CACHE_PATH).
712
- if attn_metadata.num_prefills > 0:
713
- # Prefll
714
- torch._dynamo.mark_dynamic(token_ids, 1)
715
- torch._dynamo.mark_dynamic(position_ids, 1)
716
- torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
717
- else:
718
- # Decode
719
- torch._dynamo.mark_dynamic(token_ids, 0)
720
- torch._dynamo.mark_dynamic(position_ids, 0)
721
- torch._dynamo.mark_dynamic(input_lens, 0)
722
- torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
723
- torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
724
- torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
725
- torch._dynamo.mark_dynamic(t, 0)
726
- torch._dynamo.mark_dynamic(p, 0)
727
- return self.model(token_ids, position_ids, attn_metadata, input_lens,
728
- t, p, num_samples, kv_caches)
729
-
730
-
711
def _get_padded_prefill_len(x: int) -> int:
712
# NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
713
# length to be a multiple of 16. We pad the prompt length to the nearest