@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0c2fa50b84
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[TPU] Use mark_dynamic only for dummy run (#7634)

Woosuk Kwon committed Aug 18, 2024 at 00:18 UTC 0c2fa50b84dddd4866313ac37074255d29a94055
1 file changed +28 -48
vllm/worker/tpu_model_runner.py
+28 -48
@@ -144,7 +144,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
144 )
145 model = model.eval()
146 xm.wait_device_ops()
147 - self.model = CompiledModelWrapper(model)
147 + model = ModelWrapper(model)
148 + self.model = torch.compile(model,
149 + backend="openxla",
150 + fullgraph=True,
151 + dynamic=False)
152
153 def _dummy_run(
154 self,
@@ -206,9 +210,31 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
210 )
211 t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
212 p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
213 + num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
214
215 + # NOTE(woosuk): There are two stages of compilation: torch.compile and
216 + # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
217 + # overhead by reusing the FX graph for different shapes.
218 + # However, the XLA graph will still require static shapes and needs to
219 + # be re-compiled for every different shapes. This overhead is inevitable
220 + # in the first run, but can be skipped afterwards as we cache the XLA
221 + # graphs in the disk (VLLM_XLA_CACHE_PATH).
222 + if is_prompt:
223 + # Prefll
224 + torch._dynamo.mark_dynamic(token_ids, 1)
225 + torch._dynamo.mark_dynamic(position_ids, 1)
226 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
227 + else:
228 + # Decode
229 + torch._dynamo.mark_dynamic(token_ids, 0)
230 + torch._dynamo.mark_dynamic(position_ids, 0)
231 + torch._dynamo.mark_dynamic(input_lens, 0)
232 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
233 + torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
234 + torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
235 + torch._dynamo.mark_dynamic(t, 0)
236 + torch._dynamo.mark_dynamic(p, 0)
237 # Dummy run.
211 - num_samples = _MAX_NUM_SAMPLES if is_prompt else 1
238 self.model(token_ids, position_ids, attn_metadata, input_lens, t, p,
239 num_samples, kv_caches)
240
@@ -682,52 +708,6 @@ class ModelWrapper(nn.Module):
708 return next_token_ids
709
710
685 -class CompiledModelWrapper:
686 -
687 - def __init__(self, model: nn.Module):
688 - model = ModelWrapper(model)
689 - self.model = torch.compile(model,
690 - backend="openxla",
691 - fullgraph=True,
692 - dynamic=False)
693 -
694 - def __call__(
695 - self,
696 - token_ids: torch.Tensor,
697 - position_ids: torch.Tensor,
698 - attn_metadata: AttentionMetadata,
699 - input_lens: torch.Tensor,
700 - t: torch.Tensor,
701 - p: torch.Tensor,
702 - num_samples: int,
703 - kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
704 - ) -> torch.Tensor:
705 - # NOTE(woosuk): There are two stages of compilation: torch.compile and
706 - # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
707 - # overhead by reusing the FX graph for different shapes.
708 - # However, the XLA graph will still require static shapes and needs to
709 - # be re-compiled for every different shapes. This overhead is inevitable
710 - # in the first run, but can be skipped afterwards as we cache the XLA
711 - # graphs in the disk (VLLM_XLA_CACHE_PATH).
712 - if attn_metadata.num_prefills > 0:
713 - # Prefll
714 - torch._dynamo.mark_dynamic(token_ids, 1)
715 - torch._dynamo.mark_dynamic(position_ids, 1)
716 - torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
717 - else:
718 - # Decode
719 - torch._dynamo.mark_dynamic(token_ids, 0)
720 - torch._dynamo.mark_dynamic(position_ids, 0)
721 - torch._dynamo.mark_dynamic(input_lens, 0)
722 - torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
723 - torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
724 - torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
725 - torch._dynamo.mark_dynamic(t, 0)
726 - torch._dynamo.mark_dynamic(p, 0)
727 - return self.model(token_ids, position_ids, attn_metadata, input_lens,
728 - t, p, num_samples, kv_caches)
729 -
730 -
711 def _get_padded_prefill_len(x: int) -> int:
712 # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
713 # length to be a multiple of 16. We pad the prompt length to the nearest