@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 90bab18f24
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Use mark_dynamic to reduce compilation time (#7340)

Woosuk Kwon committed Aug 10, 2024 at 18:12 UTC 90bab18f24dce1967282fbb1ebcd2c9aecc67d30
3 files changed +50 -16
Dockerfile.tpu
+1 -1
@@ -1,4 +1,4 @@
1 -ARG NIGHTLY_DATE="20240726"
1 +ARG NIGHTLY_DATE="20240808"
2 ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
4 FROM $BASE_IMAGE
docs/source/getting_started/tpu-installation.rst
+2 -2
@@ -56,7 +56,7 @@ First, install the dependencies:
56 $ pip uninstall torch torch-xla -y
57
58 $ # Install PyTorch and PyTorch XLA.
59 - $ export DATE="+20240726"
59 + $ export DATE="+20240808"
60 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62
@@ -65,7 +65,7 @@ First, install the dependencies:
65 $ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
66
67 $ # Install other build dependencies.
68 - $ pip install packaging aiohttp
68 + $ pip install -r requirements-tpu.txt
69
70
71 Next, build vLLM from source. This will only take a few seconds:
vllm/worker/tpu_model_runner.py
+47 -13
@@ -147,19 +147,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
147 )
148 model = model.eval()
149 xm.wait_device_ops()
150 -
151 - model = ModelWrapper(model)
152 - # NOTE(woosuk): There are two stages of compilation: torch.compile and
153 - # XLA compilation. Setting dynamic=True can reduce the torch.compile
154 - # overhead by reusing the FX graph for different shapes.
155 - # However, the XLA graph will still require static shapes and needs to
156 - # be re-compiled for every different shapes. This overhead is inevitable
157 - # in the first run, but can be skipped afterwards as we cache the XLA
158 - # graphs in the disk (VLLM_XLA_CACHE_PATH).
159 - self.model = torch.compile(model,
160 - backend="openxla",
161 - fullgraph=True,
162 - dynamic=True)
150 + self.model = CompiledModelWrapper(model)
151
152 def _dummy_run(
153 self,
@@ -697,6 +685,52 @@ class ModelWrapper(nn.Module):
685 return next_token_ids
686
687
688 +class CompiledModelWrapper:
689 +
690 + def __init__(self, model: nn.Module):
691 + model = ModelWrapper(model)
692 + self.model = torch.compile(model,
693 + backend="openxla",
694 + fullgraph=True,
695 + dynamic=False)
696 +
697 + def __call__(
698 + self,
699 + token_ids: torch.Tensor,
700 + position_ids: torch.Tensor,
701 + attn_metadata: AttentionMetadata,
702 + input_lens: torch.Tensor,
703 + t: torch.Tensor,
704 + p: torch.Tensor,
705 + num_samples: int,
706 + kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
707 + ) -> torch.Tensor:
708 + # NOTE(woosuk): There are two stages of compilation: torch.compile and
709 + # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
710 + # overhead by reusing the FX graph for different shapes.
711 + # However, the XLA graph will still require static shapes and needs to
712 + # be re-compiled for every different shapes. This overhead is inevitable
713 + # in the first run, but can be skipped afterwards as we cache the XLA
714 + # graphs in the disk (VLLM_XLA_CACHE_PATH).
715 + if attn_metadata.num_prefills > 0:
716 + # Prefll
717 + torch._dynamo.mark_dynamic(token_ids, 1)
718 + torch._dynamo.mark_dynamic(position_ids, 1)
719 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
720 + else:
721 + # Decode
722 + torch._dynamo.mark_dynamic(token_ids, 0)
723 + torch._dynamo.mark_dynamic(position_ids, 0)
724 + torch._dynamo.mark_dynamic(input_lens, 0)
725 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
726 + torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
727 + torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
728 + torch._dynamo.mark_dynamic(t, 0)
729 + torch._dynamo.mark_dynamic(p, 0)
730 + return self.model(token_ids, position_ids, attn_metadata, input_lens,
731 + t, p, num_samples, kv_caches)
732 +
733 +
734 def _get_padded_prefill_len(x: int) -> int:
735 # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
736 # length to be a multiple of 16. We pad the prompt length to the nearest