A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[TPU] Use mark_dynamic to reduce compilation time (#7340)
Woosuk Kwon committed
Aug 10, 2024 at 18:12 UTC
90bab18f24dce1967282fbb1ebcd2c9aecc67d30
3 files changed
+50
-16
Dockerfile.tpu
+1
-1
@@ -1,4 +1,4 @@
1
-ARG NIGHTLY_DATE="20240726"
1
+ARG NIGHTLY_DATE="20240808"
2
ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
4
FROM $BASE_IMAGE
docs/source/getting_started/tpu-installation.rst
+2
-2
@@ -56,7 +56,7 @@ First, install the dependencies:
56
$ pip uninstall torch torch-xla -y
57
58
$ # Install PyTorch and PyTorch XLA.
59
- $ export DATE="+20240726"
59
+ $ export DATE="+20240808"
60
$ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61
$ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62
@@ -65,7 +65,7 @@ First, install the dependencies:
65
$ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
66
67
$ # Install other build dependencies.
68
- $ pip install packaging aiohttp
68
+ $ pip install -r requirements-tpu.txt
69
70
71
Next, build vLLM from source. This will only take a few seconds:
vllm/worker/tpu_model_runner.py
+47
-13
@@ -147,19 +147,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
147
)
148
model = model.eval()
149
xm.wait_device_ops()
150
-
151
- model = ModelWrapper(model)
152
- # NOTE(woosuk): There are two stages of compilation: torch.compile and
153
- # XLA compilation. Setting dynamic=True can reduce the torch.compile
154
- # overhead by reusing the FX graph for different shapes.
155
- # However, the XLA graph will still require static shapes and needs to
156
- # be re-compiled for every different shapes. This overhead is inevitable
157
- # in the first run, but can be skipped afterwards as we cache the XLA
158
- # graphs in the disk (VLLM_XLA_CACHE_PATH).
159
- self.model = torch.compile(model,
160
- backend="openxla",
161
- fullgraph=True,
162
- dynamic=True)
150
+ self.model = CompiledModelWrapper(model)
151
152
def _dummy_run(
153
self,
@@ -697,6 +685,52 @@ class ModelWrapper(nn.Module):
685
return next_token_ids
686
687
688
+class CompiledModelWrapper:
689
+
690
+ def __init__(self, model: nn.Module):
691
+ model = ModelWrapper(model)
692
+ self.model = torch.compile(model,
693
+ backend="openxla",
694
+ fullgraph=True,
695
+ dynamic=False)
696
+
697
+ def __call__(
698
+ self,
699
+ token_ids: torch.Tensor,
700
+ position_ids: torch.Tensor,
701
+ attn_metadata: AttentionMetadata,
702
+ input_lens: torch.Tensor,
703
+ t: torch.Tensor,
704
+ p: torch.Tensor,
705
+ num_samples: int,
706
+ kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
707
+ ) -> torch.Tensor:
708
+ # NOTE(woosuk): There are two stages of compilation: torch.compile and
709
+ # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
710
+ # overhead by reusing the FX graph for different shapes.
711
+ # However, the XLA graph will still require static shapes and needs to
712
+ # be re-compiled for every different shapes. This overhead is inevitable
713
+ # in the first run, but can be skipped afterwards as we cache the XLA
714
+ # graphs in the disk (VLLM_XLA_CACHE_PATH).
715
+ if attn_metadata.num_prefills > 0:
716
+ # Prefll
717
+ torch._dynamo.mark_dynamic(token_ids, 1)
718
+ torch._dynamo.mark_dynamic(position_ids, 1)
719
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
720
+ else:
721
+ # Decode
722
+ torch._dynamo.mark_dynamic(token_ids, 0)
723
+ torch._dynamo.mark_dynamic(position_ids, 0)
724
+ torch._dynamo.mark_dynamic(input_lens, 0)
725
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
726
+ torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
727
+ torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
728
+ torch._dynamo.mark_dynamic(t, 0)
729
+ torch._dynamo.mark_dynamic(p, 0)
730
+ return self.model(token_ids, position_ids, attn_metadata, input_lens,
731
+ t, p, num_samples, kv_caches)
732
+
733
+
734
def _get_padded_prefill_len(x: int) -> int:
735
# NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
736
# length to be a multiple of 16. We pad the prompt length to the nearest