@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fad5576c58
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[TPU] Reduce compilation time & Upgrade PyTorch XLA version (#6856)

Woosuk Kwon committed Jul 27, 2024 at 10:28 UTC fad5576c58864a6c2cf528f67e60e03a949b3dac
6 files changed +24 -7
Dockerfile.tpu
+1 -1
@@ -1,4 +1,4 @@
1 -ARG NIGHTLY_DATE="20240713"
1 +ARG NIGHTLY_DATE="20240726"
2 ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
4 FROM $BASE_IMAGE
docs/source/getting_started/tpu-installation.rst
+8 -1
@@ -56,7 +56,7 @@ First, install the dependencies:
56 $ pip uninstall torch torch-xla -y
57
58 $ # Install PyTorch and PyTorch XLA.
59 - $ export DATE="+20240713"
59 + $ export DATE="+20240726"
60 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61 $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62
@@ -75,6 +75,13 @@ Next, build vLLM from source. This will only take a few seconds:
75 $ VLLM_TARGET_DEVICE="tpu" python setup.py develop
76
77
78 +.. note::
79 +
80 + Since TPU relies on XLA which requires static shapes, vLLM bucketizes the possible input shapes and compiles an XLA graph for each different shape.
81 + The compilation time may take 20~30 minutes in the first run.
82 + However, the compilation time reduces to ~5 minutes afterwards because the XLA graphs are cached in the disk (in :code:`VLLM_XLA_CACHE_PATH` or :code:`~/.cache/vllm/xla_cache` by default).
83 +
84 +
85 .. tip::
86
87 If you encounter the following error:
vllm/attention/backends/pallas.py
-1
@@ -3,7 +3,6 @@ from typing import Any, Dict, List, Optional, Tuple, Type
3
4 import torch
5 import torch_xla.experimental.custom_kernel # Required to register custom ops.
6 -import torch_xla.experimental.dynamo_set_buffer_donor
6
7 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8 AttentionMetadata, AttentionType)
vllm/distributed/device_communicators/tpu_communicator.py
+2 -1
@@ -6,6 +6,7 @@ from vllm.platforms import current_platform
6
7 if current_platform.is_tpu():
8 import torch_xla.core.xla_model as xm
9 + import torch_xla.runtime as xr
10 from torch_xla._internal import pjrt
11
12
@@ -20,7 +21,7 @@ class TpuCommunicator:
21 local_rank = dist.get_rank(group)
22 world_size = dist.get_world_size(group)
23 pjrt.initialize_multiprocess(local_rank, world_size)
23 - xm._init_world_size_ordinal()
24 + xr._init_world_size_ordinal()
25
26 def all_reduce(self, x: torch.Tensor) -> torch.Tensor:
27 return xm.all_reduce(xm.REDUCE_SUM, x)
vllm/worker/tpu_model_runner.py
+13 -2
@@ -7,6 +7,7 @@ import numpy as np
7 import torch
8 import torch.nn as nn
9 import torch_xla.core.xla_model as xm
10 +import torch_xla.runtime as xr
11
12 from vllm.attention import AttentionMetadata, get_attn_backend
13 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
@@ -127,7 +128,7 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
128 # determine the order of concatenating the output tensors.
129 # As a workaround, we use the xm's rank assignment only when loading
130 # the embedding weights.
130 - xm_tp_rank = xm.get_ordinal()
131 + xm_tp_rank = xr.global_ordinal()
132 with patch(
133 "vllm.model_executor.layers.vocab_parallel_embedding."
134 "get_tensor_model_parallel_rank",
@@ -146,7 +147,17 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
147 xm.wait_device_ops()
148
149 model = ModelWrapper(model)
149 - self.model = torch.compile(model, backend="openxla", fullgraph=True)
150 + # NOTE(woosuk): There are two stages of compilation: torch.compile and
151 + # XLA compilation. Setting dynamic=True can reduce the torch.compile
152 + # overhead by reusing the FX graph for different shapes.
153 + # However, the XLA graph will still require static shapes and needs to
154 + # be re-compiled for every different shapes. This overhead is inevitable
155 + # in the first run, but can be skipped afterwards as we cache the XLA
156 + # graphs in the disk (VLLM_XLA_CACHE_PATH).
157 + self.model = torch.compile(model,
158 + backend="openxla",
159 + fullgraph=True,
160 + dynamic=True)
161
162 def _dummy_run(
163 self,
vllm/worker/tpu_worker.py
-1
@@ -3,7 +3,6 @@ from typing import List, Optional, Tuple, Union
3
4 import torch
5 import torch_xla.core.xla_model as xm
6 -import torch_xla.experimental.dynamo_set_buffer_donor # noqa: F401
6 import torch_xla.runtime as xr
7
8 import vllm.envs as envs