A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Hardware] Initial TPU integration (#5292)
Woosuk Kwon committed
Jun 12, 2024 at 11:53 UTC
1a8bfd92d5f35d638e3cfc8c4cd1779aeda0adfb
22 files changed
+1322
-28
Dockerfile.tpu
new
+19
@@ -0,0 +1,19 @@
1
+ARG NIGHTLY_DATE="20240601"
2
+ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3
+
4
+FROM $BASE_IMAGE
5
+
6
+WORKDIR /workspace
7
+COPY . /workspace/vllm
8
+
9
+ENV VLLM_TARGET_DEVICE="tpu"
10
+# Install aiohttp separately to avoid build errors.
11
+RUN pip install aiohttp
12
+# Install the TPU and Pallas dependencies.
13
+RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
14
+RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
15
+
16
+# Build vLLM.
17
+RUN cd /workspace/vllm && python setup.py develop
18
+
19
+CMD ["/bin/bash"]
benchmarks/benchmark_latency.py
+1
-1
@@ -189,7 +189,7 @@ if __name__ == '__main__':
189
"--device",
190
type=str,
191
default="cuda",
192
- choices=["cuda", "cpu"],
192
+ choices=["cuda", "cpu", "tpu"],
193
help='device type for vLLM execution, supporting CUDA and CPU.')
194
parser.add_argument('--block-size',
195
type=int,
benchmarks/benchmark_throughput.py
+1
-1
@@ -346,7 +346,7 @@ if __name__ == "__main__":
346
"--device",
347
type=str,
348
default="cuda",
349
- choices=["cuda", "cpu"],
349
+ choices=["cuda", "cpu", "tpu"],
350
help='device type for vLLM execution, supporting CUDA and CPU.')
351
parser.add_argument(
352
"--enable-prefix-caching",
docs/source/getting_started/tpu-installation.rst
new
+75
@@ -0,0 +1,75 @@
1
+.. _installation_tpu:
2
+
3
+Installation with TPU
4
+=====================
5
+
6
+vLLM supports Google Cloud TPUs using PyTorch XLA.
7
+
8
+Requirements
9
+------------
10
+
11
+* Google Cloud TPU VM (single host)
12
+* TPU versions: v5e, v5p, v4
13
+* Python: 3.10
14
+
15
+Installation options:
16
+
17
+1. :ref:`Build a docker image with Dockerfile <build_docker_tpu>`.
18
+2. :ref:`Build from source <build_from_source_tpu>`.
19
+
20
+.. _build_docker_tpu:
21
+
22
+Build a docker image with :code:`Dockerfile.tpu`
23
+------------------------------------------------
24
+
25
+`Dockerfile.tpu <https://github.com/vllm-project/vllm/blob/main/Dockerfile.tpu>`_ is provided to build a docker image with TPU support.
26
+
27
+.. code-block:: console
28
+
29
+ $ docker build -f Dockerfile.tpu -t vllm-tpu .
30
+
31
+
32
+You can run the docker image with the following command:
33
+
34
+.. code-block:: console
35
+
36
+ $ # Make sure to add `--privileged --net host --shm-size=16G`.
37
+ $ docker run --privileged --net host --shm-size=16G -it vllm-tpu
38
+
39
+
40
+.. _build_from_source_tpu:
41
+
42
+Build from source
43
+-----------------
44
+
45
+You can also build and install the TPU backend from source.
46
+
47
+First, install the dependencies:
48
+
49
+.. code-block:: console
50
+
51
+ $ # (Recommended) Create a new conda environment.
52
+ $ conda create -n myenv python=3.10 -y
53
+ $ conda activate myenv
54
+
55
+ $ # Clean up the existing torch and torch-xla packages.
56
+ $ pip uninstall torch torch-xla -y
57
+
58
+ $ # Install PyTorch and PyTorch XLA.
59
+ $ export DATE="+20240601"
60
+ $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61
+ $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62
+
63
+ $ # Install JAX and Pallas.
64
+ $ pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
65
+ $ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
66
+
67
+ $ # Install other build dependencies.
68
+ $ pip install packaging aiohttp
69
+
70
+
71
+Next, build vLLM from source. This will only take a few seconds:
72
+
73
+.. code-block:: console
74
+
75
+ $ VLLM_TARGET_DEVICE="tpu" python setup.py develop
docs/source/index.rst
+2
-1
@@ -63,8 +63,9 @@ Documentation
63
64
getting_started/installation
65
getting_started/amd-installation
66
- getting_started/neuron-installation
66
getting_started/cpu-installation
67
+ getting_started/neuron-installation
68
+ getting_started/tpu-installation
69
getting_started/quickstart
70
getting_started/debugging
71
getting_started/examples/examples_index
requirements-tpu.txt
new
+7
@@ -0,0 +1,7 @@
1
+# Common dependencies
2
+-r requirements-common.txt
3
+
4
+# Dependencies for TPU
5
+# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6
+# You can install the dependencies in Dockerfile.tpu.
7
+triton # To avoid import errors
setup.py
+17
-5
@@ -206,9 +206,9 @@ class cmake_build_ext(build_ext):
206
207
208
def _is_cuda() -> bool:
209
- return VLLM_TARGET_DEVICE == "cuda" \
210
- and torch.version.cuda is not None \
211
- and not _is_neuron()
209
+ has_cuda = torch.version.cuda is not None
210
+ return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
211
+ and not (_is_neuron() or _is_tpu()))
212
213
214
def _is_hip() -> bool:
@@ -225,10 +225,18 @@ def _is_neuron() -> bool:
225
return torch_neuronx_installed or VLLM_TARGET_DEVICE == "neuron"
226
227
228
+def _is_tpu() -> bool:
229
+ return VLLM_TARGET_DEVICE == "tpu"
230
+
231
+
232
def _is_cpu() -> bool:
233
return VLLM_TARGET_DEVICE == "cpu"
234
235
236
+def _build_custom_ops() -> bool:
237
+ return _is_cuda() or _is_hip() or _is_cpu()
238
+
239
+
240
def _install_punica() -> bool:
241
return envs.VLLM_INSTALL_PUNICA_KERNELS
242
@@ -325,6 +333,8 @@ def get_vllm_version() -> str:
333
if neuron_version != MAIN_CUDA_VERSION:
334
neuron_version_str = neuron_version.replace(".", "")[:3]
335
version += f"+neuron{neuron_version_str}"
336
+ elif _is_tpu():
337
+ version += "+tpu"
338
elif _is_cpu():
339
version += "+cpu"
340
else:
@@ -372,6 +382,8 @@ def get_requirements() -> List[str]:
382
requirements = _read_requirements("requirements-rocm.txt")
383
elif _is_neuron():
384
requirements = _read_requirements("requirements-neuron.txt")
385
+ elif _is_tpu():
386
+ requirements = _read_requirements("requirements-tpu.txt")
387
elif _is_cpu():
388
requirements = _read_requirements("requirements-cpu.txt")
389
else:
@@ -385,7 +397,7 @@ ext_modules = []
397
if _is_cuda() or _is_hip():
398
ext_modules.append(CMakeExtension(name="vllm._moe_C"))
399
388
-if not _is_neuron():
400
+if _build_custom_ops():
401
ext_modules.append(CMakeExtension(name="vllm._C"))
402
403
if _install_punica():
@@ -428,6 +440,6 @@ setup(
440
extras_require={
441
"tensorizer": ["tensorizer>=2.9.0"],
442
},
431
- cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
443
+ cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
444
package_data=package_data,
445
)
vllm/attention/backends/pallas.py
new
+232
@@ -0,0 +1,232 @@
1
+from dataclasses import dataclass
2
+from typing import Any, Dict, List, Optional, Tuple, Type
3
+
4
+import torch
5
+import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
+import torch_xla.experimental.dynamo_set_buffer_donor
7
+
8
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
+ AttentionMetadata)
10
+
11
+
12
+class PallasAttentionBackend(AttentionBackend):
13
+
14
+ @staticmethod
15
+ def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
16
+ return PallasAttentionBackendImpl
17
+
18
+ @staticmethod
19
+ def make_metadata(*args, **kwargs) -> "PallasMetadata":
20
+ return PallasMetadata(*args, **kwargs)
21
+
22
+ @staticmethod
23
+ def get_kv_cache_shape(
24
+ num_blocks: int,
25
+ block_size: int,
26
+ num_kv_heads: int,
27
+ head_size: int,
28
+ ) -> Tuple[int, ...]:
29
+ return (num_kv_heads, num_blocks, block_size, head_size)
30
+
31
+ @staticmethod
32
+ def swap_blocks(
33
+ src_kv_cache: torch.Tensor,
34
+ dst_kv_cache: torch.Tensor,
35
+ src_to_dst: Dict[int, int],
36
+ ) -> None:
37
+ raise NotImplementedError("swap_blocks is not implemented.")
38
+
39
+ @staticmethod
40
+ def copy_blocks(
41
+ kv_caches: List[torch.Tensor],
42
+ src_to_dists: Dict[int, List[int]],
43
+ ) -> None:
44
+ # TODO(woosuk): Implement this.
45
+ raise NotImplementedError("copy_blocks is not implemented.")
46
+
47
+
48
+@dataclass
49
+class PallasMetadata(AttentionMetadata):
50
+
51
+ # Currently, input sequences can only contain all prefills
52
+ # or all decoding.
53
+ block_tables: Optional[torch.Tensor]
54
+ context_lens: Optional[torch.Tensor]
55
+
56
+ @property
57
+ def prefill_metadata(self) -> Optional["PallasMetadata"]:
58
+ if self.num_prefills == 0:
59
+ return None
60
+
61
+ assert self.num_decode_tokens == 0
62
+ assert self.block_tables is None
63
+ assert self.context_lens is None
64
+ return self
65
+
66
+ @property
67
+ def decode_metadata(self) -> Optional["PallasMetadata"]:
68
+ if self.num_decode_tokens == 0:
69
+ return None
70
+
71
+ assert self.num_prefills == 0
72
+ assert self.num_prefill_tokens == 0
73
+ assert self.block_tables is not None
74
+ assert self.context_lens is not None
75
+ return self
76
+
77
+
78
+class PallasAttentionBackendImpl(AttentionImpl):
79
+
80
+ def __init__(
81
+ self,
82
+ num_heads: int,
83
+ head_size: int,
84
+ scale: float,
85
+ num_kv_heads: int,
86
+ alibi_slopes: Optional[List[float]],
87
+ sliding_window: Optional[int],
88
+ kv_cache_dtype: str,
89
+ blocksparse_params: Optional[Dict[str, Any]] = None,
90
+ ) -> None:
91
+ self.num_heads = num_heads
92
+ self.head_size = head_size
93
+ self.scale = float(scale)
94
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
95
+
96
+ assert self.num_heads % self.num_kv_heads == 0
97
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
98
+ if head_size % 128 != 0:
99
+ raise NotImplementedError("Head size must be a multiple of 128.")
100
+ if alibi_slopes is not None:
101
+ raise NotImplementedError("Alibi slopes is not supported.")
102
+ if sliding_window is not None:
103
+ raise NotImplementedError("Sliding window is not supported.")
104
+ if kv_cache_dtype != "auto":
105
+ raise NotImplementedError("FP8 KV cache dtype is not supported.")
106
+ if blocksparse_params is not None:
107
+ raise NotImplementedError("Blocksparse is not supported.")
108
+
109
+ if torch_xla.tpu.version() < 4:
110
+ raise NotImplementedError("TPU version must be 4 or higher.")
111
+
112
+ self.megacore_mode = None
113
+ tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
114
+ if not tpu_type.endswith("lite"):
115
+ if self.num_kv_heads % 2 == 0:
116
+ self.megacore_mode = "kv_head"
117
+ else:
118
+ # NOTE(woosuk): If the batch size is not a multiple of 2, the
119
+ # megacore mode will be None.
120
+ self.megacore_mode = "batch"
121
+
122
+ def forward(
123
+ self,
124
+ query: torch.Tensor,
125
+ key: torch.Tensor,
126
+ value: torch.Tensor,
127
+ kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
128
+ attn_metadata: PallasMetadata,
129
+ kv_scale: float = 1.0,
130
+ ) -> torch.Tensor:
131
+ """Forward pass with Pallas attention.
132
+
133
+ Args:
134
+ query: shape = [batch_size, seq_len, num_heads * head_size]
135
+ key: shape = [batch_size, seq_len, num_kv_heads * head_size]
136
+ value: shape = [batch_size, seq_len, num_kv_heads * head_size]
137
+ key_cache = [num_kv_heads, num_blocks, block_size, head_size]
138
+ value_cache = [num_kv_heads, num_blocks, block_size, head_size]
139
+ attn_metadata: Metadata for attention.
140
+ Returns:
141
+ shape = [batch_size, seq_len, num_heads * head_size]
142
+ """
143
+ assert kv_scale == 1.0
144
+ batch_size, seq_len, hidden_size = query.shape
145
+ query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
146
+ key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
147
+ value = value.view(batch_size, seq_len, self.num_kv_heads,
148
+ self.head_size)
149
+
150
+ if kv_cache[0] is not None:
151
+ slot_mapping = attn_metadata.slot_mapping
152
+ key_cache, value_cache = kv_cache
153
+ write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
154
+
155
+ query = query * self.scale
156
+ if attn_metadata.num_prefills > 0:
157
+ assert seq_len % 16 == 0, (
158
+ "Pallas FlashAttention kernel requires seq_len to be a "
159
+ f"multiple of 16 but got {seq_len}")
160
+
161
+ # Handle GQA/MQA.
162
+ if self.num_kv_heads != self.num_heads:
163
+ key = key.repeat_interleave(self.num_queries_per_kv, dim=-2)
164
+ key = key.view(batch_size, seq_len, self.num_heads,
165
+ self.head_size)
166
+ value = value.repeat_interleave(self.num_queries_per_kv,
167
+ dim=-2)
168
+ value = value.view(batch_size, seq_len, self.num_heads,
169
+ self.head_size)
170
+ # FlashAttention requires [batch_size, num_heads, seq_len, d_model]
171
+ # while the input is [batch_size, seq_len, num_heads, d_model].
172
+ # Permute the input to match the required format.
173
+ output = torch.ops.xla.flash_attention(
174
+ query.permute(0, 2, 1, 3),
175
+ key.permute(0, 2, 1, 3),
176
+ value.permute(0, 2, 1, 3),
177
+ True,
178
+ )
179
+ output = output.permute(0, 2, 1, 3)
180
+ else:
181
+ # Decoding run.
182
+ assert kv_cache is not None
183
+
184
+ pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
185
+ if self.megacore_mode == "batch" and batch_size % 2 != 0:
186
+ megacore_mode = None
187
+ else:
188
+ megacore_mode = self.megacore_mode
189
+
190
+ # NOTE(woosuk): A temporary workaround to avoid the error:
191
+ # "xla::paged_attention() Expected a value of type 'str' for
192
+ # argument 'megacore_mode' but instead found type 'NoneType'."
193
+ if megacore_mode is not None:
194
+ output = torch.ops.xla.paged_attention(
195
+ query.squeeze(dim=1),
196
+ key_cache,
197
+ value_cache,
198
+ attn_metadata.context_lens,
199
+ attn_metadata.block_tables,
200
+ pages_per_compute_block,
201
+ megacore_mode=megacore_mode,
202
+ )
203
+ else:
204
+ output = torch.ops.xla.paged_attention(
205
+ query.squeeze(dim=1),
206
+ key_cache,
207
+ value_cache,
208
+ attn_metadata.context_lens,
209
+ attn_metadata.block_tables,
210
+ pages_per_compute_block,
211
+ )
212
+
213
+ # Reshape the output tensor.
214
+ return output.reshape(batch_size, seq_len, hidden_size)
215
+
216
+
217
+def write_to_kv_cache(
218
+ key: torch.Tensor,
219
+ value: torch.Tensor,
220
+ key_cache: torch.Tensor,
221
+ value_cache: torch.Tensor,
222
+ slot_mapping: torch.Tensor,
223
+) -> None:
224
+ torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True)
225
+ torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True)
226
+
227
+ key = key.flatten(0, 2)
228
+ value = value.flatten(0, 2)
229
+ key_cache = key_cache.flatten(0, 2)
230
+ value_cache = value_cache.flatten(0, 2)
231
+ key_cache.index_copy_(0, slot_mapping, key)
232
+ value_cache.index_copy_(0, slot_mapping, value)
vllm/attention/selector.py
+11
-2
@@ -7,7 +7,7 @@ import torch
7
import vllm.envs as envs
8
from vllm.attention.backends.abstract import AttentionBackend
9
from vllm.logger import init_logger
10
-from vllm.utils import is_cpu, is_hip
10
+from vllm.utils import is_cpu, is_hip, is_tpu
11
12
logger = init_logger(__name__)
13
@@ -18,6 +18,7 @@ class _Backend(enum.Enum):
18
ROCM_FLASH = enum.auto()
19
TORCH_SDPA = enum.auto()
20
FLASHINFER = enum.auto()
21
+ PALLAS = enum.auto()
22
23
24
@lru_cache(maxsize=None)
@@ -66,6 +67,10 @@ def get_attn_backend(
67
"Please make sure --enforce-eager is set.")
68
from vllm.attention.backends.flashinfer import FlashInferBackend
69
return FlashInferBackend
70
+ elif backend == _Backend.PALLAS:
71
+ logger.info("Using Pallas backend.")
72
+ from vllm.attention.backends.pallas import PallasAttentionBackend
73
+ return PallasAttentionBackend
74
else:
75
raise ValueError("Invalid attention backend.")
76
@@ -80,7 +85,6 @@ def which_attn_to_use(
85
block_size: int,
86
) -> _Backend:
87
"""Returns which flash attention backend to use."""
83
-
88
# Default case.
89
selected_backend = _Backend.FLASH_ATTN
90
@@ -100,6 +104,11 @@ def which_attn_to_use(
104
logger.info("Cannot use %s backend on CPU.", selected_backend)
105
return _Backend.TORCH_SDPA
106
107
+ if is_tpu():
108
+ if selected_backend != _Backend.PALLAS:
109
+ logger.info("Cannot use %s backend on TPU.", selected_backend)
110
+ return _Backend.PALLAS
111
+
112
if is_hip():
113
# AMD GPUs.
114
selected_backend = (_Backend.ROCM_FLASH if selected_backend
vllm/config.py
+5
-1
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
11
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12
from vllm.model_executor.models import ModelRegistry
13
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
-from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14
+from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron, is_tpu
15
16
if TYPE_CHECKING:
17
from ray.util.placement_group import PlacementGroup
@@ -748,6 +748,8 @@ class DeviceConfig:
748
# Automated device type detection
749
if is_neuron():
750
self.device_type = "neuron"
751
+ elif is_tpu():
752
+ self.device_type = "tpu"
753
elif is_cpu():
754
self.device_type = "cpu"
755
else:
@@ -761,6 +763,8 @@ class DeviceConfig:
763
# Some device types require processing inputs on CPU
764
if self.device_type in ["neuron"]:
765
self.device = torch.device("cpu")
766
+ elif self.device_type in ["tpu"]:
767
+ self.device = None
768
else:
769
# Set device with device type
770
self.device = torch.device(self.device_type)
vllm/engine/arg_utils.py
+1
-1
@@ -504,7 +504,7 @@ class EngineArgs:
504
parser.add_argument("--device",
505
type=str,
506
default=EngineArgs.device,
507
- choices=["auto", "cuda", "neuron", "cpu"],
507
+ choices=["auto", "cuda", "neuron", "cpu", "tpu"],
508
help='Device type for vLLM execution.')
509
510
# Related to Vision-language models such as llava
vllm/engine/async_llm_engine.py
+3
@@ -375,6 +375,9 @@ class AsyncLLMEngine:
375
if engine_config.device_config.device_type == "neuron":
376
from vllm.executor.neuron_executor import NeuronExecutorAsync
377
executor_class = NeuronExecutorAsync
378
+ elif engine_config.device_config.device_type == "tpu":
379
+ from vllm.executor.tpu_executor import TPUExecutorAsync
380
+ executor_class = TPUExecutorAsync
381
elif engine_config.device_config.device_type == "cpu":
382
assert distributed_executor_backend is None, (
383
"Distributed execution is not supported with the CPU backend.")
vllm/engine/llm_engine.py
+3
@@ -341,6 +341,9 @@ class LLMEngine:
341
if engine_config.device_config.device_type == "neuron":
342
from vllm.executor.neuron_executor import NeuronExecutor
343
executor_class = NeuronExecutor
344
+ elif engine_config.device_config.device_type == "tpu":
345
+ from vllm.executor.tpu_executor import TPUExecutor
346
+ executor_class = TPUExecutor
347
elif engine_config.device_config.device_type == "cpu":
348
from vllm.executor.cpu_executor import CPUExecutor
349
executor_class = CPUExecutor
vllm/envs.py
+6
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
27
VLLM_TRACE_FUNCTION: int = 0
28
VLLM_ATTENTION_BACKEND: Optional[str] = None
29
VLLM_CPU_KVCACHE_SPACE: int = 0
30
+ VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
31
VLLM_USE_RAY_COMPILED_DAG: bool = False
32
VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
33
VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -217,6 +218,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
218
# Default is 5 seconds
219
"VLLM_IMAGE_FETCH_TIMEOUT":
220
lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
221
+
222
+ # Path to the XLA persistent cache directory.
223
+ # Only used for XLA devices such as TPUs.
224
+ "VLLM_XLA_CACHE_PATH":
225
+ lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
226
}
227
228
# end-env-vars-definition
vllm/executor/tpu_executor.py
new
+101
@@ -0,0 +1,101 @@
1
+from typing import List, Set, Tuple
2
+
3
+import torch
4
+
5
+from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6
+from vllm.logger import init_logger
7
+from vllm.lora.request import LoRARequest
8
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
9
+from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10
+ make_async)
11
+
12
+logger = init_logger(__name__)
13
+
14
+
15
+class TPUExecutor(ExecutorBase):
16
+
17
+ def _init_executor(self) -> None:
18
+ assert not self.scheduler_config.chunked_prefill_enabled, (
19
+ "Chunked prefill is not yet supported for TPU backend")
20
+ assert not self.speculative_config, (
21
+ "Speculative decoding is not yet supported for TPU backend")
22
+ if self.model_config.dtype in (torch.float16, torch.float32):
23
+ logger.warning(
24
+ "The TPU backend currently does not support %s. "
25
+ "Using bfloat16 instead.", self.model_config.dtype)
26
+ self.model_config.dtype = torch.bfloat16
27
+
28
+ # Instantiate the worker and load the model to the device.
29
+ self._init_worker()
30
+
31
+ def _init_worker(self):
32
+ from vllm.worker.tpu_worker import TPUWorker
33
+
34
+ assert self.parallel_config.world_size == 1, (
35
+ "TPUExecutor currently only supports a single TPU chip.")
36
+ distributed_init_method = get_distributed_init_method(
37
+ get_ip(), get_open_port())
38
+ self.driver_worker = TPUWorker(
39
+ self.model_config,
40
+ self.parallel_config,
41
+ self.scheduler_config,
42
+ self.device_config,
43
+ self.cache_config,
44
+ self.load_config,
45
+ self.vision_language_config,
46
+ local_rank=0,
47
+ rank=0,
48
+ distributed_init_method=distributed_init_method,
49
+ )
50
+ self.driver_worker.init_device()
51
+ self.driver_worker.load_model()
52
+
53
+ def initialize_cache(
54
+ self,
55
+ num_gpu_blocks: int,
56
+ num_cpu_blocks: int,
57
+ ) -> None:
58
+ """Initialize the KV cache by invoking the underlying worker."""
59
+ # NOTE: This is logged in the executor because there can be >1 worker
60
+ # with other executors. We could log in the engine level, but work
61
+ # remains to abstract away the device for non-GPU configurations.
62
+ logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
63
+ num_cpu_blocks)
64
+ self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
65
+
66
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
67
+ """Determine the number of available KV blocks by invoking the
68
+ underlying worker.
69
+ """
70
+ return self.driver_worker.determine_num_available_blocks()
71
+
72
+ def execute_model(
73
+ self,
74
+ execute_model_req: ExecuteModelRequest,
75
+ ) -> List[SamplerOutput]:
76
+ output = self.driver_worker.execute_model(execute_model_req)
77
+ return output
78
+
79
+ def add_lora(self, lora_request: LoRARequest) -> bool:
80
+ raise NotImplementedError("LoRA is not implemented for TPU backend.")
81
+
82
+ def remove_lora(self, lora_id: int) -> bool:
83
+ raise NotImplementedError("LoRA is not implemented for TPU backend.")
84
+
85
+ def list_loras(self) -> Set[int]:
86
+ raise NotImplementedError("LoRA is not implemented for TPU backend.")
87
+
88
+ def check_health(self) -> None:
89
+ # TPUExecutor will always be healthy as long as it's running.
90
+ return
91
+
92
+
93
+class TPUExecutorAsync(TPUExecutor, ExecutorAsyncBase):
94
+
95
+ async def execute_model_async(
96
+ self,
97
+ sexecute_model_req: ExecuteModelRequest,
98
+ ) -> SamplerOutput:
99
+ output = await make_async(self.driver_worker.execute_model
100
+ )(sexecute_model_req)
101
+ return output
vllm/model_executor/custom_op.py
+3
-1
@@ -1,6 +1,6 @@
1
import torch.nn as nn
2
3
-from vllm.utils import is_cpu, is_hip
3
+from vllm.utils import is_cpu, is_hip, is_tpu
4
5
6
class CustomOp(nn.Module):
@@ -56,5 +56,7 @@ class CustomOp(nn.Module):
56
return self.forward_hip
57
elif is_cpu():
58
return self.forward_cpu
59
+ elif is_tpu():
60
+ return self.forward_tpu
61
else:
62
return self.forward_cuda
vllm/model_executor/layers/rotary_embedding.py
+74
-3
@@ -28,6 +28,7 @@ import torch
28
import torch.nn as nn
29
30
from vllm.model_executor.custom_op import CustomOp
31
+from vllm.utils import is_tpu
32
33
34
def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -43,6 +44,19 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
44
return x.flatten(-2)
45
46
47
+def _apply_rotary_emb(
48
+ x: torch.Tensor,
49
+ freqs_cis: torch.Tensor,
50
+) -> torch.Tensor:
51
+ x_ = torch.view_as_complex(
52
+ torch.stack(torch.chunk(x.transpose(1, 2).float(), 2, dim=-1), dim=-1))
53
+ x_out = torch.view_as_real(x_ * freqs_cis).type_as(x)
54
+ x_out = torch.cat(torch.chunk(x_out, 2, dim=-1), dim=-2)
55
+ x_out = x_out.reshape(x_out.shape[0], x_out.shape[1], x_out.shape[2],
56
+ -1).transpose(1, 2)
57
+ return x_out
58
+
59
+
60
class RotaryEmbedding(CustomOp):
61
"""Original rotary positional embedding."""
62
@@ -64,8 +78,14 @@ class RotaryEmbedding(CustomOp):
78
self.dtype = dtype
79
80
cache = self._compute_cos_sin_cache()
67
- cache = cache.to(dtype)
68
- self.register_buffer("cos_sin_cache", cache, persistent=False)
81
+ self.use_native2 = is_tpu() and is_neox_style
82
+ if not self.use_native2:
83
+ cache = cache.to(dtype)
84
+ self.register_buffer("cos_sin_cache", cache, persistent=False)
85
+ else:
86
+ cos, sin = cache.chunk(2, dim=-1)
87
+ freqs_cis = cos + 1j * sin
88
+ self.register_buffer("freqs_cis", freqs_cis, persistent=False)
89
90
def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
91
"""Compute the inverse frequency."""
@@ -100,7 +120,11 @@ class RotaryEmbedding(CustomOp):
120
key: torch.Tensor,
121
offsets: Optional[torch.Tensor] = None,
122
) -> Tuple[torch.Tensor, torch.Tensor]:
103
- """PyTorch-native implementation equivalent to forward()."""
123
+ """A PyTorch-native implementation equivalent to forward().
124
+
125
+ This method mimics the implementation of the custom CUDA kernel
126
+ used in `forward_cuda()`.
127
+ """
128
query = query.view(*query.shape[:-1], -1, self.head_size)
129
key = key.view(*key.shape[:-1], -1, self.head_size)
130
@@ -138,6 +162,42 @@ class RotaryEmbedding(CustomOp):
162
key = key.flatten(-2)
163
return query, key
164
165
+ def forward_native2(
166
+ self,
167
+ positions: torch.Tensor,
168
+ query: torch.Tensor,
169
+ key: torch.Tensor,
170
+ offsets: Optional[torch.Tensor] = None,
171
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
172
+ """Another PyTorch-native implementation of forward().
173
+
174
+ This method might perform better than `forward_native()` when compiled.
175
+ """
176
+ if positions.dim() == 1:
177
+ batch_size = 1
178
+ seq_len = positions.shape[0]
179
+ else:
180
+ batch_size, seq_len = positions.shape
181
+ if offsets is not None:
182
+ positions = positions + offsets
183
+ freqs_cis = self.freqs_cis.index_select(0, positions.flatten())
184
+ freqs_cis = freqs_cis.view(batch_size, 1, seq_len, -1)
185
+
186
+ query_shape = query.shape
187
+ query = query.view(batch_size, seq_len, -1, self.head_size)
188
+ query_rot = query[..., :self.rotary_dim]
189
+ query_pass = query[..., self.rotary_dim:]
190
+ query_rot = _apply_rotary_emb(query_rot, freqs_cis)
191
+ query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
192
+
193
+ key_shape = key.shape
194
+ key = key.view(batch_size, seq_len, -1, self.head_size)
195
+ key_rot = key[..., :self.rotary_dim]
196
+ key_pass = key[..., self.rotary_dim:]
197
+ key_rot = _apply_rotary_emb(key_rot, freqs_cis)
198
+ key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
199
+ return query, key
200
+
201
def forward_cuda(
202
self,
203
positions: torch.Tensor,
@@ -161,6 +221,17 @@ class RotaryEmbedding(CustomOp):
221
self.cos_sin_cache, self.is_neox_style)
222
return query, key
223
224
+ def forward_tpu(
225
+ self,
226
+ positions: torch.Tensor,
227
+ query: torch.Tensor,
228
+ key: torch.Tensor,
229
+ offsets: Optional[torch.Tensor] = None,
230
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
231
+ forward_fn = (self.forward_native2
232
+ if self.use_native2 else self.forward_native)
233
+ return forward_fn(positions, query, key, offsets)
234
+
235
def extra_repr(self) -> str:
236
s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
237
s += f", max_position_embeddings={self.max_position_embeddings}"
vllm/model_executor/model_loader/loader.py
+21
-6
@@ -34,6 +34,7 @@ from vllm.model_executor.model_loader.weight_utils import (
34
pt_weights_iterator, safetensors_weights_iterator)
35
from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36
from vllm.model_executor.utils import set_weight_attrs
37
+from vllm.utils import is_tpu
38
39
logger = init_logger(__name__)
40
@@ -227,12 +228,26 @@ class DefaultModelLoader(BaseModelLoader):
228
if self.load_config.load_format == LoadFormat.NPCACHE:
229
# Currently np_cache only support *.bin checkpoints
230
assert use_safetensors is False
230
- return np_cache_weights_iterator(model_name_or_path,
231
- self.load_config.download_dir,
232
- hf_folder, hf_weights_files)
233
- if use_safetensors:
234
- return safetensors_weights_iterator(hf_weights_files)
235
- return pt_weights_iterator(hf_weights_files)
231
+ weights_iterator = np_cache_weights_iterator(
232
+ model_name_or_path, self.load_config.download_dir, hf_folder,
233
+ hf_weights_files)
234
+ elif use_safetensors:
235
+ weights_iterator = safetensors_weights_iterator(hf_weights_files)
236
+ else:
237
+ weights_iterator = pt_weights_iterator(hf_weights_files)
238
+
239
+ if is_tpu():
240
+ # In PyTorch XLA, we should call `xm.mark_step` frequently so that
241
+ # not too many ops are accumulated in the XLA program.
242
+ import torch_xla.core.xla_model as xm
243
+
244
+ def _xla_weights_iterator(iterator: Generator):
245
+ for weights in iterator:
246
+ yield weights
247
+ xm.mark_step()
248
+
249
+ weights_iterator = _xla_weights_iterator(weights_iterator)
250
+ return weights_iterator
251
252
def load_model(self, *, model_config: ModelConfig,
253
device_config: DeviceConfig,
vllm/utils.py
+14
@@ -146,6 +146,15 @@ def is_neuron() -> bool:
146
return transformers_neuronx is not None
147
148
149
+@lru_cache(maxsize=None)
150
+def is_tpu() -> bool:
151
+ try:
152
+ import libtpu
153
+ except ImportError:
154
+ libtpu = None
155
+ return libtpu is not None
156
+
157
+
158
@lru_cache(maxsize=None)
159
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
160
"""Returns the maximum shared memory per thread block in bytes."""
@@ -546,6 +555,11 @@ def maybe_expand_dim(tensor: torch.Tensor,
555
return tensor
556
557
558
+def get_dtype_size(dtype: torch.dtype) -> int:
559
+ """Get the size of the data type in bytes."""
560
+ return torch.tensor([], dtype=dtype).element_size()
561
+
562
+
563
def merge_dicts(dict1: Dict[Any, List[Any]],
564
dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
565
"""Merge 2 dicts that have key -> List of items.
vllm/worker/cache_engine.py
+3
-6
@@ -6,7 +6,8 @@ import torch
6
from vllm.attention import get_attn_backend
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_pin_memory_available
9
+from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size,
10
+ is_pin_memory_available)
11
12
logger = init_logger(__name__)
13
@@ -108,9 +109,5 @@ class CacheEngine:
109
dtype = model_config.dtype
110
else:
111
dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
111
- dtype_size = _get_dtype_size(dtype)
112
+ dtype_size = get_dtype_size(dtype)
113
return dtype_size * total
113
-
114
-
115
-def _get_dtype_size(dtype: torch.dtype) -> int:
116
- return torch.tensor([], dtype=dtype).element_size()
vllm/worker/tpu_model_runner.py
new
+525
@@ -0,0 +1,525 @@
1
+import time
2
+from typing import List, Optional, Tuple
3
+
4
+import numpy as np
5
+import torch
6
+import torch.nn as nn
7
+import torch_xla.core.xla_model as xm
8
+
9
+from vllm.attention import AttentionMetadata, get_attn_backend
10
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
12
+from vllm.logger import init_logger
13
+from vllm.model_executor.model_loader import get_model
14
+from vllm.model_executor.sampling_metadata import SamplingMetadata
15
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
16
+ SamplerOutput, SequenceGroupMetadata,
17
+ SequenceOutput)
18
+from vllm.utils import make_tensor_with_pad
19
+
20
+logger = init_logger(__name__)
21
+
22
+_PAD_SLOT_ID = 0 # FIXME(woosuk)
23
+
24
+
25
+class TPUModelRunner:
26
+
27
+ def __init__(
28
+ self,
29
+ model_config: ModelConfig,
30
+ parallel_config: ParallelConfig,
31
+ scheduler_config: SchedulerConfig,
32
+ device_config: DeviceConfig,
33
+ cache_config: CacheConfig,
34
+ load_config: LoadConfig,
35
+ vision_language_config: Optional[VisionLanguageConfig] = None,
36
+ ):
37
+ self.model_config = model_config
38
+ self.parallel_config = parallel_config
39
+ self.scheduler_config = scheduler_config
40
+ self.device_config = device_config
41
+ self.cache_config = cache_config
42
+ self.load_config = load_config
43
+ self.vision_language_config = vision_language_config
44
+
45
+ self.block_size = self.cache_config.block_size
46
+ self.max_num_blocks_per_seq = (self.model_config.max_model_len //
47
+ self.block_size)
48
+ self.block_tables = np.zeros(
49
+ (self.scheduler_config.max_num_seqs, self.max_num_blocks_per_seq),
50
+ dtype=np.int32)
51
+ self.attn_backend = get_attn_backend(
52
+ self.model_config.get_num_attention_heads(self.parallel_config),
53
+ self.model_config.get_head_size(),
54
+ self.model_config.get_num_kv_heads(self.parallel_config),
55
+ self.model_config.get_sliding_window(),
56
+ self.model_config.dtype,
57
+ self.cache_config.cache_dtype,
58
+ self.block_size,
59
+ False,
60
+ )
61
+
62
+ def load_model(self) -> None:
63
+ self.device = self.device_config.device
64
+
65
+ model = get_model(
66
+ model_config=self.model_config,
67
+ load_config=self.load_config,
68
+ device_config=self.device_config,
69
+ parallel_config=self.parallel_config,
70
+ cache_config=self.cache_config,
71
+ scheduler_config=self.scheduler_config,
72
+ vision_language_config=self.vision_language_config,
73
+ lora_config=None,
74
+ )
75
+ xm.wait_device_ops()
76
+
77
+ model = ModelWrapper(model)
78
+ self.model = torch.compile(model, backend="openxla", fullgraph=True)
79
+
80
+ def _dummy_run(
81
+ self,
82
+ batch_size: int,
83
+ seq_len: int,
84
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
85
+ is_prompt: bool,
86
+ ) -> None:
87
+ if is_prompt:
88
+ seq_len = (seq_len + 15) // 16 * 16
89
+ token_ids = torch.zeros((batch_size, seq_len),
90
+ dtype=torch.int32,
91
+ device=self.device)
92
+ position_ids = torch.zeros((batch_size, seq_len),
93
+ dtype=torch.int32,
94
+ device=self.device)
95
+ slot_mapping = torch.zeros((batch_size, seq_len),
96
+ dtype=torch.int64,
97
+ device=self.device)
98
+ attn_metadata = self.attn_backend.make_metadata(
99
+ num_prefills=batch_size,
100
+ num_prefill_tokens=batch_size * seq_len,
101
+ num_decode_tokens=0,
102
+ slot_mapping=slot_mapping,
103
+ block_tables=None,
104
+ context_lens=None,
105
+ )
106
+ input_lens = torch.ones((batch_size, ),
107
+ dtype=torch.int32,
108
+ device=self.device)
109
+ else:
110
+ assert seq_len == 1
111
+ token_ids = torch.zeros((batch_size, seq_len),
112
+ dtype=torch.int32,
113
+ device=self.device)
114
+ position_ids = torch.zeros((batch_size, seq_len),
115
+ dtype=torch.int32,
116
+ device=self.device)
117
+ slot_mapping = torch.zeros((batch_size, seq_len),
118
+ dtype=torch.int64,
119
+ device=self.device)
120
+ block_tables = torch.zeros(
121
+ (batch_size, self.max_num_blocks_per_seq),
122
+ dtype=torch.int32,
123
+ device=self.device)
124
+ context_lens = torch.ones((batch_size, ),
125
+ dtype=torch.int32,
126
+ device=self.device)
127
+ input_lens = torch.ones((batch_size, ),
128
+ dtype=torch.int32,
129
+ device=self.device)
130
+ attn_metadata = self.attn_backend.make_metadata(
131
+ num_prefills=0,
132
+ num_prefill_tokens=0,
133
+ num_decode_tokens=batch_size * seq_len,
134
+ slot_mapping=slot_mapping,
135
+ block_tables=block_tables,
136
+ context_lens=context_lens,
137
+ )
138
+ t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
139
+ p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
140
+
141
+ # Dummy run.
142
+ self.model(token_ids, position_ids, kv_caches, attn_metadata,
143
+ input_lens, t, p)
144
+
145
+ def warmup_model(
146
+ self,
147
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
148
+ ) -> None:
149
+ # Prefill
150
+ logger.info("Compiling the model with different input shapes...")
151
+ start = time.time()
152
+ for batch_size in [1]:
153
+ seq_len = 16
154
+ while True:
155
+ self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=True)
156
+ xm.wait_device_ops()
157
+ logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
158
+
159
+ if seq_len >= self.model_config.max_model_len:
160
+ break
161
+ num_tokens = batch_size * seq_len
162
+ if num_tokens >= self.scheduler_config.max_num_batched_tokens:
163
+ break
164
+ seq_len = seq_len * 2
165
+
166
+ end = time.time()
167
+ logger.info("Compilation for prefill done in %.2f s.", end - start)
168
+
169
+ # Decode
170
+ start = time.time()
171
+ seq_len = 1
172
+ batch_size = 1
173
+ while True:
174
+ self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
175
+ xm.wait_device_ops()
176
+ logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
177
+
178
+ if batch_size >= self.scheduler_config.max_num_seqs:
179
+ break
180
+ batch_size = batch_size + 16 if batch_size >= 16 else batch_size * 2
181
+
182
+ end = time.time()
183
+ logger.info("Compilation for decode done in %.2f s.", end - start)
184
+
185
+ def _prepare_prompt(
186
+ self,
187
+ seq_group_metadata_list: List[SequenceGroupMetadata],
188
+ ):
189
+ assert len(seq_group_metadata_list) > 0
190
+ input_tokens: List[List[int]] = []
191
+ input_positions: List[List[int]] = []
192
+ prompt_lens: List[int] = []
193
+ slot_mapping: List[List[int]] = []
194
+
195
+ for seq_group_metadata in seq_group_metadata_list:
196
+ assert seq_group_metadata.is_prompt
197
+ seq_ids = list(seq_group_metadata.seq_data.keys())
198
+ assert len(seq_ids) == 1
199
+ seq_id = seq_ids[0]
200
+
201
+ seq_data = seq_group_metadata.seq_data[seq_id]
202
+ # Could include output tokens when a request is preempted.
203
+ prompt_tokens = seq_data.get_token_ids()
204
+ prompt_len = len(prompt_tokens)
205
+ prompt_lens.append(prompt_len)
206
+
207
+ input_tokens.append(prompt_tokens)
208
+ input_positions.append(list(range(prompt_len)))
209
+
210
+ assert seq_group_metadata.block_tables is not None
211
+ block_table = seq_group_metadata.block_tables[seq_id]
212
+ slot_mapping.append([])
213
+ for i in range(prompt_len):
214
+ block_number = block_table[i // self.block_size]
215
+ block_offset = i % self.block_size
216
+ slot = block_number * self.block_size + block_offset
217
+ slot_mapping[-1].append(slot)
218
+
219
+ assert len(prompt_lens) > 0
220
+ num_prefills = len(prompt_lens)
221
+ num_prefill_tokens = sum(prompt_lens)
222
+
223
+ # Add paddings to make the shape [batch_size, max_prompt_len] where
224
+ # max_prompt_len is smallest power of 2 that is greater than or equal
225
+ # to the maximum prompt length.
226
+ # We need the 2D input shape because the Pallas FlashAttention kernel
227
+ # does not support packed 1D inputs.
228
+ # We pad the seq_len to powers of 2 to reduce the compilation overhead.
229
+ max_prompt_len = _get_padded_prefill_len(max(prompt_lens))
230
+ input_tokens = make_tensor_with_pad(input_tokens,
231
+ max_prompt_len,
232
+ pad=0,
233
+ dtype=torch.int32,
234
+ device=self.device)
235
+ input_positions = make_tensor_with_pad(input_positions,
236
+ max_prompt_len,
237
+ pad=0,
238
+ dtype=torch.int32,
239
+ device=self.device)
240
+ slot_mapping = make_tensor_with_pad(slot_mapping,
241
+ max_prompt_len,
242
+ pad=_PAD_SLOT_ID,
243
+ dtype=torch.int64,
244
+ device=self.device)
245
+ prompt_lens = torch.tensor(prompt_lens,
246
+ dtype=torch.int32,
247
+ device=self.device)
248
+ attn_metadata = self.attn_backend.make_metadata(
249
+ num_prefills=num_prefills,
250
+ num_prefill_tokens=num_prefill_tokens, # NOTE: This is not used.
251
+ num_decode_tokens=0,
252
+ slot_mapping=slot_mapping,
253
+ block_tables=None,
254
+ context_lens=None,
255
+ )
256
+ return input_tokens, input_positions, attn_metadata, prompt_lens
257
+
258
+ def _prepare_decode(
259
+ self,
260
+ seq_group_metadata_list: List[SequenceGroupMetadata],
261
+ ):
262
+ assert len(seq_group_metadata_list) > 0
263
+ input_tokens: List[List[int]] = []
264
+ input_positions: List[List[int]] = []
265
+ slot_mapping: List[List[int]] = []
266
+ context_lens: List[int] = []
267
+ num_seq_groups = len(seq_group_metadata_list)
268
+ batch_size = _get_padded_batch_size(num_seq_groups)
269
+
270
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
271
+ assert not seq_group_metadata.is_prompt
272
+
273
+ seq_ids = list(seq_group_metadata.seq_data.keys())
274
+
275
+ for seq_id in seq_ids:
276
+ seq_data = seq_group_metadata.seq_data[seq_id]
277
+ generation_token = seq_data.get_last_token_id()
278
+ input_tokens.append([generation_token])
279
+
280
+ seq_len = seq_data.get_len()
281
+ position = seq_len - 1
282
+ input_positions.append([position])
283
+ context_lens.append(seq_len)
284
+
285
+ assert seq_group_metadata.block_tables is not None
286
+ block_table = seq_group_metadata.block_tables[seq_id]
287
+ self.block_tables[i, :len(block_table)] = block_table
288
+
289
+ block_number = block_table[position // self.block_size]
290
+ block_offset = position % self.block_size
291
+ slot = block_number * self.block_size + block_offset
292
+ slot_mapping.append([slot])
293
+
294
+ num_paddings = batch_size - num_seq_groups
295
+ input_tokens = input_tokens + [[0]] * num_paddings
296
+ input_positions = input_positions + [[0]] * num_paddings
297
+ slot_mapping = slot_mapping + [[_PAD_SLOT_ID]] * num_paddings
298
+ context_lens = context_lens + [0] * num_paddings
299
+
300
+ input_tokens = torch.tensor(input_tokens,
301
+ dtype=torch.int32,
302
+ device=self.device)
303
+ input_positions = torch.tensor(input_positions,
304
+ dtype=torch.int32,
305
+ device=self.device)
306
+ slot_mapping = torch.tensor(slot_mapping,
307
+ dtype=torch.int64,
308
+ device=self.device)
309
+ context_lens = torch.tensor(context_lens,
310
+ dtype=torch.int32,
311
+ device=self.device)
312
+ block_tables = torch.tensor(self.block_tables[:batch_size],
313
+ dtype=torch.int32,
314
+ device=self.device)
315
+ input_lens = torch.tensor([1] * batch_size,
316
+ dtype=torch.int32,
317
+ device=self.device)
318
+ attn_metadata = self.attn_backend.make_metadata(
319
+ num_prefills=0,
320
+ num_prefill_tokens=0,
321
+ num_decode_tokens=batch_size,
322
+ slot_mapping=slot_mapping,
323
+ block_tables=block_tables,
324
+ context_lens=context_lens,
325
+ )
326
+ return input_tokens, input_positions, attn_metadata, input_lens
327
+
328
+ def _prepare_sample(
329
+ self,
330
+ seq_group_metadata_list: List[SequenceGroupMetadata],
331
+ padded_batch_size: int,
332
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
333
+ assert len(seq_group_metadata_list) > 0
334
+ t = []
335
+ p = []
336
+ for seq_group_metadata in seq_group_metadata_list:
337
+ assert seq_group_metadata.sampling_params is not None
338
+ sampling_params = seq_group_metadata.sampling_params
339
+
340
+ t.append(sampling_params.temperature
341
+ if sampling_params.temperature >= 1e-5 else 1e-5)
342
+ p.append(sampling_params.top_p)
343
+ num_paddings = padded_batch_size - len(seq_group_metadata_list)
344
+ t += [1.0] * num_paddings
345
+ p += [1.0] * num_paddings
346
+
347
+ t = torch.tensor(t, dtype=torch.float32, device=self.device)
348
+ p = torch.tensor(p, dtype=torch.float32, device=self.device)
349
+ return t, p
350
+
351
+ def prepare_inputs(
352
+ self,
353
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
354
+ ):
355
+ assert seq_group_metadata_list is not None
356
+ assert len(seq_group_metadata_list) > 0
357
+ # NOTE: We assume that all sequences in the group are all prompts or
358
+ # all decodes.
359
+ if seq_group_metadata_list[0].is_prompt:
360
+ inputs = self._prepare_prompt(seq_group_metadata_list)
361
+ else:
362
+ inputs = self._prepare_decode(seq_group_metadata_list)
363
+ padded_batch_size = inputs[0].shape[0]
364
+ sample_inputs = self._prepare_sample(seq_group_metadata_list,
365
+ padded_batch_size)
366
+ return inputs + sample_inputs
367
+
368
+ def _execute_model(
369
+ self,
370
+ seq_group_metadata_list: List[SequenceGroupMetadata],
371
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
372
+ ) -> List[CompletionSequenceGroupOutput]:
373
+ inputs = self.prepare_inputs(seq_group_metadata_list)
374
+ next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
375
+ *inputs[2:])
376
+ next_token_ids = next_token_ids.cpu().tolist()
377
+
378
+ i = 0
379
+ sampler_outputs = []
380
+ for seq_group_metadata in seq_group_metadata_list:
381
+ seq_outputs = []
382
+ seq_ids = list(seq_group_metadata.seq_data.keys())
383
+ for seq_id in seq_ids:
384
+ next_token_id = next_token_ids[i]
385
+ seq_outputs.append(
386
+ SequenceOutput(seq_id, next_token_id,
387
+ {next_token_id: Logprob(0.0)}))
388
+ i += 1
389
+ sampler_outputs.append(
390
+ CompletionSequenceGroupOutput(seq_outputs, None))
391
+ return sampler_outputs
392
+
393
+ def execute_model(
394
+ self,
395
+ seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
396
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
397
+ ) -> SamplerOutput:
398
+ assert seq_group_metadata_list is not None
399
+ if seq_group_metadata_list[0].is_prompt:
400
+ # NOTE(woosuk): To reduce the compilation time, we only compile the
401
+ # prefill inputs with batch size 1. Because the scheduler is not
402
+ # aware of this limitation, we need to handle batch size > 1
403
+ # internally by calling the model multiple times and concatenating
404
+ # the outputs.
405
+ # FIXME(woosuk): This is a temporary hack to not change the existing
406
+ # scheduler. We need to fix this in the future.
407
+ sampler_outputs = []
408
+ for seq_group_metadata in seq_group_metadata_list:
409
+ sampler_outputs += self._execute_model([seq_group_metadata],
410
+ kv_caches)
411
+ else:
412
+ sampler_outputs = self._execute_model(seq_group_metadata_list,
413
+ kv_caches)
414
+ return SamplerOutput(sampler_outputs)
415
+
416
+
417
+class ModelWrapper(nn.Module):
418
+
419
+ def __init__(self, model: nn.Module):
420
+ super().__init__()
421
+ self.model = model.eval()
422
+
423
+ def forward(
424
+ self,
425
+ token_ids: torch.Tensor,
426
+ position_ids: torch.Tensor,
427
+ kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
428
+ attn_metadata: AttentionMetadata,
429
+ input_lens: torch.Tensor,
430
+ t: torch.Tensor,
431
+ p: torch.Tensor,
432
+ ) -> torch.Tensor:
433
+ """Executes the forward pass of the model and samples the next token.
434
+
435
+ Args:
436
+ token_ids: The input token IDs of shape [batch_size, seq_len].
437
+ position_ids: The input position IDs of shape [batch_size, seq_len].
438
+ kv_caches: The key and value caches. They can be None during the
439
+ memory profiling at initialization.
440
+ attn_metadata: The Pallas attention metadata.
441
+ input_lens: The actual input lengths of shape [batch_size].
442
+ t: The sampling temperature of shape [batch_size].
443
+ p: The top-p probability of shape [batch_size].
444
+ """
445
+ batch_size, seq_len = token_ids.shape
446
+ # Calculate the positions to sample from.
447
+ base_indicies = torch.arange(
448
+ batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
449
+ logits_indices = base_indicies + input_lens - 1
450
+
451
+ # FIXME(woosuk): This is a temporary hack to avoid using the existing
452
+ # sampler and sampling metadata.
453
+ sampling_metadata = SamplingMetadata(
454
+ seq_groups=[],
455
+ selected_token_indices=logits_indices,
456
+ categorized_sample_indices={},
457
+ num_prompts=attn_metadata.num_prefills,
458
+ )
459
+
460
+ # Skip this in memory profiling at initialization.
461
+ if kv_caches[0][0] is not None:
462
+ # index_copy_(slot_mapping) only works when the inserted dimension
463
+ # is 0. However, the KV cache in the Pallas backend has the shape
464
+ # [num_kv_heads, num_blocks, block_size, head_size]. To make it
465
+ # work, we need to flatten the first three dimensions and modify
466
+ # the slot_mapping accordingly.
467
+ num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape
468
+ slot_mapping = attn_metadata.slot_mapping
469
+ slot_mapping = slot_mapping.flatten()
470
+ head_indicies = torch.arange(0,
471
+ num_kv_heads,
472
+ device=slot_mapping.device,
473
+ dtype=slot_mapping.dtype)
474
+ head_indicies *= block_size * num_blocks
475
+ slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view(
476
+ -1, num_kv_heads)
477
+ slot_mapping = slot_mapping + head_indicies.view(1, -1)
478
+ slot_mapping = slot_mapping.flatten()
479
+ attn_metadata.slot_mapping = slot_mapping
480
+
481
+ hidden_states = self.model(
482
+ token_ids,
483
+ position_ids,
484
+ kv_caches,
485
+ attn_metadata,
486
+ )
487
+ hidden_states = hidden_states.flatten(0, 1)
488
+ logits = self.model.compute_logits(hidden_states, sampling_metadata)
489
+
490
+ logits = logits / t.unsqueeze(dim=1)
491
+ # FIXME(woosuk): Disabled top-p sampling since it's too slow.
492
+ # logits = _apply_top_p(logits, p.unsqueeze(dim=1))
493
+ probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
494
+ # FIXME(woosuk): best_of > 1 is not supported.
495
+ next_token_ids = torch.multinomial(probs, num_samples=1).squeeze(dim=1)
496
+ return next_token_ids
497
+
498
+
499
+def _get_padded_prefill_len(x: int) -> int:
500
+ # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
501
+ # length to be a multiple of 16. We pad the prompt length to the nearest
502
+ # multiple of 16. This is also good for performance.
503
+ if x <= 16:
504
+ return 16
505
+ return 1 << (x - 1).bit_length()
506
+
507
+
508
+def _get_padded_batch_size(batch_size: int) -> int:
509
+ if batch_size <= 2:
510
+ return batch_size
511
+ elif batch_size <= 4:
512
+ return 4
513
+ elif batch_size <= 8:
514
+ return 8
515
+ else:
516
+ return ((batch_size + 15) // 16) * 16
517
+
518
+
519
+def _apply_top_p(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor:
520
+ logits_sorted = torch.sort(logits, dim=-1, descending=True).values
521
+ sorted_cum_probs = torch.cumsum(logits_sorted.softmax(dim=-1), dim=-1)
522
+ cutoff_index = torch.sum(sorted_cum_probs < p, dim=-1, keepdim=True)
523
+ cutoff_logit = torch.gather(logits_sorted, -1, cutoff_index)
524
+ logits = logits.masked_fill_(logits < cutoff_logit, -float("inf"))
525
+ return logits
vllm/worker/tpu_worker.py
new
+198
@@ -0,0 +1,198 @@
1
+import os
2
+from typing import List, Optional, Tuple
3
+
4
+import torch
5
+import torch_xla.core.xla_model as xm
6
+import torch_xla.runtime as xr
7
+
8
+import vllm.envs as envs
9
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10
+ ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
+from vllm.distributed import (ensure_model_parallel_initialized,
12
+ init_distributed_environment)
13
+from vllm.logger import init_logger
14
+from vllm.model_executor import set_random_seed
15
+from vllm.sequence import ExecuteModelRequest, SamplerOutput
16
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
17
+from vllm.worker.tpu_model_runner import TPUModelRunner
18
+from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19
+
20
+logger = init_logger(__name__)
21
+
22
+
23
+class TPUWorker(LoraNotSupportedWorkerBase):
24
+
25
+ def __init__(
26
+ self,
27
+ model_config: ModelConfig,
28
+ parallel_config: ParallelConfig,
29
+ scheduler_config: SchedulerConfig,
30
+ device_config: DeviceConfig,
31
+ cache_config: CacheConfig,
32
+ load_config: LoadConfig,
33
+ vision_language_config: Optional[VisionLanguageConfig],
34
+ local_rank: int,
35
+ rank: int,
36
+ distributed_init_method: str,
37
+ ) -> None:
38
+ self.model_config = model_config
39
+ self.parallel_config = parallel_config
40
+ self.scheduler_config = scheduler_config
41
+ self.device_config = device_config
42
+ self.cache_config = cache_config
43
+ self.load_config = load_config
44
+ self.vision_language_config = vision_language_config
45
+ self.local_rank = local_rank
46
+ self.rank = rank
47
+ self.distributed_init_method = distributed_init_method
48
+
49
+ assert self.device_config.device_type == "tpu"
50
+ if self.cache_config.cache_dtype == "auto":
51
+ self.cache_dtype = self.model_config.dtype
52
+ else:
53
+ self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
54
+ self.cache_config.cache_dtype]
55
+
56
+ self.model_runner = TPUModelRunner(model_config, parallel_config,
57
+ scheduler_config, device_config,
58
+ cache_config, load_config,
59
+ vision_language_config)
60
+
61
+ def init_device(self) -> None:
62
+ os.environ["PJRT_DEVICE"] = "TPU"
63
+ self.device = xm.xla_device()
64
+ self.device_config.device = self.device
65
+ torch.set_grad_enabled(False)
66
+ torch.set_default_dtype(self.model_config.dtype)
67
+
68
+ # NOTE(woosuk): This is just a hack to initialize the TP group.
69
+ # This cannot perform the actual communication ops.
70
+ init_distributed_environment(
71
+ world_size=self.parallel_config.world_size,
72
+ rank=self.rank,
73
+ local_rank=self.local_rank,
74
+ distributed_init_method=self.distributed_init_method,
75
+ backend="gloo",
76
+ )
77
+ ensure_model_parallel_initialized(
78
+ self.parallel_config.tensor_parallel_size,
79
+ self.parallel_config.pipeline_parallel_size)
80
+
81
+ # Set random seed.
82
+ set_random_seed(self.model_config.seed)
83
+ xm.set_rng_state(self.model_config.seed, self.device)
84
+
85
+ # Increase the cache size limit, which is the maximum number of
86
+ # dynamo graphs that can be compiled.
87
+ # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and
88
+ # 30-40 graphs for decode. 128 is an arbitrary safe number.
89
+ torch._dynamo.config.cache_size_limit = 128
90
+ # Use persistent cache to avoid XLA recompilation.
91
+ # NOTE(woosuk): This does not completely eliminate the recompilation
92
+ # overhead because dynamo does not cache the compiled results.
93
+ xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
94
+ readonly=False)
95
+
96
+ def load_model(self):
97
+ self.model_runner.load_model()
98
+
99
+ def determine_num_available_blocks(self) -> Tuple[int, int]:
100
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
101
+ head_size = self.model_config.get_head_size()
102
+ num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
103
+
104
+ kv_caches = [(None, None) for _ in range(num_layers)]
105
+ self.model_runner._dummy_run(
106
+ batch_size=1,
107
+ seq_len=self.scheduler_config.max_num_batched_tokens,
108
+ kv_caches=kv_caches,
109
+ is_prompt=True,
110
+ )
111
+ # Synchronize before measuring the memory usage.
112
+ xm.wait_device_ops()
113
+
114
+ m = xm.get_memory_info(self.device)
115
+ program_size = 1024 * 1024 * 1024 # 1GB
116
+ free_bytes = max(m["bytes_limit"] - m["bytes_used"] - program_size, 0)
117
+ kv_cache_bytes = int(free_bytes *
118
+ self.cache_config.gpu_memory_utilization)
119
+ kv_cache_dtype_btyes = get_dtype_size(self.cache_dtype)
120
+ block_size = self.cache_config.block_size
121
+ num_tpu_blocks = (kv_cache_bytes //
122
+ (kv_cache_dtype_btyes * block_size * num_layers * 2 *
123
+ head_size * num_kv_heads))
124
+ num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
125
+ return num_tpu_blocks, 0
126
+
127
+ def initialize_cache(
128
+ self,
129
+ num_gpu_blocks: int,
130
+ num_cpu_blocks: int,
131
+ ) -> None:
132
+ self.cache_config.num_gpu_blocks = num_gpu_blocks
133
+ self.cache_config.num_cpu_blocks = num_cpu_blocks
134
+ self.block_size = self.cache_config.block_size
135
+
136
+ dtype = self.cache_dtype
137
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
138
+ num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
139
+ head_size = self.model_config.get_head_size()
140
+
141
+ self.tpu_cache = []
142
+ tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
143
+ num_gpu_blocks, self.block_size, num_kv_heads, head_size)
144
+ for _ in range(num_layers):
145
+ key_cache = torch.zeros(tpu_cache_shape,
146
+ dtype=dtype,
147
+ device=self.device)
148
+ value_cache = torch.zeros_like(key_cache)
149
+ self.tpu_cache.append((key_cache, value_cache))
150
+ self._warmup_model()
151
+
152
+ def _warmup_model(self) -> None:
153
+ # FIXME(woosuk): Here we are abusing `enforce_eager` which is defined
154
+ # for CUDA graphs. We should refactor this part.
155
+ if not self.model_config.enforce_eager:
156
+ # Warm up the model with all possible input shapes so that
157
+ # compilation never happens during the actual execution.
158
+ # This may take ~30 mins for the first run and ~20 mins for the
159
+ # subsequent runs.
160
+ # If `enforce_eager` is True, the ahead-of-time compilation is
161
+ # skipped and the compilation happens during the actual execution,
162
+ # which is bad for performance but useful for development.
163
+ self.model_runner.warmup_model(self.tpu_cache)
164
+
165
+ def get_cache_block_size_bytes(self) -> int:
166
+ head_size = self.model_config.get_head_size()
167
+ num_heads = self.model_config.get_num_kv_heads(self.parallel_config)
168
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
169
+
170
+ key_cache_block = self.cache_config.block_size * num_heads * head_size
171
+ value_cache_block = key_cache_block
172
+ total = num_layers * (key_cache_block + value_cache_block)
173
+ dtype_size = get_dtype_size(self.cache_dtype)
174
+ return dtype_size * total
175
+
176
+ def execute_model(
177
+ self,
178
+ execute_model_req: Optional[ExecuteModelRequest] = None
179
+ ) -> List[SamplerOutput]:
180
+ if execute_model_req is None:
181
+ return []
182
+
183
+ seq_group_metadata_list = execute_model_req.seq_group_metadata_list
184
+ num_seq_groups = len(seq_group_metadata_list)
185
+ if num_seq_groups == 0:
186
+ return []
187
+
188
+ # Currently, TPUWorker does not support swapping.
189
+ # TODO(woosuk): Support block copying.
190
+ assert len(execute_model_req.blocks_to_swap_in) == 0, (
191
+ "Swapping is not supported for the TPU backend.")
192
+ assert len(execute_model_req.blocks_to_swap_out) == 0, (
193
+ "Swapping is not supported for the TPU backend.")
194
+ assert len(execute_model_req.blocks_to_copy) == 0
195
+
196
+ output = self.model_runner.execute_model(seq_group_metadata_list,
197
+ self.tpu_cache)
198
+ return [output]