@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1a8bfd92d5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Hardware] Initial TPU integration (#5292)

Woosuk Kwon committed Jun 12, 2024 at 11:53 UTC 1a8bfd92d5f35d638e3cfc8c4cd1779aeda0adfb
22 files changed +1322 -28
Dockerfile.tpu new
+19
@@ -0,0 +1,19 @@
1 +ARG NIGHTLY_DATE="20240601"
2 +ARG BASE_IMAGE="us-central1-docker.pkg.dev/tpu-pytorch-releases/docker/xla:nightly_3.10_tpuvm_$NIGHTLY_DATE"
3 +
4 +FROM $BASE_IMAGE
5 +
6 +WORKDIR /workspace
7 +COPY . /workspace/vllm
8 +
9 +ENV VLLM_TARGET_DEVICE="tpu"
10 +# Install aiohttp separately to avoid build errors.
11 +RUN pip install aiohttp
12 +# Install the TPU and Pallas dependencies.
13 +RUN pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
14 +RUN pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
15 +
16 +# Build vLLM.
17 +RUN cd /workspace/vllm && python setup.py develop
18 +
19 +CMD ["/bin/bash"]
benchmarks/benchmark_latency.py
+1 -1
@@ -189,7 +189,7 @@ if __name__ == '__main__':
189 "--device",
190 type=str,
191 default="cuda",
192 - choices=["cuda", "cpu"],
192 + choices=["cuda", "cpu", "tpu"],
193 help='device type for vLLM execution, supporting CUDA and CPU.')
194 parser.add_argument('--block-size',
195 type=int,
benchmarks/benchmark_throughput.py
+1 -1
@@ -346,7 +346,7 @@ if __name__ == "__main__":
346 "--device",
347 type=str,
348 default="cuda",
349 - choices=["cuda", "cpu"],
349 + choices=["cuda", "cpu", "tpu"],
350 help='device type for vLLM execution, supporting CUDA and CPU.')
351 parser.add_argument(
352 "--enable-prefix-caching",
docs/source/getting_started/tpu-installation.rst new
+75
@@ -0,0 +1,75 @@
1 +.. _installation_tpu:
2 +
3 +Installation with TPU
4 +=====================
5 +
6 +vLLM supports Google Cloud TPUs using PyTorch XLA.
7 +
8 +Requirements
9 +------------
10 +
11 +* Google Cloud TPU VM (single host)
12 +* TPU versions: v5e, v5p, v4
13 +* Python: 3.10
14 +
15 +Installation options:
16 +
17 +1. :ref:`Build a docker image with Dockerfile <build_docker_tpu>`.
18 +2. :ref:`Build from source <build_from_source_tpu>`.
19 +
20 +.. _build_docker_tpu:
21 +
22 +Build a docker image with :code:`Dockerfile.tpu`
23 +------------------------------------------------
24 +
25 +`Dockerfile.tpu <https://github.com/vllm-project/vllm/blob/main/Dockerfile.tpu>`_ is provided to build a docker image with TPU support.
26 +
27 +.. code-block:: console
28 +
29 + $ docker build -f Dockerfile.tpu -t vllm-tpu .
30 +
31 +
32 +You can run the docker image with the following command:
33 +
34 +.. code-block:: console
35 +
36 + $ # Make sure to add `--privileged --net host --shm-size=16G`.
37 + $ docker run --privileged --net host --shm-size=16G -it vllm-tpu
38 +
39 +
40 +.. _build_from_source_tpu:
41 +
42 +Build from source
43 +-----------------
44 +
45 +You can also build and install the TPU backend from source.
46 +
47 +First, install the dependencies:
48 +
49 +.. code-block:: console
50 +
51 + $ # (Recommended) Create a new conda environment.
52 + $ conda create -n myenv python=3.10 -y
53 + $ conda activate myenv
54 +
55 + $ # Clean up the existing torch and torch-xla packages.
56 + $ pip uninstall torch torch-xla -y
57 +
58 + $ # Install PyTorch and PyTorch XLA.
59 + $ export DATE="+20240601"
60 + $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch-nightly${DATE}-cp310-cp310-linux_x86_64.whl
61 + $ pip install https://storage.googleapis.com/pytorch-xla-releases/wheels/tpuvm/torch_xla-nightly${DATE}-cp310-cp310-linux_x86_64.whl
62 +
63 + $ # Install JAX and Pallas.
64 + $ pip install torch_xla[tpu] -f https://storage.googleapis.com/libtpu-releases/index.html
65 + $ pip install torch_xla[pallas] -f https://storage.googleapis.com/jax-releases/jax_nightly_releases.html -f https://storage.googleapis.com/jax-releases/jaxlib_nightly_releases.html
66 +
67 + $ # Install other build dependencies.
68 + $ pip install packaging aiohttp
69 +
70 +
71 +Next, build vLLM from source. This will only take a few seconds:
72 +
73 +.. code-block:: console
74 +
75 + $ VLLM_TARGET_DEVICE="tpu" python setup.py develop
docs/source/index.rst
+2 -1
@@ -63,8 +63,9 @@ Documentation
63
64 getting_started/installation
65 getting_started/amd-installation
66 - getting_started/neuron-installation
66 getting_started/cpu-installation
67 + getting_started/neuron-installation
68 + getting_started/tpu-installation
69 getting_started/quickstart
70 getting_started/debugging
71 getting_started/examples/examples_index
requirements-tpu.txt new
+7
@@ -0,0 +1,7 @@
1 +# Common dependencies
2 +-r requirements-common.txt
3 +
4 +# Dependencies for TPU
5 +# Currently, the TPU backend uses a nightly version of PyTorch XLA.
6 +# You can install the dependencies in Dockerfile.tpu.
7 +triton # To avoid import errors
setup.py
+17 -5
@@ -206,9 +206,9 @@ class cmake_build_ext(build_ext):
206
207
208 def _is_cuda() -> bool:
209 - return VLLM_TARGET_DEVICE == "cuda" \
210 - and torch.version.cuda is not None \
211 - and not _is_neuron()
209 + has_cuda = torch.version.cuda is not None
210 + return (VLLM_TARGET_DEVICE == "cuda" and has_cuda
211 + and not (_is_neuron() or _is_tpu()))
212
213
214 def _is_hip() -> bool:
@@ -225,10 +225,18 @@ def _is_neuron() -> bool:
225 return torch_neuronx_installed or VLLM_TARGET_DEVICE == "neuron"
226
227
228 +def _is_tpu() -> bool:
229 + return VLLM_TARGET_DEVICE == "tpu"
230 +
231 +
232 def _is_cpu() -> bool:
233 return VLLM_TARGET_DEVICE == "cpu"
234
235
236 +def _build_custom_ops() -> bool:
237 + return _is_cuda() or _is_hip() or _is_cpu()
238 +
239 +
240 def _install_punica() -> bool:
241 return envs.VLLM_INSTALL_PUNICA_KERNELS
242
@@ -325,6 +333,8 @@ def get_vllm_version() -> str:
333 if neuron_version != MAIN_CUDA_VERSION:
334 neuron_version_str = neuron_version.replace(".", "")[:3]
335 version += f"+neuron{neuron_version_str}"
336 + elif _is_tpu():
337 + version += "+tpu"
338 elif _is_cpu():
339 version += "+cpu"
340 else:
@@ -372,6 +382,8 @@ def get_requirements() -> List[str]:
382 requirements = _read_requirements("requirements-rocm.txt")
383 elif _is_neuron():
384 requirements = _read_requirements("requirements-neuron.txt")
385 + elif _is_tpu():
386 + requirements = _read_requirements("requirements-tpu.txt")
387 elif _is_cpu():
388 requirements = _read_requirements("requirements-cpu.txt")
389 else:
@@ -385,7 +397,7 @@ ext_modules = []
397 if _is_cuda() or _is_hip():
398 ext_modules.append(CMakeExtension(name="vllm._moe_C"))
399
388 -if not _is_neuron():
400 +if _build_custom_ops():
401 ext_modules.append(CMakeExtension(name="vllm._C"))
402
403 if _install_punica():
@@ -428,6 +440,6 @@ setup(
440 extras_require={
441 "tensorizer": ["tensorizer>=2.9.0"],
442 },
431 - cmdclass={"build_ext": cmake_build_ext} if not _is_neuron() else {},
443 + cmdclass={"build_ext": cmake_build_ext} if _build_custom_ops() else {},
444 package_data=package_data,
445 )
vllm/attention/backends/pallas.py new
+232
@@ -0,0 +1,232 @@
1 +from dataclasses import dataclass
2 +from typing import Any, Dict, List, Optional, Tuple, Type
3 +
4 +import torch
5 +import torch_xla.experimental.custom_kernel # Required to register custom ops.
6 +import torch_xla.experimental.dynamo_set_buffer_donor
7 +
8 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 + AttentionMetadata)
10 +
11 +
12 +class PallasAttentionBackend(AttentionBackend):
13 +
14 + @staticmethod
15 + def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
16 + return PallasAttentionBackendImpl
17 +
18 + @staticmethod
19 + def make_metadata(*args, **kwargs) -> "PallasMetadata":
20 + return PallasMetadata(*args, **kwargs)
21 +
22 + @staticmethod
23 + def get_kv_cache_shape(
24 + num_blocks: int,
25 + block_size: int,
26 + num_kv_heads: int,
27 + head_size: int,
28 + ) -> Tuple[int, ...]:
29 + return (num_kv_heads, num_blocks, block_size, head_size)
30 +
31 + @staticmethod
32 + def swap_blocks(
33 + src_kv_cache: torch.Tensor,
34 + dst_kv_cache: torch.Tensor,
35 + src_to_dst: Dict[int, int],
36 + ) -> None:
37 + raise NotImplementedError("swap_blocks is not implemented.")
38 +
39 + @staticmethod
40 + def copy_blocks(
41 + kv_caches: List[torch.Tensor],
42 + src_to_dists: Dict[int, List[int]],
43 + ) -> None:
44 + # TODO(woosuk): Implement this.
45 + raise NotImplementedError("copy_blocks is not implemented.")
46 +
47 +
48 +@dataclass
49 +class PallasMetadata(AttentionMetadata):
50 +
51 + # Currently, input sequences can only contain all prefills
52 + # or all decoding.
53 + block_tables: Optional[torch.Tensor]
54 + context_lens: Optional[torch.Tensor]
55 +
56 + @property
57 + def prefill_metadata(self) -> Optional["PallasMetadata"]:
58 + if self.num_prefills == 0:
59 + return None
60 +
61 + assert self.num_decode_tokens == 0
62 + assert self.block_tables is None
63 + assert self.context_lens is None
64 + return self
65 +
66 + @property
67 + def decode_metadata(self) -> Optional["PallasMetadata"]:
68 + if self.num_decode_tokens == 0:
69 + return None
70 +
71 + assert self.num_prefills == 0
72 + assert self.num_prefill_tokens == 0
73 + assert self.block_tables is not None
74 + assert self.context_lens is not None
75 + return self
76 +
77 +
78 +class PallasAttentionBackendImpl(AttentionImpl):
79 +
80 + def __init__(
81 + self,
82 + num_heads: int,
83 + head_size: int,
84 + scale: float,
85 + num_kv_heads: int,
86 + alibi_slopes: Optional[List[float]],
87 + sliding_window: Optional[int],
88 + kv_cache_dtype: str,
89 + blocksparse_params: Optional[Dict[str, Any]] = None,
90 + ) -> None:
91 + self.num_heads = num_heads
92 + self.head_size = head_size
93 + self.scale = float(scale)
94 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
95 +
96 + assert self.num_heads % self.num_kv_heads == 0
97 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
98 + if head_size % 128 != 0:
99 + raise NotImplementedError("Head size must be a multiple of 128.")
100 + if alibi_slopes is not None:
101 + raise NotImplementedError("Alibi slopes is not supported.")
102 + if sliding_window is not None:
103 + raise NotImplementedError("Sliding window is not supported.")
104 + if kv_cache_dtype != "auto":
105 + raise NotImplementedError("FP8 KV cache dtype is not supported.")
106 + if blocksparse_params is not None:
107 + raise NotImplementedError("Blocksparse is not supported.")
108 +
109 + if torch_xla.tpu.version() < 4:
110 + raise NotImplementedError("TPU version must be 4 or higher.")
111 +
112 + self.megacore_mode = None
113 + tpu_type = torch_xla.tpu.get_tpu_env()["TYPE"].lower()
114 + if not tpu_type.endswith("lite"):
115 + if self.num_kv_heads % 2 == 0:
116 + self.megacore_mode = "kv_head"
117 + else:
118 + # NOTE(woosuk): If the batch size is not a multiple of 2, the
119 + # megacore mode will be None.
120 + self.megacore_mode = "batch"
121 +
122 + def forward(
123 + self,
124 + query: torch.Tensor,
125 + key: torch.Tensor,
126 + value: torch.Tensor,
127 + kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
128 + attn_metadata: PallasMetadata,
129 + kv_scale: float = 1.0,
130 + ) -> torch.Tensor:
131 + """Forward pass with Pallas attention.
132 +
133 + Args:
134 + query: shape = [batch_size, seq_len, num_heads * head_size]
135 + key: shape = [batch_size, seq_len, num_kv_heads * head_size]
136 + value: shape = [batch_size, seq_len, num_kv_heads * head_size]
137 + key_cache = [num_kv_heads, num_blocks, block_size, head_size]
138 + value_cache = [num_kv_heads, num_blocks, block_size, head_size]
139 + attn_metadata: Metadata for attention.
140 + Returns:
141 + shape = [batch_size, seq_len, num_heads * head_size]
142 + """
143 + assert kv_scale == 1.0
144 + batch_size, seq_len, hidden_size = query.shape
145 + query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
146 + key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
147 + value = value.view(batch_size, seq_len, self.num_kv_heads,
148 + self.head_size)
149 +
150 + if kv_cache[0] is not None:
151 + slot_mapping = attn_metadata.slot_mapping
152 + key_cache, value_cache = kv_cache
153 + write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
154 +
155 + query = query * self.scale
156 + if attn_metadata.num_prefills > 0:
157 + assert seq_len % 16 == 0, (
158 + "Pallas FlashAttention kernel requires seq_len to be a "
159 + f"multiple of 16 but got {seq_len}")
160 +
161 + # Handle GQA/MQA.
162 + if self.num_kv_heads != self.num_heads:
163 + key = key.repeat_interleave(self.num_queries_per_kv, dim=-2)
164 + key = key.view(batch_size, seq_len, self.num_heads,
165 + self.head_size)
166 + value = value.repeat_interleave(self.num_queries_per_kv,
167 + dim=-2)
168 + value = value.view(batch_size, seq_len, self.num_heads,
169 + self.head_size)
170 + # FlashAttention requires [batch_size, num_heads, seq_len, d_model]
171 + # while the input is [batch_size, seq_len, num_heads, d_model].
172 + # Permute the input to match the required format.
173 + output = torch.ops.xla.flash_attention(
174 + query.permute(0, 2, 1, 3),
175 + key.permute(0, 2, 1, 3),
176 + value.permute(0, 2, 1, 3),
177 + True,
178 + )
179 + output = output.permute(0, 2, 1, 3)
180 + else:
181 + # Decoding run.
182 + assert kv_cache is not None
183 +
184 + pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
185 + if self.megacore_mode == "batch" and batch_size % 2 != 0:
186 + megacore_mode = None
187 + else:
188 + megacore_mode = self.megacore_mode
189 +
190 + # NOTE(woosuk): A temporary workaround to avoid the error:
191 + # "xla::paged_attention() Expected a value of type 'str' for
192 + # argument 'megacore_mode' but instead found type 'NoneType'."
193 + if megacore_mode is not None:
194 + output = torch.ops.xla.paged_attention(
195 + query.squeeze(dim=1),
196 + key_cache,
197 + value_cache,
198 + attn_metadata.context_lens,
199 + attn_metadata.block_tables,
200 + pages_per_compute_block,
201 + megacore_mode=megacore_mode,
202 + )
203 + else:
204 + output = torch.ops.xla.paged_attention(
205 + query.squeeze(dim=1),
206 + key_cache,
207 + value_cache,
208 + attn_metadata.context_lens,
209 + attn_metadata.block_tables,
210 + pages_per_compute_block,
211 + )
212 +
213 + # Reshape the output tensor.
214 + return output.reshape(batch_size, seq_len, hidden_size)
215 +
216 +
217 +def write_to_kv_cache(
218 + key: torch.Tensor,
219 + value: torch.Tensor,
220 + key_cache: torch.Tensor,
221 + value_cache: torch.Tensor,
222 + slot_mapping: torch.Tensor,
223 +) -> None:
224 + torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True)
225 + torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True)
226 +
227 + key = key.flatten(0, 2)
228 + value = value.flatten(0, 2)
229 + key_cache = key_cache.flatten(0, 2)
230 + value_cache = value_cache.flatten(0, 2)
231 + key_cache.index_copy_(0, slot_mapping, key)
232 + value_cache.index_copy_(0, slot_mapping, value)
vllm/attention/selector.py
+11 -2
@@ -7,7 +7,7 @@ import torch
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import AttentionBackend
9 from vllm.logger import init_logger
10 -from vllm.utils import is_cpu, is_hip
10 +from vllm.utils import is_cpu, is_hip, is_tpu
11
12 logger = init_logger(__name__)
13
@@ -18,6 +18,7 @@ class _Backend(enum.Enum):
18 ROCM_FLASH = enum.auto()
19 TORCH_SDPA = enum.auto()
20 FLASHINFER = enum.auto()
21 + PALLAS = enum.auto()
22
23
24 @lru_cache(maxsize=None)
@@ -66,6 +67,10 @@ def get_attn_backend(
67 "Please make sure --enforce-eager is set.")
68 from vllm.attention.backends.flashinfer import FlashInferBackend
69 return FlashInferBackend
70 + elif backend == _Backend.PALLAS:
71 + logger.info("Using Pallas backend.")
72 + from vllm.attention.backends.pallas import PallasAttentionBackend
73 + return PallasAttentionBackend
74 else:
75 raise ValueError("Invalid attention backend.")
76
@@ -80,7 +85,6 @@ def which_attn_to_use(
85 block_size: int,
86 ) -> _Backend:
87 """Returns which flash attention backend to use."""
83 -
88 # Default case.
89 selected_backend = _Backend.FLASH_ATTN
90
@@ -100,6 +104,11 @@ def which_attn_to_use(
104 logger.info("Cannot use %s backend on CPU.", selected_backend)
105 return _Backend.TORCH_SDPA
106
107 + if is_tpu():
108 + if selected_backend != _Backend.PALLAS:
109 + logger.info("Cannot use %s backend on TPU.", selected_backend)
110 + return _Backend.PALLAS
111 +
112 if is_hip():
113 # AMD GPUs.
114 selected_backend = (_Backend.ROCM_FLASH if selected_backend
vllm/config.py
+5 -1
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
11 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
12 from vllm.model_executor.models import ModelRegistry
13 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 -from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
14 +from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron, is_tpu
15
16 if TYPE_CHECKING:
17 from ray.util.placement_group import PlacementGroup
@@ -748,6 +748,8 @@ class DeviceConfig:
748 # Automated device type detection
749 if is_neuron():
750 self.device_type = "neuron"
751 + elif is_tpu():
752 + self.device_type = "tpu"
753 elif is_cpu():
754 self.device_type = "cpu"
755 else:
@@ -761,6 +763,8 @@ class DeviceConfig:
763 # Some device types require processing inputs on CPU
764 if self.device_type in ["neuron"]:
765 self.device = torch.device("cpu")
766 + elif self.device_type in ["tpu"]:
767 + self.device = None
768 else:
769 # Set device with device type
770 self.device = torch.device(self.device_type)
vllm/engine/arg_utils.py
+1 -1
@@ -504,7 +504,7 @@ class EngineArgs:
504 parser.add_argument("--device",
505 type=str,
506 default=EngineArgs.device,
507 - choices=["auto", "cuda", "neuron", "cpu"],
507 + choices=["auto", "cuda", "neuron", "cpu", "tpu"],
508 help='Device type for vLLM execution.')
509
510 # Related to Vision-language models such as llava
vllm/engine/async_llm_engine.py
+3
@@ -375,6 +375,9 @@ class AsyncLLMEngine:
375 if engine_config.device_config.device_type == "neuron":
376 from vllm.executor.neuron_executor import NeuronExecutorAsync
377 executor_class = NeuronExecutorAsync
378 + elif engine_config.device_config.device_type == "tpu":
379 + from vllm.executor.tpu_executor import TPUExecutorAsync
380 + executor_class = TPUExecutorAsync
381 elif engine_config.device_config.device_type == "cpu":
382 assert distributed_executor_backend is None, (
383 "Distributed execution is not supported with the CPU backend.")
vllm/engine/llm_engine.py
+3
@@ -341,6 +341,9 @@ class LLMEngine:
341 if engine_config.device_config.device_type == "neuron":
342 from vllm.executor.neuron_executor import NeuronExecutor
343 executor_class = NeuronExecutor
344 + elif engine_config.device_config.device_type == "tpu":
345 + from vllm.executor.tpu_executor import TPUExecutor
346 + executor_class = TPUExecutor
347 elif engine_config.device_config.device_type == "cpu":
348 from vllm.executor.cpu_executor import CPUExecutor
349 executor_class = CPUExecutor
vllm/envs.py
+6
@@ -27,6 +27,7 @@ if TYPE_CHECKING:
27 VLLM_TRACE_FUNCTION: int = 0
28 VLLM_ATTENTION_BACKEND: Optional[str] = None
29 VLLM_CPU_KVCACHE_SPACE: int = 0
30 + VLLM_XLA_CACHE_PATH: str = "~/.vllm/xla_cache/"
31 VLLM_USE_RAY_COMPILED_DAG: bool = False
32 VLLM_WORKER_MULTIPROC_METHOD: str = "spawn"
33 VLLM_IMAGE_FETCH_TIMEOUT: int = 5
@@ -217,6 +218,11 @@ environment_variables: Dict[str, Callable[[], Any]] = {
218 # Default is 5 seconds
219 "VLLM_IMAGE_FETCH_TIMEOUT":
220 lambda: int(os.getenv("VLLM_IMAGE_FETCH_TIMEOUT", "5")),
221 +
222 + # Path to the XLA persistent cache directory.
223 + # Only used for XLA devices such as TPUs.
224 + "VLLM_XLA_CACHE_PATH":
225 + lambda: os.getenv("VLLM_XLA_CACHE_PATH", "~/.vllm/xla_cache/"),
226 }
227
228 # end-env-vars-definition
vllm/executor/tpu_executor.py new
+101
@@ -0,0 +1,101 @@
1 +from typing import List, Set, Tuple
2 +
3 +import torch
4 +
5 +from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
6 +from vllm.logger import init_logger
7 +from vllm.lora.request import LoRARequest
8 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
9 +from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
10 + make_async)
11 +
12 +logger = init_logger(__name__)
13 +
14 +
15 +class TPUExecutor(ExecutorBase):
16 +
17 + def _init_executor(self) -> None:
18 + assert not self.scheduler_config.chunked_prefill_enabled, (
19 + "Chunked prefill is not yet supported for TPU backend")
20 + assert not self.speculative_config, (
21 + "Speculative decoding is not yet supported for TPU backend")
22 + if self.model_config.dtype in (torch.float16, torch.float32):
23 + logger.warning(
24 + "The TPU backend currently does not support %s. "
25 + "Using bfloat16 instead.", self.model_config.dtype)
26 + self.model_config.dtype = torch.bfloat16
27 +
28 + # Instantiate the worker and load the model to the device.
29 + self._init_worker()
30 +
31 + def _init_worker(self):
32 + from vllm.worker.tpu_worker import TPUWorker
33 +
34 + assert self.parallel_config.world_size == 1, (
35 + "TPUExecutor currently only supports a single TPU chip.")
36 + distributed_init_method = get_distributed_init_method(
37 + get_ip(), get_open_port())
38 + self.driver_worker = TPUWorker(
39 + self.model_config,
40 + self.parallel_config,
41 + self.scheduler_config,
42 + self.device_config,
43 + self.cache_config,
44 + self.load_config,
45 + self.vision_language_config,
46 + local_rank=0,
47 + rank=0,
48 + distributed_init_method=distributed_init_method,
49 + )
50 + self.driver_worker.init_device()
51 + self.driver_worker.load_model()
52 +
53 + def initialize_cache(
54 + self,
55 + num_gpu_blocks: int,
56 + num_cpu_blocks: int,
57 + ) -> None:
58 + """Initialize the KV cache by invoking the underlying worker."""
59 + # NOTE: This is logged in the executor because there can be >1 worker
60 + # with other executors. We could log in the engine level, but work
61 + # remains to abstract away the device for non-GPU configurations.
62 + logger.info("# TPU blocks: %d, # CPU blocks: %d", num_gpu_blocks,
63 + num_cpu_blocks)
64 + self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
65 +
66 + def determine_num_available_blocks(self) -> Tuple[int, int]:
67 + """Determine the number of available KV blocks by invoking the
68 + underlying worker.
69 + """
70 + return self.driver_worker.determine_num_available_blocks()
71 +
72 + def execute_model(
73 + self,
74 + execute_model_req: ExecuteModelRequest,
75 + ) -> List[SamplerOutput]:
76 + output = self.driver_worker.execute_model(execute_model_req)
77 + return output
78 +
79 + def add_lora(self, lora_request: LoRARequest) -> bool:
80 + raise NotImplementedError("LoRA is not implemented for TPU backend.")
81 +
82 + def remove_lora(self, lora_id: int) -> bool:
83 + raise NotImplementedError("LoRA is not implemented for TPU backend.")
84 +
85 + def list_loras(self) -> Set[int]:
86 + raise NotImplementedError("LoRA is not implemented for TPU backend.")
87 +
88 + def check_health(self) -> None:
89 + # TPUExecutor will always be healthy as long as it's running.
90 + return
91 +
92 +
93 +class TPUExecutorAsync(TPUExecutor, ExecutorAsyncBase):
94 +
95 + async def execute_model_async(
96 + self,
97 + sexecute_model_req: ExecuteModelRequest,
98 + ) -> SamplerOutput:
99 + output = await make_async(self.driver_worker.execute_model
100 + )(sexecute_model_req)
101 + return output
vllm/model_executor/custom_op.py
+3 -1
@@ -1,6 +1,6 @@
1 import torch.nn as nn
2
3 -from vllm.utils import is_cpu, is_hip
3 +from vllm.utils import is_cpu, is_hip, is_tpu
4
5
6 class CustomOp(nn.Module):
@@ -56,5 +56,7 @@ class CustomOp(nn.Module):
56 return self.forward_hip
57 elif is_cpu():
58 return self.forward_cpu
59 + elif is_tpu():
60 + return self.forward_tpu
61 else:
62 return self.forward_cuda
vllm/model_executor/layers/rotary_embedding.py
+74 -3
@@ -28,6 +28,7 @@ import torch
28 import torch.nn as nn
29
30 from vllm.model_executor.custom_op import CustomOp
31 +from vllm.utils import is_tpu
32
33
34 def _rotate_neox(x: torch.Tensor) -> torch.Tensor:
@@ -43,6 +44,19 @@ def _rotate_gptj(x: torch.Tensor) -> torch.Tensor:
44 return x.flatten(-2)
45
46
47 +def _apply_rotary_emb(
48 + x: torch.Tensor,
49 + freqs_cis: torch.Tensor,
50 +) -> torch.Tensor:
51 + x_ = torch.view_as_complex(
52 + torch.stack(torch.chunk(x.transpose(1, 2).float(), 2, dim=-1), dim=-1))
53 + x_out = torch.view_as_real(x_ * freqs_cis).type_as(x)
54 + x_out = torch.cat(torch.chunk(x_out, 2, dim=-1), dim=-2)
55 + x_out = x_out.reshape(x_out.shape[0], x_out.shape[1], x_out.shape[2],
56 + -1).transpose(1, 2)
57 + return x_out
58 +
59 +
60 class RotaryEmbedding(CustomOp):
61 """Original rotary positional embedding."""
62
@@ -64,8 +78,14 @@ class RotaryEmbedding(CustomOp):
78 self.dtype = dtype
79
80 cache = self._compute_cos_sin_cache()
67 - cache = cache.to(dtype)
68 - self.register_buffer("cos_sin_cache", cache, persistent=False)
81 + self.use_native2 = is_tpu() and is_neox_style
82 + if not self.use_native2:
83 + cache = cache.to(dtype)
84 + self.register_buffer("cos_sin_cache", cache, persistent=False)
85 + else:
86 + cos, sin = cache.chunk(2, dim=-1)
87 + freqs_cis = cos + 1j * sin
88 + self.register_buffer("freqs_cis", freqs_cis, persistent=False)
89
90 def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
91 """Compute the inverse frequency."""
@@ -100,7 +120,11 @@ class RotaryEmbedding(CustomOp):
120 key: torch.Tensor,
121 offsets: Optional[torch.Tensor] = None,
122 ) -> Tuple[torch.Tensor, torch.Tensor]:
103 - """PyTorch-native implementation equivalent to forward()."""
123 + """A PyTorch-native implementation equivalent to forward().
124 +
125 + This method mimics the implementation of the custom CUDA kernel
126 + used in `forward_cuda()`.
127 + """
128 query = query.view(*query.shape[:-1], -1, self.head_size)
129 key = key.view(*key.shape[:-1], -1, self.head_size)
130
@@ -138,6 +162,42 @@ class RotaryEmbedding(CustomOp):
162 key = key.flatten(-2)
163 return query, key
164
165 + def forward_native2(
166 + self,
167 + positions: torch.Tensor,
168 + query: torch.Tensor,
169 + key: torch.Tensor,
170 + offsets: Optional[torch.Tensor] = None,
171 + ) -> Tuple[torch.Tensor, torch.Tensor]:
172 + """Another PyTorch-native implementation of forward().
173 +
174 + This method might perform better than `forward_native()` when compiled.
175 + """
176 + if positions.dim() == 1:
177 + batch_size = 1
178 + seq_len = positions.shape[0]
179 + else:
180 + batch_size, seq_len = positions.shape
181 + if offsets is not None:
182 + positions = positions + offsets
183 + freqs_cis = self.freqs_cis.index_select(0, positions.flatten())
184 + freqs_cis = freqs_cis.view(batch_size, 1, seq_len, -1)
185 +
186 + query_shape = query.shape
187 + query = query.view(batch_size, seq_len, -1, self.head_size)
188 + query_rot = query[..., :self.rotary_dim]
189 + query_pass = query[..., self.rotary_dim:]
190 + query_rot = _apply_rotary_emb(query_rot, freqs_cis)
191 + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
192 +
193 + key_shape = key.shape
194 + key = key.view(batch_size, seq_len, -1, self.head_size)
195 + key_rot = key[..., :self.rotary_dim]
196 + key_pass = key[..., self.rotary_dim:]
197 + key_rot = _apply_rotary_emb(key_rot, freqs_cis)
198 + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
199 + return query, key
200 +
201 def forward_cuda(
202 self,
203 positions: torch.Tensor,
@@ -161,6 +221,17 @@ class RotaryEmbedding(CustomOp):
221 self.cos_sin_cache, self.is_neox_style)
222 return query, key
223
224 + def forward_tpu(
225 + self,
226 + positions: torch.Tensor,
227 + query: torch.Tensor,
228 + key: torch.Tensor,
229 + offsets: Optional[torch.Tensor] = None,
230 + ) -> Tuple[torch.Tensor, torch.Tensor]:
231 + forward_fn = (self.forward_native2
232 + if self.use_native2 else self.forward_native)
233 + return forward_fn(positions, query, key, offsets)
234 +
235 def extra_repr(self) -> str:
236 s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
237 s += f", max_position_embeddings={self.max_position_embeddings}"
vllm/model_executor/model_loader/loader.py
+21 -6
@@ -34,6 +34,7 @@ from vllm.model_executor.model_loader.weight_utils import (
34 pt_weights_iterator, safetensors_weights_iterator)
35 from vllm.model_executor.models.vlm_base import VisionLanguageModelBase
36 from vllm.model_executor.utils import set_weight_attrs
37 +from vllm.utils import is_tpu
38
39 logger = init_logger(__name__)
40
@@ -227,12 +228,26 @@ class DefaultModelLoader(BaseModelLoader):
228 if self.load_config.load_format == LoadFormat.NPCACHE:
229 # Currently np_cache only support *.bin checkpoints
230 assert use_safetensors is False
230 - return np_cache_weights_iterator(model_name_or_path,
231 - self.load_config.download_dir,
232 - hf_folder, hf_weights_files)
233 - if use_safetensors:
234 - return safetensors_weights_iterator(hf_weights_files)
235 - return pt_weights_iterator(hf_weights_files)
231 + weights_iterator = np_cache_weights_iterator(
232 + model_name_or_path, self.load_config.download_dir, hf_folder,
233 + hf_weights_files)
234 + elif use_safetensors:
235 + weights_iterator = safetensors_weights_iterator(hf_weights_files)
236 + else:
237 + weights_iterator = pt_weights_iterator(hf_weights_files)
238 +
239 + if is_tpu():
240 + # In PyTorch XLA, we should call `xm.mark_step` frequently so that
241 + # not too many ops are accumulated in the XLA program.
242 + import torch_xla.core.xla_model as xm
243 +
244 + def _xla_weights_iterator(iterator: Generator):
245 + for weights in iterator:
246 + yield weights
247 + xm.mark_step()
248 +
249 + weights_iterator = _xla_weights_iterator(weights_iterator)
250 + return weights_iterator
251
252 def load_model(self, *, model_config: ModelConfig,
253 device_config: DeviceConfig,
vllm/utils.py
+14
@@ -146,6 +146,15 @@ def is_neuron() -> bool:
146 return transformers_neuronx is not None
147
148
149 +@lru_cache(maxsize=None)
150 +def is_tpu() -> bool:
151 + try:
152 + import libtpu
153 + except ImportError:
154 + libtpu = None
155 + return libtpu is not None
156 +
157 +
158 @lru_cache(maxsize=None)
159 def get_max_shared_memory_bytes(gpu: int = 0) -> int:
160 """Returns the maximum shared memory per thread block in bytes."""
@@ -546,6 +555,11 @@ def maybe_expand_dim(tensor: torch.Tensor,
555 return tensor
556
557
558 +def get_dtype_size(dtype: torch.dtype) -> int:
559 + """Get the size of the data type in bytes."""
560 + return torch.tensor([], dtype=dtype).element_size()
561 +
562 +
563 def merge_dicts(dict1: Dict[Any, List[Any]],
564 dict2: Dict[Any, List[Any]]) -> Dict[Any, List[Any]]:
565 """Merge 2 dicts that have key -> List of items.
vllm/worker/cache_engine.py
+3 -6
@@ -6,7 +6,8 @@ import torch
6 from vllm.attention import get_attn_backend
7 from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, is_pin_memory_available
9 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size,
10 + is_pin_memory_available)
11
12 logger = init_logger(__name__)
13
@@ -108,9 +109,5 @@ class CacheEngine:
109 dtype = model_config.dtype
110 else:
111 dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
111 - dtype_size = _get_dtype_size(dtype)
112 + dtype_size = get_dtype_size(dtype)
113 return dtype_size * total
113 -
114 -
115 -def _get_dtype_size(dtype: torch.dtype) -> int:
116 - return torch.tensor([], dtype=dtype).element_size()
vllm/worker/tpu_model_runner.py new
+525
@@ -0,0 +1,525 @@
1 +import time
2 +from typing import List, Optional, Tuple
3 +
4 +import numpy as np
5 +import torch
6 +import torch.nn as nn
7 +import torch_xla.core.xla_model as xm
8 +
9 +from vllm.attention import AttentionMetadata, get_attn_backend
10 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
12 +from vllm.logger import init_logger
13 +from vllm.model_executor.model_loader import get_model
14 +from vllm.model_executor.sampling_metadata import SamplingMetadata
15 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
16 + SamplerOutput, SequenceGroupMetadata,
17 + SequenceOutput)
18 +from vllm.utils import make_tensor_with_pad
19 +
20 +logger = init_logger(__name__)
21 +
22 +_PAD_SLOT_ID = 0 # FIXME(woosuk)
23 +
24 +
25 +class TPUModelRunner:
26 +
27 + def __init__(
28 + self,
29 + model_config: ModelConfig,
30 + parallel_config: ParallelConfig,
31 + scheduler_config: SchedulerConfig,
32 + device_config: DeviceConfig,
33 + cache_config: CacheConfig,
34 + load_config: LoadConfig,
35 + vision_language_config: Optional[VisionLanguageConfig] = None,
36 + ):
37 + self.model_config = model_config
38 + self.parallel_config = parallel_config
39 + self.scheduler_config = scheduler_config
40 + self.device_config = device_config
41 + self.cache_config = cache_config
42 + self.load_config = load_config
43 + self.vision_language_config = vision_language_config
44 +
45 + self.block_size = self.cache_config.block_size
46 + self.max_num_blocks_per_seq = (self.model_config.max_model_len //
47 + self.block_size)
48 + self.block_tables = np.zeros(
49 + (self.scheduler_config.max_num_seqs, self.max_num_blocks_per_seq),
50 + dtype=np.int32)
51 + self.attn_backend = get_attn_backend(
52 + self.model_config.get_num_attention_heads(self.parallel_config),
53 + self.model_config.get_head_size(),
54 + self.model_config.get_num_kv_heads(self.parallel_config),
55 + self.model_config.get_sliding_window(),
56 + self.model_config.dtype,
57 + self.cache_config.cache_dtype,
58 + self.block_size,
59 + False,
60 + )
61 +
62 + def load_model(self) -> None:
63 + self.device = self.device_config.device
64 +
65 + model = get_model(
66 + model_config=self.model_config,
67 + load_config=self.load_config,
68 + device_config=self.device_config,
69 + parallel_config=self.parallel_config,
70 + cache_config=self.cache_config,
71 + scheduler_config=self.scheduler_config,
72 + vision_language_config=self.vision_language_config,
73 + lora_config=None,
74 + )
75 + xm.wait_device_ops()
76 +
77 + model = ModelWrapper(model)
78 + self.model = torch.compile(model, backend="openxla", fullgraph=True)
79 +
80 + def _dummy_run(
81 + self,
82 + batch_size: int,
83 + seq_len: int,
84 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
85 + is_prompt: bool,
86 + ) -> None:
87 + if is_prompt:
88 + seq_len = (seq_len + 15) // 16 * 16
89 + token_ids = torch.zeros((batch_size, seq_len),
90 + dtype=torch.int32,
91 + device=self.device)
92 + position_ids = torch.zeros((batch_size, seq_len),
93 + dtype=torch.int32,
94 + device=self.device)
95 + slot_mapping = torch.zeros((batch_size, seq_len),
96 + dtype=torch.int64,
97 + device=self.device)
98 + attn_metadata = self.attn_backend.make_metadata(
99 + num_prefills=batch_size,
100 + num_prefill_tokens=batch_size * seq_len,
101 + num_decode_tokens=0,
102 + slot_mapping=slot_mapping,
103 + block_tables=None,
104 + context_lens=None,
105 + )
106 + input_lens = torch.ones((batch_size, ),
107 + dtype=torch.int32,
108 + device=self.device)
109 + else:
110 + assert seq_len == 1
111 + token_ids = torch.zeros((batch_size, seq_len),
112 + dtype=torch.int32,
113 + device=self.device)
114 + position_ids = torch.zeros((batch_size, seq_len),
115 + dtype=torch.int32,
116 + device=self.device)
117 + slot_mapping = torch.zeros((batch_size, seq_len),
118 + dtype=torch.int64,
119 + device=self.device)
120 + block_tables = torch.zeros(
121 + (batch_size, self.max_num_blocks_per_seq),
122 + dtype=torch.int32,
123 + device=self.device)
124 + context_lens = torch.ones((batch_size, ),
125 + dtype=torch.int32,
126 + device=self.device)
127 + input_lens = torch.ones((batch_size, ),
128 + dtype=torch.int32,
129 + device=self.device)
130 + attn_metadata = self.attn_backend.make_metadata(
131 + num_prefills=0,
132 + num_prefill_tokens=0,
133 + num_decode_tokens=batch_size * seq_len,
134 + slot_mapping=slot_mapping,
135 + block_tables=block_tables,
136 + context_lens=context_lens,
137 + )
138 + t = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
139 + p = torch.ones((batch_size, ), dtype=torch.float32, device=self.device)
140 +
141 + # Dummy run.
142 + self.model(token_ids, position_ids, kv_caches, attn_metadata,
143 + input_lens, t, p)
144 +
145 + def warmup_model(
146 + self,
147 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
148 + ) -> None:
149 + # Prefill
150 + logger.info("Compiling the model with different input shapes...")
151 + start = time.time()
152 + for batch_size in [1]:
153 + seq_len = 16
154 + while True:
155 + self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=True)
156 + xm.wait_device_ops()
157 + logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
158 +
159 + if seq_len >= self.model_config.max_model_len:
160 + break
161 + num_tokens = batch_size * seq_len
162 + if num_tokens >= self.scheduler_config.max_num_batched_tokens:
163 + break
164 + seq_len = seq_len * 2
165 +
166 + end = time.time()
167 + logger.info("Compilation for prefill done in %.2f s.", end - start)
168 +
169 + # Decode
170 + start = time.time()
171 + seq_len = 1
172 + batch_size = 1
173 + while True:
174 + self._dummy_run(batch_size, seq_len, kv_caches, is_prompt=False)
175 + xm.wait_device_ops()
176 + logger.info("batch_size: %d, seq_len: %d", batch_size, seq_len)
177 +
178 + if batch_size >= self.scheduler_config.max_num_seqs:
179 + break
180 + batch_size = batch_size + 16 if batch_size >= 16 else batch_size * 2
181 +
182 + end = time.time()
183 + logger.info("Compilation for decode done in %.2f s.", end - start)
184 +
185 + def _prepare_prompt(
186 + self,
187 + seq_group_metadata_list: List[SequenceGroupMetadata],
188 + ):
189 + assert len(seq_group_metadata_list) > 0
190 + input_tokens: List[List[int]] = []
191 + input_positions: List[List[int]] = []
192 + prompt_lens: List[int] = []
193 + slot_mapping: List[List[int]] = []
194 +
195 + for seq_group_metadata in seq_group_metadata_list:
196 + assert seq_group_metadata.is_prompt
197 + seq_ids = list(seq_group_metadata.seq_data.keys())
198 + assert len(seq_ids) == 1
199 + seq_id = seq_ids[0]
200 +
201 + seq_data = seq_group_metadata.seq_data[seq_id]
202 + # Could include output tokens when a request is preempted.
203 + prompt_tokens = seq_data.get_token_ids()
204 + prompt_len = len(prompt_tokens)
205 + prompt_lens.append(prompt_len)
206 +
207 + input_tokens.append(prompt_tokens)
208 + input_positions.append(list(range(prompt_len)))
209 +
210 + assert seq_group_metadata.block_tables is not None
211 + block_table = seq_group_metadata.block_tables[seq_id]
212 + slot_mapping.append([])
213 + for i in range(prompt_len):
214 + block_number = block_table[i // self.block_size]
215 + block_offset = i % self.block_size
216 + slot = block_number * self.block_size + block_offset
217 + slot_mapping[-1].append(slot)
218 +
219 + assert len(prompt_lens) > 0
220 + num_prefills = len(prompt_lens)
221 + num_prefill_tokens = sum(prompt_lens)
222 +
223 + # Add paddings to make the shape [batch_size, max_prompt_len] where
224 + # max_prompt_len is smallest power of 2 that is greater than or equal
225 + # to the maximum prompt length.
226 + # We need the 2D input shape because the Pallas FlashAttention kernel
227 + # does not support packed 1D inputs.
228 + # We pad the seq_len to powers of 2 to reduce the compilation overhead.
229 + max_prompt_len = _get_padded_prefill_len(max(prompt_lens))
230 + input_tokens = make_tensor_with_pad(input_tokens,
231 + max_prompt_len,
232 + pad=0,
233 + dtype=torch.int32,
234 + device=self.device)
235 + input_positions = make_tensor_with_pad(input_positions,
236 + max_prompt_len,
237 + pad=0,
238 + dtype=torch.int32,
239 + device=self.device)
240 + slot_mapping = make_tensor_with_pad(slot_mapping,
241 + max_prompt_len,
242 + pad=_PAD_SLOT_ID,
243 + dtype=torch.int64,
244 + device=self.device)
245 + prompt_lens = torch.tensor(prompt_lens,
246 + dtype=torch.int32,
247 + device=self.device)
248 + attn_metadata = self.attn_backend.make_metadata(
249 + num_prefills=num_prefills,
250 + num_prefill_tokens=num_prefill_tokens, # NOTE: This is not used.
251 + num_decode_tokens=0,
252 + slot_mapping=slot_mapping,
253 + block_tables=None,
254 + context_lens=None,
255 + )
256 + return input_tokens, input_positions, attn_metadata, prompt_lens
257 +
258 + def _prepare_decode(
259 + self,
260 + seq_group_metadata_list: List[SequenceGroupMetadata],
261 + ):
262 + assert len(seq_group_metadata_list) > 0
263 + input_tokens: List[List[int]] = []
264 + input_positions: List[List[int]] = []
265 + slot_mapping: List[List[int]] = []
266 + context_lens: List[int] = []
267 + num_seq_groups = len(seq_group_metadata_list)
268 + batch_size = _get_padded_batch_size(num_seq_groups)
269 +
270 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
271 + assert not seq_group_metadata.is_prompt
272 +
273 + seq_ids = list(seq_group_metadata.seq_data.keys())
274 +
275 + for seq_id in seq_ids:
276 + seq_data = seq_group_metadata.seq_data[seq_id]
277 + generation_token = seq_data.get_last_token_id()
278 + input_tokens.append([generation_token])
279 +
280 + seq_len = seq_data.get_len()
281 + position = seq_len - 1
282 + input_positions.append([position])
283 + context_lens.append(seq_len)
284 +
285 + assert seq_group_metadata.block_tables is not None
286 + block_table = seq_group_metadata.block_tables[seq_id]
287 + self.block_tables[i, :len(block_table)] = block_table
288 +
289 + block_number = block_table[position // self.block_size]
290 + block_offset = position % self.block_size
291 + slot = block_number * self.block_size + block_offset
292 + slot_mapping.append([slot])
293 +
294 + num_paddings = batch_size - num_seq_groups
295 + input_tokens = input_tokens + [[0]] * num_paddings
296 + input_positions = input_positions + [[0]] * num_paddings
297 + slot_mapping = slot_mapping + [[_PAD_SLOT_ID]] * num_paddings
298 + context_lens = context_lens + [0] * num_paddings
299 +
300 + input_tokens = torch.tensor(input_tokens,
301 + dtype=torch.int32,
302 + device=self.device)
303 + input_positions = torch.tensor(input_positions,
304 + dtype=torch.int32,
305 + device=self.device)
306 + slot_mapping = torch.tensor(slot_mapping,
307 + dtype=torch.int64,
308 + device=self.device)
309 + context_lens = torch.tensor(context_lens,
310 + dtype=torch.int32,
311 + device=self.device)
312 + block_tables = torch.tensor(self.block_tables[:batch_size],
313 + dtype=torch.int32,
314 + device=self.device)
315 + input_lens = torch.tensor([1] * batch_size,
316 + dtype=torch.int32,
317 + device=self.device)
318 + attn_metadata = self.attn_backend.make_metadata(
319 + num_prefills=0,
320 + num_prefill_tokens=0,
321 + num_decode_tokens=batch_size,
322 + slot_mapping=slot_mapping,
323 + block_tables=block_tables,
324 + context_lens=context_lens,
325 + )
326 + return input_tokens, input_positions, attn_metadata, input_lens
327 +
328 + def _prepare_sample(
329 + self,
330 + seq_group_metadata_list: List[SequenceGroupMetadata],
331 + padded_batch_size: int,
332 + ) -> Tuple[torch.Tensor, torch.Tensor]:
333 + assert len(seq_group_metadata_list) > 0
334 + t = []
335 + p = []
336 + for seq_group_metadata in seq_group_metadata_list:
337 + assert seq_group_metadata.sampling_params is not None
338 + sampling_params = seq_group_metadata.sampling_params
339 +
340 + t.append(sampling_params.temperature
341 + if sampling_params.temperature >= 1e-5 else 1e-5)
342 + p.append(sampling_params.top_p)
343 + num_paddings = padded_batch_size - len(seq_group_metadata_list)
344 + t += [1.0] * num_paddings
345 + p += [1.0] * num_paddings
346 +
347 + t = torch.tensor(t, dtype=torch.float32, device=self.device)
348 + p = torch.tensor(p, dtype=torch.float32, device=self.device)
349 + return t, p
350 +
351 + def prepare_inputs(
352 + self,
353 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
354 + ):
355 + assert seq_group_metadata_list is not None
356 + assert len(seq_group_metadata_list) > 0
357 + # NOTE: We assume that all sequences in the group are all prompts or
358 + # all decodes.
359 + if seq_group_metadata_list[0].is_prompt:
360 + inputs = self._prepare_prompt(seq_group_metadata_list)
361 + else:
362 + inputs = self._prepare_decode(seq_group_metadata_list)
363 + padded_batch_size = inputs[0].shape[0]
364 + sample_inputs = self._prepare_sample(seq_group_metadata_list,
365 + padded_batch_size)
366 + return inputs + sample_inputs
367 +
368 + def _execute_model(
369 + self,
370 + seq_group_metadata_list: List[SequenceGroupMetadata],
371 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
372 + ) -> List[CompletionSequenceGroupOutput]:
373 + inputs = self.prepare_inputs(seq_group_metadata_list)
374 + next_token_ids = self.model(inputs[0], inputs[1], kv_caches,
375 + *inputs[2:])
376 + next_token_ids = next_token_ids.cpu().tolist()
377 +
378 + i = 0
379 + sampler_outputs = []
380 + for seq_group_metadata in seq_group_metadata_list:
381 + seq_outputs = []
382 + seq_ids = list(seq_group_metadata.seq_data.keys())
383 + for seq_id in seq_ids:
384 + next_token_id = next_token_ids[i]
385 + seq_outputs.append(
386 + SequenceOutput(seq_id, next_token_id,
387 + {next_token_id: Logprob(0.0)}))
388 + i += 1
389 + sampler_outputs.append(
390 + CompletionSequenceGroupOutput(seq_outputs, None))
391 + return sampler_outputs
392 +
393 + def execute_model(
394 + self,
395 + seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
396 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
397 + ) -> SamplerOutput:
398 + assert seq_group_metadata_list is not None
399 + if seq_group_metadata_list[0].is_prompt:
400 + # NOTE(woosuk): To reduce the compilation time, we only compile the
401 + # prefill inputs with batch size 1. Because the scheduler is not
402 + # aware of this limitation, we need to handle batch size > 1
403 + # internally by calling the model multiple times and concatenating
404 + # the outputs.
405 + # FIXME(woosuk): This is a temporary hack to not change the existing
406 + # scheduler. We need to fix this in the future.
407 + sampler_outputs = []
408 + for seq_group_metadata in seq_group_metadata_list:
409 + sampler_outputs += self._execute_model([seq_group_metadata],
410 + kv_caches)
411 + else:
412 + sampler_outputs = self._execute_model(seq_group_metadata_list,
413 + kv_caches)
414 + return SamplerOutput(sampler_outputs)
415 +
416 +
417 +class ModelWrapper(nn.Module):
418 +
419 + def __init__(self, model: nn.Module):
420 + super().__init__()
421 + self.model = model.eval()
422 +
423 + def forward(
424 + self,
425 + token_ids: torch.Tensor,
426 + position_ids: torch.Tensor,
427 + kv_caches: List[Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]],
428 + attn_metadata: AttentionMetadata,
429 + input_lens: torch.Tensor,
430 + t: torch.Tensor,
431 + p: torch.Tensor,
432 + ) -> torch.Tensor:
433 + """Executes the forward pass of the model and samples the next token.
434 +
435 + Args:
436 + token_ids: The input token IDs of shape [batch_size, seq_len].
437 + position_ids: The input position IDs of shape [batch_size, seq_len].
438 + kv_caches: The key and value caches. They can be None during the
439 + memory profiling at initialization.
440 + attn_metadata: The Pallas attention metadata.
441 + input_lens: The actual input lengths of shape [batch_size].
442 + t: The sampling temperature of shape [batch_size].
443 + p: The top-p probability of shape [batch_size].
444 + """
445 + batch_size, seq_len = token_ids.shape
446 + # Calculate the positions to sample from.
447 + base_indicies = torch.arange(
448 + batch_size, dtype=torch.int32, device=input_lens.device) * seq_len
449 + logits_indices = base_indicies + input_lens - 1
450 +
451 + # FIXME(woosuk): This is a temporary hack to avoid using the existing
452 + # sampler and sampling metadata.
453 + sampling_metadata = SamplingMetadata(
454 + seq_groups=[],
455 + selected_token_indices=logits_indices,
456 + categorized_sample_indices={},
457 + num_prompts=attn_metadata.num_prefills,
458 + )
459 +
460 + # Skip this in memory profiling at initialization.
461 + if kv_caches[0][0] is not None:
462 + # index_copy_(slot_mapping) only works when the inserted dimension
463 + # is 0. However, the KV cache in the Pallas backend has the shape
464 + # [num_kv_heads, num_blocks, block_size, head_size]. To make it
465 + # work, we need to flatten the first three dimensions and modify
466 + # the slot_mapping accordingly.
467 + num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape
468 + slot_mapping = attn_metadata.slot_mapping
469 + slot_mapping = slot_mapping.flatten()
470 + head_indicies = torch.arange(0,
471 + num_kv_heads,
472 + device=slot_mapping.device,
473 + dtype=slot_mapping.dtype)
474 + head_indicies *= block_size * num_blocks
475 + slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view(
476 + -1, num_kv_heads)
477 + slot_mapping = slot_mapping + head_indicies.view(1, -1)
478 + slot_mapping = slot_mapping.flatten()
479 + attn_metadata.slot_mapping = slot_mapping
480 +
481 + hidden_states = self.model(
482 + token_ids,
483 + position_ids,
484 + kv_caches,
485 + attn_metadata,
486 + )
487 + hidden_states = hidden_states.flatten(0, 1)
488 + logits = self.model.compute_logits(hidden_states, sampling_metadata)
489 +
490 + logits = logits / t.unsqueeze(dim=1)
491 + # FIXME(woosuk): Disabled top-p sampling since it's too slow.
492 + # logits = _apply_top_p(logits, p.unsqueeze(dim=1))
493 + probs = torch.softmax(logits, dim=-1, dtype=torch.float32)
494 + # FIXME(woosuk): best_of > 1 is not supported.
495 + next_token_ids = torch.multinomial(probs, num_samples=1).squeeze(dim=1)
496 + return next_token_ids
497 +
498 +
499 +def _get_padded_prefill_len(x: int) -> int:
500 + # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
501 + # length to be a multiple of 16. We pad the prompt length to the nearest
502 + # multiple of 16. This is also good for performance.
503 + if x <= 16:
504 + return 16
505 + return 1 << (x - 1).bit_length()
506 +
507 +
508 +def _get_padded_batch_size(batch_size: int) -> int:
509 + if batch_size <= 2:
510 + return batch_size
511 + elif batch_size <= 4:
512 + return 4
513 + elif batch_size <= 8:
514 + return 8
515 + else:
516 + return ((batch_size + 15) // 16) * 16
517 +
518 +
519 +def _apply_top_p(logits: torch.Tensor, p: torch.Tensor) -> torch.Tensor:
520 + logits_sorted = torch.sort(logits, dim=-1, descending=True).values
521 + sorted_cum_probs = torch.cumsum(logits_sorted.softmax(dim=-1), dim=-1)
522 + cutoff_index = torch.sum(sorted_cum_probs < p, dim=-1, keepdim=True)
523 + cutoff_logit = torch.gather(logits_sorted, -1, cutoff_index)
524 + logits = logits.masked_fill_(logits < cutoff_logit, -float("inf"))
525 + return logits
vllm/worker/tpu_worker.py new
+198
@@ -0,0 +1,198 @@
1 +import os
2 +from typing import List, Optional, Tuple
3 +
4 +import torch
5 +import torch_xla.core.xla_model as xm
6 +import torch_xla.runtime as xr
7 +
8 +import vllm.envs as envs
9 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
10 + ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 +from vllm.distributed import (ensure_model_parallel_initialized,
12 + init_distributed_environment)
13 +from vllm.logger import init_logger
14 +from vllm.model_executor import set_random_seed
15 +from vllm.sequence import ExecuteModelRequest, SamplerOutput
16 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
17 +from vllm.worker.tpu_model_runner import TPUModelRunner
18 +from vllm.worker.worker_base import LoraNotSupportedWorkerBase
19 +
20 +logger = init_logger(__name__)
21 +
22 +
23 +class TPUWorker(LoraNotSupportedWorkerBase):
24 +
25 + def __init__(
26 + self,
27 + model_config: ModelConfig,
28 + parallel_config: ParallelConfig,
29 + scheduler_config: SchedulerConfig,
30 + device_config: DeviceConfig,
31 + cache_config: CacheConfig,
32 + load_config: LoadConfig,
33 + vision_language_config: Optional[VisionLanguageConfig],
34 + local_rank: int,
35 + rank: int,
36 + distributed_init_method: str,
37 + ) -> None:
38 + self.model_config = model_config
39 + self.parallel_config = parallel_config
40 + self.scheduler_config = scheduler_config
41 + self.device_config = device_config
42 + self.cache_config = cache_config
43 + self.load_config = load_config
44 + self.vision_language_config = vision_language_config
45 + self.local_rank = local_rank
46 + self.rank = rank
47 + self.distributed_init_method = distributed_init_method
48 +
49 + assert self.device_config.device_type == "tpu"
50 + if self.cache_config.cache_dtype == "auto":
51 + self.cache_dtype = self.model_config.dtype
52 + else:
53 + self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
54 + self.cache_config.cache_dtype]
55 +
56 + self.model_runner = TPUModelRunner(model_config, parallel_config,
57 + scheduler_config, device_config,
58 + cache_config, load_config,
59 + vision_language_config)
60 +
61 + def init_device(self) -> None:
62 + os.environ["PJRT_DEVICE"] = "TPU"
63 + self.device = xm.xla_device()
64 + self.device_config.device = self.device
65 + torch.set_grad_enabled(False)
66 + torch.set_default_dtype(self.model_config.dtype)
67 +
68 + # NOTE(woosuk): This is just a hack to initialize the TP group.
69 + # This cannot perform the actual communication ops.
70 + init_distributed_environment(
71 + world_size=self.parallel_config.world_size,
72 + rank=self.rank,
73 + local_rank=self.local_rank,
74 + distributed_init_method=self.distributed_init_method,
75 + backend="gloo",
76 + )
77 + ensure_model_parallel_initialized(
78 + self.parallel_config.tensor_parallel_size,
79 + self.parallel_config.pipeline_parallel_size)
80 +
81 + # Set random seed.
82 + set_random_seed(self.model_config.seed)
83 + xm.set_rng_state(self.model_config.seed, self.device)
84 +
85 + # Increase the cache size limit, which is the maximum number of
86 + # dynamo graphs that can be compiled.
87 + # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and
88 + # 30-40 graphs for decode. 128 is an arbitrary safe number.
89 + torch._dynamo.config.cache_size_limit = 128
90 + # Use persistent cache to avoid XLA recompilation.
91 + # NOTE(woosuk): This does not completely eliminate the recompilation
92 + # overhead because dynamo does not cache the compiled results.
93 + xr.initialize_cache(os.path.expanduser(envs.VLLM_XLA_CACHE_PATH),
94 + readonly=False)
95 +
96 + def load_model(self):
97 + self.model_runner.load_model()
98 +
99 + def determine_num_available_blocks(self) -> Tuple[int, int]:
100 + num_layers = self.model_config.get_num_layers(self.parallel_config)
101 + head_size = self.model_config.get_head_size()
102 + num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
103 +
104 + kv_caches = [(None, None) for _ in range(num_layers)]
105 + self.model_runner._dummy_run(
106 + batch_size=1,
107 + seq_len=self.scheduler_config.max_num_batched_tokens,
108 + kv_caches=kv_caches,
109 + is_prompt=True,
110 + )
111 + # Synchronize before measuring the memory usage.
112 + xm.wait_device_ops()
113 +
114 + m = xm.get_memory_info(self.device)
115 + program_size = 1024 * 1024 * 1024 # 1GB
116 + free_bytes = max(m["bytes_limit"] - m["bytes_used"] - program_size, 0)
117 + kv_cache_bytes = int(free_bytes *
118 + self.cache_config.gpu_memory_utilization)
119 + kv_cache_dtype_btyes = get_dtype_size(self.cache_dtype)
120 + block_size = self.cache_config.block_size
121 + num_tpu_blocks = (kv_cache_bytes //
122 + (kv_cache_dtype_btyes * block_size * num_layers * 2 *
123 + head_size * num_kv_heads))
124 + num_tpu_blocks = (num_tpu_blocks // 8) * 8 # Round down to 8.
125 + return num_tpu_blocks, 0
126 +
127 + def initialize_cache(
128 + self,
129 + num_gpu_blocks: int,
130 + num_cpu_blocks: int,
131 + ) -> None:
132 + self.cache_config.num_gpu_blocks = num_gpu_blocks
133 + self.cache_config.num_cpu_blocks = num_cpu_blocks
134 + self.block_size = self.cache_config.block_size
135 +
136 + dtype = self.cache_dtype
137 + num_layers = self.model_config.get_num_layers(self.parallel_config)
138 + num_kv_heads = self.model_config.get_num_kv_heads(self.parallel_config)
139 + head_size = self.model_config.get_head_size()
140 +
141 + self.tpu_cache = []
142 + tpu_cache_shape = self.model_runner.attn_backend.get_kv_cache_shape(
143 + num_gpu_blocks, self.block_size, num_kv_heads, head_size)
144 + for _ in range(num_layers):
145 + key_cache = torch.zeros(tpu_cache_shape,
146 + dtype=dtype,
147 + device=self.device)
148 + value_cache = torch.zeros_like(key_cache)
149 + self.tpu_cache.append((key_cache, value_cache))
150 + self._warmup_model()
151 +
152 + def _warmup_model(self) -> None:
153 + # FIXME(woosuk): Here we are abusing `enforce_eager` which is defined
154 + # for CUDA graphs. We should refactor this part.
155 + if not self.model_config.enforce_eager:
156 + # Warm up the model with all possible input shapes so that
157 + # compilation never happens during the actual execution.
158 + # This may take ~30 mins for the first run and ~20 mins for the
159 + # subsequent runs.
160 + # If `enforce_eager` is True, the ahead-of-time compilation is
161 + # skipped and the compilation happens during the actual execution,
162 + # which is bad for performance but useful for development.
163 + self.model_runner.warmup_model(self.tpu_cache)
164 +
165 + def get_cache_block_size_bytes(self) -> int:
166 + head_size = self.model_config.get_head_size()
167 + num_heads = self.model_config.get_num_kv_heads(self.parallel_config)
168 + num_layers = self.model_config.get_num_layers(self.parallel_config)
169 +
170 + key_cache_block = self.cache_config.block_size * num_heads * head_size
171 + value_cache_block = key_cache_block
172 + total = num_layers * (key_cache_block + value_cache_block)
173 + dtype_size = get_dtype_size(self.cache_dtype)
174 + return dtype_size * total
175 +
176 + def execute_model(
177 + self,
178 + execute_model_req: Optional[ExecuteModelRequest] = None
179 + ) -> List[SamplerOutput]:
180 + if execute_model_req is None:
181 + return []
182 +
183 + seq_group_metadata_list = execute_model_req.seq_group_metadata_list
184 + num_seq_groups = len(seq_group_metadata_list)
185 + if num_seq_groups == 0:
186 + return []
187 +
188 + # Currently, TPUWorker does not support swapping.
189 + # TODO(woosuk): Support block copying.
190 + assert len(execute_model_req.blocks_to_swap_in) == 0, (
191 + "Swapping is not supported for the TPU backend.")
192 + assert len(execute_model_req.blocks_to_swap_out) == 0, (
193 + "Swapping is not supported for the TPU backend.")
194 + assert len(execute_model_req.blocks_to_copy) == 0
195 +
196 + output = self.model_runner.execute_model(seq_group_metadata_list,
197 + self.tpu_cache)
198 + return [output]