@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 45f90bcbba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[WIP] TPU V1 Support Refactored (#13049)

Alexander Matveev committed Feb 14, 2025 at 03:21 UTC 45f90bcbba76dcdf1c1cf664431138d3a9d7acf0
8 files changed +1738 -25
tests/entrypoints/llm/test_accuracy.py
+15 -5
@@ -21,10 +21,13 @@ RTOL = 0.03
21 EXPECTED_VALUE = 0.58
22
23
24 -def run_test():
24 +def run_test(more_args=None):
25 """Run the end to end accuracy test."""
26
27 - model_args = f"pretrained={MODEL_NAME},max_model_len=2048"
27 + model_args = f"pretrained={MODEL_NAME},max_model_len=4096"
28 +
29 + if more_args is not None:
30 + model_args = "{},{}".format(model_args, more_args)
31
32 results = lm_eval.simple_evaluate(
33 model="vllm",
@@ -39,14 +42,21 @@ def run_test():
42 ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
43
44
42 -@pytest.mark.skipif(not current_platform.is_cuda(),
43 - reason="V1 is currently only supported on CUDA.")
45 +@pytest.mark.skipif(not current_platform.is_cuda()
46 + and not current_platform.is_tpu(),
47 + reason="V1 is currently only supported on CUDA and TPU")
48 def test_lm_eval_accuracy_v1_engine(monkeypatch):
49 """Run with the V1 Engine."""
50
51 with monkeypatch.context() as m:
52 m.setenv("VLLM_USE_V1", "1")
49 - run_test()
53 +
54 + more_args = None
55 + if current_platform.is_tpu():
56 + # Limit compilation time for TPU V1
57 + more_args = "max_num_seqs=64"
58 +
59 + run_test(more_args)
60
61
62 def test_lm_eval_accuracy_v0_engine(monkeypatch):
tests/entrypoints/openai/correctness/test_lmeval.py
+11 -4
@@ -21,7 +21,7 @@ TASK = "gsm8k"
21 FILTER = "exact_match,strict-match"
22 RTOL = 0.03
23 EXPECTED_VALUE = 0.58
24 -DEFAULT_ARGS = ["--max-model-len", "2048", "--disable-log-requests"]
24 +DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
25 MORE_ARGS_LIST = [
26 [], # Default
27 ["--enable-chunked-prefill"], # Chunked
@@ -67,14 +67,21 @@ def run_test(more_args):
67 ), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
68
69
70 -@pytest.mark.skipif(not current_platform.is_cuda(),
71 - reason="V1 currently only supported on CUDA")
70 +@pytest.mark.skipif(not current_platform.is_cuda()
71 + and not current_platform.is_tpu(),
72 + reason="V1 currently only supported on CUDA and TPU")
73 def test_lm_eval_accuracy_v1_engine(monkeypatch):
74 """Run with the V1 Engine."""
75
76 with monkeypatch.context() as m:
77 m.setenv("VLLM_USE_V1", "1")
77 - run_test([])
78 + more_args = []
79 +
80 + # Limit compilation time for V1
81 + if current_platform.is_tpu():
82 + more_args = ["--max-num-seqs", "64"]
83 +
84 + run_test(more_args)
85
86
87 @pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
vllm/platforms/interface.py
+1
@@ -37,6 +37,7 @@ class _Backend(enum.Enum):
37 TRITON_MLA = enum.auto()
38 HPU_ATTN = enum.auto()
39 PALLAS = enum.auto()
40 + PALLAS_VLLM_V1 = enum.auto()
41 IPEX = enum.auto()
42 BLOCK_SPARSE_FLASH_ATTN = enum.auto()
43 NO_ATTENTION = enum.auto()
vllm/platforms/tpu.py
+38 -16
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Optional
4
5 import torch
6
7 +import vllm.envs as envs
8 from vllm.logger import init_logger
9
10 from .interface import Platform, PlatformEnum, _Backend
@@ -33,14 +34,20 @@ class TpuPlatform(Platform):
34 dtype: torch.dtype, kv_cache_dtype: Optional[str],
35 block_size: int, use_v1: bool,
36 use_mla: bool) -> str:
36 - if selected_backend != _Backend.PALLAS:
37 + if (selected_backend != _Backend.PALLAS
38 + and selected_backend != _Backend.PALLAS_VLLM_V1):
39 logger.info("Cannot use %s backend on TPU.", selected_backend)
38 - logger.info("Using Pallas backend.")
39 - return "vllm.attention.backends.pallas.PallasAttentionBackend"
40 +
41 + if use_v1:
42 + logger.info("Using Pallas V1 backend.")
43 + return "vllm.v1.attention.backends.pallas.PallasAttentionBackend"
44 + else:
45 + logger.info("Using Pallas backend.")
46 + return "vllm.attention.backends.pallas.PallasAttentionBackend"
47
48 @classmethod
49 def get_device_name(cls, device_id: int = 0) -> str:
43 - raise NotImplementedError
50 + return "tpu"
51
52 @classmethod
53 def get_device_total_memory(cls, device_id: int = 0) -> int:
@@ -48,7 +55,7 @@ class TpuPlatform(Platform):
55
56 @classmethod
57 def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
51 - return True
58 + return not envs.VLLM_USE_V1
59
60 @classmethod
61 def inference_mode(cls):
@@ -63,11 +70,11 @@ class TpuPlatform(Platform):
70 cache_config.block_size = 16
71
72 compilation_config = vllm_config.compilation_config
66 - if compilation_config.level == CompilationLevel.NO_COMPILATION:
67 - # TPU does not support NO_COMPILATION
73 +
74 + # TPU only supports DYNAMO_ONCE compilation level
75 + if compilation_config.level != CompilationLevel.DYNAMO_ONCE:
76 + logger.info("[TPU] Forcing DYNAMO_ONCE compilation level")
77 compilation_config.level = CompilationLevel.DYNAMO_ONCE
69 - assert compilation_config.level < CompilationLevel.PIECEWISE,\
70 - "TPU does not support Inductor."
78
79 if compilation_config.backend == "":
80 compilation_config.backend = "openxla"
@@ -75,10 +82,6 @@ class TpuPlatform(Platform):
82 assert vllm_config.speculative_config is None, \
83 "TPU does not support speculative decoding"
84
78 - assert not vllm_config.scheduler_config.chunked_prefill_enabled, (
79 - "Chunked prefill is not yet supported for TPU backend")
80 - assert not vllm_config.speculative_config, (
81 - "Speculative decoding is not yet supported for TPU backend")
85 if vllm_config.model_config.dtype in (torch.float16, torch.float32):
86 logger.warning(
87 "The TPU backend currently does not support %s. "
@@ -88,8 +91,27 @@ class TpuPlatform(Platform):
91 parallel_config = vllm_config.parallel_config
92 scheduler_config = vllm_config.scheduler_config
93 if parallel_config.worker_cls == "auto":
91 - if scheduler_config.is_multi_step:
94 + if envs.VLLM_USE_V1:
95 parallel_config.worker_cls = \
93 - "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
96 + "vllm.v1.worker.tpu_worker.TPUWorker"
97 else:
95 - parallel_config.worker_cls = "vllm.worker.tpu_worker.TPUWorker"
98 + if scheduler_config.is_multi_step:
99 + parallel_config.worker_cls = \
100 + "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
101 + else:
102 + parallel_config.worker_cls = \
103 + "vllm.worker.tpu_worker.TPUWorker"
104 +
105 + # Adjust scheduler config for V1
106 + # TODO: Add support for these
107 + if envs.VLLM_USE_V1 and vllm_config.cache_config.enable_prefix_caching:
108 + logger.warning("[V1][TPU] Disable prefix caching")
109 + vllm_config.cache_config.enable_prefix_caching = False
110 +
111 + assert not vllm_config.speculative_config, (
112 + "Speculative decoding is not yet supported for TPU backend")
113 +
114 + @classmethod
115 + def is_pin_memory_available(cls):
116 + logger.warning("Pin memory is not supported on TPU.")
117 + return False
vllm/v1/attention/backends/pallas.py new
+353
@@ -0,0 +1,353 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +from dataclasses import dataclass
4 +from typing import Any, Dict, List, Optional, Tuple, Type
5 +
6 +import torch
7 +import torch_xla.experimental.custom_kernel # Required to register custom ops.
8 +
9 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 + AttentionLayer,
11 + AttentionMetadata, AttentionType)
12 +from vllm.attention.backends.utils import CommonAttentionState
13 +
14 +
15 +class PallasAttentionBackend(AttentionBackend):
16 +
17 + @staticmethod
18 + def get_name() -> str:
19 + return "PALLAS_VLLM_V1"
20 +
21 + @staticmethod
22 + def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
23 + return PallasAttentionBackendImpl
24 +
25 + @staticmethod
26 + def get_metadata_cls() -> Type["PallasMetadata"]:
27 + return PallasMetadata
28 +
29 + @staticmethod
30 + def get_state_cls() -> Type["CommonAttentionState"]:
31 + return CommonAttentionState
32 +
33 + @staticmethod
34 + def get_kv_cache_shape(
35 + num_blocks: int,
36 + block_size: int,
37 + num_kv_heads: int,
38 + head_size: int,
39 + ) -> Tuple[int, ...]:
40 + return (num_kv_heads, num_blocks, block_size, head_size)
41 +
42 + @staticmethod
43 + def swap_blocks(
44 + src_kv_cache: torch.Tensor,
45 + dst_kv_cache: torch.Tensor,
46 + src_to_dst: torch.Tensor,
47 + ) -> None:
48 + raise RuntimeError("swap_blocks is not used for the TPU backend.")
49 +
50 + @torch.compile(backend="openxla")
51 + @staticmethod
52 + def copy_blocks(
53 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
54 + src_to_dists: Tuple[torch.Tensor, torch.Tensor],
55 + ) -> None:
56 + src_indices, dst_indices = src_to_dists
57 + for k_cache, v_cache in kv_caches:
58 + torch.ops.xla.dynamo_set_buffer_donor_(k_cache, True)
59 + k_cache[:, dst_indices] = k_cache[:, src_indices]
60 + torch.ops.xla.dynamo_set_buffer_donor_(v_cache, True)
61 + v_cache[:, dst_indices] = v_cache[:, src_indices]
62 +
63 +
64 +@dataclass
65 +class PallasMetadata(AttentionMetadata):
66 +
67 + # Currently, input sequences can only contain all prefills
68 + # or all decoding.
69 + block_tables: Optional[torch.Tensor] = None
70 + context_lens: Optional[torch.Tensor] = None
71 + effective_query_lens: Optional[torch.Tensor] = None
72 +
73 + @property
74 + def prefill_metadata(self) -> Optional["PallasMetadata"]:
75 + if self.num_prefills == 0:
76 + return None
77 +
78 + assert self.num_decode_tokens == 0
79 + return self
80 +
81 + @property
82 + def decode_metadata(self) -> Optional["PallasMetadata"]:
83 + if self.num_decode_tokens == 0:
84 + return None
85 +
86 + assert self.num_prefills == 0
87 + assert self.num_prefill_tokens == 0
88 + assert self.block_tables is not None
89 + assert self.context_lens is not None
90 + return self
91 +
92 +
93 +class PallasAttentionBackendImpl(AttentionImpl):
94 +
95 + def __init__(
96 + self,
97 + num_heads: int,
98 + head_size: int,
99 + scale: float,
100 + num_kv_heads: int,
101 + alibi_slopes: Optional[List[float]],
102 + sliding_window: Optional[int],
103 + kv_cache_dtype: str,
104 + blocksparse_params: Optional[Dict[str, Any]] = None,
105 + logits_soft_cap: Optional[float] = None,
106 + attn_type: str = AttentionType.DECODER,
107 + ) -> None:
108 + self.num_heads = num_heads
109 + self.head_size = head_size
110 + self.scale = float(scale)
111 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
112 +
113 + assert self.num_heads % self.num_kv_heads == 0
114 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
115 + if head_size % 128 != 0:
116 + raise NotImplementedError("Head size must be a multiple of 128.")
117 + if alibi_slopes is not None:
118 + raise NotImplementedError("Alibi slopes is not supported.")
119 + if sliding_window is not None:
120 + raise NotImplementedError("Sliding window is not supported.")
121 + if kv_cache_dtype != "auto":
122 + raise NotImplementedError("FP8 KV cache dtype is not supported.")
123 + if blocksparse_params is not None:
124 + raise NotImplementedError("Blocksparse is not supported.")
125 + if logits_soft_cap is not None:
126 + raise NotImplementedError(
127 + "Attention logits soft-capping is not supported.")
128 +
129 + if torch_xla.tpu.version() < 4:
130 + raise NotImplementedError("TPU version must be 4 or higher.")
131 +
132 + self.megacore_mode = None
133 + tpu_env = torch_xla.tpu.get_tpu_env()
134 + tpu_type = (tpu_env.get("ACCELERATOR_TYPE", None)
135 + or tpu_env.get("TYPE", None)
136 + or tpu_env.get("TPU_ACCELERATOR_TYPE", None))
137 + assert tpu_type is not None
138 + tpu_type = tpu_type.lower()
139 +
140 + if (("lite" not in tpu_type) and ("v6" not in tpu_type)):
141 + if self.num_kv_heads % 2 == 0:
142 + self.megacore_mode = "kv_head"
143 + else:
144 + # NOTE(woosuk): If the batch size is not a multiple of 2, the
145 + # megacore mode will be None.
146 + self.megacore_mode = "batch"
147 +
148 + if attn_type != AttentionType.DECODER:
149 + raise NotImplementedError("Encoder self-attention and "
150 + "encoder/decoder cross-attention "
151 + "are not implemented for "
152 + "PallasAttentionBackendImpl")
153 +
154 + def forward(
155 + self,
156 + layer: AttentionLayer,
157 + query: torch.Tensor,
158 + key: torch.Tensor,
159 + value: torch.Tensor,
160 + kv_cache: Tuple[torch.Tensor, torch.Tensor],
161 + attn_metadata: PallasMetadata,
162 + output: Optional[torch.Tensor] = None,
163 + ) -> torch.Tensor:
164 + """Forward pass with Pallas attention.
165 +
166 + Args:
167 + query: shape = [batch_size, seq_len, num_heads * head_size]
168 + key: shape = [batch_size, seq_len, num_kv_heads * head_size]
169 + value: shape = [batch_size, seq_len, num_kv_heads * head_size]
170 + kv_cache[0] = [num_kv_heads, num_blocks, block_size, head_size]
171 + kv_cache[1] = [num_kv_heads, num_blocks, block_size, head_size]
172 + NOTE: kv_cache[0] and kv_cache[1] will be an empty tensor
173 + with shape [0] for profiling run.
174 + attn_metadata: Metadata for attention.
175 + Returns:
176 + shape = [batch_size, seq_len, num_heads * head_size]
177 + """
178 +
179 + if attn_metadata is None:
180 + if output is None:
181 + output = torch.ones_like(query)
182 + return output
183 +
184 + assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
185 + batch_size, seq_len, hidden_size = query.shape
186 + query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
187 + key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
188 + value = value.view(batch_size, seq_len, self.num_kv_heads,
189 + self.head_size)
190 +
191 + if kv_cache[0].numel() > 0:
192 + slot_mapping = attn_metadata.slot_mapping
193 + key_cache, value_cache = kv_cache
194 + write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
195 +
196 + query = query * self.scale
197 + if attn_metadata.num_prefills > 0:
198 + if attn_metadata.block_tables is None:
199 + # Prefill without paged KV cache.
200 + assert seq_len % 16 == 0, (
201 + "Pallas FlashAttention kernel requires seq_len to be a "
202 + f"multiple of 16 but got {seq_len}")
203 +
204 + # Handle GQA/MQA.
205 + if self.num_kv_heads != self.num_heads:
206 + key = key.repeat_interleave(self.num_queries_per_kv,
207 + dim=-2)
208 + key = key.view(batch_size, seq_len, self.num_heads,
209 + self.head_size)
210 + value = value.repeat_interleave(self.num_queries_per_kv,
211 + dim=-2)
212 + value = value.view(batch_size, seq_len, self.num_heads,
213 + self.head_size)
214 + # FlashAttention kernel requires the input shape to be
215 + # [batch_size, num_heads, seq_len, d_model]
216 + # while the input is [batch_size, seq_len, num_heads, d_model].
217 + # Permute the input to match the required format.
218 + output = torch.ops.xla.flash_attention(
219 + query.permute(0, 2, 1, 3),
220 + key.permute(0, 2, 1, 3),
221 + value.permute(0, 2, 1, 3),
222 + True,
223 + )
224 + output = output.permute(0, 2, 1, 3)
225 + else:
226 + # Prefill with paged KV cache.
227 + # TODO(woosuk): Tune the below knobs.
228 + num_kv_pages_per_compute_block = 16
229 + num_queries_per_compute_block = 16
230 + assert seq_len % num_queries_per_compute_block == 0
231 + output = torch.ops.xla.multi_queries_paged_attention(
232 + query,
233 + key_cache,
234 + value_cache,
235 + attn_metadata.context_lens,
236 + attn_metadata.block_tables,
237 + attn_metadata.effective_query_lens,
238 + num_kv_pages_per_compute_block,
239 + num_queries_per_compute_block,
240 + use_kernel=True,
241 + )
242 + else:
243 + # Decoding run.
244 + assert kv_cache[0].numel() > 0
245 + query = query.squeeze(dim=1)
246 + pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
247 +
248 + assert attn_metadata.block_tables is not None
249 + assert attn_metadata.context_lens is not None
250 + # NOTE(woosuk): The PagedAttention Pallas kernel stores the entire
251 + # block table in SMEM. Therefore, if the block table is too large,
252 + # the kernel compilation will fail. To avoid this, we split the
253 + # batch dimension into smaller chunks and run the kernel multiple
254 + # times.
255 + MAX_SMEM_USAGE = 512 * 1024
256 + size_per_seq = 4 * attn_metadata.block_tables.shape[1]
257 + max_num_seq = MAX_SMEM_USAGE // size_per_seq
258 +
259 + if batch_size <= max_num_seq:
260 + output = paged_attention(
261 + query,
262 + key_cache,
263 + value_cache,
264 + attn_metadata.context_lens,
265 + attn_metadata.block_tables,
266 + pages_per_compute_block,
267 + self.megacore_mode,
268 + )
269 + else:
270 + chunk_size = max_num_seq
271 + # Make sure the chunk size is a multiple of 2.
272 + chunk_size = chunk_size // 2 * 2
273 + num_chunks = (batch_size + chunk_size - 1) // chunk_size
274 +
275 + output = torch.empty_like(query)
276 + for chunk_idx in range(num_chunks):
277 + chunk_start = chunk_idx * chunk_size
278 + chunk_end = chunk_start + chunk_size
279 + # NOTE(woosuk): We skip this line because it causes Dynamo
280 + # compilation error. Instead, we rely on the slice operation
281 + # to handle the out-of-bound case.
282 + # chunk_end = min(chunk_end, batch_size)
283 + chunk_output = paged_attention(
284 + query[chunk_start:chunk_end],
285 + key_cache,
286 + value_cache,
287 + attn_metadata.context_lens[chunk_start:chunk_end],
288 + attn_metadata.block_tables[chunk_start:chunk_end],
289 + pages_per_compute_block,
290 + self.megacore_mode,
291 + )
292 + output[chunk_start:chunk_end] = chunk_output
293 +
294 + # Reshape the output tensor.
295 + return output.reshape(batch_size, seq_len, hidden_size)
296 +
297 +
298 +def write_to_kv_cache(
299 + key: torch.Tensor,
300 + value: torch.Tensor,
301 + key_cache: torch.Tensor,
302 + value_cache: torch.Tensor,
303 + slot_mapping: torch.Tensor,
304 +) -> None:
305 + torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True)
306 + torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True)
307 +
308 + key = key.flatten(0, 2)
309 + value = value.flatten(0, 2)
310 + key_cache = key_cache.flatten(0, 2)
311 + value_cache = value_cache.flatten(0, 2)
312 + key_cache.index_copy_(0, slot_mapping, key)
313 + value_cache.index_copy_(0, slot_mapping, value)
314 +
315 +
316 +def paged_attention(
317 + query: torch.Tensor,
318 + key_cache: torch.Tensor,
319 + value_cache: torch.Tensor,
320 + context_lens: torch.Tensor,
321 + block_tables: torch.Tensor,
322 + pages_per_compute_block: int,
323 + megacore_mode: Optional[str],
324 +) -> torch.Tensor:
325 + batch_size = query.shape[0]
326 + if megacore_mode == "batch" and batch_size % 2 != 0:
327 + megacore_mode = None
328 + else:
329 + megacore_mode = megacore_mode
330 +
331 + # NOTE(woosuk): A temporary workaround to avoid the error:
332 + # "xla::paged_attention() Expected a value of type 'str' for
333 + # argument 'megacore_mode' but instead found type 'NoneType'."
334 + if megacore_mode is not None:
335 + output = torch.ops.xla.paged_attention(
336 + query,
337 + key_cache,
338 + value_cache,
339 + context_lens,
340 + block_tables,
341 + pages_per_compute_block,
342 + megacore_mode=megacore_mode,
343 + )
344 + else:
345 + output = torch.ops.xla.paged_attention(
346 + query,
347 + key_cache,
348 + value_cache,
349 + context_lens,
350 + block_tables,
351 + pages_per_compute_block,
352 + )
353 + return output
vllm/v1/worker/block_table.py
+8
@@ -61,6 +61,14 @@ class BlockTable:
61 src, :num_blocks]
62 self.num_blocks_per_row[tgt] = num_blocks
63
64 + def swap_row(self, src: int, tgt: int) -> None:
65 + num_blocks_src = self.num_blocks_per_row[src]
66 + num_blocks_tgt = self.num_blocks_per_row[tgt]
67 + self.num_blocks_per_row[src] = num_blocks_tgt
68 + self.num_blocks_per_row[tgt] = num_blocks_src
69 +
70 + self.block_table_np[[src, tgt]] = self.block_table_np[[tgt, src]]
71 +
72 def commit(self, num_reqs: int) -> None:
73 self.block_table[:num_reqs].copy_(self.block_table_cpu[:num_reqs],
74 non_blocking=True)
vllm/v1/worker/tpu_model_runner.py new
+1109
@@ -0,0 +1,1109 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +import enum
3 +import time
4 +from dataclasses import dataclass
5 +from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
6 +from unittest.mock import patch
7 +
8 +import numpy as np
9 +import torch
10 +import torch.distributed
11 +import torch.nn as nn
12 +# TPU XLA related
13 +import torch_xla.core.xla_model as xm
14 +import torch_xla.runtime as xr
15 +
16 +from vllm.attention import AttentionMetadata
17 +from vllm.attention.backends.abstract import AttentionType
18 +from vllm.attention.layer import Attention
19 +from vllm.config import VllmConfig
20 +from vllm.forward_context import set_forward_context
21 +from vllm.logger import init_logger
22 +from vllm.model_executor.model_loader import get_model
23 +from vllm.sampling_params import SamplingType
24 +from vllm.utils import LayerBlockType, cdiv, is_pin_memory_available
25 +from vllm.v1.attention.backends.pallas import (PallasAttentionBackend,
26 + PallasMetadata)
27 +from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
28 + KVCacheSpec)
29 +from vllm.v1.outputs import LogprobsTensors, ModelRunnerOutput
30 +from vllm.v1.utils import bind_kv_cache
31 +from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
32 +
33 +if TYPE_CHECKING:
34 + from vllm.v1.core.scheduler import SchedulerOutput
35 +
36 +logger = init_logger(__name__)
37 +
38 +# Here we utilize the behavior that out-of-bound index is ignored.
39 +# FIXME(woosuk): Find a more reliable way to prevent possible bugs.
40 +_PAD_SLOT_ID = 1_000_000_000
41 +
42 +
43 +class ExecutionMode(enum.Enum):
44 + PREFILL = enum.auto()
45 + DECODE = enum.auto()
46 + PREFIX_PREFILL = enum.auto()
47 +
48 + def is_prefill(self) -> bool:
49 + return self in (ExecutionMode.PREFILL, ExecutionMode.PREFIX_PREFILL)
50 +
51 +
52 +@dataclass
53 +class PromptDecodeInfo:
54 + prompt_req_ids: List[str]
55 + decode_req_ids: List[str]
56 + prompt_scheduled_tokens: List[int]
57 +
58 +
59 +@dataclass
60 +class PromptData:
61 + input_tokens: torch.Tensor
62 + input_positions: torch.Tensor
63 + attn_metadata: PallasMetadata
64 +
65 +
66 +@dataclass
67 +class DecodeData:
68 + input_tokens: Optional[torch.Tensor] = None
69 + input_positions: Optional[torch.Tensor] = None
70 + attn_metadata: Optional[PallasMetadata] = None
71 +
72 +
73 +class TPUModelRunner:
74 +
75 + def __init__(
76 + self,
77 + vllm_config: VllmConfig,
78 + device: torch.device,
79 + ):
80 + self.vllm_config = vllm_config
81 + self.model_config = vllm_config.model_config
82 + self.cache_config = vllm_config.cache_config
83 + self.lora_config = vllm_config.lora_config
84 + self.load_config = vllm_config.load_config
85 + self.parallel_config = vllm_config.parallel_config
86 + self.scheduler_config = vllm_config.scheduler_config
87 + self.speculative_config = vllm_config.speculative_config
88 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
89 + self.observability_config = vllm_config.observability_config
90 + self.device_config = vllm_config.device_config
91 +
92 + model_config = self.model_config
93 + cache_config = self.cache_config
94 + scheduler_config = self.scheduler_config
95 + parallel_config = self.parallel_config
96 + self.device = device
97 + self.pin_memory = is_pin_memory_available()
98 + self.dtype = self.model_config.dtype
99 +
100 + self.is_multimodal_model = model_config.is_multimodal_model
101 + self.sliding_window = model_config.get_sliding_window()
102 + self.block_size = cache_config.block_size
103 + self.max_model_len = model_config.max_model_len
104 + self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
105 + self.max_num_tokens = scheduler_config.max_num_batched_tokens
106 + self.max_num_reqs = scheduler_config.max_num_seqs
107 +
108 + # Model-related.
109 + self.num_attn_layers = model_config.get_num_layers_by_block_type(
110 + parallel_config, LayerBlockType.attention)
111 + self.num_query_heads = model_config.get_num_attention_heads(
112 + parallel_config)
113 + self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
114 + self.head_size = model_config.get_head_size()
115 + self.hidden_size = model_config.get_hidden_size()
116 +
117 + self.model: Optional[nn.Module] = None
118 +
119 + # Persistent batch.
120 + self.input_batch = InputBatch(
121 + max_num_reqs=self.max_num_reqs,
122 + max_model_len=self.max_model_len,
123 + max_num_blocks_per_req=self.max_num_blocks_per_req,
124 + device=self.device,
125 + pin_memory=self.pin_memory,
126 + vocab_size=self.model_config.get_vocab_size(),
127 + )
128 +
129 + # Request states.
130 + self.requests: Dict[str, CachedRequestState] = {}
131 +
132 + # req_id -> (input_id -> encoder_output)
133 + self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}
134 +
135 + # KV caches for forward pass
136 + self.kv_caches: List[Tuple[torch.Tensor, torch.Tensor]] = []
137 +
138 + # Cached torch/numpy tensors
139 + self.num_swaps = 2
140 + self.cur_swap_id = 0
141 + self.input_ids_cpu = []
142 + self.input_ids_np = []
143 + self.input_positions_cpu = []
144 + self.input_positions_np = []
145 + self.slot_mapping_cpu = []
146 + self.slot_mapping_np = []
147 + self.prompt_context_lens_cpu = []
148 + self.prompt_effective_query_lens_cpu = []
149 + self.decode_context_lens_cpu = []
150 + self.decode_context_lens_np = []
151 + for _ in range(self.num_swaps):
152 + self.input_ids_cpu.append(
153 + torch.empty(self.max_num_tokens,
154 + dtype=torch.int32,
155 + device="cpu"))
156 + self.input_ids_np.append(self.input_ids_cpu[-1].numpy())
157 +
158 + self.input_positions_cpu.append(
159 + torch.empty(self.max_num_tokens,
160 + dtype=torch.int32,
161 + device="cpu"))
162 + self.input_positions_np.append(
163 + self.input_positions_cpu[-1].numpy())
164 +
165 + self.slot_mapping_cpu.append(
166 + torch.empty(self.max_num_tokens,
167 + dtype=torch.int64,
168 + device="cpu"))
169 + self.slot_mapping_np.append(self.slot_mapping_cpu[-1].numpy())
170 +
171 + self.prompt_context_lens_cpu.append(
172 + torch.empty((1), dtype=torch.int32, device="cpu"))
173 + self.prompt_effective_query_lens_cpu.append(
174 + torch.empty((1), dtype=torch.int32, device="cpu"))
175 +
176 + self.decode_context_lens_cpu.append(
177 + torch.empty(self.max_num_tokens,
178 + dtype=torch.int32,
179 + device="cpu"))
180 + self.decode_context_lens_np.append(
181 + self.decode_context_lens_cpu[-1].numpy())
182 +
183 + # Range tensor with values [0 .. self.max_num_tokens - 1].
184 + # Used to initialize positions / context_lens / seq_lens
185 + self.arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
186 +
187 + def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
188 + """Update the cached states and the persistent batch with the scheduler
189 + output.
190 +
191 + The updated states are used by the `_prepare_inputs` function to create
192 + the input GPU tensors for the model.
193 +
194 + Returns:
195 + True if there is a new/resumed/paused/finished request in the batch.
196 + If False, we can skip copying SamplingMetadata to the GPU.
197 + """
198 + # Remove finished requests from the cached states.
199 + for req_id in scheduler_output.finished_req_ids:
200 + self.requests.pop(req_id, None)
201 +
202 + # Remove the finished requests from the persistent batch.
203 + # NOTE(woosuk): There could be an edge case where finished_req_ids and
204 + # scheduled_req_ids overlap. This happens when a request is aborted and
205 + # then resubmitted with the same ID. In this case, we treat them as two
206 + # distinct requests - clearing the cached states for the first request
207 + # and handling the second as a new request.
208 + removed_req_indices: List[int] = []
209 + for req_id in scheduler_output.finished_req_ids:
210 + req_index = self.input_batch.remove_request(req_id)
211 + if req_index is not None:
212 + removed_req_indices.append(req_index)
213 +
214 + # Remove the unscheduled requests from the persistent batch.
215 + # NOTE(woosuk): The unscheduled requests are either preempted requests
216 + # or running requests that are not scheduled in this step. We remove
217 + # them from the persistent batch but keep their cached states since
218 + # they will be scheduled again sometime in the future.
219 + scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
220 + cached_req_ids = self.input_batch.req_id_to_index.keys()
221 + unscheduled_req_ids = cached_req_ids - scheduled_req_ids
222 + # NOTE(woosuk): The persistent batch optimization assumes that
223 + # consecutive batches contain mostly the same requests. If batches
224 + # have low request overlap (e.g., alternating between two distinct
225 + # sets of requests), this optimization becomes very inefficient.
226 + for req_id in unscheduled_req_ids:
227 + req_index = self.input_batch.remove_request(req_id)
228 + assert req_index is not None
229 + removed_req_indices.append(req_index)
230 +
231 + req_ids_to_add: List[str] = []
232 + # Add new requests to the cached states.
233 + for new_req_data in scheduler_output.scheduled_new_reqs:
234 + req_id = new_req_data.req_id
235 + sampling_params = new_req_data.sampling_params
236 + if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
237 + generator = torch.Generator(device=self.device)
238 + generator.manual_seed(sampling_params.seed)
239 + else:
240 + generator = None
241 +
242 + self.requests[req_id] = CachedRequestState(
243 + req_id=req_id,
244 + prompt_token_ids=new_req_data.prompt_token_ids,
245 + prompt=new_req_data.prompt,
246 + mm_inputs=new_req_data.mm_inputs,
247 + mm_positions=new_req_data.mm_positions,
248 + sampling_params=sampling_params,
249 + generator=generator,
250 + block_ids=new_req_data.block_ids,
251 + num_computed_tokens=new_req_data.num_computed_tokens,
252 + output_token_ids=[],
253 + lora_request=new_req_data.lora_request,
254 + )
255 +
256 + req_ids_to_add.append(req_id)
257 +
258 + # Update the states of the running/resumed requests.
259 + for req_data in scheduler_output.scheduled_cached_reqs:
260 + req_id = req_data.req_id
261 + req_state = self.requests[req_id]
262 +
263 + # Update the cached states.
264 + req_state.num_computed_tokens = req_data.num_computed_tokens
265 + if not req_data.resumed_from_preemption:
266 + # Append the new blocks to the existing block IDs.
267 + req_state.block_ids.extend(req_data.new_block_ids)
268 + else:
269 + # The request is resumed from preemption.
270 + # Replace the existing block IDs with the new ones.
271 + req_state.block_ids = req_data.new_block_ids
272 +
273 + req_index = self.input_batch.req_id_to_index.get(req_id)
274 + if req_index is None:
275 + # The request is not in the persistent batch.
276 + # The request was either preempted and resumed later, or was not
277 + # scheduled in the previous step and needs to be added again.
278 + req_ids_to_add.append(req_id)
279 + continue
280 +
281 + # Update the persistent batch.
282 + self.input_batch.num_computed_tokens_cpu[req_index] = (
283 + req_data.num_computed_tokens)
284 + start_index = len(req_state.block_ids) - len(
285 + req_data.new_block_ids)
286 + self.input_batch.block_table.append_row(req_index, start_index,
287 + req_data.new_block_ids)
288 +
289 + # Add the new or resumed requests to the persistent batch.
290 + # The smaller empty indices are filled first.
291 + removed_req_indices = sorted(removed_req_indices, reverse=True)
292 + for req_id in req_ids_to_add:
293 + req_state = self.requests[req_id]
294 + if removed_req_indices:
295 + # Fill the empty index.
296 + req_index = removed_req_indices.pop()
297 + else:
298 + # Append to the end.
299 + req_index = None
300 + self.input_batch.add_request(req_state, req_index)
301 +
302 + # Condense the batched states if there are empty indices.
303 + if removed_req_indices:
304 + self.input_batch.condense(removed_req_indices)
305 + return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
306 +
307 + def swap_step(self):
308 + self.cur_swap_id = (self.cur_swap_id + 1) % self.num_swaps
309 +
310 + def get_model(self) -> nn.Module:
311 + assert self.model is not None
312 + return self.model
313 +
314 + def get_kv_cache_spec(self) -> KVCacheSpec:
315 + """
316 + Generates the KVCacheSpec by parsing the kv cache format from each
317 + Attention module in the static forward context.
318 + Returns:
319 + KVCacheSpec: A dictionary mapping layer names to their KV cache
320 + format. Layers that do not need KV cache are not included.
321 + """
322 +
323 + forward_ctx = self.vllm_config.compilation_config.static_forward_context
324 + block_size = self.vllm_config.cache_config.block_size
325 + kv_cache_spec: KVCacheSpec = {}
326 + for layer_name, attn_module in forward_ctx.items():
327 + # TODO: Support other attention modules, e.g., sliding window,
328 + # cross-attention, MLA.
329 + assert isinstance(attn_module, Attention)
330 + if attn_module.attn_type == AttentionType.DECODER:
331 + kv_cache_spec[layer_name] = FullAttentionSpec(
332 + block_size=block_size,
333 + num_kv_heads=attn_module.num_kv_heads,
334 + head_size=attn_module.head_size,
335 + dtype=attn_module.dtype,
336 + )
337 + elif attn_module.attn_type in (AttentionType.ENCODER,
338 + AttentionType.ENCODER_ONLY):
339 + # encoder-only attention does not need KV cache.
340 + continue
341 + elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
342 + raise NotImplementedError
343 + else:
344 + raise ValueError(
345 + f"Unknown attention type: {attn_module.attn_type}")
346 +
347 + return kv_cache_spec
348 +
349 + def _get_prompts_and_decodes(
350 + self,
351 + scheduler_output: "SchedulerOutput",
352 + ) -> PromptDecodeInfo:
353 + total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
354 + assert total_num_scheduled_tokens > 0
355 + num_reqs = self.input_batch.num_reqs
356 + assert num_reqs > 0
357 +
358 + # Traverse decodes first
359 + decode_req_ids = []
360 + for i in range(num_reqs):
361 + req_id = self.input_batch.req_ids[i]
362 + assert req_id is not None
363 +
364 + num_computed_tokens = self.input_batch.num_computed_tokens_cpu[i]
365 + num_prompt_tokens = self.input_batch.num_prompt_tokens[i]
366 + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
367 + req_id]
368 +
369 + if num_computed_tokens < num_prompt_tokens:
370 + # This is prompt
371 + break
372 +
373 + # This is decode
374 + assert num_scheduled_tokens == 1
375 + decode_req_ids.append(req_id)
376 +
377 + # Traverse prompts
378 + prompt_req_ids = []
379 + prompt_scheduled_tokens = []
380 + for i in range(len(decode_req_ids), num_reqs):
381 + req_id = self.input_batch.req_ids[i]
382 + assert req_id is not None
383 +
384 + num_computed_tokens = self.input_batch.num_computed_tokens_cpu[i]
385 + num_prompt_tokens = self.input_batch.num_prompt_tokens[i]
386 + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
387 + req_id]
388 +
389 + # Must be prompt
390 + assert num_computed_tokens < num_prompt_tokens
391 +
392 + prompt_req_ids.append(req_id)
393 + prompt_scheduled_tokens.append(num_scheduled_tokens)
394 +
395 + return PromptDecodeInfo(prompt_req_ids, decode_req_ids,
396 + prompt_scheduled_tokens)
397 +
398 + def _prepare_prompt(self, req_index: int,
399 + num_scheduled_tokens: int) -> PromptData:
400 + num_computed_tokens = self.input_batch.num_computed_tokens_cpu[
401 + req_index]
402 + num_prompt_tokens = self.input_batch.num_prompt_tokens[req_index]
403 +
404 + # Must be prompt
405 + assert num_computed_tokens < num_prompt_tokens
406 +
407 + # Prompt len
408 + prompt_len = num_scheduled_tokens
409 + padded_prompt_len = _get_padded_prompt_len(prompt_len)
410 + assert padded_prompt_len <= self.max_model_len
411 +
412 + # Seq len
413 + seq_len = num_computed_tokens + prompt_len
414 + padded_seq_len = num_computed_tokens + padded_prompt_len
415 +
416 + # Input tokens
417 + input_tokens_cpu = self.input_batch.token_ids_cpu_tensor[
418 + req_index, num_computed_tokens:padded_seq_len]
419 + input_tokens_cpu[prompt_len:] = 0
420 +
421 + # Input positions
422 + input_positions_np = self.input_positions_np[
423 + self.cur_swap_id][:padded_prompt_len]
424 + np.add(num_computed_tokens,
425 + self.arange_np[:padded_prompt_len],
426 + out=input_positions_np)
427 + input_positions_np[prompt_len:] = 0
428 +
429 + # Slot mapping
430 + block_table_np = \
431 + self.input_batch.block_table.get_numpy_array()
432 + block_numbers_np = block_table_np[req_index, input_positions_np //
433 + self.block_size]
434 + block_offsets_np = input_positions_np % self.block_size
435 +
436 + slot_mapping_np = self.slot_mapping_np[
437 + self.cur_swap_id][:padded_prompt_len]
438 + np.add(block_numbers_np * self.block_size,
439 + block_offsets_np,
440 + out=slot_mapping_np)
441 + slot_mapping_np[prompt_len:] = _PAD_SLOT_ID
442 +
443 + # Block table
444 + block_table_cpu = None
445 + if num_computed_tokens > 0:
446 + block_table_cpu = self.input_batch.block_table.get_cpu_tensor()
447 + block_table_cpu = block_table_cpu[req_index]
448 +
449 + # Context len
450 + self.prompt_context_lens_cpu[self.cur_swap_id][0] = 0
451 + if num_computed_tokens > 0:
452 + self.prompt_context_lens_cpu[self.cur_swap_id][0] = seq_len
453 +
454 + # Effective query len
455 + self.prompt_effective_query_lens_cpu[self.cur_swap_id][0] = prompt_len
456 +
457 + # Get final tensors
458 + input_tokens = input_tokens_cpu.reshape(1, -1).to(self.device)
459 + input_positions = self.input_positions_cpu[
460 + self.cur_swap_id][:padded_prompt_len].reshape(1,
461 + -1).to(self.device)
462 + slot_mapping = self.slot_mapping_cpu[
463 + self.cur_swap_id][:padded_prompt_len].reshape(1,
464 + -1).to(self.device)
465 + block_table = block_table_cpu.reshape(1, -1).to(
466 + self.device) if block_table_cpu is not None else None
467 +
468 + context_lens = self.prompt_context_lens_cpu[self.cur_swap_id].to(
469 + self.device)
470 + effective_query_lens = self.prompt_effective_query_lens_cpu[
471 + self.cur_swap_id].to(self.device)
472 +
473 + self.swap_step()
474 +
475 + # Attn metadata
476 + attn_metadata = PallasMetadata(
477 + num_prefills=1,
478 + num_prefill_tokens=0, # NOTE: This is not used.
479 + num_decode_tokens=0,
480 + slot_mapping=slot_mapping,
481 + multi_modal_placeholder_index_maps=None,
482 + enable_kv_scales_calculation=True,
483 + block_tables=block_table,
484 + context_lens=context_lens,
485 + effective_query_lens=effective_query_lens,
486 + )
487 +
488 + return PromptData(input_tokens, input_positions, attn_metadata)
489 +
490 + def _prepare_decode(
491 + self,
492 + decode_req_ids: List[str],
493 + ) -> DecodeData:
494 + # Batch size
495 + batch_size = len(decode_req_ids)
496 + padded_batch_size = _get_padded_batch_size(batch_size)
497 + assert padded_batch_size <= self.max_model_len
498 +
499 + # Init [0 .. batch_size - 1]
500 + req_indices_np = self.arange_np[:padded_batch_size]
501 +
502 + # Input positions
503 + input_positions_np = self.input_positions_np[
504 + self.cur_swap_id][:padded_batch_size]
505 + np.add(self.input_batch.num_computed_tokens_cpu[:padded_batch_size],
506 + 0,
507 + out=input_positions_np)
508 + input_positions_np[batch_size:] = 0
509 + input_positions_cpu = self.input_positions_cpu[
510 + self.cur_swap_id][:padded_batch_size]
511 +
512 + # Input tokens
513 + token_indices_np = (
514 + input_positions_np +
515 + req_indices_np * self.input_batch.token_ids_cpu.shape[1])
516 + input_tokens_cpu = self.input_ids_cpu[
517 + self.cur_swap_id][:padded_batch_size]
518 + torch.index_select(self.input_batch.token_ids_cpu_tensor.flatten(),
519 + 0,
520 + torch.from_numpy(token_indices_np),
521 + out=input_tokens_cpu)
522 + input_tokens_cpu[batch_size:] = 0
523 +
524 + # Slot mapping
525 + block_table_indices_np = (
526 + req_indices_np * self.max_num_blocks_per_req +
527 + input_positions_np // self.block_size)
528 +
529 + block_table_cpu = self.input_batch.block_table.get_cpu_tensor()
530 +
531 + block_numbers_np = block_table_cpu.flatten(
532 + )[block_table_indices_np].numpy()
533 +
534 + block_offsets_np = input_positions_np % self.block_size
535 +
536 + slot_mapping_np = self.slot_mapping_np[
537 + self.cur_swap_id][:padded_batch_size]
538 + np.add(block_numbers_np * self.block_size,
539 + block_offsets_np,
540 + out=slot_mapping_np)
541 + slot_mapping_np[batch_size:] = _PAD_SLOT_ID
542 +
543 + block_table_cpu = block_table_cpu[:padded_batch_size]
544 +
545 + # Context lens
546 + context_lens_np = self.decode_context_lens_np[
547 + self.cur_swap_id][:padded_batch_size]
548 + np.add(self.input_batch.num_computed_tokens_cpu[:padded_batch_size],
549 + 1,
550 + out=context_lens_np)
551 + context_lens_np[batch_size:] = 0
552 +
553 + # Get final tensors
554 + input_tokens = input_tokens_cpu.reshape(-1, 1).to(self.device)
555 + input_positions = input_positions_cpu.reshape(-1, 1).to(self.device)
556 + slot_mapping = self.slot_mapping_cpu[
557 + self.cur_swap_id][:padded_batch_size].reshape(-1,
558 + 1).to(self.device)
559 + block_table = block_table_cpu.to(self.device)
560 + context_lens = self.decode_context_lens_cpu[
561 + self.cur_swap_id][:padded_batch_size].to(self.device)
562 +
563 + self.swap_step()
564 +
565 + # Attn metadata
566 + attn_metadata = PallasMetadata(
567 + num_prefills=0,
568 + num_prefill_tokens=0,
569 + num_decode_tokens=padded_batch_size,
570 + slot_mapping=slot_mapping,
571 + multi_modal_placeholder_index_maps=None,
572 + enable_kv_scales_calculation=True,
573 + block_tables=block_table,
574 + context_lens=context_lens,
575 + effective_query_lens=None,
576 + )
577 +
578 + return DecodeData(input_tokens=input_tokens,
579 + input_positions=input_positions,
580 + attn_metadata=attn_metadata)
581 +
582 + @torch.no_grad()
583 + def execute_model(
584 + self,
585 + scheduler_output: "SchedulerOutput",
586 + ) -> ModelRunnerOutput:
587 + # Update cached state
588 + self._update_states(scheduler_output)
589 +
590 + # If necessary, swap decodes/prompts to have all decodes on the start
591 + ensure_decodes_first(self.input_batch)
592 +
593 + # Prepare prompts/decodes info
594 + pd_info = self._get_prompts_and_decodes(scheduler_output)
595 +
596 + # Init
597 + num_prompts = len(pd_info.prompt_req_ids)
598 + num_decodes = len(pd_info.decode_req_ids)
599 + decode_data = None
600 + sampled_token_ids = [0] * self.input_batch.num_reqs
601 +
602 + # Run each prompt individually
603 + is_first = True
604 + for i in range(num_prompts):
605 + req_id = pd_info.prompt_req_ids[i]
606 + req_index = num_decodes + i
607 + assert req_index == self.input_batch.req_id_to_index[
608 + req_id] # TODO: Remove
609 + req_state = self.requests[req_id]
610 + num_scheduled_tokens = pd_info.prompt_scheduled_tokens[i]
611 + prompt_len = num_scheduled_tokens
612 + seq_len = req_state.num_computed_tokens + num_scheduled_tokens
613 +
614 + # Prepare first prompt
615 + if is_first:
616 + prompt_data = self._prepare_prompt(req_index,
617 + num_scheduled_tokens)
618 + is_first = False
619 +
620 + # Run forward pass
621 + with set_forward_context(prompt_data.attn_metadata,
622 + self.vllm_config):
623 + assert self.model is not None
624 + selected_token_ids = self.model(prompt_data.input_tokens,
625 + prompt_data.input_positions,
626 + prompt_data.attn_metadata,
627 + self.kv_caches)
628 +
629 + # In parallel to TPU execution, prepare the next iteration
630 + if i < num_prompts - 1:
631 + # There is next prompt => prepare it
632 + prompt_data = self._prepare_prompt(
633 + req_index + 1, pd_info.prompt_scheduled_tokens[i + 1])
634 + elif i == num_prompts - 1 and num_decodes > 0:
635 + # There is next decode => prepare it
636 + decode_data = self._prepare_decode(pd_info.decode_req_ids)
637 +
638 + # Update cached state (if prompt is fully done)
639 + if seq_len >= len(req_state.prompt_token_ids):
640 + # Transfer sampled tokens from TPU to CPU
641 + selected_token_ids_cpu = selected_token_ids.cpu()
642 +
643 + # Get output token
644 + token_id = selected_token_ids_cpu[prompt_len - 1].item()
645 + sampled_token_ids[req_index] = token_id
646 +
647 + # Add output token to the request
648 + self.input_batch.token_ids_cpu[req_index, seq_len] = token_id
649 + self.input_batch.num_tokens[req_index] += 1
650 + req_state.output_token_ids.append(token_id)
651 +
652 + # Run decodes (a single batch)
653 + if num_decodes > 0:
654 +
655 + # Prepare decode (if was not yet prepared)
656 + if decode_data is None:
657 + decode_data = self._prepare_decode(pd_info.decode_req_ids)
658 +
659 + # Run forward pass
660 + with set_forward_context(decode_data.attn_metadata,
661 + self.vllm_config):
662 + assert self.model is not None
663 + selected_token_ids = self.model(decode_data.input_tokens,
664 + decode_data.input_positions,
665 + decode_data.attn_metadata,
666 + self.kv_caches)
667 +
668 + # Transfer sampled tokens from TPU to CPU
669 + decode_token_ids_cpu = selected_token_ids.cpu()
670 + # Convert to list
671 + decode_token_ids_list = decode_token_ids_cpu.tolist()
672 +
673 + # Update cached state for each decode request
674 + for i in range(num_decodes):
675 + req_id = pd_info.decode_req_ids[i]
676 + req_index = i
677 + assert req_index == self.input_batch.req_id_to_index[
678 + req_id] # TODO: Remove
679 + req_state = self.requests[req_id]
680 + seq_len = req_state.num_computed_tokens + 1
681 +
682 + token_id = decode_token_ids_list[i]
683 + sampled_token_ids[req_index] = token_id
684 +
685 + self.input_batch.token_ids_cpu[req_index, seq_len] = token_id
686 + self.input_batch.num_tokens[req_index] += 1
687 + req_state.output_token_ids.append(token_id)
688 +
689 + # Create output.
690 + all_req_ids = pd_info.decode_req_ids + pd_info.prompt_req_ids
691 + prompt_logprobs_dict: Dict[str, Optional[LogprobsTensors]] = {}
692 + for req_id in all_req_ids:
693 + prompt_logprobs_dict[req_id] = None
694 +
695 + model_runner_output = ModelRunnerOutput(
696 + req_ids=all_req_ids,
697 + req_id_to_index=self.input_batch.req_id_to_index,
698 + sampled_token_ids=sampled_token_ids,
699 + logprobs=None,
700 + prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
701 + )
702 +
703 + return model_runner_output
704 +
705 + def load_model(self) -> None:
706 + self.device = self.device_config.device
707 +
708 + # NOTE(woosuk): While the executor assigns the TP ranks to the worker
709 + # process, the ranks can be different from the ranks internally assigned
710 + # by the xm runtime. Therefore, there is a mismatch in the rank
711 + # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
712 + # This is not a problem in linear layers because all-reduce is
713 + # rank-agnostic. However, it matters for all-gather as the ranks
714 + # determine the order of concatenating the output tensors.
715 + # As a workaround, we use the xm's rank assignment only when loading
716 + # the embedding weights.
717 + xm_tp_rank = xr.global_ordinal()
718 + with patch(
719 + "vllm.model_executor.layers.vocab_parallel_embedding."
720 + "get_tensor_model_parallel_rank",
721 + return_value=xm_tp_rank):
722 + model = get_model(vllm_config=self.vllm_config)
723 + model = model.eval()
724 + xm.mark_step()
725 + xm.wait_device_ops()
726 + model = ModelWrapperV1(model)
727 + self.model = torch.compile(model,
728 + backend="openxla",
729 + fullgraph=True,
730 + dynamic=False)
731 +
732 + def dummy_run(
733 + self,
734 + kv_caches,
735 + num_tokens: int,
736 + seq_len: Optional[int] = None,
737 + exec_mode: Optional[ExecutionMode] = None,
738 + ) -> None:
739 + assert seq_len is not None
740 + assert exec_mode is not None
741 +
742 + exec_mode = ExecutionMode(exec_mode)
743 + if exec_mode.is_prefill():
744 + seq_len = (seq_len + 15) // 16 * 16
745 + token_ids = torch.zeros((num_tokens, seq_len),
746 + dtype=torch.int32,
747 + device=self.device)
748 + position_ids = torch.zeros((num_tokens, seq_len),
749 + dtype=torch.int32,
750 + device=self.device)
751 + slot_mapping = torch.zeros((num_tokens, seq_len),
752 + dtype=torch.int64,
753 + device=self.device)
754 + if exec_mode == ExecutionMode.PREFILL:
755 + attn_metadata = PallasMetadata(
756 + num_prefills=num_tokens,
757 + num_prefill_tokens=num_tokens * seq_len,
758 + num_decode_tokens=0,
759 + slot_mapping=slot_mapping,
760 + multi_modal_placeholder_index_maps=None,
761 + enable_kv_scales_calculation=True,
762 + block_tables=None,
763 + context_lens=None,
764 + effective_query_lens=None,
765 + )
766 +
767 + else:
768 + context_lens = torch.ones((num_tokens, ),
769 + dtype=torch.int32,
770 + device=self.device)
771 +
772 + block_tables = torch.zeros(
773 + (num_tokens, self.max_num_blocks_per_req),
774 + dtype=torch.int32,
775 + device=self.device)
776 +
777 + effective_query_lens = torch.ones_like(context_lens)
778 +
779 + attn_metadata = PallasMetadata(
780 + num_prefills=num_tokens,
781 + num_prefill_tokens=num_tokens * seq_len,
782 + num_decode_tokens=0,
783 + slot_mapping=slot_mapping,
784 + multi_modal_placeholder_index_maps=None,
785 + enable_kv_scales_calculation=True,
786 + block_tables=block_tables,
787 + context_lens=context_lens,
788 + effective_query_lens=effective_query_lens,
789 + )
790 + else:
791 + assert seq_len == 1
792 + token_ids = torch.zeros((num_tokens, seq_len),
793 + dtype=torch.int32,
794 + device=self.device)
795 + position_ids = torch.zeros((num_tokens, seq_len),
796 + dtype=torch.int32,
797 + device=self.device)
798 + slot_mapping = torch.zeros((num_tokens, seq_len),
799 + dtype=torch.int64,
800 + device=self.device)
801 + block_tables = torch.zeros(
802 + (num_tokens, self.max_num_blocks_per_req),
803 + dtype=torch.int32,
804 + device=self.device)
805 + context_lens = torch.ones((num_tokens, ),
806 + dtype=torch.int32,
807 + device=self.device)
808 + attn_metadata = PallasMetadata(
809 + num_prefills=0,
810 + num_prefill_tokens=0,
811 + num_decode_tokens=num_tokens * seq_len,
812 + slot_mapping=slot_mapping,
813 + multi_modal_placeholder_index_maps=None,
814 + enable_kv_scales_calculation=True,
815 + block_tables=block_tables,
816 + context_lens=context_lens,
817 + )
818 +
819 + # NOTE(woosuk): There are two stages of compilation: torch.compile and
820 + # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
821 + # overhead by reusing the FX graph for different shapes.
822 + # However, the XLA graph will still require static shapes and needs to
823 + # be re-compiled for every different shapes. This overhead is inevitable
824 + # in the first run, but can be skipped afterwards as we cache the XLA
825 + # graphs in the disk (VLLM_XLA_CACHE_PATH).
826 + if exec_mode.is_prefill():
827 + # Prefll
828 + torch._dynamo.mark_dynamic(token_ids, 1)
829 + torch._dynamo.mark_dynamic(position_ids, 1)
830 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
831 + else:
832 + # Decode
833 + torch._dynamo.mark_dynamic(token_ids, 0)
834 + torch._dynamo.mark_dynamic(position_ids, 0)
835 + torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
836 + torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
837 + torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
838 +
839 + with set_forward_context(attn_metadata, self.vllm_config, 0):
840 + assert self.model is not None
841 + self.model(token_ids, position_ids, attn_metadata, kv_caches)
842 +
843 + def capture_model(self) -> None:
844 + """Compile the model."""
845 +
846 + # Prefill
847 + logger.info(
848 + "Compiling the model with different input shapes for prefill:")
849 + start = time.time()
850 + for batch_size in [1]:
851 + seq_len = 16
852 + while seq_len <= self.model_config.max_model_len:
853 + self.dummy_run(self.kv_caches,
854 + batch_size,
855 + seq_len,
856 + exec_mode=ExecutionMode.PREFILL)
857 + xm.wait_device_ops()
858 + logger.info(" batch_size: %d, seq_len: %d", batch_size,
859 + seq_len)
860 + num_tokens = batch_size * seq_len
861 + if num_tokens >= self.scheduler_config.max_num_batched_tokens:
862 + break
863 + seq_len = seq_len * 2
864 +
865 + end = time.time()
866 + logger.info(" -- Compilation for prefill done in %.2f [secs].",
867 + end - start)
868 +
869 + # Prefix prefill
870 + if self.scheduler_config.enable_chunked_prefill:
871 + logger.info("Compiling the model with different input shapes for "
872 + "prefix prefill:")
873 + start = time.time()
874 + for batch_size in [1]:
875 + seq_len = 16
876 + while seq_len <= self.model_config.max_model_len:
877 + self.dummy_run(self.kv_caches,
878 + batch_size,
879 + seq_len,
880 + exec_mode=ExecutionMode.PREFIX_PREFILL)
881 + xm.wait_device_ops()
882 + logger.info(" batch_size: %d, seq_len: %d", batch_size,
883 + seq_len)
884 + num_tokens = batch_size * seq_len
885 + if (num_tokens
886 + >= self.scheduler_config.max_num_batched_tokens):
887 + break
888 + seq_len = seq_len * 2
889 + end = time.time()
890 + logger.info(
891 + " -- Compilation for prefix prefill done in %.2f [secs].",
892 + end - start)
893 +
894 + # Decode
895 + logger.info(
896 + "Compiling the model with different input shapes for decode:")
897 + start = time.time()
898 + seq_len = 1
899 + batch_size = 8 # Must be in sync with _get_padded_batch_size()
900 + while True:
901 + self.dummy_run(self.kv_caches,
902 + batch_size,
903 + seq_len,
904 + exec_mode=ExecutionMode.DECODE)
905 + xm.wait_device_ops()
906 + logger.info(" batch_size: %d, seq_len: %d", batch_size, seq_len)
907 +
908 + if batch_size >= self.scheduler_config.max_num_seqs:
909 + break
910 + batch_size = batch_size + 16 if batch_size >= 16 else batch_size * 2
911 +
912 + end = time.time()
913 + logger.info(" -- Compilation for decode done in %.2f [secs].",
914 + end - start)
915 +
916 + def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
917 + """
918 + Initialize KV cache based on `kv_cache_config`.
919 + Args:
920 + kv_cache_config: Configuration for the KV cache, including the KV
921 + cache size of each layer
922 + """
923 + if len(kv_cache_config.groups) > 1:
924 + raise NotImplementedError(
925 + "Hybrid models with more than one KV cache type are not "
926 + "supported yet.")
927 +
928 + kv_caches: Dict[str, torch.Tensor] = {}
929 +
930 + for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():
931 + tensor_config = kv_cache_config.tensors[layer_name]
932 + assert tensor_config.size % layer_spec.page_size_bytes == 0
933 + num_blocks = tensor_config.size // layer_spec.page_size_bytes
934 + if isinstance(layer_spec, FullAttentionSpec):
935 + kv_cache_shape = PallasAttentionBackend.get_kv_cache_shape(
936 + num_blocks, layer_spec.block_size, layer_spec.num_kv_heads,
937 + layer_spec.head_size)
938 + dtype = layer_spec.dtype
939 +
940 + tpu_k_cache = torch.zeros(kv_cache_shape,
941 + dtype=dtype,
942 + device=self.device)
943 + tpu_v_cache = torch.zeros_like(tpu_k_cache)
944 +
945 + kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache)
946 + else:
947 + raise NotImplementedError
948 +
949 + bind_kv_cache(
950 + kv_caches,
951 + self.vllm_config.compilation_config.static_forward_context,
952 + self.kv_caches)
953 +
954 +
955 +class ModelWrapperV1(nn.Module):
956 +
957 + def __init__(self, model: nn.Module):
958 + super().__init__()
959 + self.model = model
960 +
961 + def forward(
962 + self,
963 + token_ids: torch.Tensor,
964 + position_ids: torch.Tensor,
965 + attn_metadata: AttentionMetadata,
966 + kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
967 + ) -> torch.Tensor:
968 + """Executes the forward pass of the model and samples the next token.
969 +
970 + Args:
971 + token_ids: The input token IDs of shape [batch_size, seq_len].
972 + position_ids: The input position IDs of shape [batch_size, seq_len].
973 + attn_metadata: The Pallas attention metadata.
974 + input_lens: The actual input lengths of shape [batch_size].
975 + t: The sampling temperature of shape [batch_size].
976 + p: The top-p probability of shape [batch_size].
977 + num_samples: Number of samples to draw from each logits vector.
978 + kv_caches: The key and value caches. They can be None during the
979 + memory profiling at initialization.
980 + """
981 + # Skip this in memory profiling at initialization.
982 + if attn_metadata is not None and kv_caches[0][0].numel() > 0:
983 + # index_copy_(slot_mapping) only works when the inserted dimension
984 + # is 0. However, the KV cache in the Pallas backend has the shape
985 + # [num_kv_heads, num_blocks, block_size, head_size]. To make it
986 + # work, we need to flatten the first three dimensions and modify
987 + # the slot_mapping accordingly.
988 + num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape
989 + slot_mapping = attn_metadata.slot_mapping
990 + slot_mapping = slot_mapping.flatten()
991 + head_indicies = torch.arange(0,
992 + num_kv_heads,
993 + device=slot_mapping.device,
994 + dtype=slot_mapping.dtype)
995 + head_indicies *= block_size * num_blocks
996 + slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view(
997 + -1, num_kv_heads)
998 + slot_mapping = slot_mapping + head_indicies.view(1, -1)
999 + slot_mapping = slot_mapping.flatten()
1000 + attn_metadata.slot_mapping = slot_mapping
1001 +
1002 + assert self.model is not None
1003 + hidden_states = self.model(
1004 + token_ids,
1005 + position_ids,
1006 + kv_caches,
1007 + attn_metadata,
1008 + )
1009 +
1010 + hidden_states = hidden_states.flatten(0, 1)
1011 + logits = self.model.compute_logits(hidden_states, None)
1012 +
1013 + # Greedy sampling.
1014 + argmax_token_ids = torch.argmax(logits, dim=-1, keepdim=True)
1015 + argmax_token_ids = argmax_token_ids.squeeze(dim=-1)
1016 + return argmax_token_ids
1017 +
1018 +
1019 +def swap_positions(b: InputBatch, id_1, id_2):
1020 + assert id_1 != id_2
1021 + req_id_1 = b.req_ids[id_1]
1022 + req_id_2 = b.req_ids[id_2]
1023 + assert req_id_1 is not None
1024 + assert req_id_2 is not None
1025 + assert id_1 == b.req_id_to_index[req_id_1]
1026 + assert id_2 == b.req_id_to_index[req_id_2]
1027 +
1028 + b.req_ids[id_1], b.req_ids[id_2] = b.req_ids[id_2], b.req_ids[id_1]
1029 + b.req_id_to_index[req_id_1], b.req_id_to_index[
1030 + req_id_2] = b.req_id_to_index[req_id_2], b.req_id_to_index[req_id_1]
1031 +
1032 + ids = [id_1, id_2]
1033 + rev_ids = [id_2, id_1]
1034 + b.num_tokens[ids] = b.num_tokens[rev_ids]
1035 + b.token_ids_cpu[ids] = b.token_ids_cpu[rev_ids]
1036 + b.num_prompt_tokens[ids] = b.num_prompt_tokens[rev_ids]
1037 + b.num_computed_tokens_cpu[ids] = b.num_computed_tokens_cpu[rev_ids]
1038 +
1039 + b.block_table.swap_row(id_1, id_2)
1040 +
1041 + b.temperature_cpu[ids] = b.temperature_cpu[rev_ids]
1042 + b.top_p_cpu[ids] = b.top_p_cpu[rev_ids]
1043 + b.top_k_cpu[ids] = b.top_k_cpu[rev_ids]
1044 + b.frequency_penalties_cpu[ids] = b.frequency_penalties_cpu[rev_ids]
1045 + b.presence_penalties_cpu[ids] = b.presence_penalties_cpu[rev_ids]
1046 + b.repetition_penalties_cpu[ids] = b.repetition_penalties_cpu[rev_ids]
1047 +
1048 + b.min_tokens[id_1], b.min_tokens[id_2] = b.min_tokens[id_2], b.min_tokens[
1049 + id_1]
1050 + b.stop_token_ids[id_1], b.stop_token_ids[id_2] = b.stop_token_ids[
1051 + id_2], b.stop_token_ids[id_1]
1052 +
1053 + gen_1 = b.generators.pop(id_1, None)
1054 + gen_2 = b.generators.pop(id_2, None)
1055 + if gen_1 is not None:
1056 + b.generators[id_2] = gen_1
1057 + if gen_2 is not None:
1058 + b.generators[id_1] = gen_2
1059 +
1060 +
1061 +def ensure_decodes_first(b: InputBatch):
1062 + num_reqs = b.num_reqs
1063 + while True:
1064 + # Find the first prompt index
1065 + first_prompt_index = None
1066 + for i in range(num_reqs):
1067 + if b.num_computed_tokens_cpu[i] < b.num_prompt_tokens[i]:
1068 + first_prompt_index = i
1069 + break
1070 + if first_prompt_index is None:
1071 + break
1072 +
1073 + # Find the last decode index
1074 + last_decode_index = None
1075 + for i in reversed(range(num_reqs)):
1076 + if b.num_computed_tokens_cpu[i] >= b.num_prompt_tokens[i]:
1077 + last_decode_index = i
1078 + break
1079 + if last_decode_index is None:
1080 + break
1081 +
1082 + # Sanity
1083 + assert first_prompt_index != last_decode_index
1084 +
1085 + # Check if done
1086 + if first_prompt_index > last_decode_index:
1087 + break
1088 +
1089 + # Swap
1090 + swap_positions(b, first_prompt_index, last_decode_index)
1091 +
1092 +
1093 +def _get_padded_prompt_len(x: int) -> int:
1094 + # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
1095 + # length to be a multiple of 16. We pad the prompt length to the nearest
1096 + # multiple of 16. This is also good for performance.
1097 + if x <= 16:
1098 + return 16
1099 + return 1 << (x - 1).bit_length()
1100 +
1101 +
1102 +def _get_padded_batch_size(batch_size: int) -> int:
1103 + # The GMM Pallas kernel requires num_tokens * topk to be a multiple of 16.
1104 + # To meet this requirement in the simplest way, we set the minimal batch
1105 + # size to 8.
1106 + if batch_size <= 8:
1107 + return 8
1108 + else:
1109 + return ((batch_size + 15) // 16) * 16
vllm/v1/worker/tpu_worker.py new
+203
@@ -0,0 +1,203 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +"""A TPU worker class."""
3 +import os
4 +from typing import Dict, List, Optional
5 +
6 +import torch
7 +import torch.distributed
8 +import torch.nn as nn
9 +import torch_xla.core.xla_model as xm
10 +import torch_xla.runtime as xr
11 +
12 +import vllm.envs as envs
13 +from vllm.config import ParallelConfig, VllmConfig
14 +from vllm.distributed import (ensure_model_parallel_initialized,
15 + init_distributed_environment)
16 +from vllm.logger import init_logger
17 +from vllm.model_executor import set_random_seed
18 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19 +from vllm.v1.core.scheduler import SchedulerOutput
20 +from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
21 + KVCacheSpec)
22 +from vllm.v1.outputs import ModelRunnerOutput
23 +from vllm.v1.utils import bind_kv_cache
24 +from vllm.v1.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
25 +
26 +logger = init_logger(__name__)
27 +
28 +
29 +class TPUWorker:
30 +
31 + def __init__(
32 + self,
33 + vllm_config: VllmConfig,
34 + local_rank: int,
35 + rank: int,
36 + distributed_init_method: str,
37 + is_driver_worker: bool = False,
38 + ):
39 + self.vllm_config = vllm_config
40 + self.model_config = vllm_config.model_config
41 + self.cache_config = vllm_config.cache_config
42 + self.lora_config = vllm_config.lora_config
43 + self.load_config = vllm_config.load_config
44 + self.parallel_config = vllm_config.parallel_config
45 + self.scheduler_config = vllm_config.scheduler_config
46 + self.device_config = vllm_config.device_config
47 + self.speculative_config = vllm_config.speculative_config
48 + self.prompt_adapter_config = vllm_config.prompt_adapter_config
49 + self.observability_config = vllm_config.observability_config
50 +
51 + self.parallel_config.rank = rank
52 + self.local_rank = local_rank
53 + self.rank = rank
54 + self.distributed_init_method = distributed_init_method
55 +
56 + if self.cache_config.cache_dtype == "auto":
57 + self.cache_dtype = self.model_config.dtype
58 + else:
59 + self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
60 + self.cache_config.cache_dtype]
61 +
62 + if self.model_config.trust_remote_code:
63 + # note: lazy import to avoid importing torch before initializing
64 + from vllm.utils import init_cached_hf_modules
65 + init_cached_hf_modules()
66 +
67 + def init_device(self):
68 + os.environ["PJRT_DEVICE"] = "TPU"
69 + torch.set_grad_enabled(False)
70 + torch.set_default_dtype(self.model_config.dtype)
71 +
72 + # Initialize the distributed environment.
73 + init_tpu_worker_distributed_environment(self.parallel_config,
74 + self.rank,
75 + self.distributed_init_method,
76 + self.local_rank)
77 +
78 + # Device initialization should happen after initializing
79 + # the distributed runtime.
80 + self.device = xm.xla_device()
81 + self.device_config.device = self.device
82 +
83 + # Set random seed.
84 + set_random_seed(self.model_config.seed)
85 + xm.set_rng_state(self.model_config.seed, self.device)
86 +
87 + # Increase the cache size limit, which is the maximum number of
88 + # dynamo graphs that can be compiled.
89 + # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and
90 + # 30-40 graphs for decode. 128 is an arbitrary safe number.
91 + torch._dynamo.config.cache_size_limit = 128
92 + # Use persistent cache to avoid XLA recompilation.
93 + # NOTE(woosuk): Set per-rank cache path since different ranks
94 + # can have slightly different XLA graphs.
95 + world_size = self.parallel_config.world_size
96 + rank = xr.global_ordinal()
97 + per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH,
98 + f"tp{world_size}_rank{rank}")
99 + xr.initialize_cache(per_rank_path, readonly=False)
100 +
101 + # Init ModelRunner here, so that we have access to self.device.
102 + self.model_runner = TPUModelRunner(self.vllm_config, self.device)
103 +
104 + def determine_available_memory(self) -> int:
105 + kv_caches: Dict[str, torch.Tensor] = {}
106 + kv_cache_spec = self.model_runner.get_kv_cache_spec()
107 + for layer_name, layer_spec in kv_cache_spec.items():
108 + if isinstance(layer_spec, FullAttentionSpec):
109 + dtype = layer_spec.dtype
110 +
111 + # Use an empty tensor instead of `None`` to force Dynamo to pass
112 + # it by reference, rather by specializing on the value ``None``.
113 + tpu_k_cache = torch.tensor([], dtype=dtype, device=self.device)
114 + tpu_v_cache = torch.tensor([], dtype=dtype, device=self.device)
115 +
116 + kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache)
117 + else:
118 + raise NotImplementedError
119 +
120 + runner_kv_caches: List[torch.Tensor] = []
121 + bind_kv_cache(
122 + kv_caches,
123 + self.vllm_config.compilation_config.static_forward_context,
124 + runner_kv_caches)
125 +
126 + self.model_runner.dummy_run(
127 + runner_kv_caches,
128 + num_tokens=1,
129 + seq_len=self.scheduler_config.max_num_batched_tokens,
130 + exec_mode=ExecutionMode.PREFILL,
131 + )
132 +
133 + # Synchronize before measuring the memory usage.
134 + xm.wait_device_ops()
135 +
136 + # Get the maximum amount of memory used by the model weights and
137 + # intermediate activations.
138 + m = xm.get_memory_info(self.device)
139 + total_memory_size = m["bytes_limit"]
140 + profiled = m["peak_bytes_used"] # Weights + intermediate activations.
141 +
142 + # Calculate the TPU KV cache size based on profiling.
143 + usable_memory_size = int(total_memory_size *
144 + self.cache_config.gpu_memory_utilization)
145 + tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0)
146 +
147 + return int(tpu_kv_cache_bytes)
148 +
149 + def execute_model(
150 + self,
151 + scheduler_output: "SchedulerOutput",
152 + ) -> Optional[ModelRunnerOutput]:
153 + output = self.model_runner.execute_model(scheduler_output)
154 + return output if self.rank == 0 else None
155 +
156 + def load_model(self) -> None:
157 + self.model_runner.load_model()
158 +
159 + def compile_or_warm_up_model(self) -> None:
160 + if not self.model_config.enforce_eager:
161 + self.model_runner.capture_model()
162 +
163 + # Reset the seed to ensure that the random state is not affected by
164 + # the model initialization and profiling.
165 + set_random_seed(self.model_config.seed)
166 +
167 + def get_model(self) -> nn.Module:
168 + return self.model_runner.get_model()
169 +
170 + def get_kv_cache_spec(self) -> KVCacheSpec:
171 + return self.model_runner.get_kv_cache_spec()
172 +
173 + def initialize_cache(self, kv_cache_configs: List[KVCacheConfig]) -> None:
174 + """Allocate GPU KV cache with the specified kv_cache_config."""
175 + kv_cache_config = kv_cache_configs[self.rank]
176 + self.model_runner.initialize_kv_cache(kv_cache_config)
177 +
178 + def check_health(self) -> None:
179 + # worker will always be healthy as long as it's running.
180 + return
181 +
182 +
183 +def init_tpu_worker_distributed_environment(
184 + parallel_config: ParallelConfig,
185 + rank: int,
186 + distributed_init_method: Optional[str] = None,
187 + local_rank: int = -1,
188 +) -> None:
189 + """Initialize the distributed environment."""
190 +
191 + # NOTE(woosuk): This is just to initialize the TP group and broadcast
192 + # the input objects on CPU. The all-reduce and all-gather ops on TPU
193 + # are invoked by `xm.all_reduce` and `xm.all_gather` which use their
194 + # own context.
195 + init_distributed_environment(
196 + world_size=parallel_config.world_size,
197 + rank=rank,
198 + local_rank=local_rank,
199 + distributed_init_method=distributed_init_method,
200 + backend="gloo",
201 + )
202 + ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
203 + parallel_config.pipeline_parallel_size)