A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[WIP] TPU V1 Support Refactored (#13049)
Alexander Matveev committed
Feb 14, 2025 at 03:21 UTC
45f90bcbba76dcdf1c1cf664431138d3a9d7acf0
8 files changed
+1738
-25
tests/entrypoints/llm/test_accuracy.py
+15
-5
@@ -21,10 +21,13 @@ RTOL = 0.03
21
EXPECTED_VALUE = 0.58
22
23
24
-def run_test():
24
+def run_test(more_args=None):
25
"""Run the end to end accuracy test."""
26
27
- model_args = f"pretrained={MODEL_NAME},max_model_len=2048"
27
+ model_args = f"pretrained={MODEL_NAME},max_model_len=4096"
28
+
29
+ if more_args is not None:
30
+ model_args = "{},{}".format(model_args, more_args)
31
32
results = lm_eval.simple_evaluate(
33
model="vllm",
@@ -39,14 +42,21 @@ def run_test():
42
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
43
44
42
-@pytest.mark.skipif(not current_platform.is_cuda(),
43
- reason="V1 is currently only supported on CUDA.")
45
+@pytest.mark.skipif(not current_platform.is_cuda()
46
+ and not current_platform.is_tpu(),
47
+ reason="V1 is currently only supported on CUDA and TPU")
48
def test_lm_eval_accuracy_v1_engine(monkeypatch):
49
"""Run with the V1 Engine."""
50
51
with monkeypatch.context() as m:
52
m.setenv("VLLM_USE_V1", "1")
49
- run_test()
53
+
54
+ more_args = None
55
+ if current_platform.is_tpu():
56
+ # Limit compilation time for TPU V1
57
+ more_args = "max_num_seqs=64"
58
+
59
+ run_test(more_args)
60
61
62
def test_lm_eval_accuracy_v0_engine(monkeypatch):
tests/entrypoints/openai/correctness/test_lmeval.py
+11
-4
@@ -21,7 +21,7 @@ TASK = "gsm8k"
21
FILTER = "exact_match,strict-match"
22
RTOL = 0.03
23
EXPECTED_VALUE = 0.58
24
-DEFAULT_ARGS = ["--max-model-len", "2048", "--disable-log-requests"]
24
+DEFAULT_ARGS = ["--max-model-len", "4096", "--disable-log-requests"]
25
MORE_ARGS_LIST = [
26
[], # Default
27
["--enable-chunked-prefill"], # Chunked
@@ -67,14 +67,21 @@ def run_test(more_args):
67
), f"Expected: {EXPECTED_VALUE} | Measured: {measured_value}"
68
69
70
-@pytest.mark.skipif(not current_platform.is_cuda(),
71
- reason="V1 currently only supported on CUDA")
70
+@pytest.mark.skipif(not current_platform.is_cuda()
71
+ and not current_platform.is_tpu(),
72
+ reason="V1 currently only supported on CUDA and TPU")
73
def test_lm_eval_accuracy_v1_engine(monkeypatch):
74
"""Run with the V1 Engine."""
75
76
with monkeypatch.context() as m:
77
m.setenv("VLLM_USE_V1", "1")
77
- run_test([])
78
+ more_args = []
79
+
80
+ # Limit compilation time for V1
81
+ if current_platform.is_tpu():
82
+ more_args = ["--max-num-seqs", "64"]
83
+
84
+ run_test(more_args)
85
86
87
@pytest.mark.parametrize("more_args", MORE_ARGS_LIST)
vllm/platforms/interface.py
+1
@@ -37,6 +37,7 @@ class _Backend(enum.Enum):
37
TRITON_MLA = enum.auto()
38
HPU_ATTN = enum.auto()
39
PALLAS = enum.auto()
40
+ PALLAS_VLLM_V1 = enum.auto()
41
IPEX = enum.auto()
42
BLOCK_SPARSE_FLASH_ATTN = enum.auto()
43
NO_ATTENTION = enum.auto()
vllm/platforms/tpu.py
+38
-16
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Optional
4
5
import torch
6
7
+import vllm.envs as envs
8
from vllm.logger import init_logger
9
10
from .interface import Platform, PlatformEnum, _Backend
@@ -33,14 +34,20 @@ class TpuPlatform(Platform):
34
dtype: torch.dtype, kv_cache_dtype: Optional[str],
35
block_size: int, use_v1: bool,
36
use_mla: bool) -> str:
36
- if selected_backend != _Backend.PALLAS:
37
+ if (selected_backend != _Backend.PALLAS
38
+ and selected_backend != _Backend.PALLAS_VLLM_V1):
39
logger.info("Cannot use %s backend on TPU.", selected_backend)
38
- logger.info("Using Pallas backend.")
39
- return "vllm.attention.backends.pallas.PallasAttentionBackend"
40
+
41
+ if use_v1:
42
+ logger.info("Using Pallas V1 backend.")
43
+ return "vllm.v1.attention.backends.pallas.PallasAttentionBackend"
44
+ else:
45
+ logger.info("Using Pallas backend.")
46
+ return "vllm.attention.backends.pallas.PallasAttentionBackend"
47
48
@classmethod
49
def get_device_name(cls, device_id: int = 0) -> str:
43
- raise NotImplementedError
50
+ return "tpu"
51
52
@classmethod
53
def get_device_total_memory(cls, device_id: int = 0) -> int:
@@ -48,7 +55,7 @@ class TpuPlatform(Platform):
55
56
@classmethod
57
def is_async_output_supported(cls, enforce_eager: Optional[bool]) -> bool:
51
- return True
58
+ return not envs.VLLM_USE_V1
59
60
@classmethod
61
def inference_mode(cls):
@@ -63,11 +70,11 @@ class TpuPlatform(Platform):
70
cache_config.block_size = 16
71
72
compilation_config = vllm_config.compilation_config
66
- if compilation_config.level == CompilationLevel.NO_COMPILATION:
67
- # TPU does not support NO_COMPILATION
73
+
74
+ # TPU only supports DYNAMO_ONCE compilation level
75
+ if compilation_config.level != CompilationLevel.DYNAMO_ONCE:
76
+ logger.info("[TPU] Forcing DYNAMO_ONCE compilation level")
77
compilation_config.level = CompilationLevel.DYNAMO_ONCE
69
- assert compilation_config.level < CompilationLevel.PIECEWISE,\
70
- "TPU does not support Inductor."
78
79
if compilation_config.backend == "":
80
compilation_config.backend = "openxla"
@@ -75,10 +82,6 @@ class TpuPlatform(Platform):
82
assert vllm_config.speculative_config is None, \
83
"TPU does not support speculative decoding"
84
78
- assert not vllm_config.scheduler_config.chunked_prefill_enabled, (
79
- "Chunked prefill is not yet supported for TPU backend")
80
- assert not vllm_config.speculative_config, (
81
- "Speculative decoding is not yet supported for TPU backend")
85
if vllm_config.model_config.dtype in (torch.float16, torch.float32):
86
logger.warning(
87
"The TPU backend currently does not support %s. "
@@ -88,8 +91,27 @@ class TpuPlatform(Platform):
91
parallel_config = vllm_config.parallel_config
92
scheduler_config = vllm_config.scheduler_config
93
if parallel_config.worker_cls == "auto":
91
- if scheduler_config.is_multi_step:
94
+ if envs.VLLM_USE_V1:
95
parallel_config.worker_cls = \
93
- "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
96
+ "vllm.v1.worker.tpu_worker.TPUWorker"
97
else:
95
- parallel_config.worker_cls = "vllm.worker.tpu_worker.TPUWorker"
98
+ if scheduler_config.is_multi_step:
99
+ parallel_config.worker_cls = \
100
+ "vllm.worker.multi_step_tpu_worker.MultiStepTPUWorker"
101
+ else:
102
+ parallel_config.worker_cls = \
103
+ "vllm.worker.tpu_worker.TPUWorker"
104
+
105
+ # Adjust scheduler config for V1
106
+ # TODO: Add support for these
107
+ if envs.VLLM_USE_V1 and vllm_config.cache_config.enable_prefix_caching:
108
+ logger.warning("[V1][TPU] Disable prefix caching")
109
+ vllm_config.cache_config.enable_prefix_caching = False
110
+
111
+ assert not vllm_config.speculative_config, (
112
+ "Speculative decoding is not yet supported for TPU backend")
113
+
114
+ @classmethod
115
+ def is_pin_memory_available(cls):
116
+ logger.warning("Pin memory is not supported on TPU.")
117
+ return False
vllm/v1/attention/backends/pallas.py
new
+353
@@ -0,0 +1,353 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from dataclasses import dataclass
4
+from typing import Any, Dict, List, Optional, Tuple, Type
5
+
6
+import torch
7
+import torch_xla.experimental.custom_kernel # Required to register custom ops.
8
+
9
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
+ AttentionLayer,
11
+ AttentionMetadata, AttentionType)
12
+from vllm.attention.backends.utils import CommonAttentionState
13
+
14
+
15
+class PallasAttentionBackend(AttentionBackend):
16
+
17
+ @staticmethod
18
+ def get_name() -> str:
19
+ return "PALLAS_VLLM_V1"
20
+
21
+ @staticmethod
22
+ def get_impl_cls() -> Type["PallasAttentionBackendImpl"]:
23
+ return PallasAttentionBackendImpl
24
+
25
+ @staticmethod
26
+ def get_metadata_cls() -> Type["PallasMetadata"]:
27
+ return PallasMetadata
28
+
29
+ @staticmethod
30
+ def get_state_cls() -> Type["CommonAttentionState"]:
31
+ return CommonAttentionState
32
+
33
+ @staticmethod
34
+ def get_kv_cache_shape(
35
+ num_blocks: int,
36
+ block_size: int,
37
+ num_kv_heads: int,
38
+ head_size: int,
39
+ ) -> Tuple[int, ...]:
40
+ return (num_kv_heads, num_blocks, block_size, head_size)
41
+
42
+ @staticmethod
43
+ def swap_blocks(
44
+ src_kv_cache: torch.Tensor,
45
+ dst_kv_cache: torch.Tensor,
46
+ src_to_dst: torch.Tensor,
47
+ ) -> None:
48
+ raise RuntimeError("swap_blocks is not used for the TPU backend.")
49
+
50
+ @torch.compile(backend="openxla")
51
+ @staticmethod
52
+ def copy_blocks(
53
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
54
+ src_to_dists: Tuple[torch.Tensor, torch.Tensor],
55
+ ) -> None:
56
+ src_indices, dst_indices = src_to_dists
57
+ for k_cache, v_cache in kv_caches:
58
+ torch.ops.xla.dynamo_set_buffer_donor_(k_cache, True)
59
+ k_cache[:, dst_indices] = k_cache[:, src_indices]
60
+ torch.ops.xla.dynamo_set_buffer_donor_(v_cache, True)
61
+ v_cache[:, dst_indices] = v_cache[:, src_indices]
62
+
63
+
64
+@dataclass
65
+class PallasMetadata(AttentionMetadata):
66
+
67
+ # Currently, input sequences can only contain all prefills
68
+ # or all decoding.
69
+ block_tables: Optional[torch.Tensor] = None
70
+ context_lens: Optional[torch.Tensor] = None
71
+ effective_query_lens: Optional[torch.Tensor] = None
72
+
73
+ @property
74
+ def prefill_metadata(self) -> Optional["PallasMetadata"]:
75
+ if self.num_prefills == 0:
76
+ return None
77
+
78
+ assert self.num_decode_tokens == 0
79
+ return self
80
+
81
+ @property
82
+ def decode_metadata(self) -> Optional["PallasMetadata"]:
83
+ if self.num_decode_tokens == 0:
84
+ return None
85
+
86
+ assert self.num_prefills == 0
87
+ assert self.num_prefill_tokens == 0
88
+ assert self.block_tables is not None
89
+ assert self.context_lens is not None
90
+ return self
91
+
92
+
93
+class PallasAttentionBackendImpl(AttentionImpl):
94
+
95
+ def __init__(
96
+ self,
97
+ num_heads: int,
98
+ head_size: int,
99
+ scale: float,
100
+ num_kv_heads: int,
101
+ alibi_slopes: Optional[List[float]],
102
+ sliding_window: Optional[int],
103
+ kv_cache_dtype: str,
104
+ blocksparse_params: Optional[Dict[str, Any]] = None,
105
+ logits_soft_cap: Optional[float] = None,
106
+ attn_type: str = AttentionType.DECODER,
107
+ ) -> None:
108
+ self.num_heads = num_heads
109
+ self.head_size = head_size
110
+ self.scale = float(scale)
111
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
112
+
113
+ assert self.num_heads % self.num_kv_heads == 0
114
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
115
+ if head_size % 128 != 0:
116
+ raise NotImplementedError("Head size must be a multiple of 128.")
117
+ if alibi_slopes is not None:
118
+ raise NotImplementedError("Alibi slopes is not supported.")
119
+ if sliding_window is not None:
120
+ raise NotImplementedError("Sliding window is not supported.")
121
+ if kv_cache_dtype != "auto":
122
+ raise NotImplementedError("FP8 KV cache dtype is not supported.")
123
+ if blocksparse_params is not None:
124
+ raise NotImplementedError("Blocksparse is not supported.")
125
+ if logits_soft_cap is not None:
126
+ raise NotImplementedError(
127
+ "Attention logits soft-capping is not supported.")
128
+
129
+ if torch_xla.tpu.version() < 4:
130
+ raise NotImplementedError("TPU version must be 4 or higher.")
131
+
132
+ self.megacore_mode = None
133
+ tpu_env = torch_xla.tpu.get_tpu_env()
134
+ tpu_type = (tpu_env.get("ACCELERATOR_TYPE", None)
135
+ or tpu_env.get("TYPE", None)
136
+ or tpu_env.get("TPU_ACCELERATOR_TYPE", None))
137
+ assert tpu_type is not None
138
+ tpu_type = tpu_type.lower()
139
+
140
+ if (("lite" not in tpu_type) and ("v6" not in tpu_type)):
141
+ if self.num_kv_heads % 2 == 0:
142
+ self.megacore_mode = "kv_head"
143
+ else:
144
+ # NOTE(woosuk): If the batch size is not a multiple of 2, the
145
+ # megacore mode will be None.
146
+ self.megacore_mode = "batch"
147
+
148
+ if attn_type != AttentionType.DECODER:
149
+ raise NotImplementedError("Encoder self-attention and "
150
+ "encoder/decoder cross-attention "
151
+ "are not implemented for "
152
+ "PallasAttentionBackendImpl")
153
+
154
+ def forward(
155
+ self,
156
+ layer: AttentionLayer,
157
+ query: torch.Tensor,
158
+ key: torch.Tensor,
159
+ value: torch.Tensor,
160
+ kv_cache: Tuple[torch.Tensor, torch.Tensor],
161
+ attn_metadata: PallasMetadata,
162
+ output: Optional[torch.Tensor] = None,
163
+ ) -> torch.Tensor:
164
+ """Forward pass with Pallas attention.
165
+
166
+ Args:
167
+ query: shape = [batch_size, seq_len, num_heads * head_size]
168
+ key: shape = [batch_size, seq_len, num_kv_heads * head_size]
169
+ value: shape = [batch_size, seq_len, num_kv_heads * head_size]
170
+ kv_cache[0] = [num_kv_heads, num_blocks, block_size, head_size]
171
+ kv_cache[1] = [num_kv_heads, num_blocks, block_size, head_size]
172
+ NOTE: kv_cache[0] and kv_cache[1] will be an empty tensor
173
+ with shape [0] for profiling run.
174
+ attn_metadata: Metadata for attention.
175
+ Returns:
176
+ shape = [batch_size, seq_len, num_heads * head_size]
177
+ """
178
+
179
+ if attn_metadata is None:
180
+ if output is None:
181
+ output = torch.ones_like(query)
182
+ return output
183
+
184
+ assert layer._k_scale_float == 1.0 and layer._v_scale_float == 1.0
185
+ batch_size, seq_len, hidden_size = query.shape
186
+ query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
187
+ key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
188
+ value = value.view(batch_size, seq_len, self.num_kv_heads,
189
+ self.head_size)
190
+
191
+ if kv_cache[0].numel() > 0:
192
+ slot_mapping = attn_metadata.slot_mapping
193
+ key_cache, value_cache = kv_cache
194
+ write_to_kv_cache(key, value, key_cache, value_cache, slot_mapping)
195
+
196
+ query = query * self.scale
197
+ if attn_metadata.num_prefills > 0:
198
+ if attn_metadata.block_tables is None:
199
+ # Prefill without paged KV cache.
200
+ assert seq_len % 16 == 0, (
201
+ "Pallas FlashAttention kernel requires seq_len to be a "
202
+ f"multiple of 16 but got {seq_len}")
203
+
204
+ # Handle GQA/MQA.
205
+ if self.num_kv_heads != self.num_heads:
206
+ key = key.repeat_interleave(self.num_queries_per_kv,
207
+ dim=-2)
208
+ key = key.view(batch_size, seq_len, self.num_heads,
209
+ self.head_size)
210
+ value = value.repeat_interleave(self.num_queries_per_kv,
211
+ dim=-2)
212
+ value = value.view(batch_size, seq_len, self.num_heads,
213
+ self.head_size)
214
+ # FlashAttention kernel requires the input shape to be
215
+ # [batch_size, num_heads, seq_len, d_model]
216
+ # while the input is [batch_size, seq_len, num_heads, d_model].
217
+ # Permute the input to match the required format.
218
+ output = torch.ops.xla.flash_attention(
219
+ query.permute(0, 2, 1, 3),
220
+ key.permute(0, 2, 1, 3),
221
+ value.permute(0, 2, 1, 3),
222
+ True,
223
+ )
224
+ output = output.permute(0, 2, 1, 3)
225
+ else:
226
+ # Prefill with paged KV cache.
227
+ # TODO(woosuk): Tune the below knobs.
228
+ num_kv_pages_per_compute_block = 16
229
+ num_queries_per_compute_block = 16
230
+ assert seq_len % num_queries_per_compute_block == 0
231
+ output = torch.ops.xla.multi_queries_paged_attention(
232
+ query,
233
+ key_cache,
234
+ value_cache,
235
+ attn_metadata.context_lens,
236
+ attn_metadata.block_tables,
237
+ attn_metadata.effective_query_lens,
238
+ num_kv_pages_per_compute_block,
239
+ num_queries_per_compute_block,
240
+ use_kernel=True,
241
+ )
242
+ else:
243
+ # Decoding run.
244
+ assert kv_cache[0].numel() > 0
245
+ query = query.squeeze(dim=1)
246
+ pages_per_compute_block = 16 # TODO(woosuk): Tune this value.
247
+
248
+ assert attn_metadata.block_tables is not None
249
+ assert attn_metadata.context_lens is not None
250
+ # NOTE(woosuk): The PagedAttention Pallas kernel stores the entire
251
+ # block table in SMEM. Therefore, if the block table is too large,
252
+ # the kernel compilation will fail. To avoid this, we split the
253
+ # batch dimension into smaller chunks and run the kernel multiple
254
+ # times.
255
+ MAX_SMEM_USAGE = 512 * 1024
256
+ size_per_seq = 4 * attn_metadata.block_tables.shape[1]
257
+ max_num_seq = MAX_SMEM_USAGE // size_per_seq
258
+
259
+ if batch_size <= max_num_seq:
260
+ output = paged_attention(
261
+ query,
262
+ key_cache,
263
+ value_cache,
264
+ attn_metadata.context_lens,
265
+ attn_metadata.block_tables,
266
+ pages_per_compute_block,
267
+ self.megacore_mode,
268
+ )
269
+ else:
270
+ chunk_size = max_num_seq
271
+ # Make sure the chunk size is a multiple of 2.
272
+ chunk_size = chunk_size // 2 * 2
273
+ num_chunks = (batch_size + chunk_size - 1) // chunk_size
274
+
275
+ output = torch.empty_like(query)
276
+ for chunk_idx in range(num_chunks):
277
+ chunk_start = chunk_idx * chunk_size
278
+ chunk_end = chunk_start + chunk_size
279
+ # NOTE(woosuk): We skip this line because it causes Dynamo
280
+ # compilation error. Instead, we rely on the slice operation
281
+ # to handle the out-of-bound case.
282
+ # chunk_end = min(chunk_end, batch_size)
283
+ chunk_output = paged_attention(
284
+ query[chunk_start:chunk_end],
285
+ key_cache,
286
+ value_cache,
287
+ attn_metadata.context_lens[chunk_start:chunk_end],
288
+ attn_metadata.block_tables[chunk_start:chunk_end],
289
+ pages_per_compute_block,
290
+ self.megacore_mode,
291
+ )
292
+ output[chunk_start:chunk_end] = chunk_output
293
+
294
+ # Reshape the output tensor.
295
+ return output.reshape(batch_size, seq_len, hidden_size)
296
+
297
+
298
+def write_to_kv_cache(
299
+ key: torch.Tensor,
300
+ value: torch.Tensor,
301
+ key_cache: torch.Tensor,
302
+ value_cache: torch.Tensor,
303
+ slot_mapping: torch.Tensor,
304
+) -> None:
305
+ torch.ops.xla.dynamo_set_buffer_donor_(key_cache, True)
306
+ torch.ops.xla.dynamo_set_buffer_donor_(value_cache, True)
307
+
308
+ key = key.flatten(0, 2)
309
+ value = value.flatten(0, 2)
310
+ key_cache = key_cache.flatten(0, 2)
311
+ value_cache = value_cache.flatten(0, 2)
312
+ key_cache.index_copy_(0, slot_mapping, key)
313
+ value_cache.index_copy_(0, slot_mapping, value)
314
+
315
+
316
+def paged_attention(
317
+ query: torch.Tensor,
318
+ key_cache: torch.Tensor,
319
+ value_cache: torch.Tensor,
320
+ context_lens: torch.Tensor,
321
+ block_tables: torch.Tensor,
322
+ pages_per_compute_block: int,
323
+ megacore_mode: Optional[str],
324
+) -> torch.Tensor:
325
+ batch_size = query.shape[0]
326
+ if megacore_mode == "batch" and batch_size % 2 != 0:
327
+ megacore_mode = None
328
+ else:
329
+ megacore_mode = megacore_mode
330
+
331
+ # NOTE(woosuk): A temporary workaround to avoid the error:
332
+ # "xla::paged_attention() Expected a value of type 'str' for
333
+ # argument 'megacore_mode' but instead found type 'NoneType'."
334
+ if megacore_mode is not None:
335
+ output = torch.ops.xla.paged_attention(
336
+ query,
337
+ key_cache,
338
+ value_cache,
339
+ context_lens,
340
+ block_tables,
341
+ pages_per_compute_block,
342
+ megacore_mode=megacore_mode,
343
+ )
344
+ else:
345
+ output = torch.ops.xla.paged_attention(
346
+ query,
347
+ key_cache,
348
+ value_cache,
349
+ context_lens,
350
+ block_tables,
351
+ pages_per_compute_block,
352
+ )
353
+ return output
vllm/v1/worker/block_table.py
+8
@@ -61,6 +61,14 @@ class BlockTable:
61
src, :num_blocks]
62
self.num_blocks_per_row[tgt] = num_blocks
63
64
+ def swap_row(self, src: int, tgt: int) -> None:
65
+ num_blocks_src = self.num_blocks_per_row[src]
66
+ num_blocks_tgt = self.num_blocks_per_row[tgt]
67
+ self.num_blocks_per_row[src] = num_blocks_tgt
68
+ self.num_blocks_per_row[tgt] = num_blocks_src
69
+
70
+ self.block_table_np[[src, tgt]] = self.block_table_np[[tgt, src]]
71
+
72
def commit(self, num_reqs: int) -> None:
73
self.block_table[:num_reqs].copy_(self.block_table_cpu[:num_reqs],
74
non_blocking=True)
vllm/v1/worker/tpu_model_runner.py
new
+1109
@@ -0,0 +1,1109 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import enum
3
+import time
4
+from dataclasses import dataclass
5
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
6
+from unittest.mock import patch
7
+
8
+import numpy as np
9
+import torch
10
+import torch.distributed
11
+import torch.nn as nn
12
+# TPU XLA related
13
+import torch_xla.core.xla_model as xm
14
+import torch_xla.runtime as xr
15
+
16
+from vllm.attention import AttentionMetadata
17
+from vllm.attention.backends.abstract import AttentionType
18
+from vllm.attention.layer import Attention
19
+from vllm.config import VllmConfig
20
+from vllm.forward_context import set_forward_context
21
+from vllm.logger import init_logger
22
+from vllm.model_executor.model_loader import get_model
23
+from vllm.sampling_params import SamplingType
24
+from vllm.utils import LayerBlockType, cdiv, is_pin_memory_available
25
+from vllm.v1.attention.backends.pallas import (PallasAttentionBackend,
26
+ PallasMetadata)
27
+from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
28
+ KVCacheSpec)
29
+from vllm.v1.outputs import LogprobsTensors, ModelRunnerOutput
30
+from vllm.v1.utils import bind_kv_cache
31
+from vllm.v1.worker.gpu_input_batch import CachedRequestState, InputBatch
32
+
33
+if TYPE_CHECKING:
34
+ from vllm.v1.core.scheduler import SchedulerOutput
35
+
36
+logger = init_logger(__name__)
37
+
38
+# Here we utilize the behavior that out-of-bound index is ignored.
39
+# FIXME(woosuk): Find a more reliable way to prevent possible bugs.
40
+_PAD_SLOT_ID = 1_000_000_000
41
+
42
+
43
+class ExecutionMode(enum.Enum):
44
+ PREFILL = enum.auto()
45
+ DECODE = enum.auto()
46
+ PREFIX_PREFILL = enum.auto()
47
+
48
+ def is_prefill(self) -> bool:
49
+ return self in (ExecutionMode.PREFILL, ExecutionMode.PREFIX_PREFILL)
50
+
51
+
52
+@dataclass
53
+class PromptDecodeInfo:
54
+ prompt_req_ids: List[str]
55
+ decode_req_ids: List[str]
56
+ prompt_scheduled_tokens: List[int]
57
+
58
+
59
+@dataclass
60
+class PromptData:
61
+ input_tokens: torch.Tensor
62
+ input_positions: torch.Tensor
63
+ attn_metadata: PallasMetadata
64
+
65
+
66
+@dataclass
67
+class DecodeData:
68
+ input_tokens: Optional[torch.Tensor] = None
69
+ input_positions: Optional[torch.Tensor] = None
70
+ attn_metadata: Optional[PallasMetadata] = None
71
+
72
+
73
+class TPUModelRunner:
74
+
75
+ def __init__(
76
+ self,
77
+ vllm_config: VllmConfig,
78
+ device: torch.device,
79
+ ):
80
+ self.vllm_config = vllm_config
81
+ self.model_config = vllm_config.model_config
82
+ self.cache_config = vllm_config.cache_config
83
+ self.lora_config = vllm_config.lora_config
84
+ self.load_config = vllm_config.load_config
85
+ self.parallel_config = vllm_config.parallel_config
86
+ self.scheduler_config = vllm_config.scheduler_config
87
+ self.speculative_config = vllm_config.speculative_config
88
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
89
+ self.observability_config = vllm_config.observability_config
90
+ self.device_config = vllm_config.device_config
91
+
92
+ model_config = self.model_config
93
+ cache_config = self.cache_config
94
+ scheduler_config = self.scheduler_config
95
+ parallel_config = self.parallel_config
96
+ self.device = device
97
+ self.pin_memory = is_pin_memory_available()
98
+ self.dtype = self.model_config.dtype
99
+
100
+ self.is_multimodal_model = model_config.is_multimodal_model
101
+ self.sliding_window = model_config.get_sliding_window()
102
+ self.block_size = cache_config.block_size
103
+ self.max_model_len = model_config.max_model_len
104
+ self.max_num_blocks_per_req = cdiv(self.max_model_len, self.block_size)
105
+ self.max_num_tokens = scheduler_config.max_num_batched_tokens
106
+ self.max_num_reqs = scheduler_config.max_num_seqs
107
+
108
+ # Model-related.
109
+ self.num_attn_layers = model_config.get_num_layers_by_block_type(
110
+ parallel_config, LayerBlockType.attention)
111
+ self.num_query_heads = model_config.get_num_attention_heads(
112
+ parallel_config)
113
+ self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
114
+ self.head_size = model_config.get_head_size()
115
+ self.hidden_size = model_config.get_hidden_size()
116
+
117
+ self.model: Optional[nn.Module] = None
118
+
119
+ # Persistent batch.
120
+ self.input_batch = InputBatch(
121
+ max_num_reqs=self.max_num_reqs,
122
+ max_model_len=self.max_model_len,
123
+ max_num_blocks_per_req=self.max_num_blocks_per_req,
124
+ device=self.device,
125
+ pin_memory=self.pin_memory,
126
+ vocab_size=self.model_config.get_vocab_size(),
127
+ )
128
+
129
+ # Request states.
130
+ self.requests: Dict[str, CachedRequestState] = {}
131
+
132
+ # req_id -> (input_id -> encoder_output)
133
+ self.encoder_cache: Dict[str, Dict[int, torch.Tensor]] = {}
134
+
135
+ # KV caches for forward pass
136
+ self.kv_caches: List[Tuple[torch.Tensor, torch.Tensor]] = []
137
+
138
+ # Cached torch/numpy tensors
139
+ self.num_swaps = 2
140
+ self.cur_swap_id = 0
141
+ self.input_ids_cpu = []
142
+ self.input_ids_np = []
143
+ self.input_positions_cpu = []
144
+ self.input_positions_np = []
145
+ self.slot_mapping_cpu = []
146
+ self.slot_mapping_np = []
147
+ self.prompt_context_lens_cpu = []
148
+ self.prompt_effective_query_lens_cpu = []
149
+ self.decode_context_lens_cpu = []
150
+ self.decode_context_lens_np = []
151
+ for _ in range(self.num_swaps):
152
+ self.input_ids_cpu.append(
153
+ torch.empty(self.max_num_tokens,
154
+ dtype=torch.int32,
155
+ device="cpu"))
156
+ self.input_ids_np.append(self.input_ids_cpu[-1].numpy())
157
+
158
+ self.input_positions_cpu.append(
159
+ torch.empty(self.max_num_tokens,
160
+ dtype=torch.int32,
161
+ device="cpu"))
162
+ self.input_positions_np.append(
163
+ self.input_positions_cpu[-1].numpy())
164
+
165
+ self.slot_mapping_cpu.append(
166
+ torch.empty(self.max_num_tokens,
167
+ dtype=torch.int64,
168
+ device="cpu"))
169
+ self.slot_mapping_np.append(self.slot_mapping_cpu[-1].numpy())
170
+
171
+ self.prompt_context_lens_cpu.append(
172
+ torch.empty((1), dtype=torch.int32, device="cpu"))
173
+ self.prompt_effective_query_lens_cpu.append(
174
+ torch.empty((1), dtype=torch.int32, device="cpu"))
175
+
176
+ self.decode_context_lens_cpu.append(
177
+ torch.empty(self.max_num_tokens,
178
+ dtype=torch.int32,
179
+ device="cpu"))
180
+ self.decode_context_lens_np.append(
181
+ self.decode_context_lens_cpu[-1].numpy())
182
+
183
+ # Range tensor with values [0 .. self.max_num_tokens - 1].
184
+ # Used to initialize positions / context_lens / seq_lens
185
+ self.arange_np = np.arange(self.max_num_tokens, dtype=np.int32)
186
+
187
+ def _update_states(self, scheduler_output: "SchedulerOutput") -> bool:
188
+ """Update the cached states and the persistent batch with the scheduler
189
+ output.
190
+
191
+ The updated states are used by the `_prepare_inputs` function to create
192
+ the input GPU tensors for the model.
193
+
194
+ Returns:
195
+ True if there is a new/resumed/paused/finished request in the batch.
196
+ If False, we can skip copying SamplingMetadata to the GPU.
197
+ """
198
+ # Remove finished requests from the cached states.
199
+ for req_id in scheduler_output.finished_req_ids:
200
+ self.requests.pop(req_id, None)
201
+
202
+ # Remove the finished requests from the persistent batch.
203
+ # NOTE(woosuk): There could be an edge case where finished_req_ids and
204
+ # scheduled_req_ids overlap. This happens when a request is aborted and
205
+ # then resubmitted with the same ID. In this case, we treat them as two
206
+ # distinct requests - clearing the cached states for the first request
207
+ # and handling the second as a new request.
208
+ removed_req_indices: List[int] = []
209
+ for req_id in scheduler_output.finished_req_ids:
210
+ req_index = self.input_batch.remove_request(req_id)
211
+ if req_index is not None:
212
+ removed_req_indices.append(req_index)
213
+
214
+ # Remove the unscheduled requests from the persistent batch.
215
+ # NOTE(woosuk): The unscheduled requests are either preempted requests
216
+ # or running requests that are not scheduled in this step. We remove
217
+ # them from the persistent batch but keep their cached states since
218
+ # they will be scheduled again sometime in the future.
219
+ scheduled_req_ids = scheduler_output.num_scheduled_tokens.keys()
220
+ cached_req_ids = self.input_batch.req_id_to_index.keys()
221
+ unscheduled_req_ids = cached_req_ids - scheduled_req_ids
222
+ # NOTE(woosuk): The persistent batch optimization assumes that
223
+ # consecutive batches contain mostly the same requests. If batches
224
+ # have low request overlap (e.g., alternating between two distinct
225
+ # sets of requests), this optimization becomes very inefficient.
226
+ for req_id in unscheduled_req_ids:
227
+ req_index = self.input_batch.remove_request(req_id)
228
+ assert req_index is not None
229
+ removed_req_indices.append(req_index)
230
+
231
+ req_ids_to_add: List[str] = []
232
+ # Add new requests to the cached states.
233
+ for new_req_data in scheduler_output.scheduled_new_reqs:
234
+ req_id = new_req_data.req_id
235
+ sampling_params = new_req_data.sampling_params
236
+ if sampling_params.sampling_type == SamplingType.RANDOM_SEED:
237
+ generator = torch.Generator(device=self.device)
238
+ generator.manual_seed(sampling_params.seed)
239
+ else:
240
+ generator = None
241
+
242
+ self.requests[req_id] = CachedRequestState(
243
+ req_id=req_id,
244
+ prompt_token_ids=new_req_data.prompt_token_ids,
245
+ prompt=new_req_data.prompt,
246
+ mm_inputs=new_req_data.mm_inputs,
247
+ mm_positions=new_req_data.mm_positions,
248
+ sampling_params=sampling_params,
249
+ generator=generator,
250
+ block_ids=new_req_data.block_ids,
251
+ num_computed_tokens=new_req_data.num_computed_tokens,
252
+ output_token_ids=[],
253
+ lora_request=new_req_data.lora_request,
254
+ )
255
+
256
+ req_ids_to_add.append(req_id)
257
+
258
+ # Update the states of the running/resumed requests.
259
+ for req_data in scheduler_output.scheduled_cached_reqs:
260
+ req_id = req_data.req_id
261
+ req_state = self.requests[req_id]
262
+
263
+ # Update the cached states.
264
+ req_state.num_computed_tokens = req_data.num_computed_tokens
265
+ if not req_data.resumed_from_preemption:
266
+ # Append the new blocks to the existing block IDs.
267
+ req_state.block_ids.extend(req_data.new_block_ids)
268
+ else:
269
+ # The request is resumed from preemption.
270
+ # Replace the existing block IDs with the new ones.
271
+ req_state.block_ids = req_data.new_block_ids
272
+
273
+ req_index = self.input_batch.req_id_to_index.get(req_id)
274
+ if req_index is None:
275
+ # The request is not in the persistent batch.
276
+ # The request was either preempted and resumed later, or was not
277
+ # scheduled in the previous step and needs to be added again.
278
+ req_ids_to_add.append(req_id)
279
+ continue
280
+
281
+ # Update the persistent batch.
282
+ self.input_batch.num_computed_tokens_cpu[req_index] = (
283
+ req_data.num_computed_tokens)
284
+ start_index = len(req_state.block_ids) - len(
285
+ req_data.new_block_ids)
286
+ self.input_batch.block_table.append_row(req_index, start_index,
287
+ req_data.new_block_ids)
288
+
289
+ # Add the new or resumed requests to the persistent batch.
290
+ # The smaller empty indices are filled first.
291
+ removed_req_indices = sorted(removed_req_indices, reverse=True)
292
+ for req_id in req_ids_to_add:
293
+ req_state = self.requests[req_id]
294
+ if removed_req_indices:
295
+ # Fill the empty index.
296
+ req_index = removed_req_indices.pop()
297
+ else:
298
+ # Append to the end.
299
+ req_index = None
300
+ self.input_batch.add_request(req_state, req_index)
301
+
302
+ # Condense the batched states if there are empty indices.
303
+ if removed_req_indices:
304
+ self.input_batch.condense(removed_req_indices)
305
+ return len(unscheduled_req_ids) > 0 or len(req_ids_to_add) > 0
306
+
307
+ def swap_step(self):
308
+ self.cur_swap_id = (self.cur_swap_id + 1) % self.num_swaps
309
+
310
+ def get_model(self) -> nn.Module:
311
+ assert self.model is not None
312
+ return self.model
313
+
314
+ def get_kv_cache_spec(self) -> KVCacheSpec:
315
+ """
316
+ Generates the KVCacheSpec by parsing the kv cache format from each
317
+ Attention module in the static forward context.
318
+ Returns:
319
+ KVCacheSpec: A dictionary mapping layer names to their KV cache
320
+ format. Layers that do not need KV cache are not included.
321
+ """
322
+
323
+ forward_ctx = self.vllm_config.compilation_config.static_forward_context
324
+ block_size = self.vllm_config.cache_config.block_size
325
+ kv_cache_spec: KVCacheSpec = {}
326
+ for layer_name, attn_module in forward_ctx.items():
327
+ # TODO: Support other attention modules, e.g., sliding window,
328
+ # cross-attention, MLA.
329
+ assert isinstance(attn_module, Attention)
330
+ if attn_module.attn_type == AttentionType.DECODER:
331
+ kv_cache_spec[layer_name] = FullAttentionSpec(
332
+ block_size=block_size,
333
+ num_kv_heads=attn_module.num_kv_heads,
334
+ head_size=attn_module.head_size,
335
+ dtype=attn_module.dtype,
336
+ )
337
+ elif attn_module.attn_type in (AttentionType.ENCODER,
338
+ AttentionType.ENCODER_ONLY):
339
+ # encoder-only attention does not need KV cache.
340
+ continue
341
+ elif attn_module.attn_type == AttentionType.ENCODER_DECODER:
342
+ raise NotImplementedError
343
+ else:
344
+ raise ValueError(
345
+ f"Unknown attention type: {attn_module.attn_type}")
346
+
347
+ return kv_cache_spec
348
+
349
+ def _get_prompts_and_decodes(
350
+ self,
351
+ scheduler_output: "SchedulerOutput",
352
+ ) -> PromptDecodeInfo:
353
+ total_num_scheduled_tokens = scheduler_output.total_num_scheduled_tokens
354
+ assert total_num_scheduled_tokens > 0
355
+ num_reqs = self.input_batch.num_reqs
356
+ assert num_reqs > 0
357
+
358
+ # Traverse decodes first
359
+ decode_req_ids = []
360
+ for i in range(num_reqs):
361
+ req_id = self.input_batch.req_ids[i]
362
+ assert req_id is not None
363
+
364
+ num_computed_tokens = self.input_batch.num_computed_tokens_cpu[i]
365
+ num_prompt_tokens = self.input_batch.num_prompt_tokens[i]
366
+ num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
367
+ req_id]
368
+
369
+ if num_computed_tokens < num_prompt_tokens:
370
+ # This is prompt
371
+ break
372
+
373
+ # This is decode
374
+ assert num_scheduled_tokens == 1
375
+ decode_req_ids.append(req_id)
376
+
377
+ # Traverse prompts
378
+ prompt_req_ids = []
379
+ prompt_scheduled_tokens = []
380
+ for i in range(len(decode_req_ids), num_reqs):
381
+ req_id = self.input_batch.req_ids[i]
382
+ assert req_id is not None
383
+
384
+ num_computed_tokens = self.input_batch.num_computed_tokens_cpu[i]
385
+ num_prompt_tokens = self.input_batch.num_prompt_tokens[i]
386
+ num_scheduled_tokens = scheduler_output.num_scheduled_tokens[
387
+ req_id]
388
+
389
+ # Must be prompt
390
+ assert num_computed_tokens < num_prompt_tokens
391
+
392
+ prompt_req_ids.append(req_id)
393
+ prompt_scheduled_tokens.append(num_scheduled_tokens)
394
+
395
+ return PromptDecodeInfo(prompt_req_ids, decode_req_ids,
396
+ prompt_scheduled_tokens)
397
+
398
+ def _prepare_prompt(self, req_index: int,
399
+ num_scheduled_tokens: int) -> PromptData:
400
+ num_computed_tokens = self.input_batch.num_computed_tokens_cpu[
401
+ req_index]
402
+ num_prompt_tokens = self.input_batch.num_prompt_tokens[req_index]
403
+
404
+ # Must be prompt
405
+ assert num_computed_tokens < num_prompt_tokens
406
+
407
+ # Prompt len
408
+ prompt_len = num_scheduled_tokens
409
+ padded_prompt_len = _get_padded_prompt_len(prompt_len)
410
+ assert padded_prompt_len <= self.max_model_len
411
+
412
+ # Seq len
413
+ seq_len = num_computed_tokens + prompt_len
414
+ padded_seq_len = num_computed_tokens + padded_prompt_len
415
+
416
+ # Input tokens
417
+ input_tokens_cpu = self.input_batch.token_ids_cpu_tensor[
418
+ req_index, num_computed_tokens:padded_seq_len]
419
+ input_tokens_cpu[prompt_len:] = 0
420
+
421
+ # Input positions
422
+ input_positions_np = self.input_positions_np[
423
+ self.cur_swap_id][:padded_prompt_len]
424
+ np.add(num_computed_tokens,
425
+ self.arange_np[:padded_prompt_len],
426
+ out=input_positions_np)
427
+ input_positions_np[prompt_len:] = 0
428
+
429
+ # Slot mapping
430
+ block_table_np = \
431
+ self.input_batch.block_table.get_numpy_array()
432
+ block_numbers_np = block_table_np[req_index, input_positions_np //
433
+ self.block_size]
434
+ block_offsets_np = input_positions_np % self.block_size
435
+
436
+ slot_mapping_np = self.slot_mapping_np[
437
+ self.cur_swap_id][:padded_prompt_len]
438
+ np.add(block_numbers_np * self.block_size,
439
+ block_offsets_np,
440
+ out=slot_mapping_np)
441
+ slot_mapping_np[prompt_len:] = _PAD_SLOT_ID
442
+
443
+ # Block table
444
+ block_table_cpu = None
445
+ if num_computed_tokens > 0:
446
+ block_table_cpu = self.input_batch.block_table.get_cpu_tensor()
447
+ block_table_cpu = block_table_cpu[req_index]
448
+
449
+ # Context len
450
+ self.prompt_context_lens_cpu[self.cur_swap_id][0] = 0
451
+ if num_computed_tokens > 0:
452
+ self.prompt_context_lens_cpu[self.cur_swap_id][0] = seq_len
453
+
454
+ # Effective query len
455
+ self.prompt_effective_query_lens_cpu[self.cur_swap_id][0] = prompt_len
456
+
457
+ # Get final tensors
458
+ input_tokens = input_tokens_cpu.reshape(1, -1).to(self.device)
459
+ input_positions = self.input_positions_cpu[
460
+ self.cur_swap_id][:padded_prompt_len].reshape(1,
461
+ -1).to(self.device)
462
+ slot_mapping = self.slot_mapping_cpu[
463
+ self.cur_swap_id][:padded_prompt_len].reshape(1,
464
+ -1).to(self.device)
465
+ block_table = block_table_cpu.reshape(1, -1).to(
466
+ self.device) if block_table_cpu is not None else None
467
+
468
+ context_lens = self.prompt_context_lens_cpu[self.cur_swap_id].to(
469
+ self.device)
470
+ effective_query_lens = self.prompt_effective_query_lens_cpu[
471
+ self.cur_swap_id].to(self.device)
472
+
473
+ self.swap_step()
474
+
475
+ # Attn metadata
476
+ attn_metadata = PallasMetadata(
477
+ num_prefills=1,
478
+ num_prefill_tokens=0, # NOTE: This is not used.
479
+ num_decode_tokens=0,
480
+ slot_mapping=slot_mapping,
481
+ multi_modal_placeholder_index_maps=None,
482
+ enable_kv_scales_calculation=True,
483
+ block_tables=block_table,
484
+ context_lens=context_lens,
485
+ effective_query_lens=effective_query_lens,
486
+ )
487
+
488
+ return PromptData(input_tokens, input_positions, attn_metadata)
489
+
490
+ def _prepare_decode(
491
+ self,
492
+ decode_req_ids: List[str],
493
+ ) -> DecodeData:
494
+ # Batch size
495
+ batch_size = len(decode_req_ids)
496
+ padded_batch_size = _get_padded_batch_size(batch_size)
497
+ assert padded_batch_size <= self.max_model_len
498
+
499
+ # Init [0 .. batch_size - 1]
500
+ req_indices_np = self.arange_np[:padded_batch_size]
501
+
502
+ # Input positions
503
+ input_positions_np = self.input_positions_np[
504
+ self.cur_swap_id][:padded_batch_size]
505
+ np.add(self.input_batch.num_computed_tokens_cpu[:padded_batch_size],
506
+ 0,
507
+ out=input_positions_np)
508
+ input_positions_np[batch_size:] = 0
509
+ input_positions_cpu = self.input_positions_cpu[
510
+ self.cur_swap_id][:padded_batch_size]
511
+
512
+ # Input tokens
513
+ token_indices_np = (
514
+ input_positions_np +
515
+ req_indices_np * self.input_batch.token_ids_cpu.shape[1])
516
+ input_tokens_cpu = self.input_ids_cpu[
517
+ self.cur_swap_id][:padded_batch_size]
518
+ torch.index_select(self.input_batch.token_ids_cpu_tensor.flatten(),
519
+ 0,
520
+ torch.from_numpy(token_indices_np),
521
+ out=input_tokens_cpu)
522
+ input_tokens_cpu[batch_size:] = 0
523
+
524
+ # Slot mapping
525
+ block_table_indices_np = (
526
+ req_indices_np * self.max_num_blocks_per_req +
527
+ input_positions_np // self.block_size)
528
+
529
+ block_table_cpu = self.input_batch.block_table.get_cpu_tensor()
530
+
531
+ block_numbers_np = block_table_cpu.flatten(
532
+ )[block_table_indices_np].numpy()
533
+
534
+ block_offsets_np = input_positions_np % self.block_size
535
+
536
+ slot_mapping_np = self.slot_mapping_np[
537
+ self.cur_swap_id][:padded_batch_size]
538
+ np.add(block_numbers_np * self.block_size,
539
+ block_offsets_np,
540
+ out=slot_mapping_np)
541
+ slot_mapping_np[batch_size:] = _PAD_SLOT_ID
542
+
543
+ block_table_cpu = block_table_cpu[:padded_batch_size]
544
+
545
+ # Context lens
546
+ context_lens_np = self.decode_context_lens_np[
547
+ self.cur_swap_id][:padded_batch_size]
548
+ np.add(self.input_batch.num_computed_tokens_cpu[:padded_batch_size],
549
+ 1,
550
+ out=context_lens_np)
551
+ context_lens_np[batch_size:] = 0
552
+
553
+ # Get final tensors
554
+ input_tokens = input_tokens_cpu.reshape(-1, 1).to(self.device)
555
+ input_positions = input_positions_cpu.reshape(-1, 1).to(self.device)
556
+ slot_mapping = self.slot_mapping_cpu[
557
+ self.cur_swap_id][:padded_batch_size].reshape(-1,
558
+ 1).to(self.device)
559
+ block_table = block_table_cpu.to(self.device)
560
+ context_lens = self.decode_context_lens_cpu[
561
+ self.cur_swap_id][:padded_batch_size].to(self.device)
562
+
563
+ self.swap_step()
564
+
565
+ # Attn metadata
566
+ attn_metadata = PallasMetadata(
567
+ num_prefills=0,
568
+ num_prefill_tokens=0,
569
+ num_decode_tokens=padded_batch_size,
570
+ slot_mapping=slot_mapping,
571
+ multi_modal_placeholder_index_maps=None,
572
+ enable_kv_scales_calculation=True,
573
+ block_tables=block_table,
574
+ context_lens=context_lens,
575
+ effective_query_lens=None,
576
+ )
577
+
578
+ return DecodeData(input_tokens=input_tokens,
579
+ input_positions=input_positions,
580
+ attn_metadata=attn_metadata)
581
+
582
+ @torch.no_grad()
583
+ def execute_model(
584
+ self,
585
+ scheduler_output: "SchedulerOutput",
586
+ ) -> ModelRunnerOutput:
587
+ # Update cached state
588
+ self._update_states(scheduler_output)
589
+
590
+ # If necessary, swap decodes/prompts to have all decodes on the start
591
+ ensure_decodes_first(self.input_batch)
592
+
593
+ # Prepare prompts/decodes info
594
+ pd_info = self._get_prompts_and_decodes(scheduler_output)
595
+
596
+ # Init
597
+ num_prompts = len(pd_info.prompt_req_ids)
598
+ num_decodes = len(pd_info.decode_req_ids)
599
+ decode_data = None
600
+ sampled_token_ids = [0] * self.input_batch.num_reqs
601
+
602
+ # Run each prompt individually
603
+ is_first = True
604
+ for i in range(num_prompts):
605
+ req_id = pd_info.prompt_req_ids[i]
606
+ req_index = num_decodes + i
607
+ assert req_index == self.input_batch.req_id_to_index[
608
+ req_id] # TODO: Remove
609
+ req_state = self.requests[req_id]
610
+ num_scheduled_tokens = pd_info.prompt_scheduled_tokens[i]
611
+ prompt_len = num_scheduled_tokens
612
+ seq_len = req_state.num_computed_tokens + num_scheduled_tokens
613
+
614
+ # Prepare first prompt
615
+ if is_first:
616
+ prompt_data = self._prepare_prompt(req_index,
617
+ num_scheduled_tokens)
618
+ is_first = False
619
+
620
+ # Run forward pass
621
+ with set_forward_context(prompt_data.attn_metadata,
622
+ self.vllm_config):
623
+ assert self.model is not None
624
+ selected_token_ids = self.model(prompt_data.input_tokens,
625
+ prompt_data.input_positions,
626
+ prompt_data.attn_metadata,
627
+ self.kv_caches)
628
+
629
+ # In parallel to TPU execution, prepare the next iteration
630
+ if i < num_prompts - 1:
631
+ # There is next prompt => prepare it
632
+ prompt_data = self._prepare_prompt(
633
+ req_index + 1, pd_info.prompt_scheduled_tokens[i + 1])
634
+ elif i == num_prompts - 1 and num_decodes > 0:
635
+ # There is next decode => prepare it
636
+ decode_data = self._prepare_decode(pd_info.decode_req_ids)
637
+
638
+ # Update cached state (if prompt is fully done)
639
+ if seq_len >= len(req_state.prompt_token_ids):
640
+ # Transfer sampled tokens from TPU to CPU
641
+ selected_token_ids_cpu = selected_token_ids.cpu()
642
+
643
+ # Get output token
644
+ token_id = selected_token_ids_cpu[prompt_len - 1].item()
645
+ sampled_token_ids[req_index] = token_id
646
+
647
+ # Add output token to the request
648
+ self.input_batch.token_ids_cpu[req_index, seq_len] = token_id
649
+ self.input_batch.num_tokens[req_index] += 1
650
+ req_state.output_token_ids.append(token_id)
651
+
652
+ # Run decodes (a single batch)
653
+ if num_decodes > 0:
654
+
655
+ # Prepare decode (if was not yet prepared)
656
+ if decode_data is None:
657
+ decode_data = self._prepare_decode(pd_info.decode_req_ids)
658
+
659
+ # Run forward pass
660
+ with set_forward_context(decode_data.attn_metadata,
661
+ self.vllm_config):
662
+ assert self.model is not None
663
+ selected_token_ids = self.model(decode_data.input_tokens,
664
+ decode_data.input_positions,
665
+ decode_data.attn_metadata,
666
+ self.kv_caches)
667
+
668
+ # Transfer sampled tokens from TPU to CPU
669
+ decode_token_ids_cpu = selected_token_ids.cpu()
670
+ # Convert to list
671
+ decode_token_ids_list = decode_token_ids_cpu.tolist()
672
+
673
+ # Update cached state for each decode request
674
+ for i in range(num_decodes):
675
+ req_id = pd_info.decode_req_ids[i]
676
+ req_index = i
677
+ assert req_index == self.input_batch.req_id_to_index[
678
+ req_id] # TODO: Remove
679
+ req_state = self.requests[req_id]
680
+ seq_len = req_state.num_computed_tokens + 1
681
+
682
+ token_id = decode_token_ids_list[i]
683
+ sampled_token_ids[req_index] = token_id
684
+
685
+ self.input_batch.token_ids_cpu[req_index, seq_len] = token_id
686
+ self.input_batch.num_tokens[req_index] += 1
687
+ req_state.output_token_ids.append(token_id)
688
+
689
+ # Create output.
690
+ all_req_ids = pd_info.decode_req_ids + pd_info.prompt_req_ids
691
+ prompt_logprobs_dict: Dict[str, Optional[LogprobsTensors]] = {}
692
+ for req_id in all_req_ids:
693
+ prompt_logprobs_dict[req_id] = None
694
+
695
+ model_runner_output = ModelRunnerOutput(
696
+ req_ids=all_req_ids,
697
+ req_id_to_index=self.input_batch.req_id_to_index,
698
+ sampled_token_ids=sampled_token_ids,
699
+ logprobs=None,
700
+ prompt_logprobs_dict=prompt_logprobs_dict, # type: ignore[arg-type]
701
+ )
702
+
703
+ return model_runner_output
704
+
705
+ def load_model(self) -> None:
706
+ self.device = self.device_config.device
707
+
708
+ # NOTE(woosuk): While the executor assigns the TP ranks to the worker
709
+ # process, the ranks can be different from the ranks internally assigned
710
+ # by the xm runtime. Therefore, there is a mismatch in the rank
711
+ # assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
712
+ # This is not a problem in linear layers because all-reduce is
713
+ # rank-agnostic. However, it matters for all-gather as the ranks
714
+ # determine the order of concatenating the output tensors.
715
+ # As a workaround, we use the xm's rank assignment only when loading
716
+ # the embedding weights.
717
+ xm_tp_rank = xr.global_ordinal()
718
+ with patch(
719
+ "vllm.model_executor.layers.vocab_parallel_embedding."
720
+ "get_tensor_model_parallel_rank",
721
+ return_value=xm_tp_rank):
722
+ model = get_model(vllm_config=self.vllm_config)
723
+ model = model.eval()
724
+ xm.mark_step()
725
+ xm.wait_device_ops()
726
+ model = ModelWrapperV1(model)
727
+ self.model = torch.compile(model,
728
+ backend="openxla",
729
+ fullgraph=True,
730
+ dynamic=False)
731
+
732
+ def dummy_run(
733
+ self,
734
+ kv_caches,
735
+ num_tokens: int,
736
+ seq_len: Optional[int] = None,
737
+ exec_mode: Optional[ExecutionMode] = None,
738
+ ) -> None:
739
+ assert seq_len is not None
740
+ assert exec_mode is not None
741
+
742
+ exec_mode = ExecutionMode(exec_mode)
743
+ if exec_mode.is_prefill():
744
+ seq_len = (seq_len + 15) // 16 * 16
745
+ token_ids = torch.zeros((num_tokens, seq_len),
746
+ dtype=torch.int32,
747
+ device=self.device)
748
+ position_ids = torch.zeros((num_tokens, seq_len),
749
+ dtype=torch.int32,
750
+ device=self.device)
751
+ slot_mapping = torch.zeros((num_tokens, seq_len),
752
+ dtype=torch.int64,
753
+ device=self.device)
754
+ if exec_mode == ExecutionMode.PREFILL:
755
+ attn_metadata = PallasMetadata(
756
+ num_prefills=num_tokens,
757
+ num_prefill_tokens=num_tokens * seq_len,
758
+ num_decode_tokens=0,
759
+ slot_mapping=slot_mapping,
760
+ multi_modal_placeholder_index_maps=None,
761
+ enable_kv_scales_calculation=True,
762
+ block_tables=None,
763
+ context_lens=None,
764
+ effective_query_lens=None,
765
+ )
766
+
767
+ else:
768
+ context_lens = torch.ones((num_tokens, ),
769
+ dtype=torch.int32,
770
+ device=self.device)
771
+
772
+ block_tables = torch.zeros(
773
+ (num_tokens, self.max_num_blocks_per_req),
774
+ dtype=torch.int32,
775
+ device=self.device)
776
+
777
+ effective_query_lens = torch.ones_like(context_lens)
778
+
779
+ attn_metadata = PallasMetadata(
780
+ num_prefills=num_tokens,
781
+ num_prefill_tokens=num_tokens * seq_len,
782
+ num_decode_tokens=0,
783
+ slot_mapping=slot_mapping,
784
+ multi_modal_placeholder_index_maps=None,
785
+ enable_kv_scales_calculation=True,
786
+ block_tables=block_tables,
787
+ context_lens=context_lens,
788
+ effective_query_lens=effective_query_lens,
789
+ )
790
+ else:
791
+ assert seq_len == 1
792
+ token_ids = torch.zeros((num_tokens, seq_len),
793
+ dtype=torch.int32,
794
+ device=self.device)
795
+ position_ids = torch.zeros((num_tokens, seq_len),
796
+ dtype=torch.int32,
797
+ device=self.device)
798
+ slot_mapping = torch.zeros((num_tokens, seq_len),
799
+ dtype=torch.int64,
800
+ device=self.device)
801
+ block_tables = torch.zeros(
802
+ (num_tokens, self.max_num_blocks_per_req),
803
+ dtype=torch.int32,
804
+ device=self.device)
805
+ context_lens = torch.ones((num_tokens, ),
806
+ dtype=torch.int32,
807
+ device=self.device)
808
+ attn_metadata = PallasMetadata(
809
+ num_prefills=0,
810
+ num_prefill_tokens=0,
811
+ num_decode_tokens=num_tokens * seq_len,
812
+ slot_mapping=slot_mapping,
813
+ multi_modal_placeholder_index_maps=None,
814
+ enable_kv_scales_calculation=True,
815
+ block_tables=block_tables,
816
+ context_lens=context_lens,
817
+ )
818
+
819
+ # NOTE(woosuk): There are two stages of compilation: torch.compile and
820
+ # XLA compilation. Using `mark_dynamic` can reduce the torch.compile
821
+ # overhead by reusing the FX graph for different shapes.
822
+ # However, the XLA graph will still require static shapes and needs to
823
+ # be re-compiled for every different shapes. This overhead is inevitable
824
+ # in the first run, but can be skipped afterwards as we cache the XLA
825
+ # graphs in the disk (VLLM_XLA_CACHE_PATH).
826
+ if exec_mode.is_prefill():
827
+ # Prefll
828
+ torch._dynamo.mark_dynamic(token_ids, 1)
829
+ torch._dynamo.mark_dynamic(position_ids, 1)
830
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 1)
831
+ else:
832
+ # Decode
833
+ torch._dynamo.mark_dynamic(token_ids, 0)
834
+ torch._dynamo.mark_dynamic(position_ids, 0)
835
+ torch._dynamo.mark_dynamic(attn_metadata.slot_mapping, 0)
836
+ torch._dynamo.mark_dynamic(attn_metadata.context_lens, 0)
837
+ torch._dynamo.mark_dynamic(attn_metadata.block_tables, 0)
838
+
839
+ with set_forward_context(attn_metadata, self.vllm_config, 0):
840
+ assert self.model is not None
841
+ self.model(token_ids, position_ids, attn_metadata, kv_caches)
842
+
843
+ def capture_model(self) -> None:
844
+ """Compile the model."""
845
+
846
+ # Prefill
847
+ logger.info(
848
+ "Compiling the model with different input shapes for prefill:")
849
+ start = time.time()
850
+ for batch_size in [1]:
851
+ seq_len = 16
852
+ while seq_len <= self.model_config.max_model_len:
853
+ self.dummy_run(self.kv_caches,
854
+ batch_size,
855
+ seq_len,
856
+ exec_mode=ExecutionMode.PREFILL)
857
+ xm.wait_device_ops()
858
+ logger.info(" batch_size: %d, seq_len: %d", batch_size,
859
+ seq_len)
860
+ num_tokens = batch_size * seq_len
861
+ if num_tokens >= self.scheduler_config.max_num_batched_tokens:
862
+ break
863
+ seq_len = seq_len * 2
864
+
865
+ end = time.time()
866
+ logger.info(" -- Compilation for prefill done in %.2f [secs].",
867
+ end - start)
868
+
869
+ # Prefix prefill
870
+ if self.scheduler_config.enable_chunked_prefill:
871
+ logger.info("Compiling the model with different input shapes for "
872
+ "prefix prefill:")
873
+ start = time.time()
874
+ for batch_size in [1]:
875
+ seq_len = 16
876
+ while seq_len <= self.model_config.max_model_len:
877
+ self.dummy_run(self.kv_caches,
878
+ batch_size,
879
+ seq_len,
880
+ exec_mode=ExecutionMode.PREFIX_PREFILL)
881
+ xm.wait_device_ops()
882
+ logger.info(" batch_size: %d, seq_len: %d", batch_size,
883
+ seq_len)
884
+ num_tokens = batch_size * seq_len
885
+ if (num_tokens
886
+ >= self.scheduler_config.max_num_batched_tokens):
887
+ break
888
+ seq_len = seq_len * 2
889
+ end = time.time()
890
+ logger.info(
891
+ " -- Compilation for prefix prefill done in %.2f [secs].",
892
+ end - start)
893
+
894
+ # Decode
895
+ logger.info(
896
+ "Compiling the model with different input shapes for decode:")
897
+ start = time.time()
898
+ seq_len = 1
899
+ batch_size = 8 # Must be in sync with _get_padded_batch_size()
900
+ while True:
901
+ self.dummy_run(self.kv_caches,
902
+ batch_size,
903
+ seq_len,
904
+ exec_mode=ExecutionMode.DECODE)
905
+ xm.wait_device_ops()
906
+ logger.info(" batch_size: %d, seq_len: %d", batch_size, seq_len)
907
+
908
+ if batch_size >= self.scheduler_config.max_num_seqs:
909
+ break
910
+ batch_size = batch_size + 16 if batch_size >= 16 else batch_size * 2
911
+
912
+ end = time.time()
913
+ logger.info(" -- Compilation for decode done in %.2f [secs].",
914
+ end - start)
915
+
916
+ def initialize_kv_cache(self, kv_cache_config: KVCacheConfig) -> None:
917
+ """
918
+ Initialize KV cache based on `kv_cache_config`.
919
+ Args:
920
+ kv_cache_config: Configuration for the KV cache, including the KV
921
+ cache size of each layer
922
+ """
923
+ if len(kv_cache_config.groups) > 1:
924
+ raise NotImplementedError(
925
+ "Hybrid models with more than one KV cache type are not "
926
+ "supported yet.")
927
+
928
+ kv_caches: Dict[str, torch.Tensor] = {}
929
+
930
+ for layer_name, layer_spec in kv_cache_config.kv_cache_spec.items():
931
+ tensor_config = kv_cache_config.tensors[layer_name]
932
+ assert tensor_config.size % layer_spec.page_size_bytes == 0
933
+ num_blocks = tensor_config.size // layer_spec.page_size_bytes
934
+ if isinstance(layer_spec, FullAttentionSpec):
935
+ kv_cache_shape = PallasAttentionBackend.get_kv_cache_shape(
936
+ num_blocks, layer_spec.block_size, layer_spec.num_kv_heads,
937
+ layer_spec.head_size)
938
+ dtype = layer_spec.dtype
939
+
940
+ tpu_k_cache = torch.zeros(kv_cache_shape,
941
+ dtype=dtype,
942
+ device=self.device)
943
+ tpu_v_cache = torch.zeros_like(tpu_k_cache)
944
+
945
+ kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache)
946
+ else:
947
+ raise NotImplementedError
948
+
949
+ bind_kv_cache(
950
+ kv_caches,
951
+ self.vllm_config.compilation_config.static_forward_context,
952
+ self.kv_caches)
953
+
954
+
955
+class ModelWrapperV1(nn.Module):
956
+
957
+ def __init__(self, model: nn.Module):
958
+ super().__init__()
959
+ self.model = model
960
+
961
+ def forward(
962
+ self,
963
+ token_ids: torch.Tensor,
964
+ position_ids: torch.Tensor,
965
+ attn_metadata: AttentionMetadata,
966
+ kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
967
+ ) -> torch.Tensor:
968
+ """Executes the forward pass of the model and samples the next token.
969
+
970
+ Args:
971
+ token_ids: The input token IDs of shape [batch_size, seq_len].
972
+ position_ids: The input position IDs of shape [batch_size, seq_len].
973
+ attn_metadata: The Pallas attention metadata.
974
+ input_lens: The actual input lengths of shape [batch_size].
975
+ t: The sampling temperature of shape [batch_size].
976
+ p: The top-p probability of shape [batch_size].
977
+ num_samples: Number of samples to draw from each logits vector.
978
+ kv_caches: The key and value caches. They can be None during the
979
+ memory profiling at initialization.
980
+ """
981
+ # Skip this in memory profiling at initialization.
982
+ if attn_metadata is not None and kv_caches[0][0].numel() > 0:
983
+ # index_copy_(slot_mapping) only works when the inserted dimension
984
+ # is 0. However, the KV cache in the Pallas backend has the shape
985
+ # [num_kv_heads, num_blocks, block_size, head_size]. To make it
986
+ # work, we need to flatten the first three dimensions and modify
987
+ # the slot_mapping accordingly.
988
+ num_kv_heads, num_blocks, block_size, _ = kv_caches[0][0].shape
989
+ slot_mapping = attn_metadata.slot_mapping
990
+ slot_mapping = slot_mapping.flatten()
991
+ head_indicies = torch.arange(0,
992
+ num_kv_heads,
993
+ device=slot_mapping.device,
994
+ dtype=slot_mapping.dtype)
995
+ head_indicies *= block_size * num_blocks
996
+ slot_mapping = slot_mapping.repeat_interleave(num_kv_heads).view(
997
+ -1, num_kv_heads)
998
+ slot_mapping = slot_mapping + head_indicies.view(1, -1)
999
+ slot_mapping = slot_mapping.flatten()
1000
+ attn_metadata.slot_mapping = slot_mapping
1001
+
1002
+ assert self.model is not None
1003
+ hidden_states = self.model(
1004
+ token_ids,
1005
+ position_ids,
1006
+ kv_caches,
1007
+ attn_metadata,
1008
+ )
1009
+
1010
+ hidden_states = hidden_states.flatten(0, 1)
1011
+ logits = self.model.compute_logits(hidden_states, None)
1012
+
1013
+ # Greedy sampling.
1014
+ argmax_token_ids = torch.argmax(logits, dim=-1, keepdim=True)
1015
+ argmax_token_ids = argmax_token_ids.squeeze(dim=-1)
1016
+ return argmax_token_ids
1017
+
1018
+
1019
+def swap_positions(b: InputBatch, id_1, id_2):
1020
+ assert id_1 != id_2
1021
+ req_id_1 = b.req_ids[id_1]
1022
+ req_id_2 = b.req_ids[id_2]
1023
+ assert req_id_1 is not None
1024
+ assert req_id_2 is not None
1025
+ assert id_1 == b.req_id_to_index[req_id_1]
1026
+ assert id_2 == b.req_id_to_index[req_id_2]
1027
+
1028
+ b.req_ids[id_1], b.req_ids[id_2] = b.req_ids[id_2], b.req_ids[id_1]
1029
+ b.req_id_to_index[req_id_1], b.req_id_to_index[
1030
+ req_id_2] = b.req_id_to_index[req_id_2], b.req_id_to_index[req_id_1]
1031
+
1032
+ ids = [id_1, id_2]
1033
+ rev_ids = [id_2, id_1]
1034
+ b.num_tokens[ids] = b.num_tokens[rev_ids]
1035
+ b.token_ids_cpu[ids] = b.token_ids_cpu[rev_ids]
1036
+ b.num_prompt_tokens[ids] = b.num_prompt_tokens[rev_ids]
1037
+ b.num_computed_tokens_cpu[ids] = b.num_computed_tokens_cpu[rev_ids]
1038
+
1039
+ b.block_table.swap_row(id_1, id_2)
1040
+
1041
+ b.temperature_cpu[ids] = b.temperature_cpu[rev_ids]
1042
+ b.top_p_cpu[ids] = b.top_p_cpu[rev_ids]
1043
+ b.top_k_cpu[ids] = b.top_k_cpu[rev_ids]
1044
+ b.frequency_penalties_cpu[ids] = b.frequency_penalties_cpu[rev_ids]
1045
+ b.presence_penalties_cpu[ids] = b.presence_penalties_cpu[rev_ids]
1046
+ b.repetition_penalties_cpu[ids] = b.repetition_penalties_cpu[rev_ids]
1047
+
1048
+ b.min_tokens[id_1], b.min_tokens[id_2] = b.min_tokens[id_2], b.min_tokens[
1049
+ id_1]
1050
+ b.stop_token_ids[id_1], b.stop_token_ids[id_2] = b.stop_token_ids[
1051
+ id_2], b.stop_token_ids[id_1]
1052
+
1053
+ gen_1 = b.generators.pop(id_1, None)
1054
+ gen_2 = b.generators.pop(id_2, None)
1055
+ if gen_1 is not None:
1056
+ b.generators[id_2] = gen_1
1057
+ if gen_2 is not None:
1058
+ b.generators[id_1] = gen_2
1059
+
1060
+
1061
+def ensure_decodes_first(b: InputBatch):
1062
+ num_reqs = b.num_reqs
1063
+ while True:
1064
+ # Find the first prompt index
1065
+ first_prompt_index = None
1066
+ for i in range(num_reqs):
1067
+ if b.num_computed_tokens_cpu[i] < b.num_prompt_tokens[i]:
1068
+ first_prompt_index = i
1069
+ break
1070
+ if first_prompt_index is None:
1071
+ break
1072
+
1073
+ # Find the last decode index
1074
+ last_decode_index = None
1075
+ for i in reversed(range(num_reqs)):
1076
+ if b.num_computed_tokens_cpu[i] >= b.num_prompt_tokens[i]:
1077
+ last_decode_index = i
1078
+ break
1079
+ if last_decode_index is None:
1080
+ break
1081
+
1082
+ # Sanity
1083
+ assert first_prompt_index != last_decode_index
1084
+
1085
+ # Check if done
1086
+ if first_prompt_index > last_decode_index:
1087
+ break
1088
+
1089
+ # Swap
1090
+ swap_positions(b, first_prompt_index, last_decode_index)
1091
+
1092
+
1093
+def _get_padded_prompt_len(x: int) -> int:
1094
+ # NOTE(woosuk): The pallas FlashAttention kernel requires the sequence
1095
+ # length to be a multiple of 16. We pad the prompt length to the nearest
1096
+ # multiple of 16. This is also good for performance.
1097
+ if x <= 16:
1098
+ return 16
1099
+ return 1 << (x - 1).bit_length()
1100
+
1101
+
1102
+def _get_padded_batch_size(batch_size: int) -> int:
1103
+ # The GMM Pallas kernel requires num_tokens * topk to be a multiple of 16.
1104
+ # To meet this requirement in the simplest way, we set the minimal batch
1105
+ # size to 8.
1106
+ if batch_size <= 8:
1107
+ return 8
1108
+ else:
1109
+ return ((batch_size + 15) // 16) * 16
vllm/v1/worker/tpu_worker.py
new
+203
@@ -0,0 +1,203 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""A TPU worker class."""
3
+import os
4
+from typing import Dict, List, Optional
5
+
6
+import torch
7
+import torch.distributed
8
+import torch.nn as nn
9
+import torch_xla.core.xla_model as xm
10
+import torch_xla.runtime as xr
11
+
12
+import vllm.envs as envs
13
+from vllm.config import ParallelConfig, VllmConfig
14
+from vllm.distributed import (ensure_model_parallel_initialized,
15
+ init_distributed_environment)
16
+from vllm.logger import init_logger
17
+from vllm.model_executor import set_random_seed
18
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
+from vllm.v1.core.scheduler import SchedulerOutput
20
+from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
21
+ KVCacheSpec)
22
+from vllm.v1.outputs import ModelRunnerOutput
23
+from vllm.v1.utils import bind_kv_cache
24
+from vllm.v1.worker.tpu_model_runner import ExecutionMode, TPUModelRunner
25
+
26
+logger = init_logger(__name__)
27
+
28
+
29
+class TPUWorker:
30
+
31
+ def __init__(
32
+ self,
33
+ vllm_config: VllmConfig,
34
+ local_rank: int,
35
+ rank: int,
36
+ distributed_init_method: str,
37
+ is_driver_worker: bool = False,
38
+ ):
39
+ self.vllm_config = vllm_config
40
+ self.model_config = vllm_config.model_config
41
+ self.cache_config = vllm_config.cache_config
42
+ self.lora_config = vllm_config.lora_config
43
+ self.load_config = vllm_config.load_config
44
+ self.parallel_config = vllm_config.parallel_config
45
+ self.scheduler_config = vllm_config.scheduler_config
46
+ self.device_config = vllm_config.device_config
47
+ self.speculative_config = vllm_config.speculative_config
48
+ self.prompt_adapter_config = vllm_config.prompt_adapter_config
49
+ self.observability_config = vllm_config.observability_config
50
+
51
+ self.parallel_config.rank = rank
52
+ self.local_rank = local_rank
53
+ self.rank = rank
54
+ self.distributed_init_method = distributed_init_method
55
+
56
+ if self.cache_config.cache_dtype == "auto":
57
+ self.cache_dtype = self.model_config.dtype
58
+ else:
59
+ self.cache_dtype = STR_DTYPE_TO_TORCH_DTYPE[
60
+ self.cache_config.cache_dtype]
61
+
62
+ if self.model_config.trust_remote_code:
63
+ # note: lazy import to avoid importing torch before initializing
64
+ from vllm.utils import init_cached_hf_modules
65
+ init_cached_hf_modules()
66
+
67
+ def init_device(self):
68
+ os.environ["PJRT_DEVICE"] = "TPU"
69
+ torch.set_grad_enabled(False)
70
+ torch.set_default_dtype(self.model_config.dtype)
71
+
72
+ # Initialize the distributed environment.
73
+ init_tpu_worker_distributed_environment(self.parallel_config,
74
+ self.rank,
75
+ self.distributed_init_method,
76
+ self.local_rank)
77
+
78
+ # Device initialization should happen after initializing
79
+ # the distributed runtime.
80
+ self.device = xm.xla_device()
81
+ self.device_config.device = self.device
82
+
83
+ # Set random seed.
84
+ set_random_seed(self.model_config.seed)
85
+ xm.set_rng_state(self.model_config.seed, self.device)
86
+
87
+ # Increase the cache size limit, which is the maximum number of
88
+ # dynamo graphs that can be compiled.
89
+ # NOTE(woosuk): Usually, we compile 10-15 graphs for prefill and
90
+ # 30-40 graphs for decode. 128 is an arbitrary safe number.
91
+ torch._dynamo.config.cache_size_limit = 128
92
+ # Use persistent cache to avoid XLA recompilation.
93
+ # NOTE(woosuk): Set per-rank cache path since different ranks
94
+ # can have slightly different XLA graphs.
95
+ world_size = self.parallel_config.world_size
96
+ rank = xr.global_ordinal()
97
+ per_rank_path = os.path.join(envs.VLLM_XLA_CACHE_PATH,
98
+ f"tp{world_size}_rank{rank}")
99
+ xr.initialize_cache(per_rank_path, readonly=False)
100
+
101
+ # Init ModelRunner here, so that we have access to self.device.
102
+ self.model_runner = TPUModelRunner(self.vllm_config, self.device)
103
+
104
+ def determine_available_memory(self) -> int:
105
+ kv_caches: Dict[str, torch.Tensor] = {}
106
+ kv_cache_spec = self.model_runner.get_kv_cache_spec()
107
+ for layer_name, layer_spec in kv_cache_spec.items():
108
+ if isinstance(layer_spec, FullAttentionSpec):
109
+ dtype = layer_spec.dtype
110
+
111
+ # Use an empty tensor instead of `None`` to force Dynamo to pass
112
+ # it by reference, rather by specializing on the value ``None``.
113
+ tpu_k_cache = torch.tensor([], dtype=dtype, device=self.device)
114
+ tpu_v_cache = torch.tensor([], dtype=dtype, device=self.device)
115
+
116
+ kv_caches[layer_name] = (tpu_k_cache, tpu_v_cache)
117
+ else:
118
+ raise NotImplementedError
119
+
120
+ runner_kv_caches: List[torch.Tensor] = []
121
+ bind_kv_cache(
122
+ kv_caches,
123
+ self.vllm_config.compilation_config.static_forward_context,
124
+ runner_kv_caches)
125
+
126
+ self.model_runner.dummy_run(
127
+ runner_kv_caches,
128
+ num_tokens=1,
129
+ seq_len=self.scheduler_config.max_num_batched_tokens,
130
+ exec_mode=ExecutionMode.PREFILL,
131
+ )
132
+
133
+ # Synchronize before measuring the memory usage.
134
+ xm.wait_device_ops()
135
+
136
+ # Get the maximum amount of memory used by the model weights and
137
+ # intermediate activations.
138
+ m = xm.get_memory_info(self.device)
139
+ total_memory_size = m["bytes_limit"]
140
+ profiled = m["peak_bytes_used"] # Weights + intermediate activations.
141
+
142
+ # Calculate the TPU KV cache size based on profiling.
143
+ usable_memory_size = int(total_memory_size *
144
+ self.cache_config.gpu_memory_utilization)
145
+ tpu_kv_cache_bytes = max(usable_memory_size - profiled, 0)
146
+
147
+ return int(tpu_kv_cache_bytes)
148
+
149
+ def execute_model(
150
+ self,
151
+ scheduler_output: "SchedulerOutput",
152
+ ) -> Optional[ModelRunnerOutput]:
153
+ output = self.model_runner.execute_model(scheduler_output)
154
+ return output if self.rank == 0 else None
155
+
156
+ def load_model(self) -> None:
157
+ self.model_runner.load_model()
158
+
159
+ def compile_or_warm_up_model(self) -> None:
160
+ if not self.model_config.enforce_eager:
161
+ self.model_runner.capture_model()
162
+
163
+ # Reset the seed to ensure that the random state is not affected by
164
+ # the model initialization and profiling.
165
+ set_random_seed(self.model_config.seed)
166
+
167
+ def get_model(self) -> nn.Module:
168
+ return self.model_runner.get_model()
169
+
170
+ def get_kv_cache_spec(self) -> KVCacheSpec:
171
+ return self.model_runner.get_kv_cache_spec()
172
+
173
+ def initialize_cache(self, kv_cache_configs: List[KVCacheConfig]) -> None:
174
+ """Allocate GPU KV cache with the specified kv_cache_config."""
175
+ kv_cache_config = kv_cache_configs[self.rank]
176
+ self.model_runner.initialize_kv_cache(kv_cache_config)
177
+
178
+ def check_health(self) -> None:
179
+ # worker will always be healthy as long as it's running.
180
+ return
181
+
182
+
183
+def init_tpu_worker_distributed_environment(
184
+ parallel_config: ParallelConfig,
185
+ rank: int,
186
+ distributed_init_method: Optional[str] = None,
187
+ local_rank: int = -1,
188
+) -> None:
189
+ """Initialize the distributed environment."""
190
+
191
+ # NOTE(woosuk): This is just to initialize the TP group and broadcast
192
+ # the input objects on CPU. The all-reduce and all-gather ops on TPU
193
+ # are invoked by `xm.all_reduce` and `xm.all_gather` which use their
194
+ # own context.
195
+ init_distributed_environment(
196
+ world_size=parallel_config.world_size,
197
+ rank=rank,
198
+ local_rank=local_rank,
199
+ distributed_init_method=distributed_init_method,
200
+ backend="gloo",
201
+ )
202
+ ensure_model_parallel_initialized(parallel_config.tensor_parallel_size,
203
+ parallel_config.pipeline_parallel_size)