@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 63f1fde277
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Hardware][CPU] Support chunked-prefill and prefix-caching on CPU (#10355)

Signed-off-by: jiang1.li <jiang1.li@intel.com>

Li, Jiang committed Nov 20, 2024 at 18:57 UTC 63f1fde277d063fbd36ccf43cb709fafca754ed5
8 files changed +559 -369
.buildkite/run-cpu-test.sh
+8 -1
@@ -25,6 +25,7 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
25
26 function cpu_tests() {
27 set -e
28 + export NUMA_NODE=$2
29
30 # offline inference
31 docker exec cpu-test-avx2-"$NUMA_NODE" bash -c "
@@ -57,6 +58,12 @@ function cpu_tests() {
58 pytest -s -v \
59 tests/quantization/test_ipex_quant.py"
60
61 + # Run chunked-prefill and prefix-cache test
62 + docker exec cpu-test-"$NUMA_NODE" bash -c "
63 + set -e
64 + pytest -s -v -k cpu_model \
65 + tests/basic_correctness/test_chunked_prefill.py"
66 +
67 # online inference
68 docker exec cpu-test-"$NUMA_NODE" bash -c "
69 set -e
@@ -75,4 +82,4 @@ function cpu_tests() {
82
83 # All of CPU tests are expected to be finished less than 25 mins.
84 export -f cpu_tests
78 -timeout 25m bash -c "cpu_tests $CORE_RANGE"
85 +timeout 30m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
docs/source/getting_started/cpu-installation.rst
+5 -5
@@ -5,11 +5,11 @@ Installation with CPU
5
6 vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
7
8 -- Tensor Parallel (``-tp = N``)
9 -- Quantization (``INT8 W8A8, AWQ``)
10 -
11 -.. note::
12 - More advanced features on `chunked-prefill`, `prefix-caching` and `FP8 KV cache` are under development and will be available soon.
8 +- Tensor Parallel
9 +- Model Quantization (``INT8 W8A8, AWQ``)
10 +- Chunked-prefill
11 +- Prefix-caching
12 +- FP8-E5M2 KV-Caching (TODO)
13
14 Table of contents:
15
docs/source/serving/compatibility_matrix.rst
+2 -2
@@ -344,7 +344,7 @@ Feature x Hardware
344 - ✅
345 - ✅
346 - ✅
347 - - ✗
347 + - ✅
348 - ✅
349 * - :ref:`APC <apc>`
350 - `✗ <https://github.com/vllm-project/vllm/issues/3687>`__
@@ -352,7 +352,7 @@ Feature x Hardware
352 - ✅
353 - ✅
354 - ✅
355 - - ✗
355 + - ✅
356 - ✅
357 * - :ref:`LoRA <lora>`
358 - ✅
tests/basic_correctness/test_chunked_prefill.py
+62 -1
@@ -12,6 +12,7 @@ from contextlib import nullcontext
12 import pytest
13
14 from tests.kernels.utils import override_backend_env_variable
15 +from vllm.platforms import current_platform
16
17 from ..models.utils import check_logprobs_close, check_outputs_equal
18 from ..utils import multi_gpu_test
@@ -206,12 +207,14 @@ def test_models_with_fp8_kv_cache(
207 # NOTE: Increasing this in this suite will fail CI because we currently cannot
208 # reset distributed env properly. Use a value > 1 just when you test.
209 @pytest.mark.parametrize("tensor_parallel_size", [1])
210 +@pytest.mark.parametrize("dtype", ["half"])
211 def test_with_prefix_caching(
212 vllm_runner,
213 max_tokens: int,
214 enforce_eager: bool,
215 chunk_size: int,
216 tensor_parallel_size: int,
217 + dtype: str,
218 ) -> None:
219 """
220 Checks exact match decode with and without prefix caching
@@ -233,7 +236,7 @@ def test_with_prefix_caching(
236 for enable in (True, False):
237 with vllm_runner(
238 model,
236 - dtype="half",
239 + dtype=dtype,
240 max_num_batched_tokens=max_num_batched_tokens,
241 enable_chunked_prefill=True,
242 enable_prefix_caching=enable,
@@ -260,3 +263,61 @@ def test_with_prefix_caching(
263 name_0="w/o prefix caching",
264 name_1="with prefix caching",
265 )
266 +
267 +
268 +@pytest.mark.parametrize("model", ["facebook/opt-125m"])
269 +@pytest.mark.parametrize("dtype", ["bfloat16"])
270 +@pytest.mark.parametrize("max_tokens", [32])
271 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
272 +@pytest.mark.parametrize("enforce_eager", [False])
273 +@pytest.mark.parametrize("attention_backend", ["TORCH_SDPA"])
274 +@pytest.mark.cpu_model
275 +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
276 +def test_models_cpu(
277 + hf_runner,
278 + vllm_runner,
279 + example_prompts,
280 + model: str,
281 + dtype: str,
282 + max_tokens: int,
283 + chunked_prefill_token_size: int,
284 + enforce_eager: bool,
285 + attention_backend: str,
286 + monkeypatch,
287 +) -> None:
288 + test_models(
289 + hf_runner,
290 + vllm_runner,
291 + example_prompts,
292 + model,
293 + dtype,
294 + max_tokens,
295 + chunked_prefill_token_size,
296 + enforce_eager,
297 + 1,
298 + attention_backend,
299 + monkeypatch,
300 + )
301 +
302 +
303 +@pytest.mark.parametrize("max_tokens", [16])
304 +@pytest.mark.parametrize("enforce_eager", [False])
305 +@pytest.mark.parametrize("chunk_size", [30, 32])
306 +@pytest.mark.parametrize("dtype", ["bfloat16"])
307 +@pytest.mark.cpu_model
308 +@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
309 +def test_with_prefix_caching_cpu(
310 + vllm_runner,
311 + max_tokens: int,
312 + enforce_eager: bool,
313 + chunk_size: int,
314 + dtype: str,
315 +) -> None:
316 + test_with_prefix_caching(
317 + vllm_runner,
318 + max_tokens,
319 + enforce_eager,
320 + chunk_size,
321 + 1,
322 + dtype,
323 + )
vllm/attention/backends/torch_sdpa.py
+152 -37
@@ -7,18 +7,14 @@ import torch
7 from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata, AttentionType)
10 + AttentionMetadata,
11 + AttentionMetadataBuilder,
12 + AttentionType)
13 from vllm.attention.backends.utils import CommonAttentionState
14 +from vllm.attention.ops.ipex_attn import PagedAttention
15 from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13 -from vllm.platforms import current_platform
14 -
15 -if current_platform.is_cpu():
16 - try:
17 - from vllm.attention.ops.ipex_attn import PagedAttention
18 - except ImportError:
19 - from vllm.attention.ops.paged_attn import PagedAttention
20 -else:
21 - from vllm.attention.ops.paged_attn import PagedAttention
16 +from vllm.utils import make_tensor_with_pad
17 +from vllm.worker.cpu_model_runner import ModelInputForCPUBuilder
18
19
20 class TorchSDPABackend(AttentionBackend):
@@ -39,6 +35,10 @@ class TorchSDPABackend(AttentionBackend):
35 def get_state_cls() -> Type["CommonAttentionState"]:
36 return CommonAttentionState
37
38 + @staticmethod
39 + def get_builder_cls() -> Type["TorchSDPAMetadataBuilder"]:
40 + return TorchSDPAMetadataBuilder
41 +
42 @staticmethod
43 def get_kv_cache_shape(
44 num_blocks: int,
@@ -71,9 +71,15 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
71 """
72 # Currently, input sequences can only contain all prompts
73 # or all decoding. True if all sequences are prompts.
74 - is_prompt: bool
75 - slot_mapping: torch.Tensor
76 - seq_lens: Optional[List[int]]
74 + chunked_prefill: bool
75 + seq_lens: Optional[List[int]] = None # For non-chunked prefill
76 +
77 + # For chunked prefill only
78 + max_query_len: Optional[int] = None
79 + max_kv_len: Optional[int] = None
80 + query_start_loc: Optional[torch.Tensor] = None
81 + kv_start_loc: Optional[torch.Tensor] = None
82 + prefill_block_tables: Optional[torch.Tensor] = None
83
84 # Begin encoder attn & enc/dec cross-attn fields...
85 # Encoder sequence lengths representation
@@ -123,20 +129,14 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
129
130 @property
131 def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
126 - # Currently chunked prefill is not supported
127 - if self.num_decode_tokens == 0:
128 - assert self.num_prefills > 0
129 - return self
130 -
131 - return None
132 + if self.num_prefill_tokens == 0:
133 + return None
134 + return self
135
136 @property
137 def decode_metadata(self) -> Optional["TorchSDPAMetadata"]:
135 - # Currently chunked prefill is not supported
136 - if self.num_prefills > 0:
137 - assert self.num_decode_tokens == 0
138 + if self.num_decode_tokens == 0:
139 return None
139 -
140 return self
141
142 def get_seq_lens(
@@ -274,6 +274,105 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
274 raise AttributeError(f"Invalid attention type {str(attn_type)}")
275
276
277 +class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
278 +
279 + def __init__(self, input_builder: ModelInputForCPUBuilder) -> None:
280 + self.chunked_prefill = input_builder.chunked_prefill
281 + self.input_data = input_builder.input_data
282 +
283 + def build(self, seq_lens: List[int], query_lens: List[int],
284 + cuda_graph_pad_size: int, batch_size: int) -> TorchSDPAMetadata:
285 + input_data = self.input_data
286 + prefill_seq_lens = seq_lens[0:input_data.num_prefills]
287 + prefill_query_lens = query_lens[0:input_data.num_prefills]
288 + slot_mapping = torch.tensor(input_data.slot_mapping,
289 + dtype=torch.long,
290 + device="cpu")
291 +
292 + # For chunked-prefill
293 + if self.chunked_prefill and input_data.num_prefill_tokens != 0:
294 + prefill_block_tables = make_tensor_with_pad(
295 + self.input_data.prefill_block_tables,
296 + pad=0,
297 + dtype=torch.int32,
298 + device="cpu",
299 + )
300 + query_lens_tensor = torch.tensor(prefill_query_lens,
301 + dtype=torch.int32,
302 + device="cpu")
303 + kv_lens_tensor = torch.tensor(prefill_seq_lens,
304 + dtype=torch.int32,
305 + device="cpu")
306 + query_start_loc = torch.zeros(input_data.num_prefills + 1,
307 + dtype=torch.int32,
308 + device="cpu")
309 + kv_start_loc = torch.zeros(input_data.num_prefills + 1,
310 + dtype=torch.int32,
311 + device="cpu")
312 + torch.cumsum(query_lens_tensor,
313 + dim=0,
314 + dtype=torch.int32,
315 + out=query_start_loc[1:])
316 + torch.cumsum(kv_lens_tensor,
317 + dim=0,
318 + dtype=torch.int32,
319 + out=kv_start_loc[1:])
320 + max_query_len = max(prefill_query_lens)
321 + max_kv_len = max(prefill_seq_lens)
322 + else:
323 + prefill_block_tables = None
324 + query_start_loc = None
325 + kv_start_loc = None
326 + max_query_len = None
327 + max_kv_len = None
328 +
329 + # For paged attention
330 + if input_data.num_decode_tokens != 0:
331 + seq_lens_tensor = torch.tensor(
332 + input_data.seq_lens[input_data.num_prefills:],
333 + dtype=torch.int32,
334 + device="cpu",
335 + )
336 + block_tables = make_tensor_with_pad(
337 + self.input_data.decode_block_tables,
338 + pad=0,
339 + dtype=torch.int32,
340 + device="cpu",
341 + )
342 + else:
343 + block_tables = torch.tensor([])
344 + seq_lens_tensor = torch.tensor([])
345 +
346 + # For multi-modal models
347 + placeholder_index_maps = None
348 + if len(input_data.multi_modal_inputs_list) != 0:
349 + placeholder_index_maps = {
350 + modality: placeholder_map.index_map()
351 + for modality, placeholder_map in
352 + input_data.multi_modal_placeholder_maps.items()
353 + }
354 +
355 + attn_metadata = TorchSDPAMetadata(
356 + chunked_prefill=self.chunked_prefill,
357 + seq_lens=prefill_seq_lens,
358 + seq_lens_tensor=seq_lens_tensor,
359 + max_query_len=max_query_len,
360 + max_kv_len=max_kv_len,
361 + query_start_loc=query_start_loc,
362 + kv_start_loc=kv_start_loc,
363 + max_decode_seq_len=input_data.max_decode_seq_len,
364 + num_prefills=input_data.num_prefills,
365 + num_prefill_tokens=input_data.num_prefill_tokens,
366 + num_decode_tokens=input_data.num_decode_tokens,
367 + block_tables=block_tables,
368 + prefill_block_tables=prefill_block_tables,
369 + slot_mapping=slot_mapping,
370 + multi_modal_placeholder_index_maps=placeholder_index_maps,
371 + )
372 +
373 + return attn_metadata
374 +
375 +
376 class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
377
378 def __init__(
@@ -409,19 +508,35 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
508 assert key.shape[0] == num_prefill_tokens + num_decode_tokens
509 assert value.shape[0] == num_prefill_tokens + num_decode_tokens
510
511 + output = torch.empty_like(query)
512 if prefill_meta := attn_metadata.prefill_metadata:
513 assert attn_metadata.seq_lens is not None
414 - if (kv_cache.numel() == 0
415 - or prefill_meta.block_tables.numel() == 0):
416 - output = self._run_sdpa_forward(query,
417 - key,
418 - value,
419 - prefill_meta,
420 - attn_type=attn_type)
514 + if not prefill_meta.prefill_metadata.chunked_prefill: # type: ignore
515 + self._run_sdpa_forward(output,
516 + query,
517 + key,
518 + value,
519 + prefill_meta,
520 + attn_type=attn_type)
521 else:
522 # prefix-enabled attention
423 - raise RuntimeError(
424 - "Torch SDPA backend doesn't support prefix decoding.")
523 + assert not self.need_mask
524 + import intel_extension_for_pytorch.llm.modules as ipex_modules
525 + output = torch.empty_like(query)
526 + ipex_modules.PagedAttention.flash_attn_varlen_func(
527 + output[:prefill_meta.num_prefill_tokens, :, :],
528 + query[:prefill_meta.num_prefill_tokens, :, :],
529 + key_cache,
530 + value_cache,
531 + prefill_meta.query_start_loc,
532 + prefill_meta.kv_start_loc,
533 + prefill_meta.max_query_len,
534 + prefill_meta.max_kv_len,
535 + self.scale,
536 + True,
537 + prefill_meta.prefill_block_tables,
538 + self.alibi_slopes,
539 + )
540
541 if decode_meta := attn_metadata.decode_metadata:
542 assert attn_type != AttentionType.ENCODER_ONLY, (
@@ -433,8 +548,9 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
548 block_tables_arg,
549 ) = decode_meta.get_seq_len_block_table_args(attn_type)
550
436 - output = PagedAttention.forward_decode(
437 - query,
551 + PagedAttention.forward_decode(
552 + output[attn_metadata.num_prefill_tokens:, :, :],
553 + query[attn_metadata.num_prefill_tokens:, :, :],
554 key_cache,
555 value_cache,
556 block_tables_arg,
@@ -453,12 +569,13 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
569
570 def _run_sdpa_forward(
571 self,
572 + output: torch.Tensor,
573 query: torch.Tensor,
574 key: torch.Tensor,
575 value: torch.Tensor,
576 attn_metadata: TorchSDPAMetadata,
577 attn_type: AttentionType = AttentionType.DECODER,
461 - ):
578 + ) -> None:
579 if self.num_kv_heads != self.num_heads:
580 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
581 value = value.repeat_interleave(self.num_queries_per_kv, dim=1)
@@ -479,7 +596,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
596 attn_masks = [None] * len(seq_lens)
597 attn_metadata.set_attn_bias(attn_masks, attn_type)
598
482 - output = torch.empty_like(query)
599 query = query.movedim(0, query.dim() - 2)
600 key = key.movedim(0, key.dim() - 2)
601 value = value.movedim(0, value.dim() - 2)
@@ -502,7 +618,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
618 scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
619 output[start_q:end_q, :, :] = sub_out
620 start_q, start_kv = end_q, end_kv
505 - return output
621
622
623 def _make_alibi_bias(
vllm/attention/ops/ipex_attn.py
+109 -41
@@ -1,12 +1,17 @@
1 from typing import Dict, List, Optional, Tuple
2
3 -import intel_extension_for_pytorch.llm.modules as ipex_modules
3 +try:
4 + import intel_extension_for_pytorch.llm.modules as ipex_modules
5 + _use_ipex = True
6 +except ImportError:
7 + _use_ipex = False
8 +
9 import torch
10
11 from vllm import _custom_ops as ops
12
13
9 -class PagedAttention:
14 +class _PagedAttention:
15
16 @staticmethod
17 def get_supported_head_sizes() -> List[int]:
@@ -22,6 +27,105 @@ class PagedAttention:
27 ) -> Tuple[int, ...]:
28 return (2, num_blocks, block_size * num_kv_heads * head_size)
29
30 + @staticmethod
31 + def split_kv_cache(
32 + kv_cache: torch.Tensor,
33 + num_kv_heads: int,
34 + head_size: int,
35 + *args,
36 + ) -> Tuple[torch.Tensor, torch.Tensor]:
37 + x = 16 // kv_cache.element_size()
38 + num_blocks = kv_cache.shape[1]
39 +
40 + key_cache = kv_cache[0]
41 + key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x,
42 + -1, x)
43 + value_cache = kv_cache[1]
44 + value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1)
45 + return key_cache, value_cache
46 +
47 + @staticmethod
48 + def write_to_paged_cache(
49 + key: torch.Tensor,
50 + value: torch.Tensor,
51 + key_cache: torch.Tensor,
52 + value_cache: torch.Tensor,
53 + slot_mapping: torch.Tensor,
54 + kv_cache_dtype: str,
55 + k_scale: float,
56 + v_scale: float,
57 + *args,
58 + ) -> None:
59 + ops.reshape_and_cache(
60 + key,
61 + value,
62 + key_cache,
63 + value_cache,
64 + slot_mapping.flatten(),
65 + kv_cache_dtype,
66 + k_scale,
67 + v_scale,
68 + )
69 +
70 + @staticmethod
71 + def forward_decode(
72 + output: torch.Tensor,
73 + query: torch.Tensor,
74 + key_cache: torch.Tensor,
75 + value_cache: torch.Tensor,
76 + block_tables: torch.Tensor,
77 + context_lens: torch.Tensor,
78 + max_context_len: int,
79 + kv_cache_dtype: str,
80 + num_kv_heads: int,
81 + scale: float,
82 + alibi_slopes: Optional[torch.Tensor],
83 + k_scale: float,
84 + v_scale: float,
85 + *args,
86 + ) -> None:
87 + tp_rank: int = 0
88 + blocksparse_local_blocks: int = 0
89 + blocksparse_vert_stride: int = 0
90 + blocksparse_block_size: int = 64
91 + blocksparse_head_sliding_step: int = 0
92 + block_size = value_cache.shape[3]
93 +
94 + ops.paged_attention_v1(
95 + output,
96 + query,
97 + key_cache,
98 + value_cache,
99 + num_kv_heads,
100 + scale,
101 + block_tables,
102 + context_lens,
103 + block_size,
104 + max_context_len,
105 + alibi_slopes,
106 + kv_cache_dtype,
107 + k_scale,
108 + v_scale,
109 + tp_rank,
110 + blocksparse_local_blocks,
111 + blocksparse_vert_stride,
112 + blocksparse_block_size,
113 + blocksparse_head_sliding_step,
114 + )
115 +
116 + @staticmethod
117 + def copy_blocks(
118 + kv_caches: List[torch.Tensor],
119 + src_to_dists: Dict[int, List[int]],
120 + *args,
121 + ) -> None:
122 + key_caches = [kv_cache[0] for kv_cache in kv_caches]
123 + value_caches = [kv_cache[1] for kv_cache in kv_caches]
124 + ops.copy_blocks(key_caches, value_caches, src_to_dists)
125 +
126 +
127 +class _IPEXPagedAttention(_PagedAttention):
128 +
129 @staticmethod
130 def split_kv_cache(
131 kv_cache: torch.Tensor,
@@ -55,6 +159,7 @@ class PagedAttention:
159
160 @staticmethod
161 def forward_decode(
162 + output: torch.Tensor,
163 query: torch.Tensor,
164 key_cache: torch.Tensor,
165 value_cache: torch.Tensor,
@@ -68,8 +173,7 @@ class PagedAttention:
173 k_scale: float,
174 v_scale: float,
175 *args,
71 - ) -> torch.Tensor:
72 - output = torch.empty_like(query)
176 + ) -> None:
177 block_size = value_cache.shape[2]
178 head_mapping = torch.arange(
179 0,
@@ -83,41 +187,5 @@ class PagedAttention:
187 scale, block_tables, context_lens, block_size, max_context_len,
188 alibi_slopes)
189
86 - return output
87 -
88 - @staticmethod
89 - def forward_prefix(
90 - query: torch.Tensor,
91 - key: torch.Tensor,
92 - value: torch.Tensor,
93 - kv_cache_dtype: str,
94 - key_cache: torch.Tensor,
95 - value_cache: torch.Tensor,
96 - block_tables: torch.Tensor,
97 - subquery_start_loc: torch.Tensor,
98 - prompt_lens_tensor: torch.Tensor,
99 - context_lens: torch.Tensor,
100 - max_subquery_len: int,
101 - alibi_slopes: Optional[torch.Tensor],
102 - *args,
103 - ) -> torch.Tensor:
104 - raise NotImplementedError
105 -
106 - @staticmethod
107 - def swap_blocks(
108 - src_kv_cache: torch.Tensor,
109 - dst_kv_cache: torch.Tensor,
110 - src_to_dst: Dict[int, int],
111 - *args,
112 - ) -> None:
113 - raise NotImplementedError
190
115 - @staticmethod
116 - def copy_blocks(
117 - kv_caches: List[torch.Tensor],
118 - src_to_dists: Dict[int, List[int]],
119 - *args,
120 - ) -> None:
121 - key_caches = [kv_cache[0] for kv_cache in kv_caches]
122 - value_caches = [kv_cache[1] for kv_cache in kv_caches]
123 - ops.copy_blocks(key_caches, value_caches, src_to_dists)
191 +PagedAttention = _IPEXPagedAttention if _use_ipex else _PagedAttention
vllm/platforms/cpu.py
+6 -9
@@ -53,11 +53,6 @@ class CpuPlatform(Platform):
53
54 cache_config = vllm_config.cache_config
55
56 - if cache_config.enable_prefix_caching:
57 - logger.warning(
58 - "Prefix caching is not supported on CPU, disable it.")
59 - cache_config.enable_prefix_caching = False
60 -
56 kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
57
58 if kv_cache_space >= 0:
@@ -74,10 +69,12 @@ class CpuPlatform(Platform):
69 f" {kv_cache_space}, expect a positive integer value.")
70
71 scheduler_config = vllm_config.scheduler_config
77 - if scheduler_config.chunked_prefill_enabled:
78 - logger.warning(
79 - "Chunked prefill is not supported on CPU, disable it.")
80 - scheduler_config.chunked_prefill_enabled = False
72 + if ((scheduler_config.chunked_prefill_enabled
73 + or cache_config.enable_prefix_caching)
74 + and model_config.dtype == torch.half):
75 + logger.warning("Chunked-prefill on the CPU backend only does not"
76 + " support fp16 for now, cast to bf16.")
77 + model_config.dtype = torch.bfloat16
78
79 parallel_config = vllm_config.parallel_config
80 if (parallel_config.distributed_executor_backend is not None
vllm/worker/cpu_model_runner.py
+215 -273
@@ -2,8 +2,8 @@ import dataclasses
2 import weakref
3 from collections import defaultdict
4 from dataclasses import dataclass
5 -from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6 - TypeVar, Union)
5 +from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Type, TypeVar,
6 + Union)
7
8 import torch
9 from torch import nn
@@ -19,7 +19,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19 MultiModalKwargs, MultiModalPlaceholderMap)
20 from vllm.sequence import (IntermediateTensors, SequenceData,
21 SequenceGroupMetadata)
22 -from vllm.utils import make_tensor_with_pad
22 from vllm.worker.model_runner_base import (
23 ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
24 _add_attn_metadata_broadcastable_dict,
@@ -104,65 +103,223 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
103
104 class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
105
106 + class ModelInputData:
107 +
108 + def __init__(self, use_mrope: bool):
109 + self.use_mrope = use_mrope
110 + self.input_tokens: List[int] = []
111 + self.input_positions: Optional[
112 + List[int]] = [] if not self.use_mrope else None
113 + self.seq_lens: List[int] = []
114 + self.query_lens: List[int] = []
115 + self.prefill_block_tables: List[List[int]] = []
116 + self.decode_block_tables: List[List[int]] = []
117 + self.max_decode_seq_len: int = 0
118 + self.num_prefills: int = 0
119 + self.num_prefill_tokens: int = 0
120 + self.num_decode_tokens: int = 0
121 + self.slot_mapping: List[int] = []
122 + self.multi_modal_inputs_list: List[MultiModalKwargs] = []
123 + self.multi_modal_placeholder_maps: Dict[
124 + str, MultiModalPlaceholderMap] = defaultdict(
125 + MultiModalPlaceholderMap)
126 + self.input_mrope_positions: Optional[List[List[int]]] = [
127 + [] for _ in range(3)
128 + ] if self.use_mrope else None
129 +
130 def __init__(self,
131 runner: "CPUModelRunner",
132 finished_requests_ids: Optional[List[str]] = None) -> None:
133 super().__init__()
134 self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
135 self.runner = runner
136 +
137 + self.chunked_prefill = (runner.scheduler_config.chunked_prefill_enabled
138 + or runner.cache_config.enable_prefix_caching)
139 self.model_input_cls = self.runner._model_input_cls
140 self.attn_backend = self.runner.attn_backend
115 - self.sliding_window = self.runner.sliding_window
116 - self.block_size = self.runner.block_size
117 - self.device = self.runner.device
141 self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
142 + self.input_data = ModelInputForCPUBuilder.ModelInputData(
143 + self.runner.model_config.uses_mrope)
144 + self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
145 + self)
146
147 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
148 self.seq_group_metadata_list.append(seq_group_metadata)
149
150 + def set_seq_group_list(
151 + self, seq_group_metadata_list: List[SequenceGroupMetadata]):
152 + self.seq_group_metadata_list = seq_group_metadata_list
153 +
154 def build(self) -> ModelInputForCPU:
155 + self._build_input_data()
156 +
157 + input_data = self.input_data
158 + input_tokens = torch.tensor(input_data.input_tokens,
159 + dtype=torch.long,
160 + device="cpu")
161 + input_positions = torch.tensor(
162 + input_data.input_positions
163 + if not input_data.use_mrope else input_data.input_mrope_positions,
164 + dtype=torch.long,
165 + device="cpu")
166 +
167 + # For multi-modal models
168 multi_modal_kwargs = None
125 - # NOTE: We assume that all sequences in the group are all prompts or
126 - # all decodes.
127 - is_prompt = self.seq_group_metadata_list[0].is_prompt
128 - # Prepare input tensors.
129 - if is_prompt:
130 - (input_tokens, input_positions, attn_metadata, seq_lens,
131 - multi_modal_kwargs) = self._prepare_prompt(
132 - self.seq_group_metadata_list)
133 - else:
134 - (input_tokens, input_positions,
135 - attn_metadata) = self._prepare_decode(
136 - self.seq_group_metadata_list)
137 - seq_lens = None
169 + if len(input_data.multi_modal_inputs_list) != 0:
170 + multi_modal_kwargs = MultiModalKwargs.batch(
171 + input_data.multi_modal_inputs_list)
172 +
173 + attn_metadata = self.att_metadata_builder.build(
174 + input_data.seq_lens, input_data.query_lens, -1, -1)
175
176 return self.model_input_cls(
177 input_tokens=input_tokens,
178 input_positions=input_positions,
179 + seq_lens=input_data.seq_lens,
180 + query_lens=input_data.query_lens,
181 attn_metadata=attn_metadata,
182 multi_modal_kwargs=multi_modal_kwargs,
144 - # query_lens is not needed if chunked prefill is not
145 - # supported. Since CPU worker doesn't support chunked prefill
146 - # just use seq_lens instead.
147 - seq_lens=seq_lens,
148 - query_lens=seq_lens,
183 )
184
151 - def _compute_multi_modal_input(
152 - self,
153 - seq_data: SequenceData,
154 - computed_len: int,
155 - seq_group_metadata: SequenceGroupMetadata,
156 - ):
185 + def _build_input_data(self):
186 + for seq_group_metadata in self.seq_group_metadata_list:
187 + for seq_id, seq_data in seq_group_metadata.seq_data.items():
188 + if seq_group_metadata.is_prompt:
189 + self._compute_prompt_input_tokens(self.input_data,
190 + seq_group_metadata,
191 + seq_data, seq_id)
192 + if seq_group_metadata.multi_modal_data:
193 + self._compute_multi_modal_input(
194 + seq_group_metadata, seq_data)
195 + else:
196 + self._compute_decode_input_tokens(self.input_data,
197 + seq_group_metadata,
198 + seq_data, seq_id)
199 +
200 + def _compute_decode_input_tokens(self, data: ModelInputData,
201 + seq_group_metadata: SequenceGroupMetadata,
202 + seq_data: SequenceData, seq_id: int):
203 + """
204 + Compute decode input tokens, positions, block table and slot mapping.
205 + """
206 + block_size = self.runner.block_size
207 +
208 + block_table = seq_group_metadata.block_tables[seq_id]
209 + seq_len = seq_data.get_len()
210 + context_len = seq_data.get_num_computed_tokens()
211 +
212 + tokens = seq_data.get_last_token_id()
213 + token_positions = seq_len - 1
214 + block_number = block_table[token_positions // block_size]
215 + block_offset = token_positions % block_size
216 + slot = block_number * block_size + block_offset
217 +
218 + # For paged_attention kernel
219 + if self.runner.sliding_window:
220 + start_idx = max(0, seq_len - self.runner.sliding_window)
221 + start_block = start_idx // block_size
222 + start_idx = start_block * block_size
223 + seq_len = seq_len - start_idx
224 + block_table = block_table[start_block:]
225 +
226 + # For MRotaryEmbedding
227 + if data.input_positions is None:
228 + next_pos = MRotaryEmbedding.get_next_input_positions(
229 + seq_data.mrope_position_delta,
230 + context_len,
231 + seq_len,
232 + )
233 + for idx in range(3):
234 + data.input_mrope_positions[idx].extend( # type: ignore
235 + next_pos[idx])
236 + else:
237 + data.input_positions.append(token_positions) # type: ignore
238 +
239 + # Update fields
240 + data.input_tokens.append(tokens)
241 + data.max_decode_seq_len = max(data.max_decode_seq_len, seq_len)
242 + data.num_decode_tokens += 1
243 + data.slot_mapping.append(slot)
244 + data.decode_block_tables.append(block_table)
245 + data.query_lens.append(1)
246 + data.seq_lens.append(seq_len)
247 +
248 + def _compute_prompt_input_tokens(self, data: ModelInputData,
249 + seq_group_metadata: SequenceGroupMetadata,
250 + seq_data: SequenceData, seq_id: int):
251 + """
252 + Compute prompt input tokens, positions, block table and slot mapping.
253 + """
254 + token_chunk_size = seq_group_metadata.token_chunk_size
255 + block_size = self.runner.block_size
256 +
257 + block_table = seq_group_metadata.block_tables[seq_id]
258 + seq_len = seq_data.get_len()
259 + context_len = seq_data.get_num_computed_tokens()
260 + seq_len = min(seq_len, context_len + token_chunk_size)
261 +
262 + # For prefix caching
263 + prefix_cache_block_num = len(seq_group_metadata.computed_block_nums)
264 + if prefix_cache_block_num > 0:
265 + prefix_cache_len = (prefix_cache_block_num *
266 + self.runner.block_size)
267 + if prefix_cache_len <= context_len:
268 + # We already passed the cache hit region,
269 + # so do normal computation.
270 + pass
271 + elif context_len < prefix_cache_len < seq_len:
272 + # Partial hit. Compute the missing part.
273 + context_len = prefix_cache_len
274 + token_chunk_size = seq_len - context_len
275 + elif seq_len <= prefix_cache_len:
276 + # Full hit. Only compute the last token to avoid
277 + # erroneous behavior. FIXME: Ideally we should directly
278 + # mark all tokens as computed in the scheduler and do not
279 + # schedule this sequence, so this case should not happen.
280 + context_len = seq_len - 1
281 + token_chunk_size = 1
282 +
283 + tokens = seq_data.get_token_ids()
284 + tokens = tokens[context_len:seq_len]
285 + token_positions = range(context_len, seq_len)
286 +
287 + # For encoder-only models, the block_table is None,
288 + # and there is no need to initialize the slot_mapping.
289 + if block_table is not None:
290 + slot_mapping = [_PAD_SLOT_ID] * len(token_positions)
291 + for i, pos in enumerate(token_positions):
292 + block_number = block_table[pos // block_size]
293 + block_offset = pos % block_size
294 + slot = block_number * block_size + block_offset
295 + slot_mapping[i] = slot
296 + data.slot_mapping.extend(slot_mapping)
297 +
298 + # The MROPE positions are prepared in _compute_multi_modal_input
299 + if data.input_positions is not None:
300 + data.input_positions.extend(token_positions)
301 +
302 + # Update fields
303 + data.input_tokens.extend(tokens)
304 + data.num_prefills += 1
305 + data.num_prefill_tokens += len(tokens)
306 + data.query_lens.append(len(tokens))
307 + data.prefill_block_tables.append(block_table)
308 + data.seq_lens.append(seq_len)
309 +
310 + def _compute_multi_modal_input(self,
311 + seq_group_metadata: SequenceGroupMetadata,
312 + seq_data: SequenceData):
313 + computed_len = seq_data.get_num_computed_tokens()
314 + seq_len = self.input_data.seq_lens[-1]
315 +
316 # NOTE: mm_data only includes the subset of multi-modal items that
317 # intersect with the current prefill positions.
318 mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
160 - seq_group_metadata,
161 - range(computed_len, len(seq_data.get_token_ids())),
162 - )
319 + seq_group_metadata, range(computed_len, seq_len))
320
321 if not mm_data:
165 - return None, None, None
322 + return
323
324 if self.runner.mm_registry.has_processor(self.runner.model_config):
325 mm_kwargs = mm_data
@@ -173,8 +330,10 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
330 )
331
332 # special processing for mrope position deltas.
176 - mrope_positions = None
333 if self.runner.model_config.uses_mrope:
334 + assert not self.chunked_prefill, \
335 + "MROPE on CPU does not support chunked-prefill."
336 +
337 image_grid_thw = mm_kwargs.get("image_grid_thw", None)
338 video_grid_thw = mm_kwargs.get("video_grid_thw", None)
339 assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -198,226 +357,15 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
357 context_len=computed_len,
358 )
359 seq_data.mrope_position_delta = mrope_position_delta
201 - return mm_kwargs, placeholder_maps, mrope_positions
360
203 - def _prepare_prompt(
204 - self,
205 - seq_group_metadata_list: List[SequenceGroupMetadata],
206 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
207 - BatchedTensorInputs]:
208 - assert len(seq_group_metadata_list) > 0
209 - input_tokens: List[int] = []
210 - input_positions: List[int] = []
211 - input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
212 -
213 - slot_mapping: List[int] = []
214 - seq_lens: List[int] = []
215 - multi_modal_kwargs_list: List[MultiModalKwargs] = []
216 - multi_modal_placeholder_maps: Dict[
217 - str,
218 - MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
219 -
220 - for seq_group_metadata in seq_group_metadata_list:
221 - assert seq_group_metadata.is_prompt
222 - seq_ids = list(seq_group_metadata.seq_data.keys())
223 - assert len(seq_ids) == 1
224 - seq_id = seq_ids[0]
225 -
226 - seq_data = seq_group_metadata.seq_data[seq_id]
227 - prompt_tokens = seq_data.get_token_ids()
228 - computed_len = seq_data.get_num_computed_tokens()
229 - seq_len = len(prompt_tokens)
230 -
231 - seq_lens.append(seq_len) # Prompt token num
232 - input_tokens.extend(prompt_tokens) # Token ids
233 -
234 - mrope_positions = None
235 - if seq_group_metadata.multi_modal_data:
236 - (
237 - mm_kwargs,
238 - placeholder_maps,
239 - mrope_positions,
240 - ) = self._compute_multi_modal_input(seq_data, computed_len,
241 - seq_group_metadata)
242 -
243 - multi_modal_kwargs_list.append(mm_kwargs)
244 - for modality, placeholder_map in placeholder_maps.items():
245 - multi_modal_placeholder_maps[modality].extend(
246 - placeholder_map)
247 -
248 - # Token position ids
249 - # NOTE(woosuk): Here we assume that the first token in the prompt
250 - # is always the first token in the sequence.
251 - if mrope_positions:
252 - for idx in range(3):
253 - input_mrope_positions[idx].extend(mrope_positions[idx])
254 - else:
255 - input_positions.extend(list(range(computed_len, seq_len)))
256 -
257 - # Compute the slot mapping.
258 - block_table = seq_group_metadata.block_tables[seq_id]
259 - # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
260 - # where start_idx is max(0, seq_len - sliding_window).
261 - # For example, if the prompt len is 10, sliding window is 8, and
262 - # block size is 4, the first two tokens are masked and the slot
263 - # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
264 - start_idx = 0
265 - if self.sliding_window is not None:
266 - start_idx = max(0, seq_len - self.sliding_window)
267 -
268 - for i in range(computed_len, seq_len):
269 - if i < start_idx:
270 - slot_mapping.append(_PAD_SLOT_ID)
271 - continue
272 -
273 - # For encoder-only models, the block_table is None,
274 - # and there is no need to initialize the slot_mapping.
275 - if block_table is not None:
276 - block_number = block_table[i //
277 - self.block_size] # type: ignore
278 - block_offset = i % self.block_size # type: ignore
279 - slot = block_number * self.block_size + block_offset
280 - slot_mapping.append(slot)
281 -
282 - if any(input_mrope_positions):
283 - input_positions = None # type: ignore
284 - else:
285 - input_mrope_positions = None # type: ignore
361 + for i in range(3):
362 + self.input_data.input_mrope_positions[ # type: ignore
363 + i].extend(mrope_positions[i])
364
287 - num_prompt_tokens = len(input_tokens)
288 -
289 - input_tokens = torch.tensor(input_tokens,
290 - dtype=torch.long,
291 - device=self.device) # type: ignore
292 - input_positions = torch.tensor(input_positions
293 - or input_mrope_positions,
294 - dtype=torch.long,
295 - device=self.device) # type: ignore
296 - slot_mapping = torch.tensor(slot_mapping,
297 - dtype=torch.long,
298 - device=self.device) # type: ignore
299 - placeholder_index_maps = {
300 - modality: placeholder_map.index_map()
301 - for modality, placeholder_map in
302 - multi_modal_placeholder_maps.items()
303 - }
304 -
305 - attn_metadata = self.attn_backend.make_metadata(
306 - is_prompt=True,
307 - seq_lens=seq_lens,
308 - seq_lens_tensor=torch.tensor([]),
309 - max_decode_seq_len=0,
310 - num_prefills=len(seq_lens),
311 - num_prefill_tokens=num_prompt_tokens,
312 - num_decode_tokens=0,
313 - block_tables=torch.tensor([]),
314 - slot_mapping=slot_mapping,
315 - multi_modal_placeholder_index_maps=placeholder_index_maps,
316 - )
317 -
318 - multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
319 -
320 - return (input_tokens, input_positions, attn_metadata, seq_lens,
321 - multi_modal_kwargs)
322 -
323 - def _prepare_decode(
324 - self,
325 - seq_group_metadata_list: List[SequenceGroupMetadata],
326 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
327 - assert len(seq_group_metadata_list) > 0
328 - input_tokens: List[int] = []
329 - input_positions: List[int] = []
330 - input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
331 - slot_mapping: List[int] = []
332 - seq_lens: List[int] = []
333 - block_tables: List[List[int]] = []
334 -
335 - for seq_group_metadata in seq_group_metadata_list:
336 - assert not seq_group_metadata.is_prompt
337 - assert seq_group_metadata.token_chunk_size == 1
338 -
339 - seq_ids = list(seq_group_metadata.seq_data.keys())
340 -
341 - for seq_id in seq_ids:
342 - seq_data = seq_group_metadata.seq_data[seq_id]
343 - generation_token = seq_data.get_last_token_id()
344 - input_tokens.append(generation_token)
345 -
346 - seq_len = seq_data.get_len()
347 - position = seq_len - 1
348 - if seq_data.mrope_position_delta is not None:
349 - context_len = seq_data.get_num_computed_tokens()
350 - next_pos = MRotaryEmbedding.get_next_input_positions(
351 - seq_data.mrope_position_delta,
352 - context_len,
353 - seq_len,
354 - )
355 - for idx in range(3):
356 - input_mrope_positions[idx].extend(next_pos[idx])
357 - else:
358 - input_positions.append(position)
359 -
360 - seq_len = seq_len if self.sliding_window is None else min(
361 - seq_len, self.sliding_window)
362 - seq_lens.append(seq_len)
363 -
364 - block_table = seq_group_metadata.block_tables[seq_id]
365 - block_number = block_table[position // self.block_size]
366 - block_offset = position % self.block_size
367 - slot = block_number * self.block_size + block_offset
368 - slot_mapping.append(slot)
369 -
370 - if self.sliding_window is not None:
371 - sliding_window_blocks = (self.sliding_window //
372 - self.block_size)
373 - block_table = block_table[-sliding_window_blocks:]
374 - block_tables.append(block_table)
375 -
376 - if any(input_mrope_positions):
377 - input_positions = None # type: ignore
378 - else:
379 - input_mrope_positions = None # type: ignore
380 -
381 - max_decode_seq_len = max(seq_lens)
382 -
383 - input_tokens = torch.tensor(input_tokens,
384 - dtype=torch.long,
385 - device=self.device)
386 - input_positions = torch.tensor(input_positions
387 - or input_mrope_positions,
388 - dtype=torch.long,
389 - device=self.device)
390 - slot_mapping = torch.tensor(slot_mapping,
391 - dtype=torch.long,
392 - device=self.device)
393 - seq_lens_tensor = torch.tensor(seq_lens,
394 - dtype=torch.int,
395 - device=self.device)
396 -
397 - block_tables = make_tensor_with_pad(
398 - block_tables,
399 - pad=0,
400 - dtype=torch.int,
401 - device=self.device,
402 - )
403 -
404 - attn_metadata = self.attn_backend.make_metadata(
405 - is_prompt=False,
406 - slot_mapping=slot_mapping,
407 - multi_modal_placeholder_index_maps=None,
408 - seq_lens=seq_lens,
409 - seq_lens_tensor=seq_lens_tensor,
410 - max_decode_seq_len=max_decode_seq_len,
411 - num_prefill_tokens=0,
412 - num_decode_tokens=len(input_tokens),
413 - num_prefills=0,
414 - block_tables=block_tables,
415 - )
416 - return (
417 - input_tokens,
418 - input_positions,
419 - attn_metadata,
420 - )
365 + self.input_data.multi_modal_inputs_list.append(mm_kwargs)
366 + for modality, placeholder_map in placeholder_maps.items():
367 + self.input_data.multi_modal_placeholder_maps[modality].extend(
368 + placeholder_map)
369
370
371 class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
@@ -436,8 +384,6 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
384 **kwargs,
385 ):
386 ModelRunnerBase.__init__(self, vllm_config)
439 - # Currently, CPU worker doesn't support chunked prefill.
440 - assert self.scheduler_config.chunked_prefill_enabled is False
387 model_config = self.model_config
388 cache_config = self.cache_config
389
@@ -479,8 +425,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
425
426 """
427 builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
482 - for seq_group_metadata in seq_group_metadata_list:
483 - builder.add_seq_group(seq_group_metadata)
428 + builder.set_seq_group_list(seq_group_metadata_list)
429
430 return builder.build() # type: ignore
431
@@ -537,22 +482,19 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
482 "CPU worker does not support multi-step execution.")
483
484 model_executable = self.model
540 - execute_model_kwargs = {
541 - "input_ids":
542 - model_input.input_tokens,
543 - "positions":
544 - model_input.input_positions,
545 - "kv_caches":
546 - kv_caches,
547 - "attn_metadata":
548 - model_input.attn_metadata,
549 - **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
550 - device=self.device),
551 - "intermediate_tensors":
552 - intermediate_tensors,
553 - }
554 -
555 - hidden_states = model_executable(**execute_model_kwargs)
485 + multimodal_kwargs = {}
486 + if model_input.multi_modal_kwargs is not None:
487 + multimodal_kwargs = MultiModalKwargs.as_kwargs(
488 + model_input.multi_modal_kwargs, device=self.device)
489 +
490 + hidden_states = model_executable(
491 + input_ids=model_input.input_tokens,
492 + positions=model_input.input_positions,
493 + kv_caches=kv_caches,
494 + attn_metadata=model_input.attn_metadata,
495 + intermediate_tensors=intermediate_tensors,
496 + **multimodal_kwargs,
497 + )
498
499 # Compute the logits.
500 logits = self.model.compute_logits(hidden_states,