A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Hardware][CPU] Support chunked-prefill and prefix-caching on CPU (#10355)
Signed-off-by: jiang1.li <jiang1.li@intel.com>
Li, Jiang committed
Nov 20, 2024 at 18:57 UTC
63f1fde277d063fbd36ccf43cb709fafca754ed5
8 files changed
+559
-369
.buildkite/run-cpu-test.sh
+8
-1
@@ -25,6 +25,7 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
25
26
function cpu_tests() {
27
set -e
28
+ export NUMA_NODE=$2
29
30
# offline inference
31
docker exec cpu-test-avx2-"$NUMA_NODE" bash -c "
@@ -57,6 +58,12 @@ function cpu_tests() {
58
pytest -s -v \
59
tests/quantization/test_ipex_quant.py"
60
61
+ # Run chunked-prefill and prefix-cache test
62
+ docker exec cpu-test-"$NUMA_NODE" bash -c "
63
+ set -e
64
+ pytest -s -v -k cpu_model \
65
+ tests/basic_correctness/test_chunked_prefill.py"
66
+
67
# online inference
68
docker exec cpu-test-"$NUMA_NODE" bash -c "
69
set -e
@@ -75,4 +82,4 @@ function cpu_tests() {
82
83
# All of CPU tests are expected to be finished less than 25 mins.
84
export -f cpu_tests
78
-timeout 25m bash -c "cpu_tests $CORE_RANGE"
85
+timeout 30m bash -c "cpu_tests $CORE_RANGE $NUMA_NODE"
docs/source/getting_started/cpu-installation.rst
+5
-5
@@ -5,11 +5,11 @@ Installation with CPU
5
6
vLLM initially supports basic model inferencing and serving on x86 CPU platform, with data types FP32, FP16 and BF16. vLLM CPU backend supports the following vLLM features:
7
8
-- Tensor Parallel (``-tp = N``)
9
-- Quantization (``INT8 W8A8, AWQ``)
10
-
11
-.. note::
12
- More advanced features on `chunked-prefill`, `prefix-caching` and `FP8 KV cache` are under development and will be available soon.
8
+- Tensor Parallel
9
+- Model Quantization (``INT8 W8A8, AWQ``)
10
+- Chunked-prefill
11
+- Prefix-caching
12
+- FP8-E5M2 KV-Caching (TODO)
13
14
Table of contents:
15
docs/source/serving/compatibility_matrix.rst
+2
-2
@@ -344,7 +344,7 @@ Feature x Hardware
344
- ✅
345
- ✅
346
- ✅
347
- - ✗
347
+ - ✅
348
- ✅
349
* - :ref:`APC <apc>`
350
- `✗ <https://github.com/vllm-project/vllm/issues/3687>`__
@@ -352,7 +352,7 @@ Feature x Hardware
352
- ✅
353
- ✅
354
- ✅
355
- - ✗
355
+ - ✅
356
- ✅
357
* - :ref:`LoRA <lora>`
358
- ✅
tests/basic_correctness/test_chunked_prefill.py
+62
-1
@@ -12,6 +12,7 @@ from contextlib import nullcontext
12
import pytest
13
14
from tests.kernels.utils import override_backend_env_variable
15
+from vllm.platforms import current_platform
16
17
from ..models.utils import check_logprobs_close, check_outputs_equal
18
from ..utils import multi_gpu_test
@@ -206,12 +207,14 @@ def test_models_with_fp8_kv_cache(
207
# NOTE: Increasing this in this suite will fail CI because we currently cannot
208
# reset distributed env properly. Use a value > 1 just when you test.
209
@pytest.mark.parametrize("tensor_parallel_size", [1])
210
+@pytest.mark.parametrize("dtype", ["half"])
211
def test_with_prefix_caching(
212
vllm_runner,
213
max_tokens: int,
214
enforce_eager: bool,
215
chunk_size: int,
216
tensor_parallel_size: int,
217
+ dtype: str,
218
) -> None:
219
"""
220
Checks exact match decode with and without prefix caching
@@ -233,7 +236,7 @@ def test_with_prefix_caching(
236
for enable in (True, False):
237
with vllm_runner(
238
model,
236
- dtype="half",
239
+ dtype=dtype,
240
max_num_batched_tokens=max_num_batched_tokens,
241
enable_chunked_prefill=True,
242
enable_prefix_caching=enable,
@@ -260,3 +263,61 @@ def test_with_prefix_caching(
263
name_0="w/o prefix caching",
264
name_1="with prefix caching",
265
)
266
+
267
+
268
+@pytest.mark.parametrize("model", ["facebook/opt-125m"])
269
+@pytest.mark.parametrize("dtype", ["bfloat16"])
270
+@pytest.mark.parametrize("max_tokens", [32])
271
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
272
+@pytest.mark.parametrize("enforce_eager", [False])
273
+@pytest.mark.parametrize("attention_backend", ["TORCH_SDPA"])
274
+@pytest.mark.cpu_model
275
+@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
276
+def test_models_cpu(
277
+ hf_runner,
278
+ vllm_runner,
279
+ example_prompts,
280
+ model: str,
281
+ dtype: str,
282
+ max_tokens: int,
283
+ chunked_prefill_token_size: int,
284
+ enforce_eager: bool,
285
+ attention_backend: str,
286
+ monkeypatch,
287
+) -> None:
288
+ test_models(
289
+ hf_runner,
290
+ vllm_runner,
291
+ example_prompts,
292
+ model,
293
+ dtype,
294
+ max_tokens,
295
+ chunked_prefill_token_size,
296
+ enforce_eager,
297
+ 1,
298
+ attention_backend,
299
+ monkeypatch,
300
+ )
301
+
302
+
303
+@pytest.mark.parametrize("max_tokens", [16])
304
+@pytest.mark.parametrize("enforce_eager", [False])
305
+@pytest.mark.parametrize("chunk_size", [30, 32])
306
+@pytest.mark.parametrize("dtype", ["bfloat16"])
307
+@pytest.mark.cpu_model
308
+@pytest.mark.skipif(not current_platform.is_cpu(), reason="CPU only")
309
+def test_with_prefix_caching_cpu(
310
+ vllm_runner,
311
+ max_tokens: int,
312
+ enforce_eager: bool,
313
+ chunk_size: int,
314
+ dtype: str,
315
+) -> None:
316
+ test_with_prefix_caching(
317
+ vllm_runner,
318
+ max_tokens,
319
+ enforce_eager,
320
+ chunk_size,
321
+ 1,
322
+ dtype,
323
+ )
vllm/attention/backends/torch_sdpa.py
+152
-37
@@ -7,18 +7,14 @@ import torch
7
from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata, AttentionType)
10
+ AttentionMetadata,
11
+ AttentionMetadataBuilder,
12
+ AttentionType)
13
from vllm.attention.backends.utils import CommonAttentionState
14
+from vllm.attention.ops.ipex_attn import PagedAttention
15
from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13
-from vllm.platforms import current_platform
14
-
15
-if current_platform.is_cpu():
16
- try:
17
- from vllm.attention.ops.ipex_attn import PagedAttention
18
- except ImportError:
19
- from vllm.attention.ops.paged_attn import PagedAttention
20
-else:
21
- from vllm.attention.ops.paged_attn import PagedAttention
16
+from vllm.utils import make_tensor_with_pad
17
+from vllm.worker.cpu_model_runner import ModelInputForCPUBuilder
18
19
20
class TorchSDPABackend(AttentionBackend):
@@ -39,6 +35,10 @@ class TorchSDPABackend(AttentionBackend):
35
def get_state_cls() -> Type["CommonAttentionState"]:
36
return CommonAttentionState
37
38
+ @staticmethod
39
+ def get_builder_cls() -> Type["TorchSDPAMetadataBuilder"]:
40
+ return TorchSDPAMetadataBuilder
41
+
42
@staticmethod
43
def get_kv_cache_shape(
44
num_blocks: int,
@@ -71,9 +71,15 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
71
"""
72
# Currently, input sequences can only contain all prompts
73
# or all decoding. True if all sequences are prompts.
74
- is_prompt: bool
75
- slot_mapping: torch.Tensor
76
- seq_lens: Optional[List[int]]
74
+ chunked_prefill: bool
75
+ seq_lens: Optional[List[int]] = None # For non-chunked prefill
76
+
77
+ # For chunked prefill only
78
+ max_query_len: Optional[int] = None
79
+ max_kv_len: Optional[int] = None
80
+ query_start_loc: Optional[torch.Tensor] = None
81
+ kv_start_loc: Optional[torch.Tensor] = None
82
+ prefill_block_tables: Optional[torch.Tensor] = None
83
84
# Begin encoder attn & enc/dec cross-attn fields...
85
# Encoder sequence lengths representation
@@ -123,20 +129,14 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
129
130
@property
131
def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
126
- # Currently chunked prefill is not supported
127
- if self.num_decode_tokens == 0:
128
- assert self.num_prefills > 0
129
- return self
130
-
131
- return None
132
+ if self.num_prefill_tokens == 0:
133
+ return None
134
+ return self
135
136
@property
137
def decode_metadata(self) -> Optional["TorchSDPAMetadata"]:
135
- # Currently chunked prefill is not supported
136
- if self.num_prefills > 0:
137
- assert self.num_decode_tokens == 0
138
+ if self.num_decode_tokens == 0:
139
return None
139
-
140
return self
141
142
def get_seq_lens(
@@ -274,6 +274,105 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
274
raise AttributeError(f"Invalid attention type {str(attn_type)}")
275
276
277
+class TorchSDPAMetadataBuilder(AttentionMetadataBuilder[TorchSDPAMetadata]):
278
+
279
+ def __init__(self, input_builder: ModelInputForCPUBuilder) -> None:
280
+ self.chunked_prefill = input_builder.chunked_prefill
281
+ self.input_data = input_builder.input_data
282
+
283
+ def build(self, seq_lens: List[int], query_lens: List[int],
284
+ cuda_graph_pad_size: int, batch_size: int) -> TorchSDPAMetadata:
285
+ input_data = self.input_data
286
+ prefill_seq_lens = seq_lens[0:input_data.num_prefills]
287
+ prefill_query_lens = query_lens[0:input_data.num_prefills]
288
+ slot_mapping = torch.tensor(input_data.slot_mapping,
289
+ dtype=torch.long,
290
+ device="cpu")
291
+
292
+ # For chunked-prefill
293
+ if self.chunked_prefill and input_data.num_prefill_tokens != 0:
294
+ prefill_block_tables = make_tensor_with_pad(
295
+ self.input_data.prefill_block_tables,
296
+ pad=0,
297
+ dtype=torch.int32,
298
+ device="cpu",
299
+ )
300
+ query_lens_tensor = torch.tensor(prefill_query_lens,
301
+ dtype=torch.int32,
302
+ device="cpu")
303
+ kv_lens_tensor = torch.tensor(prefill_seq_lens,
304
+ dtype=torch.int32,
305
+ device="cpu")
306
+ query_start_loc = torch.zeros(input_data.num_prefills + 1,
307
+ dtype=torch.int32,
308
+ device="cpu")
309
+ kv_start_loc = torch.zeros(input_data.num_prefills + 1,
310
+ dtype=torch.int32,
311
+ device="cpu")
312
+ torch.cumsum(query_lens_tensor,
313
+ dim=0,
314
+ dtype=torch.int32,
315
+ out=query_start_loc[1:])
316
+ torch.cumsum(kv_lens_tensor,
317
+ dim=0,
318
+ dtype=torch.int32,
319
+ out=kv_start_loc[1:])
320
+ max_query_len = max(prefill_query_lens)
321
+ max_kv_len = max(prefill_seq_lens)
322
+ else:
323
+ prefill_block_tables = None
324
+ query_start_loc = None
325
+ kv_start_loc = None
326
+ max_query_len = None
327
+ max_kv_len = None
328
+
329
+ # For paged attention
330
+ if input_data.num_decode_tokens != 0:
331
+ seq_lens_tensor = torch.tensor(
332
+ input_data.seq_lens[input_data.num_prefills:],
333
+ dtype=torch.int32,
334
+ device="cpu",
335
+ )
336
+ block_tables = make_tensor_with_pad(
337
+ self.input_data.decode_block_tables,
338
+ pad=0,
339
+ dtype=torch.int32,
340
+ device="cpu",
341
+ )
342
+ else:
343
+ block_tables = torch.tensor([])
344
+ seq_lens_tensor = torch.tensor([])
345
+
346
+ # For multi-modal models
347
+ placeholder_index_maps = None
348
+ if len(input_data.multi_modal_inputs_list) != 0:
349
+ placeholder_index_maps = {
350
+ modality: placeholder_map.index_map()
351
+ for modality, placeholder_map in
352
+ input_data.multi_modal_placeholder_maps.items()
353
+ }
354
+
355
+ attn_metadata = TorchSDPAMetadata(
356
+ chunked_prefill=self.chunked_prefill,
357
+ seq_lens=prefill_seq_lens,
358
+ seq_lens_tensor=seq_lens_tensor,
359
+ max_query_len=max_query_len,
360
+ max_kv_len=max_kv_len,
361
+ query_start_loc=query_start_loc,
362
+ kv_start_loc=kv_start_loc,
363
+ max_decode_seq_len=input_data.max_decode_seq_len,
364
+ num_prefills=input_data.num_prefills,
365
+ num_prefill_tokens=input_data.num_prefill_tokens,
366
+ num_decode_tokens=input_data.num_decode_tokens,
367
+ block_tables=block_tables,
368
+ prefill_block_tables=prefill_block_tables,
369
+ slot_mapping=slot_mapping,
370
+ multi_modal_placeholder_index_maps=placeholder_index_maps,
371
+ )
372
+
373
+ return attn_metadata
374
+
375
+
376
class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
377
378
def __init__(
@@ -409,19 +508,35 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
508
assert key.shape[0] == num_prefill_tokens + num_decode_tokens
509
assert value.shape[0] == num_prefill_tokens + num_decode_tokens
510
511
+ output = torch.empty_like(query)
512
if prefill_meta := attn_metadata.prefill_metadata:
513
assert attn_metadata.seq_lens is not None
414
- if (kv_cache.numel() == 0
415
- or prefill_meta.block_tables.numel() == 0):
416
- output = self._run_sdpa_forward(query,
417
- key,
418
- value,
419
- prefill_meta,
420
- attn_type=attn_type)
514
+ if not prefill_meta.prefill_metadata.chunked_prefill: # type: ignore
515
+ self._run_sdpa_forward(output,
516
+ query,
517
+ key,
518
+ value,
519
+ prefill_meta,
520
+ attn_type=attn_type)
521
else:
522
# prefix-enabled attention
423
- raise RuntimeError(
424
- "Torch SDPA backend doesn't support prefix decoding.")
523
+ assert not self.need_mask
524
+ import intel_extension_for_pytorch.llm.modules as ipex_modules
525
+ output = torch.empty_like(query)
526
+ ipex_modules.PagedAttention.flash_attn_varlen_func(
527
+ output[:prefill_meta.num_prefill_tokens, :, :],
528
+ query[:prefill_meta.num_prefill_tokens, :, :],
529
+ key_cache,
530
+ value_cache,
531
+ prefill_meta.query_start_loc,
532
+ prefill_meta.kv_start_loc,
533
+ prefill_meta.max_query_len,
534
+ prefill_meta.max_kv_len,
535
+ self.scale,
536
+ True,
537
+ prefill_meta.prefill_block_tables,
538
+ self.alibi_slopes,
539
+ )
540
541
if decode_meta := attn_metadata.decode_metadata:
542
assert attn_type != AttentionType.ENCODER_ONLY, (
@@ -433,8 +548,9 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
548
block_tables_arg,
549
) = decode_meta.get_seq_len_block_table_args(attn_type)
550
436
- output = PagedAttention.forward_decode(
437
- query,
551
+ PagedAttention.forward_decode(
552
+ output[attn_metadata.num_prefill_tokens:, :, :],
553
+ query[attn_metadata.num_prefill_tokens:, :, :],
554
key_cache,
555
value_cache,
556
block_tables_arg,
@@ -453,12 +569,13 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
569
570
def _run_sdpa_forward(
571
self,
572
+ output: torch.Tensor,
573
query: torch.Tensor,
574
key: torch.Tensor,
575
value: torch.Tensor,
576
attn_metadata: TorchSDPAMetadata,
577
attn_type: AttentionType = AttentionType.DECODER,
461
- ):
578
+ ) -> None:
579
if self.num_kv_heads != self.num_heads:
580
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
581
value = value.repeat_interleave(self.num_queries_per_kv, dim=1)
@@ -479,7 +596,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
596
attn_masks = [None] * len(seq_lens)
597
attn_metadata.set_attn_bias(attn_masks, attn_type)
598
482
- output = torch.empty_like(query)
599
query = query.movedim(0, query.dim() - 2)
600
key = key.movedim(0, key.dim() - 2)
601
value = value.movedim(0, value.dim() - 2)
@@ -502,7 +618,6 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
618
scale=self.scale).squeeze(0).movedim(query.dim() - 2, 0)
619
output[start_q:end_q, :, :] = sub_out
620
start_q, start_kv = end_q, end_kv
505
- return output
621
622
623
def _make_alibi_bias(
vllm/attention/ops/ipex_attn.py
+109
-41
@@ -1,12 +1,17 @@
1
from typing import Dict, List, Optional, Tuple
2
3
-import intel_extension_for_pytorch.llm.modules as ipex_modules
3
+try:
4
+ import intel_extension_for_pytorch.llm.modules as ipex_modules
5
+ _use_ipex = True
6
+except ImportError:
7
+ _use_ipex = False
8
+
9
import torch
10
11
from vllm import _custom_ops as ops
12
13
9
-class PagedAttention:
14
+class _PagedAttention:
15
16
@staticmethod
17
def get_supported_head_sizes() -> List[int]:
@@ -22,6 +27,105 @@ class PagedAttention:
27
) -> Tuple[int, ...]:
28
return (2, num_blocks, block_size * num_kv_heads * head_size)
29
30
+ @staticmethod
31
+ def split_kv_cache(
32
+ kv_cache: torch.Tensor,
33
+ num_kv_heads: int,
34
+ head_size: int,
35
+ *args,
36
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
37
+ x = 16 // kv_cache.element_size()
38
+ num_blocks = kv_cache.shape[1]
39
+
40
+ key_cache = kv_cache[0]
41
+ key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x,
42
+ -1, x)
43
+ value_cache = kv_cache[1]
44
+ value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1)
45
+ return key_cache, value_cache
46
+
47
+ @staticmethod
48
+ def write_to_paged_cache(
49
+ key: torch.Tensor,
50
+ value: torch.Tensor,
51
+ key_cache: torch.Tensor,
52
+ value_cache: torch.Tensor,
53
+ slot_mapping: torch.Tensor,
54
+ kv_cache_dtype: str,
55
+ k_scale: float,
56
+ v_scale: float,
57
+ *args,
58
+ ) -> None:
59
+ ops.reshape_and_cache(
60
+ key,
61
+ value,
62
+ key_cache,
63
+ value_cache,
64
+ slot_mapping.flatten(),
65
+ kv_cache_dtype,
66
+ k_scale,
67
+ v_scale,
68
+ )
69
+
70
+ @staticmethod
71
+ def forward_decode(
72
+ output: torch.Tensor,
73
+ query: torch.Tensor,
74
+ key_cache: torch.Tensor,
75
+ value_cache: torch.Tensor,
76
+ block_tables: torch.Tensor,
77
+ context_lens: torch.Tensor,
78
+ max_context_len: int,
79
+ kv_cache_dtype: str,
80
+ num_kv_heads: int,
81
+ scale: float,
82
+ alibi_slopes: Optional[torch.Tensor],
83
+ k_scale: float,
84
+ v_scale: float,
85
+ *args,
86
+ ) -> None:
87
+ tp_rank: int = 0
88
+ blocksparse_local_blocks: int = 0
89
+ blocksparse_vert_stride: int = 0
90
+ blocksparse_block_size: int = 64
91
+ blocksparse_head_sliding_step: int = 0
92
+ block_size = value_cache.shape[3]
93
+
94
+ ops.paged_attention_v1(
95
+ output,
96
+ query,
97
+ key_cache,
98
+ value_cache,
99
+ num_kv_heads,
100
+ scale,
101
+ block_tables,
102
+ context_lens,
103
+ block_size,
104
+ max_context_len,
105
+ alibi_slopes,
106
+ kv_cache_dtype,
107
+ k_scale,
108
+ v_scale,
109
+ tp_rank,
110
+ blocksparse_local_blocks,
111
+ blocksparse_vert_stride,
112
+ blocksparse_block_size,
113
+ blocksparse_head_sliding_step,
114
+ )
115
+
116
+ @staticmethod
117
+ def copy_blocks(
118
+ kv_caches: List[torch.Tensor],
119
+ src_to_dists: Dict[int, List[int]],
120
+ *args,
121
+ ) -> None:
122
+ key_caches = [kv_cache[0] for kv_cache in kv_caches]
123
+ value_caches = [kv_cache[1] for kv_cache in kv_caches]
124
+ ops.copy_blocks(key_caches, value_caches, src_to_dists)
125
+
126
+
127
+class _IPEXPagedAttention(_PagedAttention):
128
+
129
@staticmethod
130
def split_kv_cache(
131
kv_cache: torch.Tensor,
@@ -55,6 +159,7 @@ class PagedAttention:
159
160
@staticmethod
161
def forward_decode(
162
+ output: torch.Tensor,
163
query: torch.Tensor,
164
key_cache: torch.Tensor,
165
value_cache: torch.Tensor,
@@ -68,8 +173,7 @@ class PagedAttention:
173
k_scale: float,
174
v_scale: float,
175
*args,
71
- ) -> torch.Tensor:
72
- output = torch.empty_like(query)
176
+ ) -> None:
177
block_size = value_cache.shape[2]
178
head_mapping = torch.arange(
179
0,
@@ -83,41 +187,5 @@ class PagedAttention:
187
scale, block_tables, context_lens, block_size, max_context_len,
188
alibi_slopes)
189
86
- return output
87
-
88
- @staticmethod
89
- def forward_prefix(
90
- query: torch.Tensor,
91
- key: torch.Tensor,
92
- value: torch.Tensor,
93
- kv_cache_dtype: str,
94
- key_cache: torch.Tensor,
95
- value_cache: torch.Tensor,
96
- block_tables: torch.Tensor,
97
- subquery_start_loc: torch.Tensor,
98
- prompt_lens_tensor: torch.Tensor,
99
- context_lens: torch.Tensor,
100
- max_subquery_len: int,
101
- alibi_slopes: Optional[torch.Tensor],
102
- *args,
103
- ) -> torch.Tensor:
104
- raise NotImplementedError
105
-
106
- @staticmethod
107
- def swap_blocks(
108
- src_kv_cache: torch.Tensor,
109
- dst_kv_cache: torch.Tensor,
110
- src_to_dst: Dict[int, int],
111
- *args,
112
- ) -> None:
113
- raise NotImplementedError
190
115
- @staticmethod
116
- def copy_blocks(
117
- kv_caches: List[torch.Tensor],
118
- src_to_dists: Dict[int, List[int]],
119
- *args,
120
- ) -> None:
121
- key_caches = [kv_cache[0] for kv_cache in kv_caches]
122
- value_caches = [kv_cache[1] for kv_cache in kv_caches]
123
- ops.copy_blocks(key_caches, value_caches, src_to_dists)
191
+PagedAttention = _IPEXPagedAttention if _use_ipex else _PagedAttention
vllm/platforms/cpu.py
+6
-9
@@ -53,11 +53,6 @@ class CpuPlatform(Platform):
53
54
cache_config = vllm_config.cache_config
55
56
- if cache_config.enable_prefix_caching:
57
- logger.warning(
58
- "Prefix caching is not supported on CPU, disable it.")
59
- cache_config.enable_prefix_caching = False
60
-
56
kv_cache_space = envs.VLLM_CPU_KVCACHE_SPACE
57
58
if kv_cache_space >= 0:
@@ -74,10 +69,12 @@ class CpuPlatform(Platform):
69
f" {kv_cache_space}, expect a positive integer value.")
70
71
scheduler_config = vllm_config.scheduler_config
77
- if scheduler_config.chunked_prefill_enabled:
78
- logger.warning(
79
- "Chunked prefill is not supported on CPU, disable it.")
80
- scheduler_config.chunked_prefill_enabled = False
72
+ if ((scheduler_config.chunked_prefill_enabled
73
+ or cache_config.enable_prefix_caching)
74
+ and model_config.dtype == torch.half):
75
+ logger.warning("Chunked-prefill on the CPU backend only does not"
76
+ " support fp16 for now, cast to bf16.")
77
+ model_config.dtype = torch.bfloat16
78
79
parallel_config = vllm_config.parallel_config
80
if (parallel_config.distributed_executor_backend is not None
vllm/worker/cpu_model_runner.py
+215
-273
@@ -2,8 +2,8 @@ import dataclasses
2
import weakref
3
from collections import defaultdict
4
from dataclasses import dataclass
5
-from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type,
6
- TypeVar, Union)
5
+from typing import (TYPE_CHECKING, Any, Dict, List, Optional, Type, TypeVar,
6
+ Union)
7
8
import torch
9
from torch import nn
@@ -19,7 +19,6 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensorInputs,
19
MultiModalKwargs, MultiModalPlaceholderMap)
20
from vllm.sequence import (IntermediateTensors, SequenceData,
21
SequenceGroupMetadata)
22
-from vllm.utils import make_tensor_with_pad
22
from vllm.worker.model_runner_base import (
23
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
24
_add_attn_metadata_broadcastable_dict,
@@ -104,65 +103,223 @@ class ModelInputForCPUWithSamplingMetadata(ModelInputForCPU):
103
104
class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
105
106
+ class ModelInputData:
107
+
108
+ def __init__(self, use_mrope: bool):
109
+ self.use_mrope = use_mrope
110
+ self.input_tokens: List[int] = []
111
+ self.input_positions: Optional[
112
+ List[int]] = [] if not self.use_mrope else None
113
+ self.seq_lens: List[int] = []
114
+ self.query_lens: List[int] = []
115
+ self.prefill_block_tables: List[List[int]] = []
116
+ self.decode_block_tables: List[List[int]] = []
117
+ self.max_decode_seq_len: int = 0
118
+ self.num_prefills: int = 0
119
+ self.num_prefill_tokens: int = 0
120
+ self.num_decode_tokens: int = 0
121
+ self.slot_mapping: List[int] = []
122
+ self.multi_modal_inputs_list: List[MultiModalKwargs] = []
123
+ self.multi_modal_placeholder_maps: Dict[
124
+ str, MultiModalPlaceholderMap] = defaultdict(
125
+ MultiModalPlaceholderMap)
126
+ self.input_mrope_positions: Optional[List[List[int]]] = [
127
+ [] for _ in range(3)
128
+ ] if self.use_mrope else None
129
+
130
def __init__(self,
131
runner: "CPUModelRunner",
132
finished_requests_ids: Optional[List[str]] = None) -> None:
133
super().__init__()
134
self.seq_group_metadata_list: List[SequenceGroupMetadata] = []
135
self.runner = runner
136
+
137
+ self.chunked_prefill = (runner.scheduler_config.chunked_prefill_enabled
138
+ or runner.cache_config.enable_prefix_caching)
139
self.model_input_cls = self.runner._model_input_cls
140
self.attn_backend = self.runner.attn_backend
115
- self.sliding_window = self.runner.sliding_window
116
- self.block_size = self.runner.block_size
117
- self.device = self.runner.device
141
self.multi_modal_input_mapper = self.runner.multi_modal_input_mapper
142
+ self.input_data = ModelInputForCPUBuilder.ModelInputData(
143
+ self.runner.model_config.uses_mrope)
144
+ self.att_metadata_builder = self.runner.attn_backend.get_builder_cls()(
145
+ self)
146
147
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
148
self.seq_group_metadata_list.append(seq_group_metadata)
149
150
+ def set_seq_group_list(
151
+ self, seq_group_metadata_list: List[SequenceGroupMetadata]):
152
+ self.seq_group_metadata_list = seq_group_metadata_list
153
+
154
def build(self) -> ModelInputForCPU:
155
+ self._build_input_data()
156
+
157
+ input_data = self.input_data
158
+ input_tokens = torch.tensor(input_data.input_tokens,
159
+ dtype=torch.long,
160
+ device="cpu")
161
+ input_positions = torch.tensor(
162
+ input_data.input_positions
163
+ if not input_data.use_mrope else input_data.input_mrope_positions,
164
+ dtype=torch.long,
165
+ device="cpu")
166
+
167
+ # For multi-modal models
168
multi_modal_kwargs = None
125
- # NOTE: We assume that all sequences in the group are all prompts or
126
- # all decodes.
127
- is_prompt = self.seq_group_metadata_list[0].is_prompt
128
- # Prepare input tensors.
129
- if is_prompt:
130
- (input_tokens, input_positions, attn_metadata, seq_lens,
131
- multi_modal_kwargs) = self._prepare_prompt(
132
- self.seq_group_metadata_list)
133
- else:
134
- (input_tokens, input_positions,
135
- attn_metadata) = self._prepare_decode(
136
- self.seq_group_metadata_list)
137
- seq_lens = None
169
+ if len(input_data.multi_modal_inputs_list) != 0:
170
+ multi_modal_kwargs = MultiModalKwargs.batch(
171
+ input_data.multi_modal_inputs_list)
172
+
173
+ attn_metadata = self.att_metadata_builder.build(
174
+ input_data.seq_lens, input_data.query_lens, -1, -1)
175
176
return self.model_input_cls(
177
input_tokens=input_tokens,
178
input_positions=input_positions,
179
+ seq_lens=input_data.seq_lens,
180
+ query_lens=input_data.query_lens,
181
attn_metadata=attn_metadata,
182
multi_modal_kwargs=multi_modal_kwargs,
144
- # query_lens is not needed if chunked prefill is not
145
- # supported. Since CPU worker doesn't support chunked prefill
146
- # just use seq_lens instead.
147
- seq_lens=seq_lens,
148
- query_lens=seq_lens,
183
)
184
151
- def _compute_multi_modal_input(
152
- self,
153
- seq_data: SequenceData,
154
- computed_len: int,
155
- seq_group_metadata: SequenceGroupMetadata,
156
- ):
185
+ def _build_input_data(self):
186
+ for seq_group_metadata in self.seq_group_metadata_list:
187
+ for seq_id, seq_data in seq_group_metadata.seq_data.items():
188
+ if seq_group_metadata.is_prompt:
189
+ self._compute_prompt_input_tokens(self.input_data,
190
+ seq_group_metadata,
191
+ seq_data, seq_id)
192
+ if seq_group_metadata.multi_modal_data:
193
+ self._compute_multi_modal_input(
194
+ seq_group_metadata, seq_data)
195
+ else:
196
+ self._compute_decode_input_tokens(self.input_data,
197
+ seq_group_metadata,
198
+ seq_data, seq_id)
199
+
200
+ def _compute_decode_input_tokens(self, data: ModelInputData,
201
+ seq_group_metadata: SequenceGroupMetadata,
202
+ seq_data: SequenceData, seq_id: int):
203
+ """
204
+ Compute decode input tokens, positions, block table and slot mapping.
205
+ """
206
+ block_size = self.runner.block_size
207
+
208
+ block_table = seq_group_metadata.block_tables[seq_id]
209
+ seq_len = seq_data.get_len()
210
+ context_len = seq_data.get_num_computed_tokens()
211
+
212
+ tokens = seq_data.get_last_token_id()
213
+ token_positions = seq_len - 1
214
+ block_number = block_table[token_positions // block_size]
215
+ block_offset = token_positions % block_size
216
+ slot = block_number * block_size + block_offset
217
+
218
+ # For paged_attention kernel
219
+ if self.runner.sliding_window:
220
+ start_idx = max(0, seq_len - self.runner.sliding_window)
221
+ start_block = start_idx // block_size
222
+ start_idx = start_block * block_size
223
+ seq_len = seq_len - start_idx
224
+ block_table = block_table[start_block:]
225
+
226
+ # For MRotaryEmbedding
227
+ if data.input_positions is None:
228
+ next_pos = MRotaryEmbedding.get_next_input_positions(
229
+ seq_data.mrope_position_delta,
230
+ context_len,
231
+ seq_len,
232
+ )
233
+ for idx in range(3):
234
+ data.input_mrope_positions[idx].extend( # type: ignore
235
+ next_pos[idx])
236
+ else:
237
+ data.input_positions.append(token_positions) # type: ignore
238
+
239
+ # Update fields
240
+ data.input_tokens.append(tokens)
241
+ data.max_decode_seq_len = max(data.max_decode_seq_len, seq_len)
242
+ data.num_decode_tokens += 1
243
+ data.slot_mapping.append(slot)
244
+ data.decode_block_tables.append(block_table)
245
+ data.query_lens.append(1)
246
+ data.seq_lens.append(seq_len)
247
+
248
+ def _compute_prompt_input_tokens(self, data: ModelInputData,
249
+ seq_group_metadata: SequenceGroupMetadata,
250
+ seq_data: SequenceData, seq_id: int):
251
+ """
252
+ Compute prompt input tokens, positions, block table and slot mapping.
253
+ """
254
+ token_chunk_size = seq_group_metadata.token_chunk_size
255
+ block_size = self.runner.block_size
256
+
257
+ block_table = seq_group_metadata.block_tables[seq_id]
258
+ seq_len = seq_data.get_len()
259
+ context_len = seq_data.get_num_computed_tokens()
260
+ seq_len = min(seq_len, context_len + token_chunk_size)
261
+
262
+ # For prefix caching
263
+ prefix_cache_block_num = len(seq_group_metadata.computed_block_nums)
264
+ if prefix_cache_block_num > 0:
265
+ prefix_cache_len = (prefix_cache_block_num *
266
+ self.runner.block_size)
267
+ if prefix_cache_len <= context_len:
268
+ # We already passed the cache hit region,
269
+ # so do normal computation.
270
+ pass
271
+ elif context_len < prefix_cache_len < seq_len:
272
+ # Partial hit. Compute the missing part.
273
+ context_len = prefix_cache_len
274
+ token_chunk_size = seq_len - context_len
275
+ elif seq_len <= prefix_cache_len:
276
+ # Full hit. Only compute the last token to avoid
277
+ # erroneous behavior. FIXME: Ideally we should directly
278
+ # mark all tokens as computed in the scheduler and do not
279
+ # schedule this sequence, so this case should not happen.
280
+ context_len = seq_len - 1
281
+ token_chunk_size = 1
282
+
283
+ tokens = seq_data.get_token_ids()
284
+ tokens = tokens[context_len:seq_len]
285
+ token_positions = range(context_len, seq_len)
286
+
287
+ # For encoder-only models, the block_table is None,
288
+ # and there is no need to initialize the slot_mapping.
289
+ if block_table is not None:
290
+ slot_mapping = [_PAD_SLOT_ID] * len(token_positions)
291
+ for i, pos in enumerate(token_positions):
292
+ block_number = block_table[pos // block_size]
293
+ block_offset = pos % block_size
294
+ slot = block_number * block_size + block_offset
295
+ slot_mapping[i] = slot
296
+ data.slot_mapping.extend(slot_mapping)
297
+
298
+ # The MROPE positions are prepared in _compute_multi_modal_input
299
+ if data.input_positions is not None:
300
+ data.input_positions.extend(token_positions)
301
+
302
+ # Update fields
303
+ data.input_tokens.extend(tokens)
304
+ data.num_prefills += 1
305
+ data.num_prefill_tokens += len(tokens)
306
+ data.query_lens.append(len(tokens))
307
+ data.prefill_block_tables.append(block_table)
308
+ data.seq_lens.append(seq_len)
309
+
310
+ def _compute_multi_modal_input(self,
311
+ seq_group_metadata: SequenceGroupMetadata,
312
+ seq_data: SequenceData):
313
+ computed_len = seq_data.get_num_computed_tokens()
314
+ seq_len = self.input_data.seq_lens[-1]
315
+
316
# NOTE: mm_data only includes the subset of multi-modal items that
317
# intersect with the current prefill positions.
318
mm_data, placeholder_maps = MultiModalPlaceholderMap.from_seq_group(
160
- seq_group_metadata,
161
- range(computed_len, len(seq_data.get_token_ids())),
162
- )
319
+ seq_group_metadata, range(computed_len, seq_len))
320
321
if not mm_data:
165
- return None, None, None
322
+ return
323
324
if self.runner.mm_registry.has_processor(self.runner.model_config):
325
mm_kwargs = mm_data
@@ -173,8 +330,10 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
330
)
331
332
# special processing for mrope position deltas.
176
- mrope_positions = None
333
if self.runner.model_config.uses_mrope:
334
+ assert not self.chunked_prefill, \
335
+ "MROPE on CPU does not support chunked-prefill."
336
+
337
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
338
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
339
assert image_grid_thw is not None or video_grid_thw is not None, (
@@ -198,226 +357,15 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
357
context_len=computed_len,
358
)
359
seq_data.mrope_position_delta = mrope_position_delta
201
- return mm_kwargs, placeholder_maps, mrope_positions
360
203
- def _prepare_prompt(
204
- self,
205
- seq_group_metadata_list: List[SequenceGroupMetadata],
206
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
207
- BatchedTensorInputs]:
208
- assert len(seq_group_metadata_list) > 0
209
- input_tokens: List[int] = []
210
- input_positions: List[int] = []
211
- input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
212
-
213
- slot_mapping: List[int] = []
214
- seq_lens: List[int] = []
215
- multi_modal_kwargs_list: List[MultiModalKwargs] = []
216
- multi_modal_placeholder_maps: Dict[
217
- str,
218
- MultiModalPlaceholderMap] = defaultdict(MultiModalPlaceholderMap)
219
-
220
- for seq_group_metadata in seq_group_metadata_list:
221
- assert seq_group_metadata.is_prompt
222
- seq_ids = list(seq_group_metadata.seq_data.keys())
223
- assert len(seq_ids) == 1
224
- seq_id = seq_ids[0]
225
-
226
- seq_data = seq_group_metadata.seq_data[seq_id]
227
- prompt_tokens = seq_data.get_token_ids()
228
- computed_len = seq_data.get_num_computed_tokens()
229
- seq_len = len(prompt_tokens)
230
-
231
- seq_lens.append(seq_len) # Prompt token num
232
- input_tokens.extend(prompt_tokens) # Token ids
233
-
234
- mrope_positions = None
235
- if seq_group_metadata.multi_modal_data:
236
- (
237
- mm_kwargs,
238
- placeholder_maps,
239
- mrope_positions,
240
- ) = self._compute_multi_modal_input(seq_data, computed_len,
241
- seq_group_metadata)
242
-
243
- multi_modal_kwargs_list.append(mm_kwargs)
244
- for modality, placeholder_map in placeholder_maps.items():
245
- multi_modal_placeholder_maps[modality].extend(
246
- placeholder_map)
247
-
248
- # Token position ids
249
- # NOTE(woosuk): Here we assume that the first token in the prompt
250
- # is always the first token in the sequence.
251
- if mrope_positions:
252
- for idx in range(3):
253
- input_mrope_positions[idx].extend(mrope_positions[idx])
254
- else:
255
- input_positions.extend(list(range(computed_len, seq_len)))
256
-
257
- # Compute the slot mapping.
258
- block_table = seq_group_metadata.block_tables[seq_id]
259
- # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
260
- # where start_idx is max(0, seq_len - sliding_window).
261
- # For example, if the prompt len is 10, sliding window is 8, and
262
- # block size is 4, the first two tokens are masked and the slot
263
- # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
264
- start_idx = 0
265
- if self.sliding_window is not None:
266
- start_idx = max(0, seq_len - self.sliding_window)
267
-
268
- for i in range(computed_len, seq_len):
269
- if i < start_idx:
270
- slot_mapping.append(_PAD_SLOT_ID)
271
- continue
272
-
273
- # For encoder-only models, the block_table is None,
274
- # and there is no need to initialize the slot_mapping.
275
- if block_table is not None:
276
- block_number = block_table[i //
277
- self.block_size] # type: ignore
278
- block_offset = i % self.block_size # type: ignore
279
- slot = block_number * self.block_size + block_offset
280
- slot_mapping.append(slot)
281
-
282
- if any(input_mrope_positions):
283
- input_positions = None # type: ignore
284
- else:
285
- input_mrope_positions = None # type: ignore
361
+ for i in range(3):
362
+ self.input_data.input_mrope_positions[ # type: ignore
363
+ i].extend(mrope_positions[i])
364
287
- num_prompt_tokens = len(input_tokens)
288
-
289
- input_tokens = torch.tensor(input_tokens,
290
- dtype=torch.long,
291
- device=self.device) # type: ignore
292
- input_positions = torch.tensor(input_positions
293
- or input_mrope_positions,
294
- dtype=torch.long,
295
- device=self.device) # type: ignore
296
- slot_mapping = torch.tensor(slot_mapping,
297
- dtype=torch.long,
298
- device=self.device) # type: ignore
299
- placeholder_index_maps = {
300
- modality: placeholder_map.index_map()
301
- for modality, placeholder_map in
302
- multi_modal_placeholder_maps.items()
303
- }
304
-
305
- attn_metadata = self.attn_backend.make_metadata(
306
- is_prompt=True,
307
- seq_lens=seq_lens,
308
- seq_lens_tensor=torch.tensor([]),
309
- max_decode_seq_len=0,
310
- num_prefills=len(seq_lens),
311
- num_prefill_tokens=num_prompt_tokens,
312
- num_decode_tokens=0,
313
- block_tables=torch.tensor([]),
314
- slot_mapping=slot_mapping,
315
- multi_modal_placeholder_index_maps=placeholder_index_maps,
316
- )
317
-
318
- multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
319
-
320
- return (input_tokens, input_positions, attn_metadata, seq_lens,
321
- multi_modal_kwargs)
322
-
323
- def _prepare_decode(
324
- self,
325
- seq_group_metadata_list: List[SequenceGroupMetadata],
326
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata]:
327
- assert len(seq_group_metadata_list) > 0
328
- input_tokens: List[int] = []
329
- input_positions: List[int] = []
330
- input_mrope_positions: List[List[int]] = [[] for _ in range(3)]
331
- slot_mapping: List[int] = []
332
- seq_lens: List[int] = []
333
- block_tables: List[List[int]] = []
334
-
335
- for seq_group_metadata in seq_group_metadata_list:
336
- assert not seq_group_metadata.is_prompt
337
- assert seq_group_metadata.token_chunk_size == 1
338
-
339
- seq_ids = list(seq_group_metadata.seq_data.keys())
340
-
341
- for seq_id in seq_ids:
342
- seq_data = seq_group_metadata.seq_data[seq_id]
343
- generation_token = seq_data.get_last_token_id()
344
- input_tokens.append(generation_token)
345
-
346
- seq_len = seq_data.get_len()
347
- position = seq_len - 1
348
- if seq_data.mrope_position_delta is not None:
349
- context_len = seq_data.get_num_computed_tokens()
350
- next_pos = MRotaryEmbedding.get_next_input_positions(
351
- seq_data.mrope_position_delta,
352
- context_len,
353
- seq_len,
354
- )
355
- for idx in range(3):
356
- input_mrope_positions[idx].extend(next_pos[idx])
357
- else:
358
- input_positions.append(position)
359
-
360
- seq_len = seq_len if self.sliding_window is None else min(
361
- seq_len, self.sliding_window)
362
- seq_lens.append(seq_len)
363
-
364
- block_table = seq_group_metadata.block_tables[seq_id]
365
- block_number = block_table[position // self.block_size]
366
- block_offset = position % self.block_size
367
- slot = block_number * self.block_size + block_offset
368
- slot_mapping.append(slot)
369
-
370
- if self.sliding_window is not None:
371
- sliding_window_blocks = (self.sliding_window //
372
- self.block_size)
373
- block_table = block_table[-sliding_window_blocks:]
374
- block_tables.append(block_table)
375
-
376
- if any(input_mrope_positions):
377
- input_positions = None # type: ignore
378
- else:
379
- input_mrope_positions = None # type: ignore
380
-
381
- max_decode_seq_len = max(seq_lens)
382
-
383
- input_tokens = torch.tensor(input_tokens,
384
- dtype=torch.long,
385
- device=self.device)
386
- input_positions = torch.tensor(input_positions
387
- or input_mrope_positions,
388
- dtype=torch.long,
389
- device=self.device)
390
- slot_mapping = torch.tensor(slot_mapping,
391
- dtype=torch.long,
392
- device=self.device)
393
- seq_lens_tensor = torch.tensor(seq_lens,
394
- dtype=torch.int,
395
- device=self.device)
396
-
397
- block_tables = make_tensor_with_pad(
398
- block_tables,
399
- pad=0,
400
- dtype=torch.int,
401
- device=self.device,
402
- )
403
-
404
- attn_metadata = self.attn_backend.make_metadata(
405
- is_prompt=False,
406
- slot_mapping=slot_mapping,
407
- multi_modal_placeholder_index_maps=None,
408
- seq_lens=seq_lens,
409
- seq_lens_tensor=seq_lens_tensor,
410
- max_decode_seq_len=max_decode_seq_len,
411
- num_prefill_tokens=0,
412
- num_decode_tokens=len(input_tokens),
413
- num_prefills=0,
414
- block_tables=block_tables,
415
- )
416
- return (
417
- input_tokens,
418
- input_positions,
419
- attn_metadata,
420
- )
365
+ self.input_data.multi_modal_inputs_list.append(mm_kwargs)
366
+ for modality, placeholder_map in placeholder_maps.items():
367
+ self.input_data.multi_modal_placeholder_maps[modality].extend(
368
+ placeholder_map)
369
370
371
class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
@@ -436,8 +384,6 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
384
**kwargs,
385
):
386
ModelRunnerBase.__init__(self, vllm_config)
439
- # Currently, CPU worker doesn't support chunked prefill.
440
- assert self.scheduler_config.chunked_prefill_enabled is False
387
model_config = self.model_config
388
cache_config = self.cache_config
389
@@ -479,8 +425,7 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
425
426
"""
427
builder = self._builder_cls(weakref.proxy(self), finished_requests_ids)
482
- for seq_group_metadata in seq_group_metadata_list:
483
- builder.add_seq_group(seq_group_metadata)
428
+ builder.set_seq_group_list(seq_group_metadata_list)
429
430
return builder.build() # type: ignore
431
@@ -537,22 +482,19 @@ class CPUModelRunner(CPUModelRunnerBase[ModelInputForCPUWithSamplingMetadata]):
482
"CPU worker does not support multi-step execution.")
483
484
model_executable = self.model
540
- execute_model_kwargs = {
541
- "input_ids":
542
- model_input.input_tokens,
543
- "positions":
544
- model_input.input_positions,
545
- "kv_caches":
546
- kv_caches,
547
- "attn_metadata":
548
- model_input.attn_metadata,
549
- **MultiModalKwargs.as_kwargs(model_input.multi_modal_kwargs or {},
550
- device=self.device),
551
- "intermediate_tensors":
552
- intermediate_tensors,
553
- }
554
-
555
- hidden_states = model_executable(**execute_model_kwargs)
485
+ multimodal_kwargs = {}
486
+ if model_input.multi_modal_kwargs is not None:
487
+ multimodal_kwargs = MultiModalKwargs.as_kwargs(
488
+ model_input.multi_modal_kwargs, device=self.device)
489
+
490
+ hidden_states = model_executable(
491
+ input_ids=model_input.input_tokens,
492
+ positions=model_input.input_positions,
493
+ kv_caches=kv_caches,
494
+ attn_metadata=model_input.attn_metadata,
495
+ intermediate_tensors=intermediate_tensors,
496
+ **multimodal_kwargs,
497
+ )
498
499
# Compute the logits.
500
logits = self.model.compute_logits(hidden_states,