A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core][2/N] Model runner refactoring part 2. Combine prepare prefill / decode to a single API (#4681)
This PR combines prepare_prompt and prepare_decode into a single API. This PR also coelsce the attn metadata for prefill/decode to a single class and allow to slice them when running attn backend. It also refactors subquery_start_loc which was not refactored in the previous PR
SangBin Cho committed
May 15, 2024 at 14:00 UTC
65bf2ac165734fb6339210c4b2b8ce68d2391b77
18 files changed
+777
-730
tests/worker/test_model_runner.py
+84
-39
@@ -58,19 +58,25 @@ def test_prepare_prompt(batch_size):
58
expected_selected_token_indices.append(selected_token_start_idx +
59
seq_len - 1)
60
selected_token_start_idx += seq_len
61
- (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
62
- _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
61
+ model_input = model_runner._prepare_model_input(seq_group_metadata_list)
62
+ input_tokens = model_input.input_tokens
63
+ input_positions = model_input.input_positions
64
+ attn_metadata = model_input.attn_metadata
65
+ return_seq_lens = model_input.seq_lens
66
+ slot_mapping = model_input.slot_mapping
67
assert return_seq_lens == seq_lens
68
assert len(slot_mapping) == len(input_tokens)
69
70
# Verify input metadata is correct for prompts.
71
device = model_runner.device
68
- assert attn_metadata.is_prompt is True
72
+ assert attn_metadata.num_prefills > 0
73
+ assert attn_metadata.num_decode_tokens == 0
74
assert torch.allclose(
75
attn_metadata.seq_lens_tensor,
76
torch.tensor(seq_lens, device=device, dtype=torch.int))
77
assert attn_metadata.seq_lens == seq_lens
73
- assert attn_metadata.max_seq_len == max(seq_lens)
78
+ assert attn_metadata.max_prefill_seq_len == max(seq_lens)
79
+ assert attn_metadata.max_decode_seq_len == 0
80
81
# Test subquery start locs.
82
start_idx = 0
@@ -79,11 +85,11 @@ def test_prepare_prompt(batch_size):
85
start_idx += seq_len
86
start_loc.append(start_idx)
87
assert torch.allclose(
82
- attn_metadata.subquery_start_loc,
88
+ attn_metadata.query_start_loc,
89
torch.tensor(start_loc, dtype=torch.int32, device=device))
90
91
# Test seq start locs. Note that for normal prefill it is
86
- # equivalent to subquery_start_loc.
92
+ # equivalent to query_start_loc.
93
start_idx = 0
94
seq_start_loc = [start_idx]
95
for seq_len in seq_lens:
@@ -123,7 +129,7 @@ def test_prepare_prompt(batch_size):
129
device=actual.device,
130
dtype=actual.dtype)
131
torch.testing.assert_close(actual, expected)
126
- assert input_tokens == input_positions
132
+ torch.allclose(input_tokens, input_positions)
133
134
actual = sampling_metadata.selected_token_indices
135
expected = torch.tensor(expected_selected_token_indices,
@@ -144,14 +150,18 @@ def test_prepare_decode_cuda_graph(batch_size):
150
enable_chunked_prefill=False,
151
)
152
147
- seq_lens = []
153
+ context_lens = []
154
seq_group_metadata_list = []
155
+ # Assume each seq group finishes prefill.
156
for i in range(batch_size):
157
# make sure all tokens fit into one block
151
- seq_len = i % (model_runner.block_size - 1) + 1
152
- seq_lens.append(seq_len)
153
- seq_data = list(range(seq_len))
158
+ context_len = i % (model_runner.block_size - 1) + 1
159
+ context_lens.append(context_len)
160
+ seq_data = list(range(context_len))
161
seq_data = SequenceData(seq_data)
162
+ seq_data.update_num_computed_tokens(context_len)
163
+ # Append one token ID since prefill is finished.
164
+ seq_data.append_token_id(1, 0)
165
seq_group_metadata = SequenceGroupMetadata(
166
request_id=f"test_{i}",
167
is_prompt=False,
@@ -162,18 +172,45 @@ def test_prepare_decode_cuda_graph(batch_size):
172
assert seq_group_metadata.token_chunk_size == 1
173
seq_group_metadata_list.append(seq_group_metadata)
174
165
- input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
166
- model_runner._prepare_decode(seq_group_metadata_list))
175
+ model_input = model_runner._prepare_model_input(seq_group_metadata_list)
176
+ input_tokens, input_positions, attn_metadata, slot_mapping = (
177
+ model_input.input_tokens, model_input.input_positions,
178
+ model_input.attn_metadata, model_input.slot_mapping)
179
assert len(slot_mapping) == len(input_tokens)
180
181
expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
182
# Verify input metadata is correct for prompts.
183
device = model_runner.device
172
- assert attn_metadata.is_prompt is False
173
- assert attn_metadata.seq_lens is None
174
- assert attn_metadata.subquery_start_loc is None
175
- assert attn_metadata.seq_start_loc is None
176
- assert attn_metadata.max_seq_len == max(seq_lens)
184
+ assert attn_metadata.num_prefills == 0
185
+ assert attn_metadata.num_prefill_tokens == 0
186
+ seq_lens = [context_len + 1 for context_len in context_lens]
187
+ # seq_lens are padded to expected_bs
188
+ for _ in range(expected_bs - len(seq_lens)):
189
+ seq_lens.append(1)
190
+ assert attn_metadata.seq_lens == seq_lens
191
+ start_idx = 0
192
+ start_loc = [start_idx]
193
+ for _ in context_lens:
194
+ # decode has only 1 token for query.
195
+ start_idx += 1
196
+ start_loc.append(start_idx)
197
+ assert torch.allclose(
198
+ attn_metadata.query_start_loc,
199
+ torch.tensor(start_loc, dtype=torch.int32, device=device))
200
+
201
+ start_idx = 0
202
+ seq_start_loc = [start_idx]
203
+ for seq_len in seq_lens:
204
+ start_idx += seq_len
205
+ seq_start_loc.append(start_idx)
206
+ assert torch.allclose(
207
+ attn_metadata.seq_start_loc,
208
+ torch.tensor(seq_start_loc, dtype=torch.int32, device=device))
209
+
210
+ assert torch.allclose(
211
+ attn_metadata.context_lens_tensor,
212
+ torch.tensor(context_lens, dtype=torch.int, device=device))
213
+ assert attn_metadata.max_decode_seq_len == max(seq_lens)
214
assert torch.allclose(
215
attn_metadata.seq_lens_tensor[:len(seq_lens)],
216
torch.tensor(seq_lens, dtype=torch.int, device=device))
@@ -185,23 +222,23 @@ def test_prepare_decode_cuda_graph(batch_size):
222
# It is padded up to
223
assert attn_metadata.block_tables.shape[1] == (
224
model_runner.get_max_block_per_batch())
188
- # Cuda graph should not be used for prerill.
225
assert attn_metadata.use_cuda_graph is True
226
227
assert len(input_tokens) == expected_bs
228
assert len(input_positions) == expected_bs
193
- assert input_tokens == input_positions
229
+ torch.allclose(input_tokens, input_positions)
230
231
# Verify Sampling
232
expected_selected_token_indices = []
233
selected_token_start_idx = 0
198
- for seq_len in seq_lens:
234
+ for _ in context_lens:
235
expected_selected_token_indices.append(selected_token_start_idx)
236
selected_token_start_idx += 1
237
sampling_metadata = SamplingMetadata.prepare(
238
seq_group_metadata_list,
239
seq_lens,
204
- query_lens=seq_lens,
240
+ # query lens is all 1 for decode.
241
+ query_lens=[1 for _ in range(len(context_lens))],
242
device=model_runner.device,
243
pin_memory=model_runner.pin_memory)
244
actual = sampling_metadata.selected_token_indices
@@ -220,15 +257,27 @@ def test_empty_seq_group():
257
enforce_eager=False,
258
)
259
seq_group_metadata_list = []
223
- input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
224
- model_runner._prepare_decode(seq_group_metadata_list))
260
+ model_input = model_runner._prepare_model_input(seq_group_metadata_list)
261
+ input_tokens, input_positions, attn_metadata, slot_mapping = (
262
+ model_input.input_tokens,
263
+ model_input.input_positions,
264
+ model_input.attn_metadata,
265
+ model_input.slot_mapping,
266
+ )
267
assert len(input_tokens) == 0
268
assert len(input_positions) == 0
269
assert attn_metadata is None
270
assert len(slot_mapping) == 0
271
230
- (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
231
- _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
272
+ model_input = model_runner._prepare_model_input(seq_group_metadata_list)
273
+ (input_tokens, input_positions, attn_metadata, slot_mapping,
274
+ return_seq_lens) = (
275
+ model_input.input_tokens,
276
+ model_input.input_positions,
277
+ model_input.attn_metadata,
278
+ model_input.slot_mapping,
279
+ model_input.seq_lens,
280
+ )
281
assert len(input_tokens) == 0
282
assert len(input_positions) == 0
283
assert attn_metadata is None
@@ -285,9 +334,11 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
334
# Add decode requests
335
for i in range(prefill_batch_size, batch_size):
336
# make sure all tokens fit into one block
288
- seq_len = i % (model_runner.block_size - 1) + 1
289
- prompt_toks = list(range(seq_len))
337
+ context_len = i % (model_runner.block_size - 1) + 1
338
+ prompt_toks = list(range(context_len))
339
seq_data = SequenceData(prompt_toks)
340
+ seq_data.append_token_id(1, 0)
341
+ seq_data.update_num_computed_tokens(context_len)
342
seq_group_metadata = SequenceGroupMetadata(
343
request_id=f"test_{i}",
344
is_prompt=False,
@@ -308,23 +359,17 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
359
assert len(attn_metadata.slot_mapping) == len(input_tokens)
360
assert len(input_positions) == len(input_tokens)
361
assert attn_metadata.num_prefills == prefill_batch_size
311
- if enforce_eager:
312
- assert attn_metadata.num_decode_tokens == decode_batch_size
313
- else:
314
- assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
315
- decode_batch_size)
362
+ assert attn_metadata.num_decode_tokens == decode_batch_size
363
assert attn_metadata.num_prefill_tokens == sum(seq_lens)
364
365
# Verify attn metadata is consistent. We don't need to test individual
366
# values here because they are tested above.
320
- prefill_meta = model_runner._prepare_prompt(
321
- prefill_metadata_list).attn_metadata
322
- decode_meta = model_runner._prepare_decode(
323
- decode_metadata_list).attn_metadata
367
+ attn_metadata = model_runner._prepare_model_input(
368
+ seq_group_metadata_list).attn_metadata
369
325
- for attr_expected, attr_actual in zip(vars(prefill_meta),
370
+ for attr_expected, attr_actual in zip(vars(attn_metadata.prefill_metadata),
371
vars(prefill_meta_actual)):
372
assert attr_expected[1] == attr_actual[1]
328
- for attr_expected, attr_actual in zip(vars(decode_meta),
373
+ for attr_expected, attr_actual in zip(vars(attn_metadata.decode_metadata),
374
vars(decode_meta_actual)):
375
assert attr_expected[1] == attr_actual[1]
vllm/attention/__init__.py
+2
-3
@@ -1,6 +1,5 @@
1
from vllm.attention.backends.abstract import (AttentionBackend,
2
- AttentionMetadata,
3
- AttentionMetadataPerStage)
2
+ AttentionMetadata)
3
from vllm.attention.layer import Attention
4
from vllm.attention.selector import get_attn_backend
5
@@ -8,6 +7,6 @@ __all__ = [
7
"Attention",
8
"AttentionBackend",
9
"AttentionMetadata",
11
- "AttentionMetadataPerStage",
10
+ "Attention",
11
"get_attn_backend",
12
]
vllm/attention/backends/abstract.py
+32
-36
@@ -21,7 +21,7 @@ class AttentionBackend(ABC):
21
22
@staticmethod
23
@abstractmethod
24
- def make_metadata(*args, **kwargs) -> "AttentionMetadataPerStage":
24
+ def make_metadata(*args, **kwargs) -> "AttentionMetadata":
25
raise NotImplementedError
26
27
@staticmethod
@@ -53,8 +53,34 @@ class AttentionBackend(ABC):
53
54
55
@dataclass
56
-class AttentionMetadataPerStage:
57
- """Attention metadata for a specific stage. I.e., prefill or decode."""
56
+class AttentionMetadata:
57
+ """Attention metadata for prefill and decode batched together."""
58
+ # Total number of prefill requests.
59
+ num_prefills: int
60
+ # Number of prefill tokens.
61
+ num_prefill_tokens: int
62
+ # Number of decode tokens. Note that it is equivalent to the number of
63
+ # decode requests.
64
+ num_decode_tokens: int
65
+ # (num_tokens,). The indices of the token slots that input tokens will be
66
+ # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
67
+ # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
68
+ # in block 0, and 1st slot in block 1, respectively.
69
+ slot_mapping: torch.Tensor
70
+
71
+ @property
72
+ @abstractmethod
73
+ def prefill_metadata(self) -> Optional["AttentionMetadata"]:
74
+ """Return the attention metadata that's required to run prefill
75
+ attention."""
76
+ pass
77
+
78
+ @property
79
+ @abstractmethod
80
+ def decode_metadata(self) -> Optional["AttentionMetadata"]:
81
+ """Return the attention metadata that's required to run decode
82
+ attention."""
83
+ pass
84
85
def asdict_zerocopy(self,
86
skip_fields: Optional[Set[str]] = None
@@ -70,40 +96,10 @@ class AttentionMetadataPerStage:
96
}
97
98
73
-T = TypeVar("T", bound=AttentionMetadataPerStage)
74
-
75
-
76
-@dataclass
77
-class AttentionMetadata(Generic[T]):
78
- """Attention metadata for prefill and decode batched together."""
79
- # Total number of prefill requests.
80
- num_prefills: int
81
- # Number of prefill tokens.
82
- num_prefill_tokens: int
83
- # Number of decode tokens. Note that it is equivalent to the number of
84
- # decode requests.
85
- num_decode_tokens: int
86
- # The attention metadata for prefill requests in a batch.
87
- # None if there's no prefill requests in a batch.
88
- prefill_metadata: Optional[T]
89
- # The attention metadata for decode requests in a batch.
90
- # None if there's no decode requests in a batch.
91
- decode_metadata: Optional[T]
92
- # (num_tokens,). The indices of the token slots that input tokens will be
93
- # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
94
- # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
95
- # in block 0, and 1st slot in block 1, respectively.
96
- slot_mapping: torch.Tensor
97
-
98
- def __post_init__(self):
99
- if self.num_prefill_tokens > 0:
100
- assert self.num_prefills > 0
101
- assert self.prefill_metadata is not None
102
- if self.num_decode_tokens > 0:
103
- assert self.decode_metadata is not None
99
+T = TypeVar("T", bound=AttentionMetadata)
100
101
106
-class AttentionImpl(ABC):
102
+class AttentionImpl(ABC, Generic[T]):
103
104
@abstractmethod
105
def __init__(
@@ -125,7 +121,7 @@ class AttentionImpl(ABC):
121
key: torch.Tensor,
122
value: torch.Tensor,
123
kv_cache: torch.Tensor,
128
- attn_metadata: AttentionMetadata,
124
+ attn_metadata: T,
125
kv_scale: float = 1.0,
126
) -> torch.Tensor:
127
raise NotImplementedError
vllm/attention/backends/flash_attn.py
+79
-16
@@ -11,8 +11,7 @@ import torch
11
from vllm_flash_attn import flash_attn_varlen_func
12
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
- AttentionMetadata,
15
- AttentionMetadataPerStage)
14
+ AttentionMetadata)
15
from vllm.attention.ops.paged_attn import (PagedAttention,
16
PagedAttentionMetadata)
17
@@ -58,8 +57,7 @@ class FlashAttentionBackend(AttentionBackend):
57
58
59
@dataclass
61
-class FlashAttentionMetadata(AttentionMetadataPerStage,
62
- PagedAttentionMetadata):
60
+class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
61
"""Metadata for FlashAttentionBackend.
62
63
NOTE: Any python object stored here is not updated when it is
@@ -67,9 +65,6 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
65
dynamically, it should be stored in tensor. The tensor has to be
66
updated from `CUDAGraphRunner.forward` API.
67
"""
70
- # Currently, input sequences can only contain all prompts
71
- # or all decoding. True if all sequences are prompts.
72
- is_prompt: bool
68
# (batch_size,). The sequence length per sequence. Sequence length means
69
# the computed tokens + new tokens None if it is a decoding.
70
seq_lens: Optional[List[int]]
@@ -84,14 +79,18 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
79
# |-------------------- seq_len ----------------------|
80
# |-- query_len ---|
81
87
- # Maximum query length in the batch.
82
+ # Maximum query length in the batch. None for decoding.
83
max_query_len: Optional[int]
89
- # Maximum sequence length in the batch.
90
- max_seq_len: Optional[int]
84
+ # Maximum sequence length among prefill batch. 0 if there are decoding
85
+ # requests only.
86
+ max_prefill_seq_len: int
87
+ # Maximum sequence length among decode batch. 0 if there are prefill
88
+ # requests only.
89
+ max_decode_seq_len: int
90
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
91
# the batch, used to index into subquery. E.g., if the subquery length
92
# is [4, 6], it is [0, 4, 10].
94
- subquery_start_loc: Optional[torch.Tensor]
93
+ query_start_loc: Optional[torch.Tensor]
94
# (batch_size + 1,). The cumulative sequence lengths of the sequences in
95
# the batch, used to index into sequence. E.g., if the sequence length is
96
# [4, 6], it is [0, 4, 10].
@@ -105,6 +104,70 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
104
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
105
use_cuda_graph: bool
106
107
+ _cached_prefill_metadata: Optional["FlashAttentionMetadata"] = None
108
+ _cached_decode_metadata: Optional["FlashAttentionMetadata"] = None
109
+
110
+ @property
111
+ def prefill_metadata(self) -> Optional["FlashAttentionMetadata"]:
112
+ if self.num_prefills == 0:
113
+ return None
114
+
115
+ if self._cached_prefill_metadata is not None:
116
+ return self._cached_prefill_metadata
117
+
118
+ assert self.seq_lens is not None
119
+ assert self.seq_lens_tensor is not None
120
+ assert self.query_start_loc is not None
121
+ assert self.context_lens_tensor is not None
122
+ assert self.block_tables is not None
123
+ assert self.seq_start_loc is not None
124
+
125
+ self._cached_prefill_metadata = FlashAttentionMetadata(
126
+ num_prefills=self.num_prefills,
127
+ num_prefill_tokens=self.num_prefill_tokens,
128
+ num_decode_tokens=0,
129
+ slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
130
+ seq_lens=self.seq_lens[:self.num_prefills],
131
+ seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
132
+ max_query_len=self.max_query_len,
133
+ max_prefill_seq_len=self.max_prefill_seq_len,
134
+ max_decode_seq_len=0,
135
+ query_start_loc=self.query_start_loc[:self.num_prefills + 1],
136
+ seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
137
+ context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
138
+ block_tables=self.block_tables[:self.num_prefills],
139
+ use_cuda_graph=False,
140
+ )
141
+ return self._cached_prefill_metadata
142
+
143
+ @property
144
+ def decode_metadata(self) -> Optional["FlashAttentionMetadata"]:
145
+ if self.num_decode_tokens == 0:
146
+ return None
147
+
148
+ if self._cached_decode_metadata is not None:
149
+ return self._cached_decode_metadata
150
+ assert self.block_tables is not None
151
+ assert self.seq_lens_tensor is not None
152
+
153
+ self._cached_decode_metadata = FlashAttentionMetadata(
154
+ num_prefills=0,
155
+ num_prefill_tokens=0,
156
+ num_decode_tokens=self.num_decode_tokens,
157
+ slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
158
+ seq_lens=None,
159
+ seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
160
+ max_query_len=None,
161
+ max_prefill_seq_len=0,
162
+ max_decode_seq_len=self.max_decode_seq_len,
163
+ query_start_loc=None,
164
+ seq_start_loc=None,
165
+ context_lens_tensor=None,
166
+ block_tables=self.block_tables[self.num_prefills:],
167
+ use_cuda_graph=self.use_cuda_graph,
168
+ )
169
+ return self._cached_decode_metadata
170
+
171
172
class FlashAttentionImpl(AttentionImpl):
173
"""
@@ -168,7 +231,7 @@ class FlashAttentionImpl(AttentionImpl):
231
key: torch.Tensor,
232
value: torch.Tensor,
233
kv_cache: torch.Tensor,
171
- attn_metadata: AttentionMetadata[FlashAttentionMetadata],
234
+ attn_metadata: FlashAttentionMetadata,
235
kv_scale: float = 1.0,
236
) -> torch.Tensor:
237
"""Forward pass with FlashAttention and PagedAttention.
@@ -228,8 +291,8 @@ class FlashAttentionImpl(AttentionImpl):
291
v=value,
292
cu_seqlens_q=prefill_meta.seq_start_loc,
293
cu_seqlens_k=prefill_meta.seq_start_loc,
231
- max_seqlen_q=prefill_meta.max_seq_len,
232
- max_seqlen_k=prefill_meta.max_seq_len,
294
+ max_seqlen_q=prefill_meta.max_prefill_seq_len,
295
+ max_seqlen_k=prefill_meta.max_prefill_seq_len,
296
softmax_scale=self.scale,
297
causal=True,
298
window_size=self.sliding_window,
@@ -249,7 +312,7 @@ class FlashAttentionImpl(AttentionImpl):
312
key_cache,
313
value_cache,
314
prefill_meta.block_tables,
252
- prefill_meta.subquery_start_loc,
315
+ prefill_meta.query_start_loc,
316
prefill_meta.seq_lens_tensor,
317
prefill_meta.context_lens_tensor,
318
prefill_meta.max_query_len,
@@ -264,7 +327,7 @@ class FlashAttentionImpl(AttentionImpl):
327
value_cache,
328
decode_meta.block_tables,
329
decode_meta.seq_lens_tensor,
267
- decode_meta.max_seq_len,
330
+ decode_meta.max_decode_seq_len,
331
self.kv_cache_dtype,
332
self.num_kv_heads,
333
self.scale,
vllm/attention/backends/flashinfer.py
+28
-10
@@ -8,8 +8,7 @@ from vllm_flash_attn import flash_attn_varlen_func
8
9
from vllm import _custom_ops as ops
10
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11
- AttentionMetadata,
12
- AttentionMetadataPerStage)
11
+ AttentionMetadata)
12
13
14
class FlashInferBackend(AttentionBackend):
@@ -56,9 +55,10 @@ class FlashInferBackend(AttentionBackend):
55
56
57
@dataclass
59
-class FlashInferMetadata(AttentionMetadataPerStage):
60
-
61
- is_prompt: bool
58
+class FlashInferMetadata(AttentionMetadata):
59
+ # Maximum sequence length among prefill batch. 0 if there are decoding
60
+ # requests only.
61
+ max_prefill_seq_len: int
62
63
use_cuda_graph: bool = False
64
@@ -67,7 +67,6 @@ class FlashInferMetadata(AttentionMetadataPerStage):
67
# Metadata for the prefill stage since we still
68
# use flash attention for prefill.
69
seq_start_loc: Optional[torch.Tensor] = None
70
- max_seq_len: Optional[int] = None
70
block_tables: Optional[torch.Tensor] = None
71
72
# Metadata for the decode stage
@@ -113,7 +112,8 @@ class FlashInferMetadata(AttentionMetadataPerStage):
112
# When using flashinfer, we are also creating the FlashInferMetadata,
113
# which will also call post_init by default, here we want to skip the
114
# post_init if it's the prefill phase.
116
- if not self.is_prompt:
115
+ if self.num_prefills == 0:
116
+ assert self.num_decode_tokens > 0
117
self.decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
118
self.workspace_buffer, "NHD")
119
self.decode_wrapper.begin_forward(
@@ -138,6 +138,24 @@ class FlashInferMetadata(AttentionMetadataPerStage):
138
skip_fields.add('decode_wrapper')
139
return super().asdict_zerocopy(skip_fields)
140
141
+ @property
142
+ def prefill_metadata(self) -> Optional["FlashInferMetadata"]:
143
+ # Currently chunked prefill is not supported
144
+ if self.num_decode_tokens == 0:
145
+ assert self.num_prefills > 0
146
+ return self
147
+
148
+ return None
149
+
150
+ @property
151
+ def decode_metadata(self) -> Optional["FlashInferMetadata"]:
152
+ # Currently chunked prefill is not supported
153
+ if self.num_prefills > 0:
154
+ assert self.num_decode_tokens == 0
155
+ return None
156
+
157
+ return self
158
+
159
160
class FlashInferImpl(AttentionImpl):
161
@@ -172,7 +190,7 @@ class FlashInferImpl(AttentionImpl):
190
key: torch.Tensor,
191
value: torch.Tensor,
192
kv_cache: Optional[torch.Tensor],
175
- attn_metadata: AttentionMetadata[FlashInferMetadata],
193
+ attn_metadata: FlashInferMetadata,
194
kv_scale: float = 1.0,
195
) -> torch.Tensor:
196
assert kv_scale == 1.0
@@ -208,8 +226,8 @@ class FlashInferImpl(AttentionImpl):
226
v=value,
227
cu_seqlens_q=prefill_meta.seq_start_loc,
228
cu_seqlens_k=prefill_meta.seq_start_loc,
211
- max_seqlen_q=prefill_meta.max_seq_len,
212
- max_seqlen_k=prefill_meta.max_seq_len,
229
+ max_seqlen_q=prefill_meta.max_prefill_seq_len,
230
+ max_seqlen_k=prefill_meta.max_prefill_seq_len,
231
softmax_scale=self.scale,
232
causal=True,
233
window_size=self.sliding_window,
vllm/attention/backends/rocm_flash_attn.py
+80
-18
@@ -6,8 +6,7 @@ import torch
6
7
import vllm.envs as envs
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
- AttentionMetadata,
10
- AttentionMetadataPerStage)
9
+ AttentionMetadata)
10
from vllm.attention.ops.paged_attn import (PagedAttention,
11
PagedAttentionMetadata)
12
from vllm.logger import init_logger
@@ -56,8 +55,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
55
56
57
@dataclass
59
-class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
60
- PagedAttentionMetadata):
58
+class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
59
"""Metadata for FlashAttentionBackend.
60
61
NOTE: Any python object stored here is not updated when it is
@@ -65,9 +63,6 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
63
dynamically, it should be stored in tensor. The tensor has to be
64
updated from `CUDAGraphRunner.forward` API.
65
"""
68
- # Currently, input sequences can only contain all prompts
69
- # or all decoding. True if all sequences are prompts.
70
- is_prompt: bool
66
# (batch_size,). The sequence length per sequence. Sequence length means
67
# the computed tokens + new tokens None if it is a decoding.
68
seq_lens: Optional[List[int]]
@@ -82,14 +77,18 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
77
# |-------------------- seq_len ----------------------|
78
# |-- query_len ---|
79
85
- # Maximum query length in the batch.
80
+ # Maximum query length in the batch. None for decoding.
81
max_query_len: Optional[int]
87
- # Maximum sequence length in the batch.
88
- max_seq_len: Optional[int]
82
+ # Maximum sequence length among prefill batch. 0 if there are decoding
83
+ # requests only.
84
+ max_prefill_seq_len: int
85
+ # Maximum sequence length among decode batch. 0 if there are prefill
86
+ # requests only.
87
+ max_decode_seq_len: int
88
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
89
# the batch, used to index into subquery. E.g., if the subquery length
90
# is [4, 6], it is [0, 4, 10].
92
- subquery_start_loc: Optional[torch.Tensor]
91
+ query_start_loc: Optional[torch.Tensor]
92
# (batch_size + 1,). The cumulative sequence lengths of the sequences in
93
# the batch, used to index into sequence. E.g., if the sequence length is
94
# [4, 6], it is [0, 4, 10].
@@ -102,6 +101,69 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
101
# (batch_size,) A tensor of context lengths (tokens that are computed
102
# so far).
103
context_lens_tensor: Optional[torch.Tensor]
104
+ _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
105
+ _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
106
+
107
+ @property
108
+ def prefill_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
109
+ if self.num_prefills == 0:
110
+ return None
111
+
112
+ if self._cached_prefill_metadata is not None:
113
+ return self._cached_prefill_metadata
114
+
115
+ assert self.seq_lens is not None
116
+ assert self.seq_lens_tensor is not None
117
+ assert self.query_start_loc is not None
118
+ assert self.context_lens_tensor is not None
119
+ assert self.block_tables is not None
120
+ assert self.seq_start_loc is not None
121
+
122
+ self._cached_prefill_metadata = ROCmFlashAttentionMetadata(
123
+ num_prefills=self.num_prefills,
124
+ num_prefill_tokens=self.num_prefill_tokens,
125
+ num_decode_tokens=0,
126
+ slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
127
+ seq_lens=self.seq_lens[:self.num_prefills],
128
+ seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
129
+ max_query_len=self.max_query_len,
130
+ max_prefill_seq_len=self.max_prefill_seq_len,
131
+ max_decode_seq_len=0,
132
+ query_start_loc=self.query_start_loc[:self.num_prefills + 1],
133
+ seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
134
+ context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
135
+ block_tables=self.block_tables[:self.num_prefills],
136
+ use_cuda_graph=False,
137
+ )
138
+ return self._cached_prefill_metadata
139
+
140
+ @property
141
+ def decode_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
142
+ if self.num_decode_tokens == 0:
143
+ return None
144
+
145
+ if self._cached_decode_metadata is not None:
146
+ return self._cached_decode_metadata
147
+ assert self.block_tables is not None
148
+ assert self.seq_lens_tensor is not None
149
+
150
+ self._cached_decode_metadata = ROCmFlashAttentionMetadata(
151
+ num_prefills=0,
152
+ num_prefill_tokens=0,
153
+ num_decode_tokens=self.num_decode_tokens,
154
+ slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
155
+ seq_lens=None,
156
+ seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
157
+ max_query_len=None,
158
+ max_prefill_seq_len=0,
159
+ max_decode_seq_len=self.max_decode_seq_len,
160
+ query_start_loc=None,
161
+ seq_start_loc=None,
162
+ context_lens_tensor=None,
163
+ block_tables=self.block_tables[self.num_prefills:],
164
+ use_cuda_graph=self.use_cuda_graph,
165
+ )
166
+ return self._cached_decode_metadata
167
168
169
class ROCmFlashAttentionImpl(AttentionImpl):
@@ -198,7 +260,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
260
key: torch.Tensor,
261
value: torch.Tensor,
262
kv_cache: torch.Tensor,
201
- attn_metadata: AttentionMetadata[ROCmFlashAttentionMetadata],
263
+ attn_metadata: ROCmFlashAttentionMetadata,
264
kv_scale: float = 1.0,
265
) -> torch.Tensor:
266
"""Forward pass with FlashAttention and PagedAttention.
@@ -266,8 +328,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
328
None,
329
prefill_meta.seq_start_loc,
330
prefill_meta.seq_start_loc,
269
- prefill_meta.max_seq_len,
270
- prefill_meta.max_seq_len,
331
+ prefill_meta.max_prefill_seq_len,
332
+ prefill_meta.max_prefill_seq_len,
333
True,
334
self.scale,
335
)
@@ -290,8 +352,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
352
v=value,
353
cu_seqlens_q=prefill_meta.seq_start_loc,
354
cu_seqlens_k=prefill_meta.seq_start_loc,
293
- max_seqlen_q=prefill_meta.max_seq_len,
294
- max_seqlen_k=prefill_meta.max_seq_len,
355
+ max_seqlen_q=prefill_meta.max_prefill_seq_len,
356
+ max_seqlen_k=prefill_meta.max_prefill_seq_len,
357
softmax_scale=self.scale,
358
causal=True,
359
)
@@ -308,7 +370,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
370
key_cache,
371
value_cache,
372
prefill_meta.block_tables,
311
- prefill_meta.subquery_start_loc,
373
+ prefill_meta.query_start_loc,
374
prefill_meta.seq_lens_tensor,
375
prefill_meta.context_lens_tensor,
376
prefill_meta.max_query_len,
@@ -324,7 +386,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
386
value_cache,
387
decode_meta.block_tables,
388
decode_meta.seq_lens_tensor,
327
- decode_meta.max_seq_len,
389
+ decode_meta.max_decode_seq_len,
390
self.kv_cache_dtype,
391
self.num_kv_heads,
392
self.scale,
vllm/attention/backends/torch_sdpa.py
+22
-6
@@ -7,8 +7,7 @@ import torch
7
from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata,
11
- AttentionMetadataPerStage)
10
+ AttentionMetadata)
11
from vllm.attention.ops.paged_attn import (PagedAttention,
12
PagedAttentionMetadata)
13
@@ -54,8 +53,7 @@ class TorchSDPABackend(AttentionBackend):
53
54
55
@dataclass
57
-class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
58
- AttentionMetadataPerStage):
56
+class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
57
"""Metadata for TorchSDPABackend.
58
"""
59
# Currently, input sequences can only contain all prompts
@@ -72,8 +70,26 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
70
# will not appear in the __repr__ and __init__
71
self.attn_bias: Optional[List[torch.Tensor]] = None
72
73
+ @property
74
+ def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
75
+ # Currently chunked prefill is not supported
76
+ if self.num_decode_tokens == 0:
77
+ assert self.num_prefills > 0
78
+ return self
79
76
-class TorchSDPABackendImpl(AttentionImpl):
80
+ return None
81
+
82
+ @property
83
+ def decode_metadata(self) -> Optional["TorchSDPAMetadata"]:
84
+ # Currently chunked prefill is not supported
85
+ if self.num_prefills > 0:
86
+ assert self.num_decode_tokens == 0
87
+ return None
88
+
89
+ return self
90
+
91
+
92
+class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
93
94
def __init__(
95
self,
@@ -200,7 +216,7 @@ class TorchSDPABackendImpl(AttentionImpl):
216
value_cache,
217
attn_metadata.block_tables,
218
attn_metadata.seq_lens_tensor,
203
- attn_metadata.max_seq_len,
219
+ attn_metadata.max_decode_seq_len,
220
self.kv_cache_dtype,
221
self.num_kv_heads,
222
self.scale,
vllm/attention/backends/xformers.py
+77
-15
@@ -9,8 +9,7 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
9
LowerTriangularMaskWithTensorBias)
10
11
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
- AttentionMetadata,
13
- AttentionMetadataPerStage)
12
+ AttentionMetadata)
13
from vllm.attention.ops.paged_attn import (PagedAttention,
14
PagedAttentionMetadata)
15
from vllm.logger import init_logger
@@ -59,7 +58,7 @@ class XFormersBackend(AttentionBackend):
58
59
60
@dataclass
62
-class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
61
+class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
62
"""Metadata for XFormersbackend.
63
64
NOTE: Any python object stored here is not updated when it is
@@ -67,9 +66,6 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
66
dynamically, it should be stored in tensor. The tensor has to be
67
updated from `CUDAGraphRunner.forward` API.
68
"""
70
- # Currently, input sequences can only contain all prompts
71
- # or all decoding. True if all sequences are prompts.
72
- is_prompt: bool
69
# (batch_size,). The sequence length per sequence. Sequence length means
70
# the computed tokens + new tokens None if it is a decoding.
71
seq_lens: Optional[List[int]]
@@ -83,15 +79,19 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
79
# |-------------------- seq_len ----------------------|
80
# |-- query_len ---|
81
86
- # Maximum query length in the batch.
82
+ # Maximum query length in the batch. None for decoding.
83
max_query_len: Optional[int]
84
# FIXME: It is for flash attn.
89
- # Maximum sequence length in the batch.
90
- max_seq_len: Optional[int]
85
+ # Maximum sequence length among prefill batch. 0 if there are decoding
86
+ # requests only.
87
+ max_prefill_seq_len: int
88
+ # Maximum sequence length among decode batch. 0 if there are prefill
89
+ # requests only.
90
+ max_decode_seq_len: int
91
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
92
# the batch, used to index into subquery. E.g., if the subquery length
93
# is [4, 6], it is [0, 4, 10].
94
- subquery_start_loc: Optional[torch.Tensor]
94
+ query_start_loc: Optional[torch.Tensor]
95
# FIXME: It is for flash attn.
96
# (batch_size + 1,). The cumulative sequence lengths of the sequences in
97
# the batch, used to index into sequence. E.g., if the sequence length is
@@ -105,6 +105,8 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
105
# Cuda-graph is currently enabled for decoding only.
106
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
107
use_cuda_graph: bool
108
+ _cached_prefill_metadata: Optional["XFormersMetadata"] = None
109
+ _cached_decode_metadata: Optional["XFormersMetadata"] = None
110
111
def __post_init__(self):
112
# Set during the execution of the first attention op.
@@ -114,8 +116,68 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
116
# will not appear in the __repr__ and __init__
117
self.attn_bias: Optional[List[AttentionBias]] = None
118
117
-
118
-class XFormersImpl(AttentionImpl):
119
+ @property
120
+ def prefill_metadata(self) -> Optional["XFormersMetadata"]:
121
+ if self.num_prefills == 0:
122
+ return None
123
+
124
+ if self._cached_prefill_metadata is not None:
125
+ return self._cached_prefill_metadata
126
+
127
+ assert self.seq_lens is not None
128
+ assert self.seq_lens_tensor is not None
129
+ assert self.query_start_loc is not None
130
+ assert self.context_lens_tensor is not None
131
+ assert self.block_tables is not None
132
+
133
+ self._cached_prefill_metadata = XFormersMetadata(
134
+ num_prefills=self.num_prefills,
135
+ num_prefill_tokens=self.num_prefill_tokens,
136
+ num_decode_tokens=0,
137
+ slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
138
+ seq_lens=self.seq_lens[:self.num_prefills],
139
+ seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
140
+ max_query_len=self.max_query_len,
141
+ max_prefill_seq_len=self.max_prefill_seq_len,
142
+ max_decode_seq_len=0,
143
+ query_start_loc=self.query_start_loc[:self.num_prefills + 1],
144
+ seq_start_loc=None,
145
+ context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
146
+ block_tables=self.block_tables[:self.num_prefills],
147
+ use_cuda_graph=False,
148
+ )
149
+ return self._cached_prefill_metadata
150
+
151
+ @property
152
+ def decode_metadata(self) -> Optional["XFormersMetadata"]:
153
+ if self.num_decode_tokens == 0:
154
+ return None
155
+
156
+ if self._cached_decode_metadata is not None:
157
+ return self._cached_decode_metadata
158
+ assert self.block_tables is not None
159
+ assert self.seq_lens_tensor is not None
160
+
161
+ self._cached_decode_metadata = XFormersMetadata(
162
+ num_prefills=0,
163
+ num_prefill_tokens=0,
164
+ num_decode_tokens=self.num_decode_tokens,
165
+ slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
166
+ seq_lens=None,
167
+ seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
168
+ max_query_len=None,
169
+ max_prefill_seq_len=0,
170
+ max_decode_seq_len=self.max_decode_seq_len,
171
+ query_start_loc=None,
172
+ seq_start_loc=None,
173
+ context_lens_tensor=None,
174
+ block_tables=self.block_tables[self.num_prefills:],
175
+ use_cuda_graph=self.use_cuda_graph,
176
+ )
177
+ return self._cached_decode_metadata
178
+
179
+
180
+class XFormersImpl(AttentionImpl[XFormersMetadata]):
181
"""
182
If the input tensors contain prompt tokens, the layout is as follows:
183
|<--------------- num_prefill_tokens ----------------->|
@@ -176,7 +238,7 @@ class XFormersImpl(AttentionImpl):
238
key: torch.Tensor,
239
value: torch.Tensor,
240
kv_cache: Optional[torch.Tensor],
179
- attn_metadata: AttentionMetadata[XFormersMetadata],
241
+ attn_metadata: "XFormersMetadata",
242
kv_scale: float = 1.0,
243
) -> torch.Tensor:
244
"""Forward pass with xFormers and PagedAttention.
@@ -244,7 +306,7 @@ class XFormersImpl(AttentionImpl):
306
key_cache,
307
value_cache,
308
prefill_meta.block_tables,
247
- prefill_meta.subquery_start_loc,
309
+ prefill_meta.query_start_loc,
310
prefill_meta.seq_lens_tensor,
311
prefill_meta.context_lens_tensor,
312
prefill_meta.max_query_len,
@@ -261,7 +323,7 @@ class XFormersImpl(AttentionImpl):
323
value_cache,
324
decode_meta.block_tables,
325
decode_meta.seq_lens_tensor,
264
- decode_meta.max_seq_len,
326
+ decode_meta.max_decode_seq_len,
327
self.kv_cache_dtype,
328
self.num_kv_heads,
329
self.scale,
vllm/attention/layer.py
+2
-3
@@ -4,8 +4,7 @@ from typing import List, Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm.attention.backends.abstract import (AttentionMetadata,
8
- AttentionMetadataPerStage)
7
+from vllm.attention.backends.abstract import AttentionMetadata
8
from vllm.attention.selector import get_attn_backend
9
from vllm.config import CacheConfig
10
@@ -57,7 +56,7 @@ class Attention(nn.Module):
56
key: torch.Tensor,
57
value: torch.Tensor,
58
kv_cache: Optional[torch.Tensor],
60
- attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
59
+ attn_metadata: AttentionMetadata,
60
kv_scale: float = 1.0,
61
) -> torch.Tensor:
62
return self.impl.forward(query, key, value, kv_cache, attn_metadata,
vllm/attention/ops/paged_attn.py
+5
-5
@@ -16,8 +16,8 @@ class PagedAttentionMetadata:
16
# (batch_size,). The length of sequences (entire tokens seen so far) per
17
# sequence.
18
seq_lens_tensor: Optional[torch.Tensor]
19
- # Maximum sequence length in the batch.
20
- max_seq_len: Optional[int]
19
+ # Maximum sequence length in the batch. 0 if it is prefill-only batch.
20
+ max_decode_seq_len: int
21
# (batch_size, max_blocks_per_seq).
22
# Block addresses per sequence. (Seq id -> list of physical block)
23
# E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
@@ -166,7 +166,7 @@ class PagedAttention:
166
key_cache: torch.Tensor,
167
value_cache: torch.Tensor,
168
block_tables: torch.Tensor,
169
- subquery_start_loc: torch.Tensor,
169
+ query_start_loc: torch.Tensor,
170
seq_lens_tensor: torch.Tensor,
171
context_lens: torch.Tensor,
172
max_query_len: int,
@@ -182,8 +182,8 @@ class PagedAttention:
182
key_cache,
183
value_cache,
184
block_tables,
185
- # subquery_start_loc is (batch_size + 1,)
186
- subquery_start_loc[:-1],
185
+ # query_start_loc is (batch_size + 1,)
186
+ query_start_loc[:-1],
187
seq_lens_tensor,
188
context_lens,
189
max_query_len,
vllm/engine/arg_utils.py
+5
@@ -618,6 +618,11 @@ class EngineArgs:
618
decoding_config = DecodingConfig(
619
guided_decoding_backend=self.guided_decoding_backend)
620
621
+ if (model_config.get_sliding_window() is not None
622
+ and scheduler_config.chunked_prefill_enabled):
623
+ raise ValueError(
624
+ "Chunked prefill is not supported with sliding window.")
625
+
626
return EngineConfig(model_config=model_config,
627
cache_config=cache_config,
628
parallel_config=parallel_config,
vllm/model_executor/layers/rejection_sampler.py
+1
@@ -122,6 +122,7 @@ class RejectionSampler(nn.Module):
122
draft_token_ids,
123
bonus_token_ids,
124
)
125
+
126
return output_token_ids
127
128
def _batch_modified_rejection_sampling(
vllm/sequence.py
+2
-1
@@ -654,8 +654,9 @@ class SequenceGroupMetadata:
654
return self.lora_request.lora_int_id if self.lora_request else 0
655
656
@property
657
- def token_chunk_size(self) -> Optional[int]:
657
+ def token_chunk_size(self) -> int:
658
"""Return the number of tokens to be processed (chunk size)."""
659
+ assert self._token_chunk_size is not None
660
return self._token_chunk_size
661
662
vllm/spec_decode/batch_expansion.py
+16
-7
@@ -293,21 +293,30 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
293
prompt_token_ids = seq_data.get_prompt_token_ids()
294
new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
295
296
+ new_seq_data_dict = {
297
+ target_seq_id:
298
+ SequenceData(
299
+ prompt_token_ids=prompt_token_ids,
300
+ output_token_ids=new_output_token_ids,
301
+ ),
302
+ }
303
+ # This is a hack. Technically, spec decoding should compute
304
+ # num_lookahead slots at one shot, but instead, it expands the batch
305
+ # and evaluate one by one right now. context_len is seq_len - 1 because
306
+ # the kv cache is filled by a previous batch in the batch expansion.
307
+ for data in new_seq_data_dict.values():
308
+ data.update_num_computed_tokens(data.get_len() - 1)
309
+
310
return SequenceGroupMetadata(
311
request_id=seq_group_metadata.request_id,
312
is_prompt=seq_group_metadata.is_prompt,
299
- seq_data={
300
- target_seq_id:
301
- SequenceData(
302
- prompt_token_ids=prompt_token_ids,
303
- output_token_ids=new_output_token_ids,
304
- ),
305
- },
313
+ seq_data=new_seq_data_dict,
314
sampling_params=seq_group_metadata.sampling_params,
315
block_tables={
316
target_seq_id: seq_group_metadata.block_tables[seq_id],
317
},
318
lora_request=None,
319
+ token_chunk_size=1,
320
)
321
322
def _split_scoring_output(
vllm/spec_decode/multi_step_worker.py
+1
@@ -114,6 +114,7 @@ class MultiStepWorker(Worker):
114
token_logprob = seq_output.logprobs[token_id]
115
116
seq.append_token_id(token_id, token_logprob.logprob)
117
+ seq.update_num_computed_tokens(1)
118
119
def _shallow_copy_inputs(
120
self, seq_group_metadata_list: List[SequenceGroupMetadata]
vllm/worker/cpu_model_runner.py
+3
-7
@@ -159,12 +159,10 @@ class CPUModelRunner:
159
is_prompt=True,
160
seq_lens=seq_lens,
161
seq_lens_tensor=None,
162
- max_seq_len=None,
162
+ max_decode_seq_len=None,
163
num_prefills=len(seq_lens),
164
num_prefill_tokens=num_prompt_tokens,
165
num_decode_tokens=0,
166
- prefill_metadata=None,
167
- decode_metadata=None,
166
block_tables=torch.tensor([]),
167
slot_mapping=slot_mapping,
168
)
@@ -213,7 +211,7 @@ class CPUModelRunner:
211
block_table = block_table[-sliding_window_blocks:]
212
block_tables.append(block_table)
213
216
- max_seq_len = max(seq_lens)
214
+ max_decode_seq_len = max(seq_lens)
215
216
input_tokens = torch.tensor(input_tokens,
217
dtype=torch.long,
@@ -243,12 +241,10 @@ class CPUModelRunner:
241
slot_mapping=slot_mapping,
242
seq_lens=seq_lens,
243
seq_lens_tensor=seq_lens_tensor,
246
- max_seq_len=max_seq_len,
244
+ max_decode_seq_len=max_decode_seq_len,
245
num_prefill_tokens=0,
246
num_decode_tokens=len(input_tokens),
247
num_prefills=0,
250
- prefill_metadata=None,
251
- decode_metadata=None,
248
block_tables=block_tables,
249
)
250
return (
vllm/worker/embedding_model_runner.py
+15
-115
@@ -13,7 +13,7 @@ from vllm.lora.request import LoRARequest
13
from vllm.model_executor.pooling_metadata import PoolingMetadata
14
from vllm.pooling_params import PoolingParams
15
from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16
-from vllm.worker.model_runner import BatchType, ModelRunner
16
+from vllm.worker.model_runner import ModelRunner
17
18
logger = init_logger(__name__)
19
@@ -88,85 +88,24 @@ class EmbeddingModelRunner(ModelRunner):
88
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89
Set[LoRARequest], LoRAMapping, torch.Tensor]:
90
if self.is_driver_worker:
91
- prefill_reqs = []
92
- decode_reqs = []
93
- for seq_group_meta in seq_group_metadata_list:
94
- if seq_group_meta.is_prompt:
95
- prefill_reqs.append(seq_group_meta)
96
- else:
97
- decode_reqs.append(seq_group_meta)
98
-
91
# Prepare input tensors.
92
(
93
input_tokens,
94
input_positions,
103
- prefill_attn_metadata,
104
- prompt_lens,
105
- subquery_lens,
106
- lora_index_mapping,
107
- lora_prompt_mapping,
95
+ attn_metadata,
96
+ seq_lens,
97
+ _,
98
+ lora_mapping,
99
lora_requests,
100
multi_modal_input,
101
slot_mapping,
111
- ) = self._prepare_prompt(prefill_reqs)
112
- (
113
- decode_input_tokens,
114
- decode_input_positions,
115
- decode_attn_metadata,
116
- decode_lora_index_mapping,
117
- decode_lora_prompt_mapping,
118
- decode_lora_requests,
119
- decode_slot_mapping,
120
- ) = self._prepare_decode(decode_reqs)
121
-
102
+ num_prefill_tokens,
103
+ num_decode_tokens,
104
+ num_prefills,
105
+ ) = self._prepare_model_input(seq_group_metadata_list)
106
# Prepare PoolingMetadata
107
pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
124
- prompt_lens)
125
-
126
- if not self.scheduler_config.chunked_prefill_enabled:
127
- assert (len(prefill_reqs) and len(decode_reqs)) == 0
128
-
129
- num_prefills = len(prompt_lens)
130
- num_prefill_tokens = len(input_tokens)
131
- num_decode_tokens = len(decode_input_tokens)
132
-
133
- # Coalesce tensors. Note that attn_metadata is currently not
134
- # coalesced for simplicity.
135
- input_tokens.extend(decode_input_tokens)
136
- input_positions.extend(decode_input_positions)
137
- slot_mapping.extend(decode_slot_mapping)
138
- lora_index_mapping.extend(decode_lora_index_mapping)
139
- lora_prompt_mapping.extend(decode_lora_prompt_mapping)
140
- lora_requests.update(decode_lora_requests)
141
-
142
- input_tokens = torch.tensor(input_tokens,
143
- dtype=torch.long,
144
- device=self.device)
145
- input_positions = torch.tensor(input_positions,
146
- dtype=torch.long,
147
- device=self.device)
148
- slot_mapping = torch.tensor(slot_mapping,
149
- dtype=torch.long,
150
- device=self.device)
151
-
152
- if self.lora_config:
153
- lora_mapping = LoRAMapping(
154
- lora_index_mapping,
155
- lora_prompt_mapping,
156
- )
157
- else:
158
- lora_mapping = None
159
-
160
- # Broadcast the metadata.
161
- # If batch contains both prefill and decode, it sends 2 broadcasts.
162
- # If it only contains 1 type, it triggers a single broadcast.
163
- if (prefill_attn_metadata is not None
164
- and decode_attn_metadata is not None):
165
- batch_type = BatchType.MIXED
166
- elif prefill_attn_metadata is not None:
167
- batch_type = BatchType.PREFILL
168
- else:
169
- batch_type = BatchType.DECODE
108
+ seq_lens)
109
110
metadata_dict = {
111
"input_tokens": input_tokens,
@@ -178,65 +117,26 @@ class EmbeddingModelRunner(ModelRunner):
117
"num_decode_tokens": num_decode_tokens,
118
"slot_mapping": slot_mapping,
119
"num_prefills": num_prefills,
181
- "batch_type": batch_type,
120
}
183
- if prefill_attn_metadata is not None:
184
- metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
185
- else:
186
- assert decode_attn_metadata is not None
187
- metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
121
+ if attn_metadata:
122
+ metadata_dict.update(attn_metadata.asdict_zerocopy())
123
broadcast_tensor_dict(metadata_dict, src=0)
189
-
190
- # Broadcast decode attn metadata for mixed batch type.
191
- # The additional broadcast costs 300us overhead on 4 A10 GPUs.
192
- # We can potentially reduce the overhead by coelescing tensors.
193
- if batch_type == BatchType.MIXED:
194
- assert decode_attn_metadata is not None
195
- metadata_dict = decode_attn_metadata.asdict_zerocopy()
196
- broadcast_tensor_dict(metadata_dict, src=0)
124
else:
125
metadata_dict = broadcast_tensor_dict(src=0)
126
input_tokens = metadata_dict.pop("input_tokens")
127
input_positions = metadata_dict.pop("input_positions")
201
- slot_mapping = metadata_dict.pop("slot_mapping")
202
- num_prefills = metadata_dict.pop("num_prefills")
128
lora_mapping = metadata_dict.pop("lora_mapping")
129
lora_requests = metadata_dict.pop("lora_requests")
130
multi_modal_input = metadata_dict.pop("multi_modal_input")
206
- num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
207
- num_decode_tokens = metadata_dict.pop("num_decode_tokens")
208
- batch_type = metadata_dict.pop("batch_type")
209
-
210
- # Create an attention metadata.
211
- prefill_attn_metadata = None
212
- decode_attn_metadata = None
213
- if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
214
- prefill_attn_metadata = self.attn_backend.make_metadata(
131
+ if metadata_dict:
132
+ attn_metadata = self.attn_backend.make_metadata(
133
**metadata_dict)
134
else:
217
- decode_attn_metadata = self.attn_backend.make_metadata(
218
- **metadata_dict)
219
-
135
+ attn_metadata = None
136
pooling_metadata = PoolingMetadata(seq_groups=None,
137
seq_data=None,
138
prompt_lens=None)
139
224
- # if it is a mixed batch, decode attn_metadata is broadcasted
225
- # separately.
226
- if batch_type == BatchType.MIXED:
227
- metadata_dict = broadcast_tensor_dict(src=0)
228
- decode_attn_metadata = self.attn_backend.make_metadata(
229
- **metadata_dict)
230
-
231
- attn_metadata = AttentionMetadata(
232
- num_prefills=num_prefills,
233
- slot_mapping=slot_mapping,
234
- num_prefill_tokens=num_prefill_tokens,
235
- num_decode_tokens=num_decode_tokens,
236
- prefill_metadata=prefill_attn_metadata,
237
- decode_metadata=decode_attn_metadata,
238
- )
239
-
140
return (input_tokens, input_positions, attn_metadata, pooling_metadata,
141
lora_requests, lora_mapping, multi_modal_input)
142
vllm/worker/model_runner.py
+323
-449
@@ -1,13 +1,11 @@
1
import time
2
-from enum import IntEnum
2
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
3
4
import numpy as np
5
import torch
6
import torch.nn as nn
7
9
-from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
10
- get_attn_backend)
8
+from vllm.attention import AttentionMetadata, get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
ModelConfig, ParallelConfig, SchedulerConfig,
11
VisionLanguageConfig)
@@ -37,66 +35,38 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
35
]
36
37
40
-class PreparePromptMetadata(NamedTuple):
41
- input_tokens: List[int]
42
- input_positions: List[int]
43
- attn_metadata: Optional[AttentionMetadataPerStage]
38
+class ModelInput(NamedTuple):
39
+ input_tokens: torch.Tensor
40
+ input_positions: torch.Tensor
41
+ attn_metadata: Optional[AttentionMetadata]
42
seq_lens: List[int]
43
query_lens: List[int]
46
- lora_index_mapping: List[int]
47
- lora_prompt_mapping: List[int]
44
+ lora_mapping: Optional[LoRAMapping]
45
lora_requests: Set[LoRARequest]
46
multi_modal_input: Optional[torch.Tensor]
50
- slot_mapping: List[int]
47
+ slot_mapping: torch.Tensor
48
+ num_prefill_tokens: int
49
+ num_decode_tokens: int
50
+ num_prefills: int
51
52
@classmethod
53
- def empty(cls):
54
- return PreparePromptMetadata(
55
- input_tokens=[],
56
- input_positions=[],
53
+ def empty(cls, device):
54
+ return ModelInput(
55
+ input_tokens=torch.empty(0, device=device),
56
+ input_positions=torch.empty(0, device=device),
57
attn_metadata=None,
58
seq_lens=[],
59
query_lens=[],
60
- lora_index_mapping=[],
61
- lora_prompt_mapping=[],
60
+ lora_mapping=None,
61
lora_requests=set(),
62
multi_modal_input=None,
64
- slot_mapping=[],
65
- )
66
-
67
-
68
-class PrepareDecodeMetadata(NamedTuple):
69
- input_tokens: List[int]
70
- input_positions: List[int]
71
- attn_metadata: Optional[AttentionMetadata]
72
- lora_index_mapping: List[int]
73
- lora_prompt_mapping: List[int]
74
- lora_requests: Set[LoRARequest]
75
- slot_mapping: List[int]
76
-
77
- @classmethod
78
- def empty(cls):
79
- return PrepareDecodeMetadata(
80
- input_tokens=[],
81
- input_positions=[],
82
- attn_metadata=None,
83
- lora_index_mapping=[],
84
- lora_prompt_mapping=[],
85
- lora_requests=set(),
86
- slot_mapping=[],
63
+ slot_mapping=torch.empty(0, device=device),
64
+ num_prefill_tokens=0,
65
+ num_decode_tokens=0,
66
+ num_prefills=0,
67
)
68
69
90
-# How batches are constructed.
91
-class BatchType(IntEnum):
92
- # Every batch is prefill.
93
- PREFILL = 0
94
- # Every batch is decode.
95
- DECODE = 1
96
- # Batch is a mixture of prefill and decode.
97
- MIXED = 2
98
-
99
-
70
class ModelRunner:
71
72
def __init__(
@@ -216,10 +186,22 @@ class ModelRunner:
186
block_size = self.block_size
187
return (self.max_seq_len_to_capture + block_size - 1) // block_size
188
219
- def _prepare_prompt(
189
+ def _prepare_model_input(
190
self,
191
seq_group_metadata_list: List[SequenceGroupMetadata],
222
- ) -> PreparePromptMetadata:
192
+ ) -> ModelInput:
193
+ """Prepare the model input based on a given sequence group.
194
+
195
+ The API assumes seq_group_metadata_list is sorted by prefill -> decode.
196
+
197
+ The result tensors and data structure also batches input in prefill
198
+ -> decode order. For example,
199
+
200
+ - input_tokens[:num_prefill_tokens] contains prefill tokens.
201
+ - input_tokens[num_prefill_tokens:] contains decode tokens.
202
+
203
+ If cuda graph is required, this API automatically pads inputs.
204
+ """
205
input_tokens: List[int] = []
206
input_positions: List[int] = []
207
slot_mapping: List[int] = []
@@ -228,212 +210,16 @@ class ModelRunner:
210
lora_requests: Set[LoRARequest] = set()
211
212
seq_lens: List[int] = []
213
+ prefill_seq_lens: List[int] = []
214
+ decode_seq_lens: List[int] = []
215
context_lens: List[int] = []
216
query_lens: List[int] = []
233
- prefix_block_tables: List[List[int]] = []
234
- multi_modal_input_list: List[torch.Tensor] = []
235
-
236
- if len(seq_group_metadata_list) == 0:
237
- return PreparePromptMetadata.empty()
238
-
239
- for seq_group_metadata in seq_group_metadata_list:
240
- assert seq_group_metadata.is_prompt
241
- seq_ids = list(seq_group_metadata.seq_data.keys())
242
- assert len(seq_ids) == 1
243
- seq_id = seq_ids[0]
244
-
245
- computed_block_nums = seq_group_metadata.computed_block_nums
246
- if (self.scheduler_config is not None
247
- and self.scheduler_config.chunked_prefill_enabled
248
- and not (computed_block_nums is None
249
- or computed_block_nums == [])):
250
- raise RuntimeError(
251
- "chunked prefill cannot be used with prefix caching "
252
- "now.")
253
-
254
- token_chunk_size = seq_group_metadata.token_chunk_size
255
- seq_data = seq_group_metadata.seq_data[seq_id]
256
- context_len = seq_data.get_num_computed_tokens()
257
- # We should use get_len here because in case of preemption
258
- # it contains output tokens.
259
- seq_len = min(seq_data.get_len(), context_len + token_chunk_size)
260
- prompt_tokens = seq_data.get_token_ids()[context_len:seq_len]
261
- seq_lens.append(seq_len)
262
-
263
- # NOTE: This only works for oooooooxxx style attention.
264
- if computed_block_nums is not None and len(
265
- computed_block_nums) > 0 and self.sliding_window is None:
266
- # Prefix is not supported with sliding_window
267
- context_len = len(computed_block_nums) * self.block_size
268
- prompt_tokens = prompt_tokens[context_len:]
269
- prefix_block_tables.append(computed_block_nums)
270
- elif self.scheduler_config.chunked_prefill_enabled:
271
- if seq_group_metadata.block_tables is not None:
272
- # Prefill has chunked before.
273
- block_table = seq_group_metadata.block_tables[seq_id]
274
- prefix_block_tables.append(block_table)
275
- else:
276
- # The first prefill.
277
- prefix_block_tables.append([])
278
- else:
279
- prefix_block_tables.append([])
280
- # Right now, prefill start is always 0. However, this
281
- # assumption can be changed once chunked prefill is introduced.
282
- assert context_len == 0
283
-
284
- # actual prompt lens
285
- context_lens.append(context_len)
286
- query_lens.append(seq_len - context_len)
287
-
288
- input_tokens.extend(prompt_tokens)
289
- # NOTE(woosuk): Here we assume that the first token in the prompt
290
- # is always the first token in the sequence.
291
- input_positions.extend(list(range(context_len, seq_len)))
292
- lora_id = seq_group_metadata.lora_int_id
293
-
294
- if lora_id > 0:
295
- lora_requests.add(seq_group_metadata.lora_request)
296
-
297
- lora_index_mapping += [lora_id] * (seq_len - context_len)
298
- lora_prompt_mapping.extend([lora_id] * (
299
- seq_len - context_len if seq_group_metadata.sampling_params
300
- and seq_group_metadata.sampling_params.prompt_logprobs else 1))
301
-
302
- if seq_group_metadata.multi_modal_data:
303
- multi_modal_input_list.append(
304
- seq_group_metadata.multi_modal_data.data)
305
-
306
- if _is_block_tables_empty(seq_group_metadata.block_tables):
307
- # During memory profiling, the block tables are not initialized
308
- # yet. In this case, we just use a dummy slot mapping.
309
- # In embeddings, the block tables are {seq_id: None}.
310
- slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
311
- continue
312
-
313
- # Compute the slot mapping.
314
- block_table = seq_group_metadata.block_tables[seq_id]
315
-
316
- # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
317
- # where start_idx is max(0, seq_len - sliding_window).
318
- # For example, if the prompt len is 10, sliding window is 8, and
319
- # block size is 4, the first two tokens are masked and the slot
320
- # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
321
- start_idx = 0
322
- if self.sliding_window is not None:
323
- assert context_len == 0, (
324
- "Prefix caching is currently not supported with "
325
- "sliding window attention")
326
- start_idx = max(0, seq_len - self.sliding_window)
327
-
328
- for i in range(context_len, seq_len):
329
- if i < start_idx:
330
- slot_mapping.append(_PAD_SLOT_ID)
331
- continue
332
-
333
- block_number = block_table[i // self.block_size]
334
- block_offset = i % self.block_size
335
- slot = block_number * self.block_size + block_offset
336
- slot_mapping.append(slot)
337
-
338
- max_query_len = max(query_lens)
339
- max_seq_len = max(seq_lens)
340
- assert max_query_len > 0
341
-
342
- context_lens_tensor = torch.tensor(context_lens,
343
- dtype=torch.int,
344
- device=self.device)
345
-
346
- if multi_modal_input_list:
347
- assert self.vision_language_config, (
348
- "Multi-modal inputs are only supported by "
349
- "vision language models.")
350
- multi_modal_input = torch.cat(multi_modal_input_list,
351
- dim=0).to(self.device)
352
- else:
353
- multi_modal_input = None
354
-
355
- # Prepare prefix block tables
356
- max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
357
- block_tables = make_tensor_with_pad(
358
- prefix_block_tables,
359
- max_len=max_prompt_block_table_len,
360
- pad=0,
361
- dtype=torch.int,
362
- device=self.device,
363
- )
364
-
365
- # Query length can be shorter than key (i.e., prompt) when prefill
366
- # is chunked or prefix cached.
367
- query_lens_tensor = torch.tensor(query_lens,
368
- dtype=torch.long,
369
- device=self.device)
370
- subquery_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
371
- dtype=torch.int32,
372
- device=self.device)
373
-
374
- seq_lens_tensor = torch.tensor(seq_lens,
375
- dtype=torch.int,
376
- device=self.device)
377
- seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
378
- dtype=torch.int32,
379
- device=self.device)
380
-
381
- torch.cumsum(query_lens_tensor,
382
- dim=0,
383
- dtype=subquery_start_loc.dtype,
384
- out=subquery_start_loc[1:])
385
-
386
- torch.cumsum(seq_lens_tensor,
387
- dim=0,
388
- dtype=seq_start_loc.dtype,
389
- out=seq_start_loc[1:])
390
-
391
- if self.attn_backend.get_name() == "flashinfer":
392
- attn_metadata = self.attn_backend.make_metadata(
393
- is_prompt=True,
394
- use_cuda_graph=False,
395
- seq_start_loc=seq_start_loc,
396
- max_seq_len=max_seq_len,
397
- block_tables=block_tables)
398
- else:
399
- attn_metadata = self.attn_backend.make_metadata(
400
- is_prompt=True,
401
- seq_lens=seq_lens,
402
- seq_lens_tensor=seq_lens_tensor,
403
- max_query_len=max_query_len,
404
- max_seq_len=max_seq_len,
405
- subquery_start_loc=subquery_start_loc,
406
- seq_start_loc=seq_start_loc,
407
- context_lens_tensor=context_lens_tensor,
408
- block_tables=block_tables,
409
- use_cuda_graph=False,
410
- )
411
-
412
- return PreparePromptMetadata(
413
- input_tokens=input_tokens,
414
- input_positions=input_positions,
415
- attn_metadata=attn_metadata,
416
- seq_lens=seq_lens,
417
- query_lens=query_lens,
418
- lora_index_mapping=lora_index_mapping,
419
- lora_prompt_mapping=lora_prompt_mapping,
420
- lora_requests=lora_requests,
421
- multi_modal_input=multi_modal_input,
422
- slot_mapping=slot_mapping,
423
- )
424
-
425
- def _prepare_decode(
426
- self,
427
- seq_group_metadata_list: List[SequenceGroupMetadata],
428
- ) -> PrepareDecodeMetadata:
429
- input_tokens: List[int] = []
430
- input_positions: List[int] = []
431
- slot_mapping: List[int] = []
432
- seq_lens: List[int] = []
217
block_tables: List[List[int]] = []
434
- lora_index_mapping: List[int] = []
435
- lora_prompt_mapping: List[int] = []
436
- lora_requests: Set[LoRARequest] = set()
218
+ multi_modal_input_list: List[torch.Tensor] = []
219
+ decode_only = True
220
+ num_prefills = 0
221
+ num_prefill_tokens = 0
222
+ num_decode_tokens = 0
223
224
# The following fields are only for flashinfer
225
# Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
@@ -454,60 +240,186 @@ class ModelRunner:
240
paged_kv_last_page_len: List[int] = []
241
242
if len(seq_group_metadata_list) == 0:
457
- return PrepareDecodeMetadata.empty()
243
+ return ModelInput.empty(self.device)
244
245
for seq_group_metadata in seq_group_metadata_list:
460
- assert not seq_group_metadata.is_prompt
461
- assert seq_group_metadata.token_chunk_size == 1
462
-
246
seq_ids = list(seq_group_metadata.seq_data.keys())
464
- lora_id = seq_group_metadata.lora_int_id
465
-
466
- if lora_id > 0:
467
- lora_requests.add(seq_group_metadata.lora_request)
247
+ is_prompt = seq_group_metadata.is_prompt
248
249
for seq_id in seq_ids:
250
+ computed_block_nums = seq_group_metadata.computed_block_nums
251
+ if (self.scheduler_config is not None
252
+ and self.scheduler_config.chunked_prefill_enabled
253
+ and not (computed_block_nums is None
254
+ or computed_block_nums == [])):
255
+ raise RuntimeError(
256
+ "chunked prefill cannot be used with prefix caching "
257
+ "now.")
258
+
259
seq_data = seq_group_metadata.seq_data[seq_id]
471
- generation_token = seq_data.get_last_token_id()
472
- input_tokens.append(generation_token)
260
+ if is_prompt:
261
+ context_len = seq_data.get_num_computed_tokens()
262
+ else:
263
+ # get_num_computed_tokens is incorrect for spec decoding.
264
+ # So, we should have a special logic here.
265
+ # TODO(sang): Fix it.
266
+ context_len = seq_data.get_len() - 1
267
+
268
+ seq_len = min(
269
+ seq_data.get_len(),
270
+ context_len + seq_group_metadata.token_chunk_size)
271
+ if is_prompt:
272
+ tokens = seq_data.get_token_ids()[context_len:seq_len]
273
+ else:
274
+ # Optimization. get_token_ids requires the entire copy of
275
+ # tokens.
276
+ tokens = [seq_data.get_last_token_id()]
277
+
278
+ # Prefix cache was hit.
279
+ # Prefix is not supported with sliding_window
280
+ prefix_cache_hit = (computed_block_nums is not None
281
+ and len(computed_block_nums) > 0
282
+ and self.sliding_window is None
283
+ and is_prompt)
284
+
285
+ # TODO(sang): Combine chunked prefill and prefix caching by
286
+ # only allowing multiple of block_size chunk size.
287
+ # NOTE: This only works for oooooooxxx style attention.
288
+ if prefix_cache_hit:
289
+ assert computed_block_nums is not None
290
+ context_len = len(computed_block_nums) * self.block_size
291
+ tokens = tokens[context_len:]
292
+ if self.attn_backend.get_name() == "flash-attn":
293
+ # NOTE(woosuk): For flash-attn, the block table should
294
+ # include the entries for the incoming prefill tokens.
295
+ # TODO(woosuk): This is a temporary fix. We should
296
+ # provide a unified interface for different backends.
297
+ block_table = seq_group_metadata.block_tables[seq_id]
298
+ else:
299
+ block_table = computed_block_nums
300
+ elif (self.scheduler_config.chunked_prefill_enabled
301
+ or not is_prompt):
302
+ if seq_group_metadata.block_tables is not None:
303
+ # chunked prefill or decode
304
+ block_table = seq_group_metadata.block_tables[seq_id]
305
+ if self.sliding_window is not None:
306
+ # chunked prefill doesn't support sliding window.
307
+ assert (not self.scheduler_config.
308
+ chunked_prefill_enabled)
309
+ sliding_window_blocks = (self.sliding_window //
310
+ self.block_size)
311
+ block_table = block_table[-sliding_window_blocks:]
312
+
313
+ if self.attn_backend.get_name() == "flashinfer":
314
+ paged_kv_indices.extend(block_table)
315
+ paged_kv_indptr.append(paged_kv_indptr[-1] +
316
+ len(block_table))
317
+ last_page_len = seq_data.get_len(
318
+ ) % self.block_size
319
+ if last_page_len == 0:
320
+ last_page_len = self.block_size
321
+ paged_kv_last_page_len.append(last_page_len)
322
+ else:
323
+ # Only happens when memory profiling runs.
324
+ block_table = []
325
+ else:
326
+ # Prefill without chunked prefill or memory profiling.
327
+ block_table = []
328
+ block_tables.append(block_table)
329
474
- seq_len = seq_data.get_len()
475
- position = seq_len - 1
476
- input_positions.append(position)
330
+ # TODO(sang): This is a hack to make sliding window work with
331
+ # paged attn. We can remove it if we make paged attn kernel
332
+ # to properly handle slinding window attn.
333
+ if (self.sliding_window is not None and not is_prompt):
334
+ seq_len = min(seq_len, self.sliding_window)
335
+ context_len = seq_len - 1
336
478
- seq_len = seq_len if self.sliding_window is None else min(
479
- seq_len, self.sliding_window)
337
seq_lens.append(seq_len)
338
+ context_lens.append(context_len)
339
+ query_len = seq_len - context_len
340
+ query_lens.append(query_len)
341
+ input_tokens.extend(tokens)
342
+ input_positions.extend(list(range(context_len, seq_len)))
343
+ lora_id = seq_group_metadata.lora_int_id
344
+
345
+ if is_prompt:
346
+ assert len(seq_ids) == 1
347
+ num_prefills += 1
348
+ num_prefill_tokens += len(tokens)
349
+ decode_only = False
350
+ prefill_seq_lens.append(seq_len)
351
+ else:
352
+ assert query_len == 1, (
353
+ "seq_len: {}, context_len: {}, query_len: {}".format(
354
+ seq_len, context_len, query_len))
355
+ num_decode_tokens += query_len
356
+ decode_seq_lens.append(seq_len)
357
+
358
+ if lora_id > 0:
359
+ lora_requests.add(seq_group_metadata.lora_request)
360
+
361
+ lora_index_mapping += [lora_id] * (seq_len - context_len)
362
+ lora_prompt_mapping.extend(
363
+ [lora_id] *
364
+ (seq_len -
365
+ context_len if seq_group_metadata.sampling_params
366
+ and seq_group_metadata.sampling_params.prompt_logprobs
367
+ else 1))
368
+
369
+ if seq_group_metadata.multi_modal_data:
370
+ multi_modal_input_list.append(
371
+ seq_group_metadata.multi_modal_data.data)
372
+
373
+ if _is_block_tables_empty(seq_group_metadata.block_tables):
374
+ # During memory profiling, the block tables are not
375
+ # initialized yet. In this case, we just use a dummy
376
+ # slot mapping.
377
+ # In embeddings, the block tables are {seq_id: None}.
378
+ slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
379
+ continue
380
381
+ # Compute the slot mapping.
382
block_table = seq_group_metadata.block_tables[seq_id]
483
- block_number = block_table[position // self.block_size]
484
- block_offset = position % self.block_size
485
- slot = block_number * self.block_size + block_offset
486
- slot_mapping.append(slot)
487
- lora_index_mapping.append(lora_id)
488
- lora_prompt_mapping.append(lora_id)
383
384
+ # Mask the [0, start_idx) tokens of the prompt with
385
+ # _PAD_SLOT_ID, where start_idx is max(0, seq_len -
386
+ # sliding_window). For example, if the prompt len is 10,
387
+ # sliding window is 8, and block size is 4, the first two
388
+ # tokens are masked and the slot mapping will be
389
+ # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
390
+ start_idx = 0
391
if self.sliding_window is not None:
491
- sliding_window_blocks = (self.sliding_window //
492
- self.block_size)
493
- block_table = block_table[-sliding_window_blocks:]
494
- block_tables.append(block_table)
392
+ if is_prompt:
393
+ assert context_len == 0, (
394
+ "Prefix caching is currently not supported with "
395
+ "sliding window attention")
396
+ # It is an optimization. When it is decoding, it is always
397
+ # 0. When prefill, we use it to not write slots to kv cache
398
+ # to save memory.
399
+ start_idx = max(0, query_len - self.sliding_window)
400
+
401
+ for i in range(context_len, seq_len):
402
+ if i < start_idx:
403
+ slot_mapping.append(_PAD_SLOT_ID)
404
+ continue
405
+
406
+ block_number = block_table[i // self.block_size]
407
+ block_offset = i % self.block_size
408
+ slot = block_number * self.block_size + block_offset
409
+ slot_mapping.append(slot)
410
496
- paged_kv_indices.extend(block_table)
497
- paged_kv_indptr.append(paged_kv_indptr[-1] + len(block_table))
498
- last_page_len = seq_data.get_len() % self.block_size
499
- if last_page_len == 0:
500
- last_page_len = self.block_size
501
- paged_kv_last_page_len.append(last_page_len)
411
+ batch_size = len(input_tokens)
412
+ max_query_len = max(query_lens)
413
+ max_prefill_seq_len = max(prefill_seq_lens, default=0)
414
+ max_decode_seq_len = max(decode_seq_lens, default=0)
415
503
- # vLLM uses cuda graph only for decoding requests.
416
+ # If cuda graph can be used, pad tensors accordingly.
417
# See `capture_model` API for more details.
505
- # For decoding requests, batch_size == input_tokens.
506
- batch_size = len(input_tokens)
507
- max_seq_len = max(seq_lens)
508
- use_captured_graph = (not self.model_config.enforce_eager
509
- and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
510
- and max_seq_len <= self.max_seq_len_to_capture)
418
+ # vLLM uses cuda graph only for decoding requests.
419
+ use_captured_graph = (
420
+ decode_only and not self.model_config.enforce_eager
421
+ and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
422
+ and max_decode_seq_len <= self.max_seq_len_to_capture)
423
if use_captured_graph:
424
graph_batch_size = _get_graph_batch_size(batch_size)
425
assert graph_batch_size >= batch_size
@@ -519,18 +431,9 @@ class ModelRunner:
431
block_tables.append([])
432
lora_index_mapping.append(0)
433
batch_size = graph_batch_size
522
-
523
- seq_lens_tensor = torch.tensor(seq_lens,
524
- dtype=torch.int,
525
- device=self.device)
434
+ num_decode_tokens = batch_size
435
436
if use_captured_graph:
528
- # When using cuda-graph all these tensors should be
529
- # padded.
530
- assert seq_lens_tensor.shape[0] == len(input_tokens)
531
- assert seq_lens_tensor.shape[0] == len(input_positions)
532
- assert seq_lens_tensor.shape[0] == len(slot_mapping)
533
-
437
# The shape of graph_block_tables is
438
# [max batch size, max context len // block size].
439
input_block_tables = self.graph_block_tables[:batch_size]
@@ -548,6 +451,57 @@ class ModelRunner:
451
dtype=torch.int,
452
device=self.device,
453
)
454
+ assert max_query_len > 0, ("query_lens: {}".format(query_lens))
455
+
456
+ context_lens_tensor = torch.tensor(context_lens,
457
+ dtype=torch.int,
458
+ device=self.device)
459
+
460
+ if multi_modal_input_list:
461
+ assert self.vision_language_config, (
462
+ "Multi-modal inputs are only supported by "
463
+ "vision language models.")
464
+ multi_modal_input = torch.cat(multi_modal_input_list,
465
+ dim=0).to(self.device)
466
+ else:
467
+ multi_modal_input = None
468
+
469
+ seq_lens_tensor = torch.tensor(seq_lens,
470
+ dtype=torch.int,
471
+ device=self.device)
472
+ query_lens_tensor = torch.tensor(query_lens,
473
+ dtype=torch.long,
474
+ device=self.device)
475
+ query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
476
+ dtype=torch.int32,
477
+ device=self.device)
478
+
479
+ seq_lens_tensor = torch.tensor(seq_lens,
480
+ dtype=torch.int,
481
+ device=self.device)
482
+ seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
483
+ dtype=torch.int32,
484
+ device=self.device)
485
+
486
+ torch.cumsum(query_lens_tensor,
487
+ dim=0,
488
+ dtype=query_start_loc.dtype,
489
+ out=query_start_loc[1:])
490
+
491
+ torch.cumsum(seq_lens_tensor,
492
+ dim=0,
493
+ dtype=seq_start_loc.dtype,
494
+ out=seq_start_loc[1:])
495
+
496
+ input_tokens_tensor = torch.tensor(input_tokens,
497
+ dtype=torch.long,
498
+ device=self.device)
499
+ input_positions_tensor = torch.tensor(input_positions,
500
+ dtype=torch.long,
501
+ device=self.device)
502
+ slot_mapping_tensor = torch.tensor(slot_mapping,
503
+ dtype=torch.long,
504
+ device=self.device)
505
506
if self.attn_backend.get_name() == "flashinfer":
507
if not hasattr(self, "flashinfer_workspace_buffer"):
@@ -555,53 +509,75 @@ class ModelRunner:
509
# Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html
510
self.flashinfer_workspace_buffer = torch.empty(
511
16 * 1024 * 1024, dtype=torch.uint8, device=self.device)
558
- paged_kv_indptr = torch.tensor(paged_kv_indptr,
559
- dtype=torch.int,
560
- device=self.device)
561
- paged_kv_indices = torch.tensor(paged_kv_indices,
562
- dtype=torch.int,
563
- device=self.device)
564
- paged_kv_last_page_len = torch.tensor(paged_kv_last_page_len,
512
+ paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
513
dtype=torch.int,
514
device=self.device)
515
+ paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
516
+ dtype=torch.int,
517
+ device=self.device)
518
+ paged_kv_last_page_len_tensor = torch.tensor(
519
+ paged_kv_last_page_len, dtype=torch.int, device=self.device)
520
kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
521
self.model_config.dtype)
569
-
522
attn_metadata = self.attn_backend.make_metadata(
571
- is_prompt=False,
523
+ num_prefills=num_prefills,
524
+ slot_mapping=slot_mapping_tensor,
525
+ num_prefill_tokens=num_prefill_tokens,
526
+ num_decode_tokens=num_decode_tokens,
527
use_cuda_graph=False,
528
+ max_prefill_seq_len=max_prefill_seq_len,
529
+ block_tables=block_tables,
530
workspace_buffer=self.flashinfer_workspace_buffer,
574
- paged_kv_indptr=paged_kv_indptr,
575
- paged_kv_indices=paged_kv_indices,
576
- paged_kv_last_page_len=paged_kv_last_page_len,
531
+ paged_kv_indptr=paged_kv_indptr_tensor,
532
+ paged_kv_indices=paged_kv_indices_tensor,
533
+ paged_kv_last_page_len=paged_kv_last_page_len_tensor,
534
num_qo_heads=self.model_config.get_num_attention_heads(
535
self.parallel_config),
536
num_kv_heads=self.model_config.get_num_kv_heads(
537
self.parallel_config),
538
head_dim=self.model_config.get_head_size(),
582
- page_size=self.block_size,
539
+ page_size=16,
540
+ seq_start_loc=seq_start_loc,
541
data_type=kv_cache_dtype)
542
else:
543
attn_metadata = self.attn_backend.make_metadata(
586
- is_prompt=False,
587
- seq_lens=None,
544
+ num_prefills=num_prefills,
545
+ slot_mapping=slot_mapping_tensor,
546
+ num_prefill_tokens=num_prefill_tokens,
547
+ num_decode_tokens=num_decode_tokens,
548
+ seq_lens=seq_lens,
549
seq_lens_tensor=seq_lens_tensor,
589
- max_query_len=None,
590
- max_seq_len=max_seq_len,
591
- subquery_start_loc=None,
592
- seq_start_loc=None,
593
- context_lens_tensor=None,
550
+ max_query_len=max_query_len,
551
+ max_prefill_seq_len=max_prefill_seq_len,
552
+ max_decode_seq_len=max_decode_seq_len,
553
+ query_start_loc=query_start_loc,
554
+ seq_start_loc=seq_start_loc,
555
+ context_lens_tensor=context_lens_tensor,
556
block_tables=block_tables,
557
use_cuda_graph=use_captured_graph,
558
)
597
- return PrepareDecodeMetadata(
598
- input_tokens=input_tokens,
599
- input_positions=input_positions,
559
+
560
+ if self.lora_config:
561
+ lora_mapping = LoRAMapping(
562
+ lora_index_mapping,
563
+ lora_prompt_mapping,
564
+ )
565
+ else:
566
+ lora_mapping = None
567
+
568
+ return ModelInput(
569
+ input_tokens=input_tokens_tensor,
570
+ input_positions=input_positions_tensor,
571
attn_metadata=attn_metadata,
601
- lora_index_mapping=lora_index_mapping,
602
- lora_prompt_mapping=lora_prompt_mapping,
572
+ seq_lens=seq_lens,
573
+ query_lens=query_lens,
574
+ lora_mapping=lora_mapping,
575
lora_requests=lora_requests,
604
- slot_mapping=slot_mapping,
576
+ multi_modal_input=multi_modal_input,
577
+ slot_mapping=slot_mapping_tensor,
578
+ num_prefill_tokens=num_prefill_tokens,
579
+ num_decode_tokens=num_decode_tokens,
580
+ num_prefills=num_prefills,
581
)
582
583
def prepare_input_tensors(
@@ -610,85 +586,25 @@ class ModelRunner:
586
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
587
Set[LoRARequest], LoRAMapping, torch.Tensor]:
588
if self.is_driver_worker:
613
- prefill_reqs = []
614
- decode_reqs = []
615
- for seq_group_meta in seq_group_metadata_list:
616
- if seq_group_meta.is_prompt:
617
- prefill_reqs.append(seq_group_meta)
618
- else:
619
- decode_reqs.append(seq_group_meta)
620
-
589
# Prepare input tensors.
590
(
591
input_tokens,
592
input_positions,
625
- prefill_attn_metadata,
593
+ attn_metadata,
594
seq_lens,
595
query_lens,
628
- lora_index_mapping,
629
- lora_prompt_mapping,
596
+ lora_mapping,
597
lora_requests,
598
multi_modal_input,
599
slot_mapping,
633
- ) = self._prepare_prompt(prefill_reqs)
634
- (
635
- decode_input_tokens,
636
- decode_input_positions,
637
- decode_attn_metadata,
638
- decode_lora_index_mapping,
639
- decode_lora_prompt_mapping,
640
- decode_lora_requests,
641
- decode_slot_mapping,
642
- ) = self._prepare_decode(decode_reqs)
600
+ num_prefill_tokens,
601
+ num_decode_tokens,
602
+ num_prefills,
603
+ ) = self._prepare_model_input(seq_group_metadata_list)
604
sampling_metadata = SamplingMetadata.prepare(
605
seq_group_metadata_list, seq_lens, query_lens, self.device,
606
self.pin_memory)
607
647
- if not self.scheduler_config.chunked_prefill_enabled:
648
- assert (len(prefill_reqs) and len(decode_reqs)) == 0
649
-
650
- num_prefills = len(seq_lens)
651
- num_prefill_tokens = len(input_tokens)
652
- num_decode_tokens = len(decode_input_tokens)
653
-
654
- # Coalesce tensors. Note that attn_metadata is currently not
655
- # coalesced for simplicity.
656
- input_tokens.extend(decode_input_tokens)
657
- input_positions.extend(decode_input_positions)
658
- slot_mapping.extend(decode_slot_mapping)
659
- lora_index_mapping.extend(decode_lora_index_mapping)
660
- lora_prompt_mapping.extend(decode_lora_prompt_mapping)
661
- lora_requests.update(decode_lora_requests)
662
-
663
- input_tokens = torch.tensor(input_tokens,
664
- dtype=torch.long,
665
- device=self.device)
666
- input_positions = torch.tensor(input_positions,
667
- dtype=torch.long,
668
- device=self.device)
669
- slot_mapping = torch.tensor(slot_mapping,
670
- dtype=torch.long,
671
- device=self.device)
672
-
673
- if self.lora_config:
674
- lora_mapping = LoRAMapping(
675
- lora_index_mapping,
676
- lora_prompt_mapping,
677
- )
678
- else:
679
- lora_mapping = None
680
-
681
- # Broadcast the metadata.
682
- # If batch contains both prefill and decode, it sends 2 broadcasts.
683
- # If it only contains 1 type, it triggers a single broadcast.
684
- if (prefill_attn_metadata is not None
685
- and decode_attn_metadata is not None):
686
- batch_type = BatchType.MIXED
687
- elif prefill_attn_metadata is not None:
688
- batch_type = BatchType.PREFILL
689
- else:
690
- batch_type = BatchType.DECODE
691
-
608
metadata_dict = {
609
"input_tokens": input_tokens,
610
"input_positions": input_positions,
@@ -701,46 +617,24 @@ class ModelRunner:
617
"num_decode_tokens": num_decode_tokens,
618
"slot_mapping": slot_mapping,
619
"num_prefills": num_prefills,
704
- "batch_type": batch_type,
620
}
706
- if prefill_attn_metadata is not None:
707
- metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
708
- else:
709
- assert decode_attn_metadata is not None
710
- metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
621
+ if attn_metadata:
622
+ metadata_dict.update(attn_metadata.asdict_zerocopy())
623
broadcast_tensor_dict(metadata_dict, src=0)
712
-
713
- # Broadcast decode attn metadata for mixed batch type.
714
- # The additional broadcast costs 300us overhead on 4 A10 GPUs.
715
- # We can potentially reduce the overhead by coelescing tensors.
716
- if batch_type == BatchType.MIXED:
717
- assert decode_attn_metadata is not None
718
- metadata_dict = decode_attn_metadata.asdict_zerocopy()
719
- broadcast_tensor_dict(metadata_dict, src=0)
624
else:
625
metadata_dict = broadcast_tensor_dict(src=0)
626
input_tokens = metadata_dict.pop("input_tokens")
627
input_positions = metadata_dict.pop("input_positions")
724
- slot_mapping = metadata_dict.pop("slot_mapping")
725
- num_prefills = metadata_dict.pop("num_prefills")
628
selected_token_indices = metadata_dict.pop(
629
"selected_token_indices")
630
lora_mapping = metadata_dict.pop("lora_mapping")
631
lora_requests = metadata_dict.pop("lora_requests")
632
multi_modal_input = metadata_dict.pop("multi_modal_input")
731
- num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
732
- num_decode_tokens = metadata_dict.pop("num_decode_tokens")
733
- batch_type = metadata_dict.pop("batch_type")
734
-
735
- # Create an attention metadata.
736
- prefill_attn_metadata = None
737
- decode_attn_metadata = None
738
- if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
739
- prefill_attn_metadata = self.attn_backend.make_metadata(
633
+ if metadata_dict:
634
+ attn_metadata = self.attn_backend.make_metadata(
635
**metadata_dict)
636
else:
742
- decode_attn_metadata = self.attn_backend.make_metadata(
743
- **metadata_dict)
637
+ attn_metadata = None
638
sampling_metadata = SamplingMetadata(
639
seq_groups=None,
640
selected_token_indices=selected_token_indices,
@@ -748,22 +642,6 @@ class ModelRunner:
642
num_prompts=0,
643
)
644
751
- # if it is a mixed batch, decode attn_metadata is broadcasted
752
- # separately.
753
- if batch_type == BatchType.MIXED:
754
- metadata_dict = broadcast_tensor_dict(src=0)
755
- decode_attn_metadata = self.attn_backend.make_metadata(
756
- **metadata_dict)
757
-
758
- attn_metadata = AttentionMetadata(
759
- num_prefills=num_prefills,
760
- slot_mapping=slot_mapping,
761
- num_prefill_tokens=num_prefill_tokens,
762
- num_decode_tokens=num_decode_tokens,
763
- prefill_metadata=prefill_attn_metadata,
764
- decode_metadata=decode_attn_metadata,
765
- )
766
-
645
return (input_tokens, input_positions, attn_metadata,
646
sampling_metadata, lora_requests, lora_mapping,
647
multi_modal_input)
@@ -954,26 +832,22 @@ class ModelRunner:
832
# memory usage of CUDA graph.
833
for batch_size in reversed(batch_size_capture_list):
834
# Create dummy attn_metadata.
957
- decode_metadata = self.attn_backend.make_metadata(
958
- is_prompt=False,
835
+ attn_metadata = self.attn_backend.make_metadata(
836
+ num_prefills=0,
837
+ num_prefill_tokens=0,
838
+ num_decode_tokens=batch_size,
839
+ slot_mapping=slot_mapping[:batch_size],
840
seq_lens=None,
841
seq_lens_tensor=seq_lens[:batch_size],
842
max_query_len=None,
962
- max_seq_len=self.max_seq_len_to_capture,
963
- subquery_start_loc=None,
843
+ max_prefill_seq_len=0,
844
+ max_decode_seq_len=self.max_seq_len_to_capture,
845
+ query_start_loc=None,
846
seq_start_loc=None,
847
context_lens_tensor=None,
848
block_tables=block_tables[:batch_size],
849
use_cuda_graph=True,
850
)
969
- attn_metadata = AttentionMetadata(
970
- num_prefills=0,
971
- num_prefill_tokens=0,
972
- num_decode_tokens=batch_size,
973
- slot_mapping=slot_mapping[:batch_size],
974
- prefill_metadata=None,
975
- decode_metadata=decode_metadata,
976
- )
851
852
if self.lora_config:
853
lora_mapping = LoRAMapping(