@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65bf2ac165
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core][2/N] Model runner refactoring part 2. Combine prepare prefill / decode to a single API (#4681)

This PR combines prepare_prompt and prepare_decode into a single API. This PR also coelsce the attn metadata for prefill/decode to a single class and allow to slice them when running attn backend. It also refactors subquery_start_loc which was not refactored in the previous PR

SangBin Cho committed May 15, 2024 at 14:00 UTC 65bf2ac165734fb6339210c4b2b8ce68d2391b77
18 files changed +777 -730
tests/worker/test_model_runner.py
+84 -39
@@ -58,19 +58,25 @@ def test_prepare_prompt(batch_size):
58 expected_selected_token_indices.append(selected_token_start_idx +
59 seq_len - 1)
60 selected_token_start_idx += seq_len
61 - (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
62 - _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
61 + model_input = model_runner._prepare_model_input(seq_group_metadata_list)
62 + input_tokens = model_input.input_tokens
63 + input_positions = model_input.input_positions
64 + attn_metadata = model_input.attn_metadata
65 + return_seq_lens = model_input.seq_lens
66 + slot_mapping = model_input.slot_mapping
67 assert return_seq_lens == seq_lens
68 assert len(slot_mapping) == len(input_tokens)
69
70 # Verify input metadata is correct for prompts.
71 device = model_runner.device
68 - assert attn_metadata.is_prompt is True
72 + assert attn_metadata.num_prefills > 0
73 + assert attn_metadata.num_decode_tokens == 0
74 assert torch.allclose(
75 attn_metadata.seq_lens_tensor,
76 torch.tensor(seq_lens, device=device, dtype=torch.int))
77 assert attn_metadata.seq_lens == seq_lens
73 - assert attn_metadata.max_seq_len == max(seq_lens)
78 + assert attn_metadata.max_prefill_seq_len == max(seq_lens)
79 + assert attn_metadata.max_decode_seq_len == 0
80
81 # Test subquery start locs.
82 start_idx = 0
@@ -79,11 +85,11 @@ def test_prepare_prompt(batch_size):
85 start_idx += seq_len
86 start_loc.append(start_idx)
87 assert torch.allclose(
82 - attn_metadata.subquery_start_loc,
88 + attn_metadata.query_start_loc,
89 torch.tensor(start_loc, dtype=torch.int32, device=device))
90
91 # Test seq start locs. Note that for normal prefill it is
86 - # equivalent to subquery_start_loc.
92 + # equivalent to query_start_loc.
93 start_idx = 0
94 seq_start_loc = [start_idx]
95 for seq_len in seq_lens:
@@ -123,7 +129,7 @@ def test_prepare_prompt(batch_size):
129 device=actual.device,
130 dtype=actual.dtype)
131 torch.testing.assert_close(actual, expected)
126 - assert input_tokens == input_positions
132 + torch.allclose(input_tokens, input_positions)
133
134 actual = sampling_metadata.selected_token_indices
135 expected = torch.tensor(expected_selected_token_indices,
@@ -144,14 +150,18 @@ def test_prepare_decode_cuda_graph(batch_size):
150 enable_chunked_prefill=False,
151 )
152
147 - seq_lens = []
153 + context_lens = []
154 seq_group_metadata_list = []
155 + # Assume each seq group finishes prefill.
156 for i in range(batch_size):
157 # make sure all tokens fit into one block
151 - seq_len = i % (model_runner.block_size - 1) + 1
152 - seq_lens.append(seq_len)
153 - seq_data = list(range(seq_len))
158 + context_len = i % (model_runner.block_size - 1) + 1
159 + context_lens.append(context_len)
160 + seq_data = list(range(context_len))
161 seq_data = SequenceData(seq_data)
162 + seq_data.update_num_computed_tokens(context_len)
163 + # Append one token ID since prefill is finished.
164 + seq_data.append_token_id(1, 0)
165 seq_group_metadata = SequenceGroupMetadata(
166 request_id=f"test_{i}",
167 is_prompt=False,
@@ -162,18 +172,45 @@ def test_prepare_decode_cuda_graph(batch_size):
172 assert seq_group_metadata.token_chunk_size == 1
173 seq_group_metadata_list.append(seq_group_metadata)
174
165 - input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
166 - model_runner._prepare_decode(seq_group_metadata_list))
175 + model_input = model_runner._prepare_model_input(seq_group_metadata_list)
176 + input_tokens, input_positions, attn_metadata, slot_mapping = (
177 + model_input.input_tokens, model_input.input_positions,
178 + model_input.attn_metadata, model_input.slot_mapping)
179 assert len(slot_mapping) == len(input_tokens)
180
181 expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
182 # Verify input metadata is correct for prompts.
183 device = model_runner.device
172 - assert attn_metadata.is_prompt is False
173 - assert attn_metadata.seq_lens is None
174 - assert attn_metadata.subquery_start_loc is None
175 - assert attn_metadata.seq_start_loc is None
176 - assert attn_metadata.max_seq_len == max(seq_lens)
184 + assert attn_metadata.num_prefills == 0
185 + assert attn_metadata.num_prefill_tokens == 0
186 + seq_lens = [context_len + 1 for context_len in context_lens]
187 + # seq_lens are padded to expected_bs
188 + for _ in range(expected_bs - len(seq_lens)):
189 + seq_lens.append(1)
190 + assert attn_metadata.seq_lens == seq_lens
191 + start_idx = 0
192 + start_loc = [start_idx]
193 + for _ in context_lens:
194 + # decode has only 1 token for query.
195 + start_idx += 1
196 + start_loc.append(start_idx)
197 + assert torch.allclose(
198 + attn_metadata.query_start_loc,
199 + torch.tensor(start_loc, dtype=torch.int32, device=device))
200 +
201 + start_idx = 0
202 + seq_start_loc = [start_idx]
203 + for seq_len in seq_lens:
204 + start_idx += seq_len
205 + seq_start_loc.append(start_idx)
206 + assert torch.allclose(
207 + attn_metadata.seq_start_loc,
208 + torch.tensor(seq_start_loc, dtype=torch.int32, device=device))
209 +
210 + assert torch.allclose(
211 + attn_metadata.context_lens_tensor,
212 + torch.tensor(context_lens, dtype=torch.int, device=device))
213 + assert attn_metadata.max_decode_seq_len == max(seq_lens)
214 assert torch.allclose(
215 attn_metadata.seq_lens_tensor[:len(seq_lens)],
216 torch.tensor(seq_lens, dtype=torch.int, device=device))
@@ -185,23 +222,23 @@ def test_prepare_decode_cuda_graph(batch_size):
222 # It is padded up to
223 assert attn_metadata.block_tables.shape[1] == (
224 model_runner.get_max_block_per_batch())
188 - # Cuda graph should not be used for prerill.
225 assert attn_metadata.use_cuda_graph is True
226
227 assert len(input_tokens) == expected_bs
228 assert len(input_positions) == expected_bs
193 - assert input_tokens == input_positions
229 + torch.allclose(input_tokens, input_positions)
230
231 # Verify Sampling
232 expected_selected_token_indices = []
233 selected_token_start_idx = 0
198 - for seq_len in seq_lens:
234 + for _ in context_lens:
235 expected_selected_token_indices.append(selected_token_start_idx)
236 selected_token_start_idx += 1
237 sampling_metadata = SamplingMetadata.prepare(
238 seq_group_metadata_list,
239 seq_lens,
204 - query_lens=seq_lens,
240 + # query lens is all 1 for decode.
241 + query_lens=[1 for _ in range(len(context_lens))],
242 device=model_runner.device,
243 pin_memory=model_runner.pin_memory)
244 actual = sampling_metadata.selected_token_indices
@@ -220,15 +257,27 @@ def test_empty_seq_group():
257 enforce_eager=False,
258 )
259 seq_group_metadata_list = []
223 - input_tokens, input_positions, attn_metadata, _, _, _, slot_mapping = (
224 - model_runner._prepare_decode(seq_group_metadata_list))
260 + model_input = model_runner._prepare_model_input(seq_group_metadata_list)
261 + input_tokens, input_positions, attn_metadata, slot_mapping = (
262 + model_input.input_tokens,
263 + model_input.input_positions,
264 + model_input.attn_metadata,
265 + model_input.slot_mapping,
266 + )
267 assert len(input_tokens) == 0
268 assert len(input_positions) == 0
269 assert attn_metadata is None
270 assert len(slot_mapping) == 0
271
230 - (input_tokens, input_positions, attn_metadata, return_seq_lens, _, _, _, _,
231 - _, slot_mapping) = (model_runner._prepare_prompt(seq_group_metadata_list))
272 + model_input = model_runner._prepare_model_input(seq_group_metadata_list)
273 + (input_tokens, input_positions, attn_metadata, slot_mapping,
274 + return_seq_lens) = (
275 + model_input.input_tokens,
276 + model_input.input_positions,
277 + model_input.attn_metadata,
278 + model_input.slot_mapping,
279 + model_input.seq_lens,
280 + )
281 assert len(input_tokens) == 0
282 assert len(input_positions) == 0
283 assert attn_metadata is None
@@ -285,9 +334,11 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
334 # Add decode requests
335 for i in range(prefill_batch_size, batch_size):
336 # make sure all tokens fit into one block
288 - seq_len = i % (model_runner.block_size - 1) + 1
289 - prompt_toks = list(range(seq_len))
337 + context_len = i % (model_runner.block_size - 1) + 1
338 + prompt_toks = list(range(context_len))
339 seq_data = SequenceData(prompt_toks)
340 + seq_data.append_token_id(1, 0)
341 + seq_data.update_num_computed_tokens(context_len)
342 seq_group_metadata = SequenceGroupMetadata(
343 request_id=f"test_{i}",
344 is_prompt=False,
@@ -308,23 +359,17 @@ def test_hybrid_batches(batch_size, enforce_eager, distributed_init):
359 assert len(attn_metadata.slot_mapping) == len(input_tokens)
360 assert len(input_positions) == len(input_tokens)
361 assert attn_metadata.num_prefills == prefill_batch_size
311 - if enforce_eager:
312 - assert attn_metadata.num_decode_tokens == decode_batch_size
313 - else:
314 - assert attn_metadata.num_decode_tokens == _get_graph_batch_size(
315 - decode_batch_size)
362 + assert attn_metadata.num_decode_tokens == decode_batch_size
363 assert attn_metadata.num_prefill_tokens == sum(seq_lens)
364
365 # Verify attn metadata is consistent. We don't need to test individual
366 # values here because they are tested above.
320 - prefill_meta = model_runner._prepare_prompt(
321 - prefill_metadata_list).attn_metadata
322 - decode_meta = model_runner._prepare_decode(
323 - decode_metadata_list).attn_metadata
367 + attn_metadata = model_runner._prepare_model_input(
368 + seq_group_metadata_list).attn_metadata
369
325 - for attr_expected, attr_actual in zip(vars(prefill_meta),
370 + for attr_expected, attr_actual in zip(vars(attn_metadata.prefill_metadata),
371 vars(prefill_meta_actual)):
372 assert attr_expected[1] == attr_actual[1]
328 - for attr_expected, attr_actual in zip(vars(decode_meta),
373 + for attr_expected, attr_actual in zip(vars(attn_metadata.decode_metadata),
374 vars(decode_meta_actual)):
375 assert attr_expected[1] == attr_actual[1]
vllm/attention/__init__.py
+2 -3
@@ -1,6 +1,5 @@
1 from vllm.attention.backends.abstract import (AttentionBackend,
2 - AttentionMetadata,
3 - AttentionMetadataPerStage)
2 + AttentionMetadata)
3 from vllm.attention.layer import Attention
4 from vllm.attention.selector import get_attn_backend
5
@@ -8,6 +7,6 @@ __all__ = [
7 "Attention",
8 "AttentionBackend",
9 "AttentionMetadata",
11 - "AttentionMetadataPerStage",
10 + "Attention",
11 "get_attn_backend",
12 ]
vllm/attention/backends/abstract.py
+32 -36
@@ -21,7 +21,7 @@ class AttentionBackend(ABC):
21
22 @staticmethod
23 @abstractmethod
24 - def make_metadata(*args, **kwargs) -> "AttentionMetadataPerStage":
24 + def make_metadata(*args, **kwargs) -> "AttentionMetadata":
25 raise NotImplementedError
26
27 @staticmethod
@@ -53,8 +53,34 @@ class AttentionBackend(ABC):
53
54
55 @dataclass
56 -class AttentionMetadataPerStage:
57 - """Attention metadata for a specific stage. I.e., prefill or decode."""
56 +class AttentionMetadata:
57 + """Attention metadata for prefill and decode batched together."""
58 + # Total number of prefill requests.
59 + num_prefills: int
60 + # Number of prefill tokens.
61 + num_prefill_tokens: int
62 + # Number of decode tokens. Note that it is equivalent to the number of
63 + # decode requests.
64 + num_decode_tokens: int
65 + # (num_tokens,). The indices of the token slots that input tokens will be
66 + # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
67 + # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
68 + # in block 0, and 1st slot in block 1, respectively.
69 + slot_mapping: torch.Tensor
70 +
71 + @property
72 + @abstractmethod
73 + def prefill_metadata(self) -> Optional["AttentionMetadata"]:
74 + """Return the attention metadata that's required to run prefill
75 + attention."""
76 + pass
77 +
78 + @property
79 + @abstractmethod
80 + def decode_metadata(self) -> Optional["AttentionMetadata"]:
81 + """Return the attention metadata that's required to run decode
82 + attention."""
83 + pass
84
85 def asdict_zerocopy(self,
86 skip_fields: Optional[Set[str]] = None
@@ -70,40 +96,10 @@ class AttentionMetadataPerStage:
96 }
97
98
73 -T = TypeVar("T", bound=AttentionMetadataPerStage)
74 -
75 -
76 -@dataclass
77 -class AttentionMetadata(Generic[T]):
78 - """Attention metadata for prefill and decode batched together."""
79 - # Total number of prefill requests.
80 - num_prefills: int
81 - # Number of prefill tokens.
82 - num_prefill_tokens: int
83 - # Number of decode tokens. Note that it is equivalent to the number of
84 - # decode requests.
85 - num_decode_tokens: int
86 - # The attention metadata for prefill requests in a batch.
87 - # None if there's no prefill requests in a batch.
88 - prefill_metadata: Optional[T]
89 - # The attention metadata for decode requests in a batch.
90 - # None if there's no decode requests in a batch.
91 - decode_metadata: Optional[T]
92 - # (num_tokens,). The indices of the token slots that input tokens will be
93 - # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
94 - # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
95 - # in block 0, and 1st slot in block 1, respectively.
96 - slot_mapping: torch.Tensor
97 -
98 - def __post_init__(self):
99 - if self.num_prefill_tokens > 0:
100 - assert self.num_prefills > 0
101 - assert self.prefill_metadata is not None
102 - if self.num_decode_tokens > 0:
103 - assert self.decode_metadata is not None
99 +T = TypeVar("T", bound=AttentionMetadata)
100
101
106 -class AttentionImpl(ABC):
102 +class AttentionImpl(ABC, Generic[T]):
103
104 @abstractmethod
105 def __init__(
@@ -125,7 +121,7 @@ class AttentionImpl(ABC):
121 key: torch.Tensor,
122 value: torch.Tensor,
123 kv_cache: torch.Tensor,
128 - attn_metadata: AttentionMetadata,
124 + attn_metadata: T,
125 kv_scale: float = 1.0,
126 ) -> torch.Tensor:
127 raise NotImplementedError
vllm/attention/backends/flash_attn.py
+79 -16
@@ -11,8 +11,7 @@ import torch
11 from vllm_flash_attn import flash_attn_varlen_func
12
13 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14 - AttentionMetadata,
15 - AttentionMetadataPerStage)
14 + AttentionMetadata)
15 from vllm.attention.ops.paged_attn import (PagedAttention,
16 PagedAttentionMetadata)
17
@@ -58,8 +57,7 @@ class FlashAttentionBackend(AttentionBackend):
57
58
59 @dataclass
61 -class FlashAttentionMetadata(AttentionMetadataPerStage,
62 - PagedAttentionMetadata):
60 +class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
61 """Metadata for FlashAttentionBackend.
62
63 NOTE: Any python object stored here is not updated when it is
@@ -67,9 +65,6 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
65 dynamically, it should be stored in tensor. The tensor has to be
66 updated from `CUDAGraphRunner.forward` API.
67 """
70 - # Currently, input sequences can only contain all prompts
71 - # or all decoding. True if all sequences are prompts.
72 - is_prompt: bool
68 # (batch_size,). The sequence length per sequence. Sequence length means
69 # the computed tokens + new tokens None if it is a decoding.
70 seq_lens: Optional[List[int]]
@@ -84,14 +79,18 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
79 # |-------------------- seq_len ----------------------|
80 # |-- query_len ---|
81
87 - # Maximum query length in the batch.
82 + # Maximum query length in the batch. None for decoding.
83 max_query_len: Optional[int]
89 - # Maximum sequence length in the batch.
90 - max_seq_len: Optional[int]
84 + # Maximum sequence length among prefill batch. 0 if there are decoding
85 + # requests only.
86 + max_prefill_seq_len: int
87 + # Maximum sequence length among decode batch. 0 if there are prefill
88 + # requests only.
89 + max_decode_seq_len: int
90 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
91 # the batch, used to index into subquery. E.g., if the subquery length
92 # is [4, 6], it is [0, 4, 10].
94 - subquery_start_loc: Optional[torch.Tensor]
93 + query_start_loc: Optional[torch.Tensor]
94 # (batch_size + 1,). The cumulative sequence lengths of the sequences in
95 # the batch, used to index into sequence. E.g., if the sequence length is
96 # [4, 6], it is [0, 4, 10].
@@ -105,6 +104,70 @@ class FlashAttentionMetadata(AttentionMetadataPerStage,
104 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
105 use_cuda_graph: bool
106
107 + _cached_prefill_metadata: Optional["FlashAttentionMetadata"] = None
108 + _cached_decode_metadata: Optional["FlashAttentionMetadata"] = None
109 +
110 + @property
111 + def prefill_metadata(self) -> Optional["FlashAttentionMetadata"]:
112 + if self.num_prefills == 0:
113 + return None
114 +
115 + if self._cached_prefill_metadata is not None:
116 + return self._cached_prefill_metadata
117 +
118 + assert self.seq_lens is not None
119 + assert self.seq_lens_tensor is not None
120 + assert self.query_start_loc is not None
121 + assert self.context_lens_tensor is not None
122 + assert self.block_tables is not None
123 + assert self.seq_start_loc is not None
124 +
125 + self._cached_prefill_metadata = FlashAttentionMetadata(
126 + num_prefills=self.num_prefills,
127 + num_prefill_tokens=self.num_prefill_tokens,
128 + num_decode_tokens=0,
129 + slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
130 + seq_lens=self.seq_lens[:self.num_prefills],
131 + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
132 + max_query_len=self.max_query_len,
133 + max_prefill_seq_len=self.max_prefill_seq_len,
134 + max_decode_seq_len=0,
135 + query_start_loc=self.query_start_loc[:self.num_prefills + 1],
136 + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
137 + context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
138 + block_tables=self.block_tables[:self.num_prefills],
139 + use_cuda_graph=False,
140 + )
141 + return self._cached_prefill_metadata
142 +
143 + @property
144 + def decode_metadata(self) -> Optional["FlashAttentionMetadata"]:
145 + if self.num_decode_tokens == 0:
146 + return None
147 +
148 + if self._cached_decode_metadata is not None:
149 + return self._cached_decode_metadata
150 + assert self.block_tables is not None
151 + assert self.seq_lens_tensor is not None
152 +
153 + self._cached_decode_metadata = FlashAttentionMetadata(
154 + num_prefills=0,
155 + num_prefill_tokens=0,
156 + num_decode_tokens=self.num_decode_tokens,
157 + slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
158 + seq_lens=None,
159 + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
160 + max_query_len=None,
161 + max_prefill_seq_len=0,
162 + max_decode_seq_len=self.max_decode_seq_len,
163 + query_start_loc=None,
164 + seq_start_loc=None,
165 + context_lens_tensor=None,
166 + block_tables=self.block_tables[self.num_prefills:],
167 + use_cuda_graph=self.use_cuda_graph,
168 + )
169 + return self._cached_decode_metadata
170 +
171
172 class FlashAttentionImpl(AttentionImpl):
173 """
@@ -168,7 +231,7 @@ class FlashAttentionImpl(AttentionImpl):
231 key: torch.Tensor,
232 value: torch.Tensor,
233 kv_cache: torch.Tensor,
171 - attn_metadata: AttentionMetadata[FlashAttentionMetadata],
234 + attn_metadata: FlashAttentionMetadata,
235 kv_scale: float = 1.0,
236 ) -> torch.Tensor:
237 """Forward pass with FlashAttention and PagedAttention.
@@ -228,8 +291,8 @@ class FlashAttentionImpl(AttentionImpl):
291 v=value,
292 cu_seqlens_q=prefill_meta.seq_start_loc,
293 cu_seqlens_k=prefill_meta.seq_start_loc,
231 - max_seqlen_q=prefill_meta.max_seq_len,
232 - max_seqlen_k=prefill_meta.max_seq_len,
294 + max_seqlen_q=prefill_meta.max_prefill_seq_len,
295 + max_seqlen_k=prefill_meta.max_prefill_seq_len,
296 softmax_scale=self.scale,
297 causal=True,
298 window_size=self.sliding_window,
@@ -249,7 +312,7 @@ class FlashAttentionImpl(AttentionImpl):
312 key_cache,
313 value_cache,
314 prefill_meta.block_tables,
252 - prefill_meta.subquery_start_loc,
315 + prefill_meta.query_start_loc,
316 prefill_meta.seq_lens_tensor,
317 prefill_meta.context_lens_tensor,
318 prefill_meta.max_query_len,
@@ -264,7 +327,7 @@ class FlashAttentionImpl(AttentionImpl):
327 value_cache,
328 decode_meta.block_tables,
329 decode_meta.seq_lens_tensor,
267 - decode_meta.max_seq_len,
330 + decode_meta.max_decode_seq_len,
331 self.kv_cache_dtype,
332 self.num_kv_heads,
333 self.scale,
vllm/attention/backends/flashinfer.py
+28 -10
@@ -8,8 +8,7 @@ from vllm_flash_attn import flash_attn_varlen_func
8
9 from vllm import _custom_ops as ops
10 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
11 - AttentionMetadata,
12 - AttentionMetadataPerStage)
11 + AttentionMetadata)
12
13
14 class FlashInferBackend(AttentionBackend):
@@ -56,9 +55,10 @@ class FlashInferBackend(AttentionBackend):
55
56
57 @dataclass
59 -class FlashInferMetadata(AttentionMetadataPerStage):
60 -
61 - is_prompt: bool
58 +class FlashInferMetadata(AttentionMetadata):
59 + # Maximum sequence length among prefill batch. 0 if there are decoding
60 + # requests only.
61 + max_prefill_seq_len: int
62
63 use_cuda_graph: bool = False
64
@@ -67,7 +67,6 @@ class FlashInferMetadata(AttentionMetadataPerStage):
67 # Metadata for the prefill stage since we still
68 # use flash attention for prefill.
69 seq_start_loc: Optional[torch.Tensor] = None
70 - max_seq_len: Optional[int] = None
70 block_tables: Optional[torch.Tensor] = None
71
72 # Metadata for the decode stage
@@ -113,7 +112,8 @@ class FlashInferMetadata(AttentionMetadataPerStage):
112 # When using flashinfer, we are also creating the FlashInferMetadata,
113 # which will also call post_init by default, here we want to skip the
114 # post_init if it's the prefill phase.
116 - if not self.is_prompt:
115 + if self.num_prefills == 0:
116 + assert self.num_decode_tokens > 0
117 self.decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
118 self.workspace_buffer, "NHD")
119 self.decode_wrapper.begin_forward(
@@ -138,6 +138,24 @@ class FlashInferMetadata(AttentionMetadataPerStage):
138 skip_fields.add('decode_wrapper')
139 return super().asdict_zerocopy(skip_fields)
140
141 + @property
142 + def prefill_metadata(self) -> Optional["FlashInferMetadata"]:
143 + # Currently chunked prefill is not supported
144 + if self.num_decode_tokens == 0:
145 + assert self.num_prefills > 0
146 + return self
147 +
148 + return None
149 +
150 + @property
151 + def decode_metadata(self) -> Optional["FlashInferMetadata"]:
152 + # Currently chunked prefill is not supported
153 + if self.num_prefills > 0:
154 + assert self.num_decode_tokens == 0
155 + return None
156 +
157 + return self
158 +
159
160 class FlashInferImpl(AttentionImpl):
161
@@ -172,7 +190,7 @@ class FlashInferImpl(AttentionImpl):
190 key: torch.Tensor,
191 value: torch.Tensor,
192 kv_cache: Optional[torch.Tensor],
175 - attn_metadata: AttentionMetadata[FlashInferMetadata],
193 + attn_metadata: FlashInferMetadata,
194 kv_scale: float = 1.0,
195 ) -> torch.Tensor:
196 assert kv_scale == 1.0
@@ -208,8 +226,8 @@ class FlashInferImpl(AttentionImpl):
226 v=value,
227 cu_seqlens_q=prefill_meta.seq_start_loc,
228 cu_seqlens_k=prefill_meta.seq_start_loc,
211 - max_seqlen_q=prefill_meta.max_seq_len,
212 - max_seqlen_k=prefill_meta.max_seq_len,
229 + max_seqlen_q=prefill_meta.max_prefill_seq_len,
230 + max_seqlen_k=prefill_meta.max_prefill_seq_len,
231 softmax_scale=self.scale,
232 causal=True,
233 window_size=self.sliding_window,
vllm/attention/backends/rocm_flash_attn.py
+80 -18
@@ -6,8 +6,7 @@ import torch
6
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 - AttentionMetadata,
10 - AttentionMetadataPerStage)
9 + AttentionMetadata)
10 from vllm.attention.ops.paged_attn import (PagedAttention,
11 PagedAttentionMetadata)
12 from vllm.logger import init_logger
@@ -56,8 +55,7 @@ class ROCmFlashAttentionBackend(AttentionBackend):
55
56
57 @dataclass
59 -class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
60 - PagedAttentionMetadata):
58 +class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
59 """Metadata for FlashAttentionBackend.
60
61 NOTE: Any python object stored here is not updated when it is
@@ -65,9 +63,6 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
63 dynamically, it should be stored in tensor. The tensor has to be
64 updated from `CUDAGraphRunner.forward` API.
65 """
68 - # Currently, input sequences can only contain all prompts
69 - # or all decoding. True if all sequences are prompts.
70 - is_prompt: bool
66 # (batch_size,). The sequence length per sequence. Sequence length means
67 # the computed tokens + new tokens None if it is a decoding.
68 seq_lens: Optional[List[int]]
@@ -82,14 +77,18 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
77 # |-------------------- seq_len ----------------------|
78 # |-- query_len ---|
79
85 - # Maximum query length in the batch.
80 + # Maximum query length in the batch. None for decoding.
81 max_query_len: Optional[int]
87 - # Maximum sequence length in the batch.
88 - max_seq_len: Optional[int]
82 + # Maximum sequence length among prefill batch. 0 if there are decoding
83 + # requests only.
84 + max_prefill_seq_len: int
85 + # Maximum sequence length among decode batch. 0 if there are prefill
86 + # requests only.
87 + max_decode_seq_len: int
88 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
89 # the batch, used to index into subquery. E.g., if the subquery length
90 # is [4, 6], it is [0, 4, 10].
92 - subquery_start_loc: Optional[torch.Tensor]
91 + query_start_loc: Optional[torch.Tensor]
92 # (batch_size + 1,). The cumulative sequence lengths of the sequences in
93 # the batch, used to index into sequence. E.g., if the sequence length is
94 # [4, 6], it is [0, 4, 10].
@@ -102,6 +101,69 @@ class ROCmFlashAttentionMetadata(AttentionMetadataPerStage,
101 # (batch_size,) A tensor of context lengths (tokens that are computed
102 # so far).
103 context_lens_tensor: Optional[torch.Tensor]
104 + _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
105 + _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
106 +
107 + @property
108 + def prefill_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
109 + if self.num_prefills == 0:
110 + return None
111 +
112 + if self._cached_prefill_metadata is not None:
113 + return self._cached_prefill_metadata
114 +
115 + assert self.seq_lens is not None
116 + assert self.seq_lens_tensor is not None
117 + assert self.query_start_loc is not None
118 + assert self.context_lens_tensor is not None
119 + assert self.block_tables is not None
120 + assert self.seq_start_loc is not None
121 +
122 + self._cached_prefill_metadata = ROCmFlashAttentionMetadata(
123 + num_prefills=self.num_prefills,
124 + num_prefill_tokens=self.num_prefill_tokens,
125 + num_decode_tokens=0,
126 + slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
127 + seq_lens=self.seq_lens[:self.num_prefills],
128 + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
129 + max_query_len=self.max_query_len,
130 + max_prefill_seq_len=self.max_prefill_seq_len,
131 + max_decode_seq_len=0,
132 + query_start_loc=self.query_start_loc[:self.num_prefills + 1],
133 + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
134 + context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
135 + block_tables=self.block_tables[:self.num_prefills],
136 + use_cuda_graph=False,
137 + )
138 + return self._cached_prefill_metadata
139 +
140 + @property
141 + def decode_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
142 + if self.num_decode_tokens == 0:
143 + return None
144 +
145 + if self._cached_decode_metadata is not None:
146 + return self._cached_decode_metadata
147 + assert self.block_tables is not None
148 + assert self.seq_lens_tensor is not None
149 +
150 + self._cached_decode_metadata = ROCmFlashAttentionMetadata(
151 + num_prefills=0,
152 + num_prefill_tokens=0,
153 + num_decode_tokens=self.num_decode_tokens,
154 + slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
155 + seq_lens=None,
156 + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
157 + max_query_len=None,
158 + max_prefill_seq_len=0,
159 + max_decode_seq_len=self.max_decode_seq_len,
160 + query_start_loc=None,
161 + seq_start_loc=None,
162 + context_lens_tensor=None,
163 + block_tables=self.block_tables[self.num_prefills:],
164 + use_cuda_graph=self.use_cuda_graph,
165 + )
166 + return self._cached_decode_metadata
167
168
169 class ROCmFlashAttentionImpl(AttentionImpl):
@@ -198,7 +260,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
260 key: torch.Tensor,
261 value: torch.Tensor,
262 kv_cache: torch.Tensor,
201 - attn_metadata: AttentionMetadata[ROCmFlashAttentionMetadata],
263 + attn_metadata: ROCmFlashAttentionMetadata,
264 kv_scale: float = 1.0,
265 ) -> torch.Tensor:
266 """Forward pass with FlashAttention and PagedAttention.
@@ -266,8 +328,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
328 None,
329 prefill_meta.seq_start_loc,
330 prefill_meta.seq_start_loc,
269 - prefill_meta.max_seq_len,
270 - prefill_meta.max_seq_len,
331 + prefill_meta.max_prefill_seq_len,
332 + prefill_meta.max_prefill_seq_len,
333 True,
334 self.scale,
335 )
@@ -290,8 +352,8 @@ class ROCmFlashAttentionImpl(AttentionImpl):
352 v=value,
353 cu_seqlens_q=prefill_meta.seq_start_loc,
354 cu_seqlens_k=prefill_meta.seq_start_loc,
293 - max_seqlen_q=prefill_meta.max_seq_len,
294 - max_seqlen_k=prefill_meta.max_seq_len,
355 + max_seqlen_q=prefill_meta.max_prefill_seq_len,
356 + max_seqlen_k=prefill_meta.max_prefill_seq_len,
357 softmax_scale=self.scale,
358 causal=True,
359 )
@@ -308,7 +370,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
370 key_cache,
371 value_cache,
372 prefill_meta.block_tables,
311 - prefill_meta.subquery_start_loc,
373 + prefill_meta.query_start_loc,
374 prefill_meta.seq_lens_tensor,
375 prefill_meta.context_lens_tensor,
376 prefill_meta.max_query_len,
@@ -324,7 +386,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
386 value_cache,
387 decode_meta.block_tables,
388 decode_meta.seq_lens_tensor,
327 - decode_meta.max_seq_len,
389 + decode_meta.max_decode_seq_len,
390 self.kv_cache_dtype,
391 self.num_kv_heads,
392 self.scale,
vllm/attention/backends/torch_sdpa.py
+22 -6
@@ -7,8 +7,7 @@ import torch
7 from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata,
11 - AttentionMetadataPerStage)
10 + AttentionMetadata)
11 from vllm.attention.ops.paged_attn import (PagedAttention,
12 PagedAttentionMetadata)
13
@@ -54,8 +53,7 @@ class TorchSDPABackend(AttentionBackend):
53
54
55 @dataclass
57 -class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
58 - AttentionMetadataPerStage):
56 +class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
57 """Metadata for TorchSDPABackend.
58 """
59 # Currently, input sequences can only contain all prompts
@@ -72,8 +70,26 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata,
70 # will not appear in the __repr__ and __init__
71 self.attn_bias: Optional[List[torch.Tensor]] = None
72
73 + @property
74 + def prefill_metadata(self) -> Optional["TorchSDPAMetadata"]:
75 + # Currently chunked prefill is not supported
76 + if self.num_decode_tokens == 0:
77 + assert self.num_prefills > 0
78 + return self
79
76 -class TorchSDPABackendImpl(AttentionImpl):
80 + return None
81 +
82 + @property
83 + def decode_metadata(self) -> Optional["TorchSDPAMetadata"]:
84 + # Currently chunked prefill is not supported
85 + if self.num_prefills > 0:
86 + assert self.num_decode_tokens == 0
87 + return None
88 +
89 + return self
90 +
91 +
92 +class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
93
94 def __init__(
95 self,
@@ -200,7 +216,7 @@ class TorchSDPABackendImpl(AttentionImpl):
216 value_cache,
217 attn_metadata.block_tables,
218 attn_metadata.seq_lens_tensor,
203 - attn_metadata.max_seq_len,
219 + attn_metadata.max_decode_seq_len,
220 self.kv_cache_dtype,
221 self.num_kv_heads,
222 self.scale,
vllm/attention/backends/xformers.py
+77 -15
@@ -9,8 +9,7 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
9 LowerTriangularMaskWithTensorBias)
10
11 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 - AttentionMetadata,
13 - AttentionMetadataPerStage)
12 + AttentionMetadata)
13 from vllm.attention.ops.paged_attn import (PagedAttention,
14 PagedAttentionMetadata)
15 from vllm.logger import init_logger
@@ -59,7 +58,7 @@ class XFormersBackend(AttentionBackend):
58
59
60 @dataclass
62 -class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
61 +class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
62 """Metadata for XFormersbackend.
63
64 NOTE: Any python object stored here is not updated when it is
@@ -67,9 +66,6 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
66 dynamically, it should be stored in tensor. The tensor has to be
67 updated from `CUDAGraphRunner.forward` API.
68 """
70 - # Currently, input sequences can only contain all prompts
71 - # or all decoding. True if all sequences are prompts.
72 - is_prompt: bool
69 # (batch_size,). The sequence length per sequence. Sequence length means
70 # the computed tokens + new tokens None if it is a decoding.
71 seq_lens: Optional[List[int]]
@@ -83,15 +79,19 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
79 # |-------------------- seq_len ----------------------|
80 # |-- query_len ---|
81
86 - # Maximum query length in the batch.
82 + # Maximum query length in the batch. None for decoding.
83 max_query_len: Optional[int]
84 # FIXME: It is for flash attn.
89 - # Maximum sequence length in the batch.
90 - max_seq_len: Optional[int]
85 + # Maximum sequence length among prefill batch. 0 if there are decoding
86 + # requests only.
87 + max_prefill_seq_len: int
88 + # Maximum sequence length among decode batch. 0 if there are prefill
89 + # requests only.
90 + max_decode_seq_len: int
91 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
92 # the batch, used to index into subquery. E.g., if the subquery length
93 # is [4, 6], it is [0, 4, 10].
94 - subquery_start_loc: Optional[torch.Tensor]
94 + query_start_loc: Optional[torch.Tensor]
95 # FIXME: It is for flash attn.
96 # (batch_size + 1,). The cumulative sequence lengths of the sequences in
97 # the batch, used to index into sequence. E.g., if the sequence length is
@@ -105,6 +105,8 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
105 # Cuda-graph is currently enabled for decoding only.
106 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
107 use_cuda_graph: bool
108 + _cached_prefill_metadata: Optional["XFormersMetadata"] = None
109 + _cached_decode_metadata: Optional["XFormersMetadata"] = None
110
111 def __post_init__(self):
112 # Set during the execution of the first attention op.
@@ -114,8 +116,68 @@ class XFormersMetadata(AttentionMetadataPerStage, PagedAttentionMetadata):
116 # will not appear in the __repr__ and __init__
117 self.attn_bias: Optional[List[AttentionBias]] = None
118
117 -
118 -class XFormersImpl(AttentionImpl):
119 + @property
120 + def prefill_metadata(self) -> Optional["XFormersMetadata"]:
121 + if self.num_prefills == 0:
122 + return None
123 +
124 + if self._cached_prefill_metadata is not None:
125 + return self._cached_prefill_metadata
126 +
127 + assert self.seq_lens is not None
128 + assert self.seq_lens_tensor is not None
129 + assert self.query_start_loc is not None
130 + assert self.context_lens_tensor is not None
131 + assert self.block_tables is not None
132 +
133 + self._cached_prefill_metadata = XFormersMetadata(
134 + num_prefills=self.num_prefills,
135 + num_prefill_tokens=self.num_prefill_tokens,
136 + num_decode_tokens=0,
137 + slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
138 + seq_lens=self.seq_lens[:self.num_prefills],
139 + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
140 + max_query_len=self.max_query_len,
141 + max_prefill_seq_len=self.max_prefill_seq_len,
142 + max_decode_seq_len=0,
143 + query_start_loc=self.query_start_loc[:self.num_prefills + 1],
144 + seq_start_loc=None,
145 + context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
146 + block_tables=self.block_tables[:self.num_prefills],
147 + use_cuda_graph=False,
148 + )
149 + return self._cached_prefill_metadata
150 +
151 + @property
152 + def decode_metadata(self) -> Optional["XFormersMetadata"]:
153 + if self.num_decode_tokens == 0:
154 + return None
155 +
156 + if self._cached_decode_metadata is not None:
157 + return self._cached_decode_metadata
158 + assert self.block_tables is not None
159 + assert self.seq_lens_tensor is not None
160 +
161 + self._cached_decode_metadata = XFormersMetadata(
162 + num_prefills=0,
163 + num_prefill_tokens=0,
164 + num_decode_tokens=self.num_decode_tokens,
165 + slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
166 + seq_lens=None,
167 + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
168 + max_query_len=None,
169 + max_prefill_seq_len=0,
170 + max_decode_seq_len=self.max_decode_seq_len,
171 + query_start_loc=None,
172 + seq_start_loc=None,
173 + context_lens_tensor=None,
174 + block_tables=self.block_tables[self.num_prefills:],
175 + use_cuda_graph=self.use_cuda_graph,
176 + )
177 + return self._cached_decode_metadata
178 +
179 +
180 +class XFormersImpl(AttentionImpl[XFormersMetadata]):
181 """
182 If the input tensors contain prompt tokens, the layout is as follows:
183 |<--------------- num_prefill_tokens ----------------->|
@@ -176,7 +238,7 @@ class XFormersImpl(AttentionImpl):
238 key: torch.Tensor,
239 value: torch.Tensor,
240 kv_cache: Optional[torch.Tensor],
179 - attn_metadata: AttentionMetadata[XFormersMetadata],
241 + attn_metadata: "XFormersMetadata",
242 kv_scale: float = 1.0,
243 ) -> torch.Tensor:
244 """Forward pass with xFormers and PagedAttention.
@@ -244,7 +306,7 @@ class XFormersImpl(AttentionImpl):
306 key_cache,
307 value_cache,
308 prefill_meta.block_tables,
247 - prefill_meta.subquery_start_loc,
309 + prefill_meta.query_start_loc,
310 prefill_meta.seq_lens_tensor,
311 prefill_meta.context_lens_tensor,
312 prefill_meta.max_query_len,
@@ -261,7 +323,7 @@ class XFormersImpl(AttentionImpl):
323 value_cache,
324 decode_meta.block_tables,
325 decode_meta.seq_lens_tensor,
264 - decode_meta.max_seq_len,
326 + decode_meta.max_decode_seq_len,
327 self.kv_cache_dtype,
328 self.num_kv_heads,
329 self.scale,
vllm/attention/layer.py
+2 -3
@@ -4,8 +4,7 @@ from typing import List, Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.attention.backends.abstract import (AttentionMetadata,
8 - AttentionMetadataPerStage)
7 +from vllm.attention.backends.abstract import AttentionMetadata
8 from vllm.attention.selector import get_attn_backend
9 from vllm.config import CacheConfig
10
@@ -57,7 +56,7 @@ class Attention(nn.Module):
56 key: torch.Tensor,
57 value: torch.Tensor,
58 kv_cache: Optional[torch.Tensor],
60 - attn_metadata: AttentionMetadata[AttentionMetadataPerStage],
59 + attn_metadata: AttentionMetadata,
60 kv_scale: float = 1.0,
61 ) -> torch.Tensor:
62 return self.impl.forward(query, key, value, kv_cache, attn_metadata,
vllm/attention/ops/paged_attn.py
+5 -5
@@ -16,8 +16,8 @@ class PagedAttentionMetadata:
16 # (batch_size,). The length of sequences (entire tokens seen so far) per
17 # sequence.
18 seq_lens_tensor: Optional[torch.Tensor]
19 - # Maximum sequence length in the batch.
20 - max_seq_len: Optional[int]
19 + # Maximum sequence length in the batch. 0 if it is prefill-only batch.
20 + max_decode_seq_len: int
21 # (batch_size, max_blocks_per_seq).
22 # Block addresses per sequence. (Seq id -> list of physical block)
23 # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
@@ -166,7 +166,7 @@ class PagedAttention:
166 key_cache: torch.Tensor,
167 value_cache: torch.Tensor,
168 block_tables: torch.Tensor,
169 - subquery_start_loc: torch.Tensor,
169 + query_start_loc: torch.Tensor,
170 seq_lens_tensor: torch.Tensor,
171 context_lens: torch.Tensor,
172 max_query_len: int,
@@ -182,8 +182,8 @@ class PagedAttention:
182 key_cache,
183 value_cache,
184 block_tables,
185 - # subquery_start_loc is (batch_size + 1,)
186 - subquery_start_loc[:-1],
185 + # query_start_loc is (batch_size + 1,)
186 + query_start_loc[:-1],
187 seq_lens_tensor,
188 context_lens,
189 max_query_len,
vllm/engine/arg_utils.py
+5
@@ -618,6 +618,11 @@ class EngineArgs:
618 decoding_config = DecodingConfig(
619 guided_decoding_backend=self.guided_decoding_backend)
620
621 + if (model_config.get_sliding_window() is not None
622 + and scheduler_config.chunked_prefill_enabled):
623 + raise ValueError(
624 + "Chunked prefill is not supported with sliding window.")
625 +
626 return EngineConfig(model_config=model_config,
627 cache_config=cache_config,
628 parallel_config=parallel_config,
vllm/model_executor/layers/rejection_sampler.py
+1
@@ -122,6 +122,7 @@ class RejectionSampler(nn.Module):
122 draft_token_ids,
123 bonus_token_ids,
124 )
125 +
126 return output_token_ids
127
128 def _batch_modified_rejection_sampling(
vllm/sequence.py
+2 -1
@@ -654,8 +654,9 @@ class SequenceGroupMetadata:
654 return self.lora_request.lora_int_id if self.lora_request else 0
655
656 @property
657 - def token_chunk_size(self) -> Optional[int]:
657 + def token_chunk_size(self) -> int:
658 """Return the number of tokens to be processed (chunk size)."""
659 + assert self._token_chunk_size is not None
660 return self._token_chunk_size
661
662
vllm/spec_decode/batch_expansion.py
+16 -7
@@ -293,21 +293,30 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
293 prompt_token_ids = seq_data.get_prompt_token_ids()
294 new_output_token_ids = [*seq_data.get_output_token_ids(), *token_ids]
295
296 + new_seq_data_dict = {
297 + target_seq_id:
298 + SequenceData(
299 + prompt_token_ids=prompt_token_ids,
300 + output_token_ids=new_output_token_ids,
301 + ),
302 + }
303 + # This is a hack. Technically, spec decoding should compute
304 + # num_lookahead slots at one shot, but instead, it expands the batch
305 + # and evaluate one by one right now. context_len is seq_len - 1 because
306 + # the kv cache is filled by a previous batch in the batch expansion.
307 + for data in new_seq_data_dict.values():
308 + data.update_num_computed_tokens(data.get_len() - 1)
309 +
310 return SequenceGroupMetadata(
311 request_id=seq_group_metadata.request_id,
312 is_prompt=seq_group_metadata.is_prompt,
299 - seq_data={
300 - target_seq_id:
301 - SequenceData(
302 - prompt_token_ids=prompt_token_ids,
303 - output_token_ids=new_output_token_ids,
304 - ),
305 - },
313 + seq_data=new_seq_data_dict,
314 sampling_params=seq_group_metadata.sampling_params,
315 block_tables={
316 target_seq_id: seq_group_metadata.block_tables[seq_id],
317 },
318 lora_request=None,
319 + token_chunk_size=1,
320 )
321
322 def _split_scoring_output(
vllm/spec_decode/multi_step_worker.py
+1
@@ -114,6 +114,7 @@ class MultiStepWorker(Worker):
114 token_logprob = seq_output.logprobs[token_id]
115
116 seq.append_token_id(token_id, token_logprob.logprob)
117 + seq.update_num_computed_tokens(1)
118
119 def _shallow_copy_inputs(
120 self, seq_group_metadata_list: List[SequenceGroupMetadata]
vllm/worker/cpu_model_runner.py
+3 -7
@@ -159,12 +159,10 @@ class CPUModelRunner:
159 is_prompt=True,
160 seq_lens=seq_lens,
161 seq_lens_tensor=None,
162 - max_seq_len=None,
162 + max_decode_seq_len=None,
163 num_prefills=len(seq_lens),
164 num_prefill_tokens=num_prompt_tokens,
165 num_decode_tokens=0,
166 - prefill_metadata=None,
167 - decode_metadata=None,
166 block_tables=torch.tensor([]),
167 slot_mapping=slot_mapping,
168 )
@@ -213,7 +211,7 @@ class CPUModelRunner:
211 block_table = block_table[-sliding_window_blocks:]
212 block_tables.append(block_table)
213
216 - max_seq_len = max(seq_lens)
214 + max_decode_seq_len = max(seq_lens)
215
216 input_tokens = torch.tensor(input_tokens,
217 dtype=torch.long,
@@ -243,12 +241,10 @@ class CPUModelRunner:
241 slot_mapping=slot_mapping,
242 seq_lens=seq_lens,
243 seq_lens_tensor=seq_lens_tensor,
246 - max_seq_len=max_seq_len,
244 + max_decode_seq_len=max_decode_seq_len,
245 num_prefill_tokens=0,
246 num_decode_tokens=len(input_tokens),
247 num_prefills=0,
250 - prefill_metadata=None,
251 - decode_metadata=None,
248 block_tables=block_tables,
249 )
250 return (
vllm/worker/embedding_model_runner.py
+15 -115
@@ -13,7 +13,7 @@ from vllm.lora.request import LoRARequest
13 from vllm.model_executor.pooling_metadata import PoolingMetadata
14 from vllm.pooling_params import PoolingParams
15 from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16 -from vllm.worker.model_runner import BatchType, ModelRunner
16 +from vllm.worker.model_runner import ModelRunner
17
18 logger = init_logger(__name__)
19
@@ -88,85 +88,24 @@ class EmbeddingModelRunner(ModelRunner):
88 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89 Set[LoRARequest], LoRAMapping, torch.Tensor]:
90 if self.is_driver_worker:
91 - prefill_reqs = []
92 - decode_reqs = []
93 - for seq_group_meta in seq_group_metadata_list:
94 - if seq_group_meta.is_prompt:
95 - prefill_reqs.append(seq_group_meta)
96 - else:
97 - decode_reqs.append(seq_group_meta)
98 -
91 # Prepare input tensors.
92 (
93 input_tokens,
94 input_positions,
103 - prefill_attn_metadata,
104 - prompt_lens,
105 - subquery_lens,
106 - lora_index_mapping,
107 - lora_prompt_mapping,
95 + attn_metadata,
96 + seq_lens,
97 + _,
98 + lora_mapping,
99 lora_requests,
100 multi_modal_input,
101 slot_mapping,
111 - ) = self._prepare_prompt(prefill_reqs)
112 - (
113 - decode_input_tokens,
114 - decode_input_positions,
115 - decode_attn_metadata,
116 - decode_lora_index_mapping,
117 - decode_lora_prompt_mapping,
118 - decode_lora_requests,
119 - decode_slot_mapping,
120 - ) = self._prepare_decode(decode_reqs)
121 -
102 + num_prefill_tokens,
103 + num_decode_tokens,
104 + num_prefills,
105 + ) = self._prepare_model_input(seq_group_metadata_list)
106 # Prepare PoolingMetadata
107 pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
124 - prompt_lens)
125 -
126 - if not self.scheduler_config.chunked_prefill_enabled:
127 - assert (len(prefill_reqs) and len(decode_reqs)) == 0
128 -
129 - num_prefills = len(prompt_lens)
130 - num_prefill_tokens = len(input_tokens)
131 - num_decode_tokens = len(decode_input_tokens)
132 -
133 - # Coalesce tensors. Note that attn_metadata is currently not
134 - # coalesced for simplicity.
135 - input_tokens.extend(decode_input_tokens)
136 - input_positions.extend(decode_input_positions)
137 - slot_mapping.extend(decode_slot_mapping)
138 - lora_index_mapping.extend(decode_lora_index_mapping)
139 - lora_prompt_mapping.extend(decode_lora_prompt_mapping)
140 - lora_requests.update(decode_lora_requests)
141 -
142 - input_tokens = torch.tensor(input_tokens,
143 - dtype=torch.long,
144 - device=self.device)
145 - input_positions = torch.tensor(input_positions,
146 - dtype=torch.long,
147 - device=self.device)
148 - slot_mapping = torch.tensor(slot_mapping,
149 - dtype=torch.long,
150 - device=self.device)
151 -
152 - if self.lora_config:
153 - lora_mapping = LoRAMapping(
154 - lora_index_mapping,
155 - lora_prompt_mapping,
156 - )
157 - else:
158 - lora_mapping = None
159 -
160 - # Broadcast the metadata.
161 - # If batch contains both prefill and decode, it sends 2 broadcasts.
162 - # If it only contains 1 type, it triggers a single broadcast.
163 - if (prefill_attn_metadata is not None
164 - and decode_attn_metadata is not None):
165 - batch_type = BatchType.MIXED
166 - elif prefill_attn_metadata is not None:
167 - batch_type = BatchType.PREFILL
168 - else:
169 - batch_type = BatchType.DECODE
108 + seq_lens)
109
110 metadata_dict = {
111 "input_tokens": input_tokens,
@@ -178,65 +117,26 @@ class EmbeddingModelRunner(ModelRunner):
117 "num_decode_tokens": num_decode_tokens,
118 "slot_mapping": slot_mapping,
119 "num_prefills": num_prefills,
181 - "batch_type": batch_type,
120 }
183 - if prefill_attn_metadata is not None:
184 - metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
185 - else:
186 - assert decode_attn_metadata is not None
187 - metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
121 + if attn_metadata:
122 + metadata_dict.update(attn_metadata.asdict_zerocopy())
123 broadcast_tensor_dict(metadata_dict, src=0)
189 -
190 - # Broadcast decode attn metadata for mixed batch type.
191 - # The additional broadcast costs 300us overhead on 4 A10 GPUs.
192 - # We can potentially reduce the overhead by coelescing tensors.
193 - if batch_type == BatchType.MIXED:
194 - assert decode_attn_metadata is not None
195 - metadata_dict = decode_attn_metadata.asdict_zerocopy()
196 - broadcast_tensor_dict(metadata_dict, src=0)
124 else:
125 metadata_dict = broadcast_tensor_dict(src=0)
126 input_tokens = metadata_dict.pop("input_tokens")
127 input_positions = metadata_dict.pop("input_positions")
201 - slot_mapping = metadata_dict.pop("slot_mapping")
202 - num_prefills = metadata_dict.pop("num_prefills")
128 lora_mapping = metadata_dict.pop("lora_mapping")
129 lora_requests = metadata_dict.pop("lora_requests")
130 multi_modal_input = metadata_dict.pop("multi_modal_input")
206 - num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
207 - num_decode_tokens = metadata_dict.pop("num_decode_tokens")
208 - batch_type = metadata_dict.pop("batch_type")
209 -
210 - # Create an attention metadata.
211 - prefill_attn_metadata = None
212 - decode_attn_metadata = None
213 - if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
214 - prefill_attn_metadata = self.attn_backend.make_metadata(
131 + if metadata_dict:
132 + attn_metadata = self.attn_backend.make_metadata(
133 **metadata_dict)
134 else:
217 - decode_attn_metadata = self.attn_backend.make_metadata(
218 - **metadata_dict)
219 -
135 + attn_metadata = None
136 pooling_metadata = PoolingMetadata(seq_groups=None,
137 seq_data=None,
138 prompt_lens=None)
139
224 - # if it is a mixed batch, decode attn_metadata is broadcasted
225 - # separately.
226 - if batch_type == BatchType.MIXED:
227 - metadata_dict = broadcast_tensor_dict(src=0)
228 - decode_attn_metadata = self.attn_backend.make_metadata(
229 - **metadata_dict)
230 -
231 - attn_metadata = AttentionMetadata(
232 - num_prefills=num_prefills,
233 - slot_mapping=slot_mapping,
234 - num_prefill_tokens=num_prefill_tokens,
235 - num_decode_tokens=num_decode_tokens,
236 - prefill_metadata=prefill_attn_metadata,
237 - decode_metadata=decode_attn_metadata,
238 - )
239 -
140 return (input_tokens, input_positions, attn_metadata, pooling_metadata,
141 lora_requests, lora_mapping, multi_modal_input)
142
vllm/worker/model_runner.py
+323 -449
@@ -1,13 +1,11 @@
1 import time
2 -from enum import IntEnum
2 from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
3
4 import numpy as np
5 import torch
6 import torch.nn as nn
7
9 -from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
10 - get_attn_backend)
8 +from vllm.attention import AttentionMetadata, get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 ModelConfig, ParallelConfig, SchedulerConfig,
11 VisionLanguageConfig)
@@ -37,66 +35,38 @@ _BATCH_SIZES_TO_CAPTURE = [1, 2, 4] + [
35 ]
36
37
40 -class PreparePromptMetadata(NamedTuple):
41 - input_tokens: List[int]
42 - input_positions: List[int]
43 - attn_metadata: Optional[AttentionMetadataPerStage]
38 +class ModelInput(NamedTuple):
39 + input_tokens: torch.Tensor
40 + input_positions: torch.Tensor
41 + attn_metadata: Optional[AttentionMetadata]
42 seq_lens: List[int]
43 query_lens: List[int]
46 - lora_index_mapping: List[int]
47 - lora_prompt_mapping: List[int]
44 + lora_mapping: Optional[LoRAMapping]
45 lora_requests: Set[LoRARequest]
46 multi_modal_input: Optional[torch.Tensor]
50 - slot_mapping: List[int]
47 + slot_mapping: torch.Tensor
48 + num_prefill_tokens: int
49 + num_decode_tokens: int
50 + num_prefills: int
51
52 @classmethod
53 - def empty(cls):
54 - return PreparePromptMetadata(
55 - input_tokens=[],
56 - input_positions=[],
53 + def empty(cls, device):
54 + return ModelInput(
55 + input_tokens=torch.empty(0, device=device),
56 + input_positions=torch.empty(0, device=device),
57 attn_metadata=None,
58 seq_lens=[],
59 query_lens=[],
60 - lora_index_mapping=[],
61 - lora_prompt_mapping=[],
60 + lora_mapping=None,
61 lora_requests=set(),
62 multi_modal_input=None,
64 - slot_mapping=[],
65 - )
66 -
67 -
68 -class PrepareDecodeMetadata(NamedTuple):
69 - input_tokens: List[int]
70 - input_positions: List[int]
71 - attn_metadata: Optional[AttentionMetadata]
72 - lora_index_mapping: List[int]
73 - lora_prompt_mapping: List[int]
74 - lora_requests: Set[LoRARequest]
75 - slot_mapping: List[int]
76 -
77 - @classmethod
78 - def empty(cls):
79 - return PrepareDecodeMetadata(
80 - input_tokens=[],
81 - input_positions=[],
82 - attn_metadata=None,
83 - lora_index_mapping=[],
84 - lora_prompt_mapping=[],
85 - lora_requests=set(),
86 - slot_mapping=[],
63 + slot_mapping=torch.empty(0, device=device),
64 + num_prefill_tokens=0,
65 + num_decode_tokens=0,
66 + num_prefills=0,
67 )
68
69
90 -# How batches are constructed.
91 -class BatchType(IntEnum):
92 - # Every batch is prefill.
93 - PREFILL = 0
94 - # Every batch is decode.
95 - DECODE = 1
96 - # Batch is a mixture of prefill and decode.
97 - MIXED = 2
98 -
99 -
70 class ModelRunner:
71
72 def __init__(
@@ -216,10 +186,22 @@ class ModelRunner:
186 block_size = self.block_size
187 return (self.max_seq_len_to_capture + block_size - 1) // block_size
188
219 - def _prepare_prompt(
189 + def _prepare_model_input(
190 self,
191 seq_group_metadata_list: List[SequenceGroupMetadata],
222 - ) -> PreparePromptMetadata:
192 + ) -> ModelInput:
193 + """Prepare the model input based on a given sequence group.
194 +
195 + The API assumes seq_group_metadata_list is sorted by prefill -> decode.
196 +
197 + The result tensors and data structure also batches input in prefill
198 + -> decode order. For example,
199 +
200 + - input_tokens[:num_prefill_tokens] contains prefill tokens.
201 + - input_tokens[num_prefill_tokens:] contains decode tokens.
202 +
203 + If cuda graph is required, this API automatically pads inputs.
204 + """
205 input_tokens: List[int] = []
206 input_positions: List[int] = []
207 slot_mapping: List[int] = []
@@ -228,212 +210,16 @@ class ModelRunner:
210 lora_requests: Set[LoRARequest] = set()
211
212 seq_lens: List[int] = []
213 + prefill_seq_lens: List[int] = []
214 + decode_seq_lens: List[int] = []
215 context_lens: List[int] = []
216 query_lens: List[int] = []
233 - prefix_block_tables: List[List[int]] = []
234 - multi_modal_input_list: List[torch.Tensor] = []
235 -
236 - if len(seq_group_metadata_list) == 0:
237 - return PreparePromptMetadata.empty()
238 -
239 - for seq_group_metadata in seq_group_metadata_list:
240 - assert seq_group_metadata.is_prompt
241 - seq_ids = list(seq_group_metadata.seq_data.keys())
242 - assert len(seq_ids) == 1
243 - seq_id = seq_ids[0]
244 -
245 - computed_block_nums = seq_group_metadata.computed_block_nums
246 - if (self.scheduler_config is not None
247 - and self.scheduler_config.chunked_prefill_enabled
248 - and not (computed_block_nums is None
249 - or computed_block_nums == [])):
250 - raise RuntimeError(
251 - "chunked prefill cannot be used with prefix caching "
252 - "now.")
253 -
254 - token_chunk_size = seq_group_metadata.token_chunk_size
255 - seq_data = seq_group_metadata.seq_data[seq_id]
256 - context_len = seq_data.get_num_computed_tokens()
257 - # We should use get_len here because in case of preemption
258 - # it contains output tokens.
259 - seq_len = min(seq_data.get_len(), context_len + token_chunk_size)
260 - prompt_tokens = seq_data.get_token_ids()[context_len:seq_len]
261 - seq_lens.append(seq_len)
262 -
263 - # NOTE: This only works for oooooooxxx style attention.
264 - if computed_block_nums is not None and len(
265 - computed_block_nums) > 0 and self.sliding_window is None:
266 - # Prefix is not supported with sliding_window
267 - context_len = len(computed_block_nums) * self.block_size
268 - prompt_tokens = prompt_tokens[context_len:]
269 - prefix_block_tables.append(computed_block_nums)
270 - elif self.scheduler_config.chunked_prefill_enabled:
271 - if seq_group_metadata.block_tables is not None:
272 - # Prefill has chunked before.
273 - block_table = seq_group_metadata.block_tables[seq_id]
274 - prefix_block_tables.append(block_table)
275 - else:
276 - # The first prefill.
277 - prefix_block_tables.append([])
278 - else:
279 - prefix_block_tables.append([])
280 - # Right now, prefill start is always 0. However, this
281 - # assumption can be changed once chunked prefill is introduced.
282 - assert context_len == 0
283 -
284 - # actual prompt lens
285 - context_lens.append(context_len)
286 - query_lens.append(seq_len - context_len)
287 -
288 - input_tokens.extend(prompt_tokens)
289 - # NOTE(woosuk): Here we assume that the first token in the prompt
290 - # is always the first token in the sequence.
291 - input_positions.extend(list(range(context_len, seq_len)))
292 - lora_id = seq_group_metadata.lora_int_id
293 -
294 - if lora_id > 0:
295 - lora_requests.add(seq_group_metadata.lora_request)
296 -
297 - lora_index_mapping += [lora_id] * (seq_len - context_len)
298 - lora_prompt_mapping.extend([lora_id] * (
299 - seq_len - context_len if seq_group_metadata.sampling_params
300 - and seq_group_metadata.sampling_params.prompt_logprobs else 1))
301 -
302 - if seq_group_metadata.multi_modal_data:
303 - multi_modal_input_list.append(
304 - seq_group_metadata.multi_modal_data.data)
305 -
306 - if _is_block_tables_empty(seq_group_metadata.block_tables):
307 - # During memory profiling, the block tables are not initialized
308 - # yet. In this case, we just use a dummy slot mapping.
309 - # In embeddings, the block tables are {seq_id: None}.
310 - slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
311 - continue
312 -
313 - # Compute the slot mapping.
314 - block_table = seq_group_metadata.block_tables[seq_id]
315 -
316 - # Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
317 - # where start_idx is max(0, seq_len - sliding_window).
318 - # For example, if the prompt len is 10, sliding window is 8, and
319 - # block size is 4, the first two tokens are masked and the slot
320 - # mapping will be [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
321 - start_idx = 0
322 - if self.sliding_window is not None:
323 - assert context_len == 0, (
324 - "Prefix caching is currently not supported with "
325 - "sliding window attention")
326 - start_idx = max(0, seq_len - self.sliding_window)
327 -
328 - for i in range(context_len, seq_len):
329 - if i < start_idx:
330 - slot_mapping.append(_PAD_SLOT_ID)
331 - continue
332 -
333 - block_number = block_table[i // self.block_size]
334 - block_offset = i % self.block_size
335 - slot = block_number * self.block_size + block_offset
336 - slot_mapping.append(slot)
337 -
338 - max_query_len = max(query_lens)
339 - max_seq_len = max(seq_lens)
340 - assert max_query_len > 0
341 -
342 - context_lens_tensor = torch.tensor(context_lens,
343 - dtype=torch.int,
344 - device=self.device)
345 -
346 - if multi_modal_input_list:
347 - assert self.vision_language_config, (
348 - "Multi-modal inputs are only supported by "
349 - "vision language models.")
350 - multi_modal_input = torch.cat(multi_modal_input_list,
351 - dim=0).to(self.device)
352 - else:
353 - multi_modal_input = None
354 -
355 - # Prepare prefix block tables
356 - max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
357 - block_tables = make_tensor_with_pad(
358 - prefix_block_tables,
359 - max_len=max_prompt_block_table_len,
360 - pad=0,
361 - dtype=torch.int,
362 - device=self.device,
363 - )
364 -
365 - # Query length can be shorter than key (i.e., prompt) when prefill
366 - # is chunked or prefix cached.
367 - query_lens_tensor = torch.tensor(query_lens,
368 - dtype=torch.long,
369 - device=self.device)
370 - subquery_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
371 - dtype=torch.int32,
372 - device=self.device)
373 -
374 - seq_lens_tensor = torch.tensor(seq_lens,
375 - dtype=torch.int,
376 - device=self.device)
377 - seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
378 - dtype=torch.int32,
379 - device=self.device)
380 -
381 - torch.cumsum(query_lens_tensor,
382 - dim=0,
383 - dtype=subquery_start_loc.dtype,
384 - out=subquery_start_loc[1:])
385 -
386 - torch.cumsum(seq_lens_tensor,
387 - dim=0,
388 - dtype=seq_start_loc.dtype,
389 - out=seq_start_loc[1:])
390 -
391 - if self.attn_backend.get_name() == "flashinfer":
392 - attn_metadata = self.attn_backend.make_metadata(
393 - is_prompt=True,
394 - use_cuda_graph=False,
395 - seq_start_loc=seq_start_loc,
396 - max_seq_len=max_seq_len,
397 - block_tables=block_tables)
398 - else:
399 - attn_metadata = self.attn_backend.make_metadata(
400 - is_prompt=True,
401 - seq_lens=seq_lens,
402 - seq_lens_tensor=seq_lens_tensor,
403 - max_query_len=max_query_len,
404 - max_seq_len=max_seq_len,
405 - subquery_start_loc=subquery_start_loc,
406 - seq_start_loc=seq_start_loc,
407 - context_lens_tensor=context_lens_tensor,
408 - block_tables=block_tables,
409 - use_cuda_graph=False,
410 - )
411 -
412 - return PreparePromptMetadata(
413 - input_tokens=input_tokens,
414 - input_positions=input_positions,
415 - attn_metadata=attn_metadata,
416 - seq_lens=seq_lens,
417 - query_lens=query_lens,
418 - lora_index_mapping=lora_index_mapping,
419 - lora_prompt_mapping=lora_prompt_mapping,
420 - lora_requests=lora_requests,
421 - multi_modal_input=multi_modal_input,
422 - slot_mapping=slot_mapping,
423 - )
424 -
425 - def _prepare_decode(
426 - self,
427 - seq_group_metadata_list: List[SequenceGroupMetadata],
428 - ) -> PrepareDecodeMetadata:
429 - input_tokens: List[int] = []
430 - input_positions: List[int] = []
431 - slot_mapping: List[int] = []
432 - seq_lens: List[int] = []
217 block_tables: List[List[int]] = []
434 - lora_index_mapping: List[int] = []
435 - lora_prompt_mapping: List[int] = []
436 - lora_requests: Set[LoRARequest] = set()
218 + multi_modal_input_list: List[torch.Tensor] = []
219 + decode_only = True
220 + num_prefills = 0
221 + num_prefill_tokens = 0
222 + num_decode_tokens = 0
223
224 # The following fields are only for flashinfer
225 # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
@@ -454,60 +240,186 @@ class ModelRunner:
240 paged_kv_last_page_len: List[int] = []
241
242 if len(seq_group_metadata_list) == 0:
457 - return PrepareDecodeMetadata.empty()
243 + return ModelInput.empty(self.device)
244
245 for seq_group_metadata in seq_group_metadata_list:
460 - assert not seq_group_metadata.is_prompt
461 - assert seq_group_metadata.token_chunk_size == 1
462 -
246 seq_ids = list(seq_group_metadata.seq_data.keys())
464 - lora_id = seq_group_metadata.lora_int_id
465 -
466 - if lora_id > 0:
467 - lora_requests.add(seq_group_metadata.lora_request)
247 + is_prompt = seq_group_metadata.is_prompt
248
249 for seq_id in seq_ids:
250 + computed_block_nums = seq_group_metadata.computed_block_nums
251 + if (self.scheduler_config is not None
252 + and self.scheduler_config.chunked_prefill_enabled
253 + and not (computed_block_nums is None
254 + or computed_block_nums == [])):
255 + raise RuntimeError(
256 + "chunked prefill cannot be used with prefix caching "
257 + "now.")
258 +
259 seq_data = seq_group_metadata.seq_data[seq_id]
471 - generation_token = seq_data.get_last_token_id()
472 - input_tokens.append(generation_token)
260 + if is_prompt:
261 + context_len = seq_data.get_num_computed_tokens()
262 + else:
263 + # get_num_computed_tokens is incorrect for spec decoding.
264 + # So, we should have a special logic here.
265 + # TODO(sang): Fix it.
266 + context_len = seq_data.get_len() - 1
267 +
268 + seq_len = min(
269 + seq_data.get_len(),
270 + context_len + seq_group_metadata.token_chunk_size)
271 + if is_prompt:
272 + tokens = seq_data.get_token_ids()[context_len:seq_len]
273 + else:
274 + # Optimization. get_token_ids requires the entire copy of
275 + # tokens.
276 + tokens = [seq_data.get_last_token_id()]
277 +
278 + # Prefix cache was hit.
279 + # Prefix is not supported with sliding_window
280 + prefix_cache_hit = (computed_block_nums is not None
281 + and len(computed_block_nums) > 0
282 + and self.sliding_window is None
283 + and is_prompt)
284 +
285 + # TODO(sang): Combine chunked prefill and prefix caching by
286 + # only allowing multiple of block_size chunk size.
287 + # NOTE: This only works for oooooooxxx style attention.
288 + if prefix_cache_hit:
289 + assert computed_block_nums is not None
290 + context_len = len(computed_block_nums) * self.block_size
291 + tokens = tokens[context_len:]
292 + if self.attn_backend.get_name() == "flash-attn":
293 + # NOTE(woosuk): For flash-attn, the block table should
294 + # include the entries for the incoming prefill tokens.
295 + # TODO(woosuk): This is a temporary fix. We should
296 + # provide a unified interface for different backends.
297 + block_table = seq_group_metadata.block_tables[seq_id]
298 + else:
299 + block_table = computed_block_nums
300 + elif (self.scheduler_config.chunked_prefill_enabled
301 + or not is_prompt):
302 + if seq_group_metadata.block_tables is not None:
303 + # chunked prefill or decode
304 + block_table = seq_group_metadata.block_tables[seq_id]
305 + if self.sliding_window is not None:
306 + # chunked prefill doesn't support sliding window.
307 + assert (not self.scheduler_config.
308 + chunked_prefill_enabled)
309 + sliding_window_blocks = (self.sliding_window //
310 + self.block_size)
311 + block_table = block_table[-sliding_window_blocks:]
312 +
313 + if self.attn_backend.get_name() == "flashinfer":
314 + paged_kv_indices.extend(block_table)
315 + paged_kv_indptr.append(paged_kv_indptr[-1] +
316 + len(block_table))
317 + last_page_len = seq_data.get_len(
318 + ) % self.block_size
319 + if last_page_len == 0:
320 + last_page_len = self.block_size
321 + paged_kv_last_page_len.append(last_page_len)
322 + else:
323 + # Only happens when memory profiling runs.
324 + block_table = []
325 + else:
326 + # Prefill without chunked prefill or memory profiling.
327 + block_table = []
328 + block_tables.append(block_table)
329
474 - seq_len = seq_data.get_len()
475 - position = seq_len - 1
476 - input_positions.append(position)
330 + # TODO(sang): This is a hack to make sliding window work with
331 + # paged attn. We can remove it if we make paged attn kernel
332 + # to properly handle slinding window attn.
333 + if (self.sliding_window is not None and not is_prompt):
334 + seq_len = min(seq_len, self.sliding_window)
335 + context_len = seq_len - 1
336
478 - seq_len = seq_len if self.sliding_window is None else min(
479 - seq_len, self.sliding_window)
337 seq_lens.append(seq_len)
338 + context_lens.append(context_len)
339 + query_len = seq_len - context_len
340 + query_lens.append(query_len)
341 + input_tokens.extend(tokens)
342 + input_positions.extend(list(range(context_len, seq_len)))
343 + lora_id = seq_group_metadata.lora_int_id
344 +
345 + if is_prompt:
346 + assert len(seq_ids) == 1
347 + num_prefills += 1
348 + num_prefill_tokens += len(tokens)
349 + decode_only = False
350 + prefill_seq_lens.append(seq_len)
351 + else:
352 + assert query_len == 1, (
353 + "seq_len: {}, context_len: {}, query_len: {}".format(
354 + seq_len, context_len, query_len))
355 + num_decode_tokens += query_len
356 + decode_seq_lens.append(seq_len)
357 +
358 + if lora_id > 0:
359 + lora_requests.add(seq_group_metadata.lora_request)
360 +
361 + lora_index_mapping += [lora_id] * (seq_len - context_len)
362 + lora_prompt_mapping.extend(
363 + [lora_id] *
364 + (seq_len -
365 + context_len if seq_group_metadata.sampling_params
366 + and seq_group_metadata.sampling_params.prompt_logprobs
367 + else 1))
368 +
369 + if seq_group_metadata.multi_modal_data:
370 + multi_modal_input_list.append(
371 + seq_group_metadata.multi_modal_data.data)
372 +
373 + if _is_block_tables_empty(seq_group_metadata.block_tables):
374 + # During memory profiling, the block tables are not
375 + # initialized yet. In this case, we just use a dummy
376 + # slot mapping.
377 + # In embeddings, the block tables are {seq_id: None}.
378 + slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
379 + continue
380
381 + # Compute the slot mapping.
382 block_table = seq_group_metadata.block_tables[seq_id]
483 - block_number = block_table[position // self.block_size]
484 - block_offset = position % self.block_size
485 - slot = block_number * self.block_size + block_offset
486 - slot_mapping.append(slot)
487 - lora_index_mapping.append(lora_id)
488 - lora_prompt_mapping.append(lora_id)
383
384 + # Mask the [0, start_idx) tokens of the prompt with
385 + # _PAD_SLOT_ID, where start_idx is max(0, seq_len -
386 + # sliding_window). For example, if the prompt len is 10,
387 + # sliding window is 8, and block size is 4, the first two
388 + # tokens are masked and the slot mapping will be
389 + # [-1, -1, 2, 3, 4, 5, 6, 7, 0, 1].
390 + start_idx = 0
391 if self.sliding_window is not None:
491 - sliding_window_blocks = (self.sliding_window //
492 - self.block_size)
493 - block_table = block_table[-sliding_window_blocks:]
494 - block_tables.append(block_table)
392 + if is_prompt:
393 + assert context_len == 0, (
394 + "Prefix caching is currently not supported with "
395 + "sliding window attention")
396 + # It is an optimization. When it is decoding, it is always
397 + # 0. When prefill, we use it to not write slots to kv cache
398 + # to save memory.
399 + start_idx = max(0, query_len - self.sliding_window)
400 +
401 + for i in range(context_len, seq_len):
402 + if i < start_idx:
403 + slot_mapping.append(_PAD_SLOT_ID)
404 + continue
405 +
406 + block_number = block_table[i // self.block_size]
407 + block_offset = i % self.block_size
408 + slot = block_number * self.block_size + block_offset
409 + slot_mapping.append(slot)
410
496 - paged_kv_indices.extend(block_table)
497 - paged_kv_indptr.append(paged_kv_indptr[-1] + len(block_table))
498 - last_page_len = seq_data.get_len() % self.block_size
499 - if last_page_len == 0:
500 - last_page_len = self.block_size
501 - paged_kv_last_page_len.append(last_page_len)
411 + batch_size = len(input_tokens)
412 + max_query_len = max(query_lens)
413 + max_prefill_seq_len = max(prefill_seq_lens, default=0)
414 + max_decode_seq_len = max(decode_seq_lens, default=0)
415
503 - # vLLM uses cuda graph only for decoding requests.
416 + # If cuda graph can be used, pad tensors accordingly.
417 # See `capture_model` API for more details.
505 - # For decoding requests, batch_size == input_tokens.
506 - batch_size = len(input_tokens)
507 - max_seq_len = max(seq_lens)
508 - use_captured_graph = (not self.model_config.enforce_eager
509 - and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
510 - and max_seq_len <= self.max_seq_len_to_capture)
418 + # vLLM uses cuda graph only for decoding requests.
419 + use_captured_graph = (
420 + decode_only and not self.model_config.enforce_eager
421 + and batch_size <= _BATCH_SIZES_TO_CAPTURE[-1]
422 + and max_decode_seq_len <= self.max_seq_len_to_capture)
423 if use_captured_graph:
424 graph_batch_size = _get_graph_batch_size(batch_size)
425 assert graph_batch_size >= batch_size
@@ -519,18 +431,9 @@ class ModelRunner:
431 block_tables.append([])
432 lora_index_mapping.append(0)
433 batch_size = graph_batch_size
522 -
523 - seq_lens_tensor = torch.tensor(seq_lens,
524 - dtype=torch.int,
525 - device=self.device)
434 + num_decode_tokens = batch_size
435
436 if use_captured_graph:
528 - # When using cuda-graph all these tensors should be
529 - # padded.
530 - assert seq_lens_tensor.shape[0] == len(input_tokens)
531 - assert seq_lens_tensor.shape[0] == len(input_positions)
532 - assert seq_lens_tensor.shape[0] == len(slot_mapping)
533 -
437 # The shape of graph_block_tables is
438 # [max batch size, max context len // block size].
439 input_block_tables = self.graph_block_tables[:batch_size]
@@ -548,6 +451,57 @@ class ModelRunner:
451 dtype=torch.int,
452 device=self.device,
453 )
454 + assert max_query_len > 0, ("query_lens: {}".format(query_lens))
455 +
456 + context_lens_tensor = torch.tensor(context_lens,
457 + dtype=torch.int,
458 + device=self.device)
459 +
460 + if multi_modal_input_list:
461 + assert self.vision_language_config, (
462 + "Multi-modal inputs are only supported by "
463 + "vision language models.")
464 + multi_modal_input = torch.cat(multi_modal_input_list,
465 + dim=0).to(self.device)
466 + else:
467 + multi_modal_input = None
468 +
469 + seq_lens_tensor = torch.tensor(seq_lens,
470 + dtype=torch.int,
471 + device=self.device)
472 + query_lens_tensor = torch.tensor(query_lens,
473 + dtype=torch.long,
474 + device=self.device)
475 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
476 + dtype=torch.int32,
477 + device=self.device)
478 +
479 + seq_lens_tensor = torch.tensor(seq_lens,
480 + dtype=torch.int,
481 + device=self.device)
482 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
483 + dtype=torch.int32,
484 + device=self.device)
485 +
486 + torch.cumsum(query_lens_tensor,
487 + dim=0,
488 + dtype=query_start_loc.dtype,
489 + out=query_start_loc[1:])
490 +
491 + torch.cumsum(seq_lens_tensor,
492 + dim=0,
493 + dtype=seq_start_loc.dtype,
494 + out=seq_start_loc[1:])
495 +
496 + input_tokens_tensor = torch.tensor(input_tokens,
497 + dtype=torch.long,
498 + device=self.device)
499 + input_positions_tensor = torch.tensor(input_positions,
500 + dtype=torch.long,
501 + device=self.device)
502 + slot_mapping_tensor = torch.tensor(slot_mapping,
503 + dtype=torch.long,
504 + device=self.device)
505
506 if self.attn_backend.get_name() == "flashinfer":
507 if not hasattr(self, "flashinfer_workspace_buffer"):
@@ -555,53 +509,75 @@ class ModelRunner:
509 # Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html
510 self.flashinfer_workspace_buffer = torch.empty(
511 16 * 1024 * 1024, dtype=torch.uint8, device=self.device)
558 - paged_kv_indptr = torch.tensor(paged_kv_indptr,
559 - dtype=torch.int,
560 - device=self.device)
561 - paged_kv_indices = torch.tensor(paged_kv_indices,
562 - dtype=torch.int,
563 - device=self.device)
564 - paged_kv_last_page_len = torch.tensor(paged_kv_last_page_len,
512 + paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
513 dtype=torch.int,
514 device=self.device)
515 + paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
516 + dtype=torch.int,
517 + device=self.device)
518 + paged_kv_last_page_len_tensor = torch.tensor(
519 + paged_kv_last_page_len, dtype=torch.int, device=self.device)
520 kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
521 self.model_config.dtype)
569 -
522 attn_metadata = self.attn_backend.make_metadata(
571 - is_prompt=False,
523 + num_prefills=num_prefills,
524 + slot_mapping=slot_mapping_tensor,
525 + num_prefill_tokens=num_prefill_tokens,
526 + num_decode_tokens=num_decode_tokens,
527 use_cuda_graph=False,
528 + max_prefill_seq_len=max_prefill_seq_len,
529 + block_tables=block_tables,
530 workspace_buffer=self.flashinfer_workspace_buffer,
574 - paged_kv_indptr=paged_kv_indptr,
575 - paged_kv_indices=paged_kv_indices,
576 - paged_kv_last_page_len=paged_kv_last_page_len,
531 + paged_kv_indptr=paged_kv_indptr_tensor,
532 + paged_kv_indices=paged_kv_indices_tensor,
533 + paged_kv_last_page_len=paged_kv_last_page_len_tensor,
534 num_qo_heads=self.model_config.get_num_attention_heads(
535 self.parallel_config),
536 num_kv_heads=self.model_config.get_num_kv_heads(
537 self.parallel_config),
538 head_dim=self.model_config.get_head_size(),
582 - page_size=self.block_size,
539 + page_size=16,
540 + seq_start_loc=seq_start_loc,
541 data_type=kv_cache_dtype)
542 else:
543 attn_metadata = self.attn_backend.make_metadata(
586 - is_prompt=False,
587 - seq_lens=None,
544 + num_prefills=num_prefills,
545 + slot_mapping=slot_mapping_tensor,
546 + num_prefill_tokens=num_prefill_tokens,
547 + num_decode_tokens=num_decode_tokens,
548 + seq_lens=seq_lens,
549 seq_lens_tensor=seq_lens_tensor,
589 - max_query_len=None,
590 - max_seq_len=max_seq_len,
591 - subquery_start_loc=None,
592 - seq_start_loc=None,
593 - context_lens_tensor=None,
550 + max_query_len=max_query_len,
551 + max_prefill_seq_len=max_prefill_seq_len,
552 + max_decode_seq_len=max_decode_seq_len,
553 + query_start_loc=query_start_loc,
554 + seq_start_loc=seq_start_loc,
555 + context_lens_tensor=context_lens_tensor,
556 block_tables=block_tables,
557 use_cuda_graph=use_captured_graph,
558 )
597 - return PrepareDecodeMetadata(
598 - input_tokens=input_tokens,
599 - input_positions=input_positions,
559 +
560 + if self.lora_config:
561 + lora_mapping = LoRAMapping(
562 + lora_index_mapping,
563 + lora_prompt_mapping,
564 + )
565 + else:
566 + lora_mapping = None
567 +
568 + return ModelInput(
569 + input_tokens=input_tokens_tensor,
570 + input_positions=input_positions_tensor,
571 attn_metadata=attn_metadata,
601 - lora_index_mapping=lora_index_mapping,
602 - lora_prompt_mapping=lora_prompt_mapping,
572 + seq_lens=seq_lens,
573 + query_lens=query_lens,
574 + lora_mapping=lora_mapping,
575 lora_requests=lora_requests,
604 - slot_mapping=slot_mapping,
576 + multi_modal_input=multi_modal_input,
577 + slot_mapping=slot_mapping_tensor,
578 + num_prefill_tokens=num_prefill_tokens,
579 + num_decode_tokens=num_decode_tokens,
580 + num_prefills=num_prefills,
581 )
582
583 def prepare_input_tensors(
@@ -610,85 +586,25 @@ class ModelRunner:
586 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
587 Set[LoRARequest], LoRAMapping, torch.Tensor]:
588 if self.is_driver_worker:
613 - prefill_reqs = []
614 - decode_reqs = []
615 - for seq_group_meta in seq_group_metadata_list:
616 - if seq_group_meta.is_prompt:
617 - prefill_reqs.append(seq_group_meta)
618 - else:
619 - decode_reqs.append(seq_group_meta)
620 -
589 # Prepare input tensors.
590 (
591 input_tokens,
592 input_positions,
625 - prefill_attn_metadata,
593 + attn_metadata,
594 seq_lens,
595 query_lens,
628 - lora_index_mapping,
629 - lora_prompt_mapping,
596 + lora_mapping,
597 lora_requests,
598 multi_modal_input,
599 slot_mapping,
633 - ) = self._prepare_prompt(prefill_reqs)
634 - (
635 - decode_input_tokens,
636 - decode_input_positions,
637 - decode_attn_metadata,
638 - decode_lora_index_mapping,
639 - decode_lora_prompt_mapping,
640 - decode_lora_requests,
641 - decode_slot_mapping,
642 - ) = self._prepare_decode(decode_reqs)
600 + num_prefill_tokens,
601 + num_decode_tokens,
602 + num_prefills,
603 + ) = self._prepare_model_input(seq_group_metadata_list)
604 sampling_metadata = SamplingMetadata.prepare(
605 seq_group_metadata_list, seq_lens, query_lens, self.device,
606 self.pin_memory)
607
647 - if not self.scheduler_config.chunked_prefill_enabled:
648 - assert (len(prefill_reqs) and len(decode_reqs)) == 0
649 -
650 - num_prefills = len(seq_lens)
651 - num_prefill_tokens = len(input_tokens)
652 - num_decode_tokens = len(decode_input_tokens)
653 -
654 - # Coalesce tensors. Note that attn_metadata is currently not
655 - # coalesced for simplicity.
656 - input_tokens.extend(decode_input_tokens)
657 - input_positions.extend(decode_input_positions)
658 - slot_mapping.extend(decode_slot_mapping)
659 - lora_index_mapping.extend(decode_lora_index_mapping)
660 - lora_prompt_mapping.extend(decode_lora_prompt_mapping)
661 - lora_requests.update(decode_lora_requests)
662 -
663 - input_tokens = torch.tensor(input_tokens,
664 - dtype=torch.long,
665 - device=self.device)
666 - input_positions = torch.tensor(input_positions,
667 - dtype=torch.long,
668 - device=self.device)
669 - slot_mapping = torch.tensor(slot_mapping,
670 - dtype=torch.long,
671 - device=self.device)
672 -
673 - if self.lora_config:
674 - lora_mapping = LoRAMapping(
675 - lora_index_mapping,
676 - lora_prompt_mapping,
677 - )
678 - else:
679 - lora_mapping = None
680 -
681 - # Broadcast the metadata.
682 - # If batch contains both prefill and decode, it sends 2 broadcasts.
683 - # If it only contains 1 type, it triggers a single broadcast.
684 - if (prefill_attn_metadata is not None
685 - and decode_attn_metadata is not None):
686 - batch_type = BatchType.MIXED
687 - elif prefill_attn_metadata is not None:
688 - batch_type = BatchType.PREFILL
689 - else:
690 - batch_type = BatchType.DECODE
691 -
608 metadata_dict = {
609 "input_tokens": input_tokens,
610 "input_positions": input_positions,
@@ -701,46 +617,24 @@ class ModelRunner:
617 "num_decode_tokens": num_decode_tokens,
618 "slot_mapping": slot_mapping,
619 "num_prefills": num_prefills,
704 - "batch_type": batch_type,
620 }
706 - if prefill_attn_metadata is not None:
707 - metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
708 - else:
709 - assert decode_attn_metadata is not None
710 - metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
621 + if attn_metadata:
622 + metadata_dict.update(attn_metadata.asdict_zerocopy())
623 broadcast_tensor_dict(metadata_dict, src=0)
712 -
713 - # Broadcast decode attn metadata for mixed batch type.
714 - # The additional broadcast costs 300us overhead on 4 A10 GPUs.
715 - # We can potentially reduce the overhead by coelescing tensors.
716 - if batch_type == BatchType.MIXED:
717 - assert decode_attn_metadata is not None
718 - metadata_dict = decode_attn_metadata.asdict_zerocopy()
719 - broadcast_tensor_dict(metadata_dict, src=0)
624 else:
625 metadata_dict = broadcast_tensor_dict(src=0)
626 input_tokens = metadata_dict.pop("input_tokens")
627 input_positions = metadata_dict.pop("input_positions")
724 - slot_mapping = metadata_dict.pop("slot_mapping")
725 - num_prefills = metadata_dict.pop("num_prefills")
628 selected_token_indices = metadata_dict.pop(
629 "selected_token_indices")
630 lora_mapping = metadata_dict.pop("lora_mapping")
631 lora_requests = metadata_dict.pop("lora_requests")
632 multi_modal_input = metadata_dict.pop("multi_modal_input")
731 - num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
732 - num_decode_tokens = metadata_dict.pop("num_decode_tokens")
733 - batch_type = metadata_dict.pop("batch_type")
734 -
735 - # Create an attention metadata.
736 - prefill_attn_metadata = None
737 - decode_attn_metadata = None
738 - if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
739 - prefill_attn_metadata = self.attn_backend.make_metadata(
633 + if metadata_dict:
634 + attn_metadata = self.attn_backend.make_metadata(
635 **metadata_dict)
636 else:
742 - decode_attn_metadata = self.attn_backend.make_metadata(
743 - **metadata_dict)
637 + attn_metadata = None
638 sampling_metadata = SamplingMetadata(
639 seq_groups=None,
640 selected_token_indices=selected_token_indices,
@@ -748,22 +642,6 @@ class ModelRunner:
642 num_prompts=0,
643 )
644
751 - # if it is a mixed batch, decode attn_metadata is broadcasted
752 - # separately.
753 - if batch_type == BatchType.MIXED:
754 - metadata_dict = broadcast_tensor_dict(src=0)
755 - decode_attn_metadata = self.attn_backend.make_metadata(
756 - **metadata_dict)
757 -
758 - attn_metadata = AttentionMetadata(
759 - num_prefills=num_prefills,
760 - slot_mapping=slot_mapping,
761 - num_prefill_tokens=num_prefill_tokens,
762 - num_decode_tokens=num_decode_tokens,
763 - prefill_metadata=prefill_attn_metadata,
764 - decode_metadata=decode_attn_metadata,
765 - )
766 -
645 return (input_tokens, input_positions, attn_metadata,
646 sampling_metadata, lora_requests, lora_mapping,
647 multi_modal_input)
@@ -954,26 +832,22 @@ class ModelRunner:
832 # memory usage of CUDA graph.
833 for batch_size in reversed(batch_size_capture_list):
834 # Create dummy attn_metadata.
957 - decode_metadata = self.attn_backend.make_metadata(
958 - is_prompt=False,
835 + attn_metadata = self.attn_backend.make_metadata(
836 + num_prefills=0,
837 + num_prefill_tokens=0,
838 + num_decode_tokens=batch_size,
839 + slot_mapping=slot_mapping[:batch_size],
840 seq_lens=None,
841 seq_lens_tensor=seq_lens[:batch_size],
842 max_query_len=None,
962 - max_seq_len=self.max_seq_len_to_capture,
963 - subquery_start_loc=None,
843 + max_prefill_seq_len=0,
844 + max_decode_seq_len=self.max_seq_len_to_capture,
845 + query_start_loc=None,
846 seq_start_loc=None,
847 context_lens_tensor=None,
848 block_tables=block_tables[:batch_size],
849 use_cuda_graph=True,
850 )
969 - attn_metadata = AttentionMetadata(
970 - num_prefills=0,
971 - num_prefill_tokens=0,
972 - num_decode_tokens=batch_size,
973 - slot_mapping=slot_mapping[:batch_size],
974 - prefill_metadata=None,
975 - decode_metadata=decode_metadata,
976 - )
851
852 if self.lora_config:
853 lora_mapping = LoRAMapping(