@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 543aa48573
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Kernel] Correctly invoke prefill & decode kernels for cross-attention (towards eventual encoder/decoder model support) (#4888)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

afeldman-nm committed Jul 8, 2024 at 13:12 UTC 543aa4857362ff385af8a9f496392b8831c42833
14 files changed +2351 -95
tests/kernels/test_attention_selector.py
+7 -7
@@ -47,32 +47,32 @@ def test_flash_attn(monkeypatch):
47 # Unsupported CUDA arch
48 with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
49 backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
50 - assert backend.name != "FLASH_ATTN"
50 + assert backend.name != STR_FLASH_ATTN_VAL
51
52 # Unsupported data type
53 backend = which_attn_to_use(8, 16, 8, None, torch.float8_e4m3fn, None, 16)
54 - assert backend.name != "FLASH_ATTN"
54 + assert backend.name != STR_FLASH_ATTN_VAL
55
56 # Unsupported kv cache data type
57 backend = which_attn_to_use(8, 16, 8, None, torch.float16, "fp8", 16)
58 - assert backend.name != "FLASH_ATTN"
58 + assert backend.name != STR_FLASH_ATTN_VAL
59
60 # Unsupported block size
61 backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 8)
62 - assert backend.name != "FLASH_ATTN"
62 + assert backend.name != STR_FLASH_ATTN_VAL
63
64 # Unsupported sliding window
65 backend = which_attn_to_use(8, 16, 8, 1, torch.float16, None, 16)
66 - assert backend.name != "FLASH_ATTN"
66 + assert backend.name != STR_FLASH_ATTN_VAL
67
68 # flash-attn is not installed
69 with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70 backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
71 - assert backend.name != "FLASH_ATTN"
71 + assert backend.name != STR_FLASH_ATTN_VAL
72
73 # Unsupported head size
74 backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
75 - assert backend.name != "FLASH_ATTN"
75 + assert backend.name != STR_FLASH_ATTN_VAL
76
77
78 def test_invalid_env(monkeypatch):
tests/kernels/test_encoder_decoder_attn.py new
+953
@@ -0,0 +1,953 @@
1 +"""
2 +Tests:
3 +
4 +* E2E test of Encoder attention + Decoder self-attention +
5 + Encoder/decoder cross-attention (collectively
6 + "encoder/decoder attention")
7 +* Confirm enc/dec models will fail for chunked prefill
8 +* Confirm enc/dec models will fail for prefix caching
9 +
10 +"""
11 +
12 +from typing import NamedTuple, Optional
13 +
14 +import pytest
15 +import torch
16 +
17 +from tests.kernels.utils import *
18 +from tests.kernels.utils import make_causal_mask, maybe_make_long_tensor
19 +from vllm.attention import Attention, AttentionMetadata
20 +from vllm.attention.backends.abstract import AttentionBackend, AttentionType
21 +from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
22 +from vllm.utils import is_hip
23 +
24 +HEAD_SIZES = [64, 256]
25 +
26 +NUM_HEADS = [1, 16]
27 +
28 +BATCH_SIZES = [1, 16]
29 +BLOCK_SIZES = [16]
30 +BACKEND_NAMES = [STR_XFORMERS_ATTN_VAL]
31 +CUDA_DEVICE = "cuda:0"
32 +
33 +MAX_DEC_SEQ_LENS = [128]
34 +MAX_ENC_SEQ_LENS = [128]
35 +
36 +# Narrow teest-cases for unsupported-scenario
37 +# tests
38 +HEAD_SIZES_FOR_UNSUPP = [HEAD_SIZES[0]]
39 +
40 +
41 +class TestPoint(NamedTuple):
42 + """
43 + Encapsulates the attributes which define a single invocation
44 + of the test_e2e_enc_dec_attn() test
45 +
46 + Attributes:
47 + num_heads: The number of heads in the model.
48 + head_size: Head dimension
49 + backend_name: Name of the backend framework used.
50 + batch_size: Number of samples per batch.
51 + block_size: Size of each block of data processed.
52 + max_dec_seq_len: Maximum sequence length for the decoder.
53 + max_enc_seq_len: Maximum sequence length for the encoder.
54 + num_blocks: Number of blocks in the model.
55 + """
56 +
57 + num_heads: int
58 + head_size: int
59 + backend_name: str
60 + batch_size: int
61 + block_size: int
62 + max_dec_seq_len: int
63 + max_enc_seq_len: int
64 + num_blocks: int
65 +
66 +
67 +class TestResources(NamedTuple):
68 + '''
69 + Encapsulates key components for performing an
70 + encoder/decoder attention test
71 +
72 + Note that
73 + (1) attn automatically selects an attention backend
74 + based on platform info & a set of canned
75 + heuristics
76 + (2) attn_backend is thus *not the same backend
77 + instance* used by attn, but rather it is
78 + intended to be a
79 + *different instance* of the *same backend class*;
80 + it is assumed that the user of TestResources
81 + will leverage attn_backend for the purpose of
82 + constructing backend-compatible attention
83 + metadata instances
84 +
85 + Attributes:
86 +
87 + * scale: 1/sqrt(d) scale factor for attn
88 + * attn_backend: implementatino of abstraction
89 + attention interface using
90 + a particular kernel library
91 + i.e. XFormers
92 + * attn: Attention layer instance
93 + * kv_cache: shared key/value cache for all attention
94 + '''
95 +
96 + scale: float
97 + attn_backend: AttentionBackend
98 + attn: Attention
99 + kv_cache: torch.Tensor
100 +
101 +
102 +def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
103 + '''
104 + Build key components for performing encoder/decoder attention test.
105 +
106 + Note that
107 + (1) The Attention instance constructed here, automatically selects
108 + an attention backend class based on platform info & a set of canned
109 + heuristics, so
110 + (2) The attention backend instance constructed here is thus *not
111 + the same backend instance* used by attn, but rather it is
112 + intended to be a *different instance* of the *same backend class*;
113 + therefore,
114 + (3) This function requires that test_pt.backend_name matches the backend
115 + class that Attention will automatically select when it is constructed.
116 +
117 +
118 + Arguments:
119 +
120 + * test_pt: TestPoint data structure; this function relies on the
121 + following fields: num_heads, head_size, num_blocks,
122 + block_size, backend_name
123 +
124 + Returns:
125 +
126 + * TestResources data structure.
127 + '''
128 +
129 + scale = float(1.0 / (test_pt.head_size**0.5))
130 + attn_backend = make_backend(test_pt.backend_name)
131 + attn = Attention(
132 + test_pt.num_heads,
133 + test_pt.head_size,
134 + scale=scale,
135 + )
136 + if test_pt.num_blocks is None or test_pt.num_heads is None:
137 + # Caller does not require a KV cache
138 + return TestResources(scale, attn_backend, attn, None)
139 +
140 + # Construct KV cache
141 + kv_cache = make_kv_cache(test_pt.num_blocks,
142 + test_pt.num_heads,
143 + test_pt.head_size,
144 + test_pt.block_size,
145 + device=CUDA_DEVICE)
146 + return TestResources(scale, attn_backend, attn, kv_cache)
147 +
148 +
149 +def _encoder_attn_setup(
150 + test_pt: TestPoint,
151 + test_rsrcs: TestResources,
152 +) -> PhaseTestParameters:
153 + '''
154 + Set up test vectors & data structures for encoder attention test.
155 +
156 + A triplet of synthetic query/key/value tensors are constructed.
157 + Given this is an encoder attention test, the key & value
158 + sequences will have the same length as the corresponding queries.
159 +
160 + The query/key/value tensors are passed to an ideal reference
161 + self-attention implementation to generate an ideal output tensor.
162 +
163 + Encoder inference does not populate the KV cache, therefore
164 + no KV cache memory mapping is constructed
165 +
166 + Arguments:
167 +
168 + * test_pt: TestPoint data structure; this function relies on the
169 + following fields: batch_size, num_heads, head_size,
170 + block_size, max_q_seq_len
171 + * test_rsrcs: TestResources data structure; this function relies on the
172 + scale field
173 +
174 +
175 + Returns:
176 +
177 + * PhaseTestParameters data structure comprising (1) packed query/key/value
178 + tensors, (2) the ideal output of attention computed using a naive
179 + implementation, and (3) KVCache field set to None
180 + '''
181 +
182 + (
183 + num_heads,
184 + head_size,
185 + _,
186 + batch_size,
187 + _,
188 + _,
189 + max_q_seq_len,
190 + _,
191 + ) = test_pt
192 +
193 + scale = test_rsrcs.scale
194 +
195 + max_kv_seq_len = max_q_seq_len
196 +
197 + # Make test tensors
198 +
199 + qkv_in, _, _ = make_qkv(batch_size,
200 + max_q_seq_len,
201 + max_kv_seq_len,
202 + num_heads,
203 + head_size,
204 + attn_type=AttentionType.ENCODER,
205 + device=CUDA_DEVICE)
206 +
207 + # Compute correct answer using naive non-causal attention
208 + # implementation
209 +
210 + ideal_output = ref_masked_attention(qkv_in.query,
211 + qkv_in.key,
212 + qkv_in.value,
213 + scale=scale,
214 + q_seq_lens=qkv_in.q_seq_lens,
215 + kv_seq_lens=qkv_in.kv_seq_lens)
216 +
217 + packed_ideal_output, _ = pack_tensor(ideal_output,
218 + qkv_in.q_seq_lens,
219 + device=CUDA_DEVICE)
220 +
221 + packed_qkv = pack_qkv(qkv_in, device=CUDA_DEVICE)
222 +
223 + return PhaseTestParameters(
224 + PackedQKVO(packed_qkv, packed_ideal_output),
225 + None # No KV cache
226 + )
227 +
228 +
229 +def _decoder_attn_setup(
230 + test_pt: TestPoint,
231 + test_rsrcs: TestResources,
232 + block_base_addr: int = 0,
233 +) -> Tuple[QKVInputs, PhaseTestParameters, PhaseTestParameters, int]:
234 + '''
235 + Set up test vectors & data structures for self-attention test.
236 +
237 + A triplet of synthetic query/key/value tensors are constructed ("baseline"
238 + query/key/value). Given this is a self-attention test, the key & value
239 + sequences will have the same length as the corresponding queries.
240 +
241 + "Prefill" query/key/value tensors are derived by masking out the last value
242 + in each baseline query/key/value. These tensors are used to test prefill &
243 + populate KV cache for a subsequent decode test.
244 +
245 + "Decode" query/key/value tensors are derived by extracting *only* the last
246 + value from each baseline query/key/value (i.e. complement of the prefill
247 + tensors.) These tensors are used to test decode, conditional on the kv cache
248 + being populated during the prefill test.
249 +
250 + The baseline query/key/value tensors are passed to an ideal reference
251 + self-attention implementation to generate a "Baseline" ideal output tensor.
252 + This tensor is split into the "Prefill" ideal output tensor (all but the
253 + last element of each output sequence) and the "Decode" ideal output tensor
254 + (*only* the last element of each output sequence); the "Prefill" and
255 + "Decode" ideal output tensors can be used to validate the prefill and decode
256 + test results, respectively.
257 +
258 + This function also constructs the self-attention KV cache memory mapping
259 + (slot mapping and block table), ensuring that the block table starts at
260 + block_base_addr
261 +
262 + Arguments:
263 +
264 + * test_pt: TestPoint data structure; this function relies on the
265 + following fields: batch_size, num_heads, head_size,
266 + block_size, max_q_seq_len
267 + * test_rsrcs: TestResources data structure; this function relies on the
268 + scale field
269 + * block_base_addr: decoder self-attention block-table base address
270 +
271 + Returns:
272 + * qkv: Unpacked (batch_size x padded_seq_len x num_heads x
273 + head_size) query/key/value tensors
274 + * Prefill-phase decoder self-attention PhaseTestParameters data structure,
275 + including (1) packed (number_of_tokens x num_heads x head_size)
276 + query/key/value tensors along with (2) ideal attention output
277 + computed using a naive implementation, and (3) memory-mapping data
278 + structures appropriate for prefill phase.
279 + * Decode-phase decoder self-attention PhaseTestParameters data structure,
280 + including (1) packed (number_of_tokens x num_heads x head_size)
281 + query/key/value tensors along with (2) ideal attention output
282 + computed using a naive implementation, and (3) memory-mapping data
283 + structures appropriate for decode phase.
284 + * max_block_idx: max physical address in decoder self-attention block-table
285 + (intended to be used as the base address for the encoder/
286 + decoder cross-attention block-table, which is not
287 + constructed in this function)
288 + '''
289 +
290 + (
291 + num_heads,
292 + head_size,
293 + _,
294 + batch_size,
295 + block_size,
296 + max_q_seq_len,
297 + _,
298 + _,
299 + ) = test_pt
300 +
301 + scale = test_rsrcs.scale
302 +
303 + max_kv_seq_len = max_q_seq_len
304 +
305 + # Build test tensors
306 +
307 + (
308 + qkv,
309 + prefill_qkv,
310 + decode_qkv,
311 + ) = make_qkv(batch_size,
312 + max_q_seq_len,
313 + max_kv_seq_len,
314 + num_heads,
315 + head_size,
316 + attn_type=AttentionType.DECODER,
317 + device=CUDA_DEVICE)
318 +
319 + # Compute correct answer using naive attention implementation
320 + # with causal attention mask
321 +
322 + causal_mask = make_causal_mask(max_q_seq_len,
323 + max_kv_seq_len).to(CUDA_DEVICE)
324 +
325 + ideal_output = ref_masked_attention(qkv.query,
326 + qkv.key,
327 + qkv.value,
328 + scale=scale,
329 + custom_mask=causal_mask,
330 + q_seq_lens=qkv.q_seq_lens,
331 + kv_seq_lens=qkv.kv_seq_lens)
332 +
333 + # Split out the prefill- & decode-phase ideal answers & pack them
334 +
335 + prefill_ideal_output = torch.zeros_like(ideal_output)
336 + decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
337 + for bdx, prefill_q_seq_len in enumerate(prefill_qkv.q_seq_lens):
338 + prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
339 + bdx, :prefill_q_seq_len]
340 + decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
341 + prefill_q_seq_len + 1)]
342 +
343 + prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
344 + prefill_qkv.q_seq_lens,
345 + device=CUDA_DEVICE)
346 + decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
347 + [1 for _ in range(batch_size)],
348 + device=CUDA_DEVICE)
349 +
350 + # Build prefill- & decode-phase data structures
351 + # for decoder self-attention. Block tables and
352 + # slot mapping must be in a format compatible
353 + # with KV caching & attention kernels
354 + #
355 + # Prefill-phase:
356 + #
357 + # * Empty block-tables tensor
358 + # * Slot-mapping with entries for prompt tokens
359 + #
360 + # Decode-phase:
361 + # * Block-tables tensor with minimum number of blocks
362 + # required by total num. tokens in the entirety of all sequences
363 + # (including both prefill & decode)
364 + # * Slot-mapping with entries for tokens that will be decoded in the
365 + # current decode iteration
366 + #
367 + # Note: the format described above is simply mirroring what ModelRunner
368 + # produces
369 +
370 + prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
371 +
372 + (
373 + decode_block_tables,
374 + slot_mapping_list,
375 + max_block_idx,
376 + ) = make_block_tables_slot_mapping(block_size,
377 + qkv.q_seq_lens,
378 + device=CUDA_DEVICE,
379 + block_base_addr=block_base_addr)
380 +
381 + (
382 + prefill_slot_mapping,
383 + decode_slot_mapping,
384 + ) = split_slot_mapping(slot_mapping_list,
385 + qkv.q_seq_lens,
386 + device=CUDA_DEVICE)
387 +
388 + prefill_pckd_qkv = pack_qkv(prefill_qkv, device=CUDA_DEVICE)
389 +
390 + decode_pckd_qkv = pack_qkv(decode_qkv, device=CUDA_DEVICE)
391 +
392 + return (
393 + qkv,
394 + PhaseTestParameters( # Prefill test params
395 + PackedQKVO(prefill_pckd_qkv, prefill_packed_ideal_output),
396 + KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
397 + PhaseTestParameters( # Decode test params
398 + PackedQKVO(decode_pckd_qkv, decode_packed_ideal_output),
399 + KVMemoryMap(decode_block_tables, decode_slot_mapping)),
400 + max_block_idx)
401 +
402 +
403 +def _enc_dec_cross_attn_setup_reuses_query(
404 + decoder_qkv: QKVInputs,
405 + encoder_test_params: PhaseTestParameters,
406 + prefill_decoder_phase_test_params: PhaseTestParameters,
407 + test_pt: TestPoint,
408 + test_rsrcs: TestResources,
409 + block_base_addr: int = 0,
410 +) -> Tuple[PhaseTestParameters, PhaseTestParameters]:
411 + '''
412 + Set up test vectors & data structures for cross-attention test.
413 +
414 + A triplet of synthetic cross-attention key/value tensors are constructed
415 + ("baseline" key/value). Given this is a cross-attention test, we assume
416 + query tensors were already synthesized for a prior self-attention test and
417 + will be reused for cross-attention. The key & value sequences generated here
418 + may have a different length than the corresponding queries (as is often
419 + the case for cross-attention between decoder and encoder sequences.)
420 +
421 + Cross attention key & value tensors do not grow during autoregressive
422 + inference; thus this function obtains a single key/value pair suitable for
423 + both prefill and decode.
424 +
425 + The "baseline" query tensor is received as an argument. The "baseline"
426 + query/key/value tensors are passed to an ideal reference cross-attention
427 + implementation to generate a "baseline" ideal output tensor. This tensor is
428 + split into the "Prefill" ideal output tensor (all but the last element of
429 + each output sequence) and the "Decode" ideal output tensor (*only* the last
430 + element of each output sequence); the "Prefill" and "Decode" ideal output
431 + tensors can be used to validate the prefill and decode test results,
432 + respectively.
433 +
434 + This function also constructs the cross-attention KV cache memory mapping
435 + (slot mapping and block table), ensuring that the block table starts at
436 + block_base_addr.
437 +
438 + Arguments:
439 +
440 + * decoder_qkv: pre-existing unpacked (batch_size x padded_seq_len x
441 + num_heads x head_size) decoder self-attention inputs;
442 + this function relies on the query and q_seq_lens
443 + fields
444 + * encoder_test_params: PhaseTestParameters data structure which was
445 + used for encoder inference; KV cache field
446 + is not used by this function
447 + * prefill_decoder_phase_test_params: PhaseTestParameters data structure
448 + used for prefill-phase decoder
449 + self-attention; all fields
450 + including KV cache required
451 + * test_pt: TestPoint data structure; this function relies on the
452 + following fields: batch_size, num_heads, head_size,
453 + block_size, max_q_seq_len
454 + * test_rsrcs: TestResources data structure; this function relies on the
455 + scale field
456 + * block_base_addr: decoder self-attention block-table base address
457 +
458 + Returns:
459 +
460 + * Prefill-phase encoder/decoder cross-attention PhaseTestParameters data
461 + structure, including (1) packed
462 + (number_of_tokens x num_heads x head_size) query/key/value tensors
463 + along with (2) ideal attention output computed using a
464 + naive implementation, and (3) memory-mapping data structures appropriate
465 + for prefill phase.
466 + * Decode-phase encoder/decoder cross-attention PhaseTestParameters data
467 + structure, including (1) packed
468 + (number_of_tokens x num_heads x head_size) query/key/value tensors
469 + along with (2) ideal attention output computed using a
470 + naive implementation, and (3) memory-mapping data structures appropriate
471 + for decode phase.
472 + '''
473 +
474 + assert encoder_test_params.packed_qkvo.packed_qkv is not None
475 + assert prefill_decoder_phase_test_params.packed_qkvo.packed_qkv is not None
476 +
477 + (
478 + num_heads,
479 + head_size,
480 + _,
481 + batch_size,
482 + block_size,
483 + max_decoder_seq_len,
484 + max_encoder_seq_len,
485 + _,
486 + ) = test_pt
487 +
488 + scale = test_rsrcs.scale
489 +
490 + decoder_query = decoder_qkv.query
491 + decoder_seq_lens = decoder_qkv.q_seq_lens
492 + encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens
493 + prefill_q_seq_lens = (
494 + prefill_decoder_phase_test_params.packed_qkvo.packed_qkv.q_seq_lens)
495 +
496 + assert prefill_q_seq_lens is not None
497 +
498 + (
499 + cross_kv,
500 + _,
501 + _,
502 + ) = make_qkv(batch_size,
503 + max_decoder_seq_len,
504 + max_encoder_seq_len,
505 + num_heads,
506 + head_size,
507 + force_kv_seq_lens=encoder_seq_lens,
508 + attn_type=AttentionType.ENCODER_DECODER,
509 + device=CUDA_DEVICE)
510 +
511 + ideal_output = ref_masked_attention(decoder_query,
512 + cross_kv.key,
513 + cross_kv.value,
514 + scale=scale,
515 + q_seq_lens=decoder_seq_lens,
516 + kv_seq_lens=cross_kv.kv_seq_lens)
517 +
518 + prefill_ideal_output = torch.zeros_like(ideal_output)
519 + decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
520 + for bdx, prefill_q_seq_len in enumerate(prefill_q_seq_lens):
521 + prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
522 + bdx, :prefill_q_seq_len]
523 + decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
524 + prefill_q_seq_len + 1)]
525 +
526 + prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
527 + prefill_q_seq_lens,
528 + device=CUDA_DEVICE)
529 + decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
530 + [1 for _ in range(batch_size)],
531 + device=CUDA_DEVICE)
532 +
533 + # Build prefill- & decode-phase data structures
534 + # for encoder/decoder cross-attention. Block tables and
535 + # slot mapping must be in a format compatible
536 + # with KV caching & attention kernels
537 + #
538 + # Whereas decoder self-attention extracts relationships between
539 + # equal-length Q/K/V sequences, which mutually grow in length
540 + # with each decoded token, cross-attention relates the Q sequence
541 + # - which grows with each new decoded token - to fixed-length
542 + # K and V sequences derived from the encoder hidden states.
543 + #
544 + # Prefill-phase:
545 + #
546 + # * Empty block-tables tensor
547 + # * Slot-mapping with as many entries as there are tokens in the encoder
548 + # prompt.
549 + #
550 + # Decode-phase:
551 + # * Block-tables tensor with minimum number of blocks to
552 + # accommodate K & V tensors which are equal in lnegth
553 + # to the encoder prompt length
554 + # * Empty slot-mapping tensor (since K & V are fixed in size,
555 + # new decoded tokens are not KV-cached and require no slot-
556 + # mapping)
557 + #
558 + # Note: the format above is simply an extension of what ModelRunner
559 + # produces for decoder-only models
560 +
561 + prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
562 + decode_slot_mapping = make_empty_slot_mapping_tensor(device=CUDA_DEVICE)
563 +
564 + (
565 + decode_block_tables,
566 + prefill_slot_mapping_list,
567 + _,
568 + ) = make_block_tables_slot_mapping(block_size,
569 + cross_kv.kv_seq_lens,
570 + block_base_addr=block_base_addr,
571 + device=CUDA_DEVICE)
572 +
573 + prefill_slot_mapping = maybe_make_long_tensor(prefill_slot_mapping_list,
574 + device=CUDA_DEVICE)
575 +
576 + # Packed key/value (query is already provided)
577 + packed_cross_kv = pack_qkv(cross_kv, device=CUDA_DEVICE)
578 +
579 + return (
580 + PhaseTestParameters( # Prefill-phase test params
581 + PackedQKVO(packed_cross_kv, prefill_packed_ideal_output),
582 + KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
583 + PhaseTestParameters( # Decode-phase test params
584 + PackedQKVO(None, decode_packed_ideal_output),
585 + KVMemoryMap(decode_block_tables, decode_slot_mapping)))
586 +
587 +
588 +def _run_encoder_attention_test(
589 + attn: Attention,
590 + encoder_test_params: PhaseTestParameters,
591 + attn_metadata: AttentionMetadata,
592 +) -> torch.Tensor:
593 + '''
594 + Run encoder attention.
595 +
596 + attn.forward() is passed attn_type=AttentionType.ENCODER in order
597 + to configure the kernel invocation for encoder attention
598 +
599 + Requires attn_metadata.num_decode_tokens == 0
600 + (There is no encoder execution in the decode-phase)
601 +
602 + Arguments:
603 +
604 + * attn: Attention wrapper instance
605 + * encoder_test_params: encoder PhaseTestParameters data structure;
606 + this function relies on the packed
607 + (number_of_tokens x num_heads x head_size)
608 + query/key/value fields
609 + * attn_metadata: attention metadata for encoder/decoder-self attention
610 +
611 + Returns:
612 + * Attention.forward() applied to packed {query,key,value} and
613 + & attn_metadata
614 + '''
615 + assert attn_metadata.num_decode_tokens == 0
616 + attn_type = AttentionType.ENCODER
617 + packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
618 + assert packed_qkv is not None
619 + return attn.forward(packed_qkv.query,
620 + packed_qkv.key,
621 + packed_qkv.value,
622 + None,
623 + attn_metadata,
624 + attn_type=attn_type)
625 +
626 +
627 +def _run_decoder_self_attention_test(
628 + test_rsrcs: TestResources,
629 + decoder_test_params: PhaseTestParameters,
630 + attn_metadata: AttentionMetadata,
631 +) -> torch.Tensor:
632 + '''
633 + Run decoder self-attention test.
634 +
635 + attn.forward() is passed attn_type=AttentionType.DECODER
636 + in order to configure the kernel invocation for decoder self-attention.
637 +
638 + Arguments:
639 +
640 + * test_rsrcs: TestResources instance; this function relies on the kv_cache
641 + and attn (Attention wrapper instance) fields
642 + * decoder_test_params: decoder PhaseTestParameters data structure;
643 + this function relies on the packed
644 + (number_of_tokens x num_heads x head_size)
645 + query/key/value fields
646 + * attn_metadata: attention metadata for decoder-self attention
647 + (contains KV cache memory-mapping)
648 +
649 + Returns:
650 + * Attention.forward() applied to packed_{query,key,value}, kv_cache
651 + & attn_metadata
652 + '''
653 + attn_type = AttentionType.DECODER
654 + attn = test_rsrcs.attn
655 + kv_cache = test_rsrcs.kv_cache
656 + packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
657 + assert packed_qkv is not None
658 + return attn.forward(packed_qkv.query,
659 + packed_qkv.key,
660 + packed_qkv.value,
661 + kv_cache,
662 + attn_metadata,
663 + attn_type=attn_type)
664 +
665 +
666 +def _run_encoder_decoder_cross_attention_test(
667 + test_rsrcs: TestResources,
668 + decoder_test_params: PhaseTestParameters,
669 + cross_test_params: Optional[PhaseTestParameters],
670 + attn_metadata: AttentionMetadata,
671 +) -> torch.Tensor:
672 + '''
673 + Run encoder/decoder cross-attention test.
674 +
675 + Via PhaseTestParameters data structures, consumes the same query utilized
676 + for decoder self-attention, plus a key/value specific to cross-attention.
677 +
678 + if cross_test_params is None or cross_test_params.packed_qkvo.packed_qkv
679 + is None, this reflects that in decode-phase cross attention there
680 + is no growth in the key and value tensors.
681 +
682 + attn.forward() is passed attn_type=AttentionType.ENCODER_DECODER
683 + in order to configure the kernel invocation for encoder/decoder cross-
684 + attention.
685 +
686 + Arguments:
687 +
688 + * test_rsrcs: TestResources instance; this function relies on the kv_cache
689 + and attn (Attention wrapper instance) fields
690 + * decoder_test_params: decoder PhaseTestParameters data structure;
691 + this function relies on the packed
692 + (number_of_tokens x num_heads x head_size)
693 + query field
694 + * cross_test_params: encoder/decoder PhaseTestParameters data structure;
695 + this function relies on the packed
696 + (number_of_tokens x num_heads x head_size)
697 + key/value fields
698 + * attn_metadata: attention metadata for encoder/decoder-self attention
699 +
700 + Returns:
701 + * Attention.forward() applied to packed_{query,key,value}, kv_cache
702 + & attn_metadata
703 + '''
704 + assert decoder_test_params.packed_qkvo.packed_qkv is not None
705 +
706 + attn_type = AttentionType.ENCODER_DECODER
707 + attn = test_rsrcs.attn
708 + kv_cache = test_rsrcs.kv_cache
709 + if cross_test_params is None:
710 + key = None
711 + value = None
712 + else:
713 + cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
714 + key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
715 + value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
716 + return attn.forward(decoder_test_params.packed_qkvo.packed_qkv.query,
717 + key,
718 + value,
719 + kv_cache,
720 + attn_metadata,
721 + attn_type=attn_type)
722 +
723 +
724 +@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
725 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
726 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
727 +@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
728 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
729 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
730 +@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
731 +@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
732 +def test_encoder_only(num_heads: int, head_size: int, backend_name: str,
733 + batch_size: int, block_size: int, max_dec_seq_len: int,
734 + max_enc_seq_len: int, monkeypatch):
735 +
736 + # Force Attention wrapper backend
737 + override_backend_env_variable(monkeypatch, backend_name)
738 +
739 + # Note: KV cache size of 4096 is arbitrary & chosen intentionally
740 + # to be more than necessary, since exceeding the kv cache size
741 + # is not part of this test
742 + test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
743 + block_size, max_dec_seq_len, max_enc_seq_len, 4096)
744 +
745 + # Attention scale factor, attention backend instance, attention wrapper
746 + # instance, KV cache init
747 + test_rsrcs = _make_test_resources(test_pt)
748 +
749 + # Construct encoder attention test params (only used
750 + # during prefill)
751 +
752 + enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
753 +
754 + # Shared prefill metadata structure
755 +
756 + prephase_attn_metadata: AttentionMetadata = make_test_metadata(
757 + test_rsrcs.attn_backend,
758 + True,
759 + None,
760 + decoder_test_params=None,
761 + encoder_test_params=enc_test_params,
762 + cross_test_params=None,
763 + device=CUDA_DEVICE)
764 +
765 + # PREFILL: encoder attention
766 +
767 + enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
768 + test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
769 +
770 + # - Is encoder attention result correct?
771 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
772 +
773 +
774 +@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
775 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
776 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
777 +@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
778 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
779 +@pytest.mark.parametrize("block_size", BLOCK_SIZES)
780 +@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
781 +@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
782 +def test_e2e_enc_dec_attn(
783 + num_heads: int,
784 + head_size: int,
785 + backend_name: str,
786 + batch_size: int,
787 + block_size: int,
788 + max_dec_seq_len: int,
789 + max_enc_seq_len: int,
790 + monkeypatch,
791 +) -> None:
792 + '''
793 + End-to-end encoder/decoder test:
794 +
795 + * Construct fake test vectors for (1) encoder attention,
796 + (2) decoder self-attention, and (3) encoder/decoder cross-attention
797 + * Construct (1) attention metadata structure with self- and cross-attention
798 + attributes for prefill-phase, and (2) an analogous attention metadata
799 + structure but for decode-phase
800 + * Test attention steps in the following order
801 +
802 + * Encoder attention
803 + * Prefill self-attention
804 + * Prefill cross-attention
805 + * Decode self-attention
806 + * Decode cross-attention
807 + * Besides being reflective of realistic use-cases, this order would
808 + exacerbate any accidental overlap in the self-/cross-attention
809 + block tables, which one hopes to avoid
810 +
811 +
812 + * Validate output correctness against ideal reference attention
813 + implementation
814 +
815 + Block tables are constructed such that cross-attention KV cache is in a
816 + higher, non-intersecting address-space than self-attention KV cache.
817 +
818 + Self- and cross-attention share the same query tensor but not the K/V
819 + tensors. Self-attention K/Vs must have the same seq len as Q while
820 + cross-attention K/Vs are allowed to differ in seq len, as is often the case
821 + for cross-attention.
822 +
823 + This test utilizes PyTest monkey patching to force the attention backend
824 + via an environment variable.
825 +
826 + Note on ROCm/HIP: currently encoder/decoder models are not supported on
827 + AMD GPUs, therefore this test simply is skipped if is_hip().
828 +
829 + Note on metadata: there is a single attention metadata structure shared by
830 + all prefill-phase attention operations (encoder, decoder, enc/dec cross),
831 + and a single one shared by all decode-phase attention operations
832 + (decoder & enc/dec cross.) This is intended to reflect the behavior
833 + of ModelRunner, which constructs a single attention metadata structure for
834 + each prefill or decode run. A realistic scenario would rely on the
835 + attention backend to utilize the appropriate attention metadata fields
836 + according to the value of attn_metadata.attention_type. Thus, this test is
837 + organized so as to confirm that the backend-under-test can handle a
838 + shared prefill attention metadata structure & a shared decode attention
839 + metadata structure.
840 + '''
841 +
842 + # Force Attention wrapper backend
843 + override_backend_env_variable(monkeypatch, backend_name)
844 +
845 + # Note: KV cache size of 4096 is arbitrary & chosen intentionally
846 + # to be more than necessary, since exceeding the kv cache size
847 + # is not part of this test
848 + test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
849 + block_size, max_dec_seq_len, max_enc_seq_len, 4096)
850 +
851 + # Attention scale factor, attention backend instance, attention wrapper
852 + # instance, KV cache init
853 + test_rsrcs = _make_test_resources(test_pt)
854 +
855 + # Construct encoder attention test params (only used
856 + # during prefill)
857 +
858 + enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
859 +
860 + # Construct Decoder self-attention prefill-phase & decode-phase
861 + # test params, including query/key/value tensors, decoder self-attention
862 + # memory-mapping. cross_block_base_addr is the uppermost address in the
863 + # decoder self-attention block-table, i.e. a base address which the
864 + # encoder/decoder cross-attention block-table may build downward toward.
865 +
866 + (
867 + dec_qkv,
868 + prephase_dec_test_params,
869 + decphase_dec_test_params,
870 + cross_block_base_addr,
871 + ) = _decoder_attn_setup(test_pt, test_rsrcs)
872 +
873 + # Construct encoder/decoder cross-attention prefill-phase & decode-phase
874 + # test params, including key/value tensors, cross-attention memory-mapping
875 +
876 + (
877 + prephase_cross_test_params,
878 + decphase_cross_test_params,
879 + ) = _enc_dec_cross_attn_setup_reuses_query(
880 + dec_qkv,
881 + enc_test_params,
882 + prephase_dec_test_params,
883 + test_pt,
884 + test_rsrcs,
885 + block_base_addr=cross_block_base_addr)
886 +
887 + # Shared prefill metadata structure
888 + assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
889 + prephase_attn_metadata: AttentionMetadata = make_test_metadata(
890 + test_rsrcs.attn_backend,
891 + True,
892 + prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
893 + decoder_test_params=prephase_dec_test_params,
894 + encoder_test_params=enc_test_params,
895 + cross_test_params=prephase_cross_test_params,
896 + device=CUDA_DEVICE)
897 +
898 + # PREFILL: encoder attention
899 +
900 + enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
901 + enc_test_params,
902 + prephase_attn_metadata)
903 +
904 + # - Is encoder attention result correct?
905 + assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
906 +
907 + # PREFILL: decoder self-attention test
908 +
909 + prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
910 + test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
911 +
912 + # - Is prefill decoder self-attention correct?
913 + assert_actual_matches_ideal(prephase_dec_test_params,
914 + prephase_dec_pckd_act_out)
915 +
916 + # PREFILL: encoder/decoder cross-attention test
917 +
918 + prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
919 + test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
920 + prephase_attn_metadata)
921 +
922 + # - Is prefill encoder/decoder cross-attention correct?
923 + assert_actual_matches_ideal(prephase_cross_test_params,
924 + prephase_cross_pckd_act_out)
925 +
926 + # DECODE: build decode-phase attention metadata
927 +
928 + decphase_attn_metadata: AttentionMetadata = make_test_metadata(
929 + test_rsrcs.attn_backend,
930 + False,
931 + dec_qkv.q_seq_lens,
932 + decoder_test_params=decphase_dec_test_params,
933 + encoder_test_params=enc_test_params,
934 + cross_test_params=decphase_cross_test_params,
935 + device=CUDA_DEVICE)
936 +
937 + # DECODE: decoder self-attention test
938 +
939 + decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
940 + test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
941 +
942 + # - Is decode-phase decoder self-attention correct?
943 + assert_actual_matches_ideal(decphase_dec_test_params,
944 + decphase_dec_pckd_act_out)
945 +
946 + # DECODE: encoder/decoder cross-attention test
947 +
948 + decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
949 + test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
950 +
951 + # - Is decode-phase encoder/decoder cross-attention correct?
952 + assert_actual_matches_ideal(decphase_cross_test_params,
953 + decphase_cross_pckd_act_out)
tests/kernels/utils.py
+920
@@ -1,12 +1,211 @@
1 """Kernel test utils"""
2
3 +import itertools
4 +import random
5 +from numbers import Number
6 +from typing import Any, List, NamedTuple, Optional, Tuple, Union
7 +
8 import pytest
9 +import torch
10 +
11 +from vllm.attention.backends.abstract import (AttentionBackend,
12 + AttentionMetadata, AttentionType)
13 +from vllm.attention.backends.xformers import XFormersBackend
14 +from vllm.utils import make_tensor_with_pad
15
16 +# String name of register which may be set in order to
17 +# force auto-selection of attention backend by Attention
18 +# wrapper
19 STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
20 +
21 +# Possible string values of STR_BACKEND_ENV_VAR
22 +# register, corresponding to possible backends
23 +STR_FLASHINFER_ATTN_VAL: str = "FLASHINFER"
24 +STR_TORCH_SDPA_ATTN_VAL: str = "TORCH_SDPA"
25 +STR_ROCM_FLASH_ATTN_VAL: str = "ROCM_FLASH"
26 +STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
27 STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
28 STR_INVALID_VAL: str = "INVALID"
29
30
31 +class QKVInputs(NamedTuple):
32 + '''
33 + Data structure for representing unpacked attention inputs,
34 + query/key/values and their sequence lengths.
35 +
36 + Attributes:
37 +
38 + * {query,key,value}: unpacked (batch_size x padded_seq_len x
39 + num_heads x head_size) attention inputs
40 + * q_seq_lens: query sequence lengths list
41 + * kv_seq_lens: shared key/value sequence lengths list
42 + '''
43 +
44 + query: torch.Tensor
45 + key: torch.Tensor
46 + value: torch.Tensor
47 + q_seq_lens: List[int]
48 + kv_seq_lens: List[int]
49 +
50 +
51 +class QKVO(NamedTuple):
52 + '''
53 + Data structure for representing unpacked attention inputs,
54 + alongside unpacked known-correct attention output
55 +
56 + Attributes:
57 +
58 + * qkv: unpacked (batch_size x padded_seq_len x
59 + num_heads x head_size) attention inputs
60 + * ideal_output: unpacked (batch_size x padded_seq_len x
61 + num_heads x head_size) known-correct attention output
62 + '''
63 +
64 + qkv: QKVInputs
65 + ideal_output: torch.Tensor
66 +
67 +
68 +class PackedQKVInputs(NamedTuple):
69 + '''
70 + Data structure for representing packed attention inputs
71 +
72 + Attributes:
73 +
74 + * {query,key,value}: packed (number_of_tokens x num_heads
75 + x head_size) attention inputs
76 + * q_start_loc_list: list of query start locations within packed tensor
77 + * kv_start_loc_list: shared list of key/value start locations within
78 + packed tensor
79 + * q_seq_lens: query sequence lengths list
80 + * kv_seq_lens: shared key/value sequence lengths list
81 + '''
82 +
83 + query: torch.Tensor
84 + key: torch.Tensor
85 + value: torch.Tensor
86 + q_start_loc_list: Optional[List[int]]
87 + kv_start_loc_list: Optional[List[int]]
88 + q_seq_lens: Optional[List[int]]
89 + kv_seq_lens: Optional[List[int]]
90 +
91 +
92 +class PackedQKVO(NamedTuple):
93 + '''
94 + Data structure for representing packed attention inputs,
95 + alongside packed known-correct attention output
96 +
97 + Attributes:
98 +
99 + * packed_qkv: packed (number_of_tokens x num_heads
100 + x head_size) attention inputs
101 + * ideal_output: packed (number_of_tokens x num_heads
102 + x head_size) known-correct attention output
103 + '''
104 +
105 + packed_qkv: Optional[PackedQKVInputs]
106 + ideal_output: torch.Tensor
107 +
108 +
109 +class KVMemoryMap(NamedTuple):
110 + '''
111 + Data structure for encapsulating KV cache memory mapping.
112 +
113 + Attributes:
114 +
115 + * block_tables: KV cache block tables
116 + * slot_mapping: mapping of sequence offset to physical address
117 + '''
118 +
119 + block_tables: torch.Tensor
120 + slot_mapping: torch.Tensor
121 +
122 +
123 +class PhaseTestParameters(NamedTuple):
124 + '''
125 + Data structure for encapsulating the test parameters
126 + for a given test "phase" (prefill or decode phase) and attention
127 + scenario (encoder, decoder-self, encoder/decoder-cross)
128 +
129 + Attributes:
130 +
131 + * packed_qkvo: packed (number_of_tokens x num_heads
132 + x head_size) attention inputs & known-correct
133 + output
134 + * kv_mmap: KV cache memory mapping, specific to this test phase &
135 + attention scenario
136 + '''
137 +
138 + packed_qkvo: PackedQKVO
139 + kv_mmap: Optional[KVMemoryMap]
140 +
141 +
142 +def maybe_make_int_tensor(
143 + _list: Optional[List[int]],
144 + device: Union[torch.device, str],
145 +) -> torch.Tensor:
146 + '''
147 + Convert Python int list to a 1D int torch.Tensor on `device`
148 +
149 + Returns:
150 +
151 + * If _list is not None: 1D int torch.Tensor on `device`
152 + * None otherwise
153 + '''
154 + return None if _list is None else torch.tensor(
155 + _list, dtype=torch.int, device=device)
156 +
157 +
158 +def maybe_make_long_tensor(
159 + _list: Optional[List[int]],
160 + device: Union[torch.device, str],
161 +) -> torch.Tensor:
162 + '''
163 + Convert Python int list to a 1D long torch.Tensor on `device`
164 +
165 + Returns:
166 +
167 + * If _list is not None: 1D long torch.Tensor on `device`
168 + * None otherwise
169 + '''
170 + return None if _list is None else torch.tensor(
171 + _list, dtype=torch.long, device=device)
172 +
173 +
174 +def maybe_max(_list: Optional[List]) -> Optional[Number]:
175 + '''
176 + Returns:
177 +
178 + * If _list is not None: max(_list)
179 + * None otherwise
180 + '''
181 + return None if _list is None else max(_list)
182 +
183 +
184 +def make_causal_mask(
185 + q_max_seq_len: int,
186 + kv_max_seq_len: int,
187 +) -> torch.Tensor:
188 + '''
189 + Create a q_max_seq_len x kv_max_seq_len causal mask
190 +
191 + Arguments:
192 +
193 + * q_max_seq_len: query max seq len
194 + * kv_max_seq_len: key/value max seq len
195 +
196 + Returns:
197 +
198 + * 2D tensor, q_max_seq_len x kv_max_seq_len
199 + '''
200 +
201 + # Create a matrix where entry (i, j) is True if i >= j
202 + mask = torch.triu(torch.ones(q_max_seq_len, kv_max_seq_len), diagonal=1)
203 + # Replace True with float('-inf') and False with 0
204 + mask = mask.masked_fill(mask == 1,
205 + float('-inf')).masked_fill(mask == 0, 0.0)
206 + return mask
207 +
208 +
209 def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
210 backend_name: str) -> None:
211 '''
@@ -20,3 +219,724 @@ def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
219 * backend_name: attention backend name to force
220 '''
221 mpatch.setenv(STR_BACKEND_ENV_VAR, backend_name)
222 +
223 +
224 +def ref_masked_attention(query: torch.Tensor,
225 + key: torch.Tensor,
226 + value: torch.Tensor,
227 + scale: float,
228 + custom_mask: Optional[torch.Tensor] = None,
229 + q_seq_lens: Optional[List] = None,
230 + kv_seq_lens: Optional[List] = None) -> torch.Tensor:
231 + '''
232 + "Golden" masked attention reference. Supports two types of masking:
233 +
234 + * Basic attention mask, utilizing {q,kv}_seq_lens args to mask out
235 + padding elements
236 + * Custom attention mask, which can force an arbitrary mask tensor, i.e.
237 + causal
238 +
239 + Arguments:
240 +
241 + * query: batch_size x q_padded_seq_len x num_heads x head_size
242 + * key: batch_size x kv_padded_seq_len x num_heads x head_size
243 + * value: batch_size x kv_padded_seq_len x num_heads x head_size
244 + * scale: Attention scale factor
245 + * custom_mask: custom attention mask; good place to inject a causal
246 + attention mask
247 + * q_seq_lens: list of unpadded query seq_lens for each batch index
248 + * kv_seq_lens: list of unpadded key/value seq_lens for each batch index
249 +
250 + Returns:
251 +
252 + * Attention result, batch_size x q_padded_seq_len x num_heads x head_size
253 + '''
254 +
255 + assert q_seq_lens is not None
256 + assert kv_seq_lens is not None
257 +
258 + batch_size = query.shape[0]
259 + assert (len(q_seq_lens) == batch_size)
260 + assert (len(kv_seq_lens) == batch_size)
261 +
262 + attn_weights = scale * torch.einsum("bqhd,bkhd->bhqk", query, key).float()
263 +
264 + # Basic attention mask, derived from seq lens
265 + if (q_seq_lens is not None) or (kv_seq_lens is not None):
266 + attn_mask = torch.zeros_like(attn_weights)
267 + if q_seq_lens is not None:
268 + for bdx, plen in enumerate(q_seq_lens):
269 + attn_mask[bdx, :, plen:, :] = -torch.inf
270 + if kv_seq_lens is not None:
271 + for bdx, plen in enumerate(kv_seq_lens):
272 + attn_mask[bdx, :, :, plen:] = -torch.inf
273 +
274 + attn_weights = attn_weights + attn_mask.float()
275 +
276 + # Custom attention mask
277 + if custom_mask is not None:
278 + attn_weights = attn_weights + custom_mask.float()
279 +
280 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
281 + out = torch.einsum("bhqk,bkhd->bqhd", attn_weights, value)
282 + return out
283 +
284 +
285 +def make_qkv(
286 + batch_size: int,
287 + max_q_seq_len: int,
288 + max_kv_seq_len: Optional[int],
289 + num_heads: int,
290 + head_size: int,
291 + device: Union[torch.device, str],
292 + force_kv_seq_lens: Optional[List[int]] = None,
293 + attn_type: AttentionType = AttentionType.ENCODER_DECODER,
294 + force_max_len: bool = False,
295 +) -> Tuple[QKVInputs, QKVInputs, QKVInputs]:
296 + '''
297 + Construct QKV test tensors for self- and cross-attention.
298 +
299 + Generates three query/key/value triplets:
300 +
301 + * "Baseline" query/key/value (for input to reference attention function)
302 + * "Prefill" query/key/value (last sequence offset zero'd out, for use as
303 + input to prefill kernel)
304 + * "Decode" query/key/value (only the last sequence offset from baseline,
305 + for use as input to decode kernel)
306 +
307 + Each Q/K/V triplet is associated with a list of q seqlens and a list of k/v
308 + seqlens
309 +
310 + Arguments:
311 +
312 + * batch_size
313 + * max_q_seq_len: max query seq len
314 + * max_kv_seq_len: max key/value seq len
315 + * num_heads
316 + * head_size
317 + * is_encoder_decoder_attn: if True, query seqlen may differ from
318 + key/value seqlen (as is often the case for cross-attention);
319 + o/w, query/key/value seqlens match at each batch index
320 + (max_kv_seq_len is unused)
321 + * force_kv_seq_lens: if not None, overrides kv sequence lengths
322 + * attn_type: encoder, decoder self, or enc/dec cross attention
323 + * force_max_len: if True, all query seqlens are max_q_seq_len; o/w query
324 + seqlens are random in [2,max_q_seq_lens]. Same for key/value seqlens
325 + and max_kv_seq_len, unless forced by is_encoder_decoder_attn=False
326 + * device: CPU or CUDA device
327 +
328 + Returns:
329 +
330 + * Overall QKVInputs structure (containing full unpacked Q/K/V tensors)
331 + * Prefill QKVInputs structure (containing all but the last sequence offset)
332 + * Decode QKVInputs structure (containing all only the last sequence offset)
333 + '''
334 +
335 + if force_max_len:
336 + q_seq_lens = [max_q_seq_len for _ in range(batch_size)]
337 + else:
338 + q_seq_lens = [
339 + random.randint(2, max_q_seq_len) for _ in range(batch_size)
340 + ]
341 + kv_seq_lens = None
342 + if force_kv_seq_lens is not None:
343 + kv_seq_lens = force_kv_seq_lens
344 + elif attn_type != AttentionType.ENCODER_DECODER:
345 + # K,V seq lens match Q for self-attention
346 + kv_seq_lens = q_seq_lens
347 + else:
348 + # K,V seq lens are distinct from Q seq lens & random
349 + assert max_kv_seq_len is not None
350 + if force_max_len:
351 + kv_seq_lens = [max_kv_seq_len] * batch_size
352 + else:
353 + kv_seq_lens = [
354 + random.randint(2, max_kv_seq_len) for _ in range(batch_size)
355 + ]
356 +
357 + query = torch.rand(
358 + (batch_size, max_q_seq_len, num_heads, head_size)).to(device)
359 + key = torch.rand(
360 + (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
361 + value = torch.rand(
362 + (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
363 +
364 + prefill_query = torch.zeros(
365 + (batch_size, max_q_seq_len, num_heads, head_size)).to(device)
366 + prefill_key = torch.zeros(
367 + (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
368 + prefill_value = torch.zeros(
369 + (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
370 +
371 + decode_query = torch.zeros(
372 + (batch_size, 1, num_heads, head_size)).to(device)
373 + decode_key = torch.zeros((batch_size, 1, num_heads, head_size)).to(device)
374 + decode_value = torch.zeros(
375 + (batch_size, 1, num_heads, head_size)).to(device)
376 +
377 + for bdx, (q_seq_len, kv_seq_len) in enumerate(zip(q_seq_lens,
378 + kv_seq_lens)):
379 + query[bdx, q_seq_len:, :, :] = 0
380 + key[bdx, kv_seq_len:, :, :] = 0
381 + value[bdx, kv_seq_len:, :, :] = 0
382 +
383 + prefill_query[bdx,
384 + 0:(q_seq_len - 1), :, :] = query[bdx,
385 + 0:(q_seq_len - 1), :, :]
386 + prefill_key[bdx,
387 + 0:(kv_seq_len - 1), :, :] = key[bdx,
388 + 0:(kv_seq_len - 1), :, :]
389 + prefill_value[bdx, 0:(kv_seq_len -
390 + 1), :, :] = value[bdx, 0:(kv_seq_len - 1), :, :]
391 +
392 + decode_query[bdx, :, :, :] = query[bdx,
393 + (q_seq_len - 1):q_seq_len, :, :]
394 + decode_key[bdx, :, :, :] = key[bdx, (kv_seq_len - 1):kv_seq_len, :, :]
395 + decode_value[bdx, :, :, :] = value[bdx,
396 + (kv_seq_len - 1):kv_seq_len, :, :]
397 +
398 + prefill_q_seq_lens = [plen - 1 for plen in q_seq_lens]
399 + prefill_kv_seq_lens = [plen - 1 for plen in kv_seq_lens]
400 +
401 + decode_q_seq_lens = [1 for _ in q_seq_lens]
402 + decode_kv_seq_lens = [1 for _ in kv_seq_lens]
403 +
404 + return (
405 + QKVInputs(
406 + query, # Overall QKV inputs
407 + key,
408 + value,
409 + q_seq_lens,
410 + kv_seq_lens),
411 + QKVInputs(
412 + prefill_query, # Prefill subset of QKV sequences
413 + prefill_key,
414 + prefill_value,
415 + prefill_q_seq_lens,
416 + prefill_kv_seq_lens),
417 + QKVInputs(
418 + decode_query, # Decode subset of KV sequences
419 + decode_key,
420 + decode_value,
421 + decode_q_seq_lens,
422 + decode_kv_seq_lens))
423 +
424 +
425 +def pack_tensor(
426 + unpacked_tensor: torch.Tensor, seq_lens: List[int],
427 + device: Union[torch.device, str]) -> Tuple[torch.Tensor, List[int]]:
428 + '''
429 + Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an
430 + unpadded number_of_tokens x num_heads x head_size tensor, where
431 + number_of_tokens = sum(seq_lens)
432 +
433 + Arguments:
434 +
435 + * unpacked_tensor: batch_size x padded_seq_len x num_heads x head_size
436 + * seq_lens: list of token counts for each seq
437 + * device: CPU or CUDA device
438 +
439 + Returns
440 +
441 + * packed_tensor: number_of_tokens x num_heads x head_size
442 + * start_loc_list: start idx of each batch elt in packed_tensor; [0] +
443 + list(itertools.accumulate(seq_lens))
444 + '''
445 +
446 + num_tok = sum(seq_lens)
447 + num_heads = unpacked_tensor.shape[-2]
448 + head_size = unpacked_tensor.shape[-1]
449 + start_loc_list = [0] + list(itertools.accumulate(seq_lens))
450 + packed_tensor = torch.zeros((num_tok, num_heads, head_size), device=device)
451 +
452 + for bdx, (seq_len, start_loc) in enumerate(zip(seq_lens, start_loc_list)):
453 +
454 + packed_tensor[start_loc:(
455 + start_loc + seq_len), :, :] = unpacked_tensor[bdx, :seq_len, :, :]
456 +
457 + return packed_tensor, start_loc_list
458 +
459 +
460 +def pack_qkv(qkv: QKVInputs, device: Union[torch.device,
461 + str]) -> PackedQKVInputs:
462 + '''
463 + Individually pack each of Q, K and V, each with dimensions batch_size x
464 + padded_seq_len x num_heads x head_size, into respective number_of_tokens x
465 + num_heads x head_size tensors.
466 +
467 + For Q, number_of_tokens = sum(q_seq_lens).
468 +
469 + For K and V, number_of_tokens = sum(kv_seq_lens)
470 +
471 + Arguments:
472 +
473 + * qkv: Unpacked (batch_size x padded_seq_len x num_heads x head_size)
474 + attention inputs
475 + * device: CPU or CUDA device
476 +
477 + Returns
478 +
479 + * Packed (number_of_tokens x num_heads x head_size) QKV inputs
480 + derived from unpacked inputs
481 + '''
482 +
483 + if qkv.query is None:
484 + packed_query = None
485 + q_start_loc_list = None
486 + else:
487 + packed_query, q_start_loc_list = pack_tensor(qkv.query,
488 + qkv.q_seq_lens,
489 + device=device)
490 + packed_key, kv_start_loc_list = pack_tensor(qkv.key,
491 + qkv.kv_seq_lens,
492 + device=device)
493 + packed_value, _ = pack_tensor(qkv.value, qkv.kv_seq_lens, device=device)
494 + return PackedQKVInputs(
495 + packed_query, packed_key, packed_value, q_start_loc_list,
496 + kv_start_loc_list,
497 + (None if q_start_loc_list is None else qkv.q_seq_lens),
498 + qkv.kv_seq_lens)
499 +
500 +
501 +def make_backend(backend_name: str) -> AttentionBackend:
502 + '''
503 + Construct the backend instance determined by the backend_name string
504 + argument.
505 +
506 + "XFORMERS" -> construct xformers backend
507 +
508 + TODO: other backends
509 +
510 + Note: at time of writing the Attention wrapper automatically selects
511 + its own backend for Attention.forward(); so the backend instance which
512 + you generate with this function is not meant to be used for *running*
513 + inference, but rather for generating compatible metadata structures
514 + using backend.make_metadata()
515 +
516 +
517 + Returns:
518 +
519 + * Backend instance
520 + '''
521 + if backend_name == STR_XFORMERS_ATTN_VAL:
522 + return XFormersBackend()
523 + raise AssertionError(
524 + f"Unrecognized backend_name {backend_name} for unit test")
525 +
526 +
527 +def _make_metadata_tensors(
528 + seq_lens: Optional[List[int]], context_lens: Optional[List[int]],
529 + encoder_seq_lens: Optional[List[int]], device: Union[torch.device, str]
530 +) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[List[int]],
531 + torch.Tensor, Optional[int]]:
532 + '''
533 + Build scalar & tensor values required to build attention metadata structure.
534 +
535 + Arguments:
536 +
537 + * seq_lens: list of token-counts for each decoder input seq
538 + * context_lens: list of context length values for each seq
539 + * encoder_seq_lens: list of token-counts for each encoder input seq
540 + * device: CPU or CUDA device
541 +
542 + Returns:
543 +
544 + * seq_lens_tensor: decoder seq_lens list, as tensor
545 + * context_lens_tensor: context_lens list, as tensor
546 + * max_context_len: max(context_lens)
547 + * max_seq_len: max(seq_lens)
548 + * seq_start_loc: start idx of each sequence
549 + * max_encoder_seq_len: encoder seq_lens list, as tensor
550 + '''
551 + seq_lens_tensor = maybe_make_int_tensor(seq_lens, device)
552 + context_lens_tensor = maybe_make_int_tensor(context_lens, device)
553 + max_context_len = maybe_max(context_lens)
554 + max_seq_len = maybe_max(seq_lens)
555 +
556 + encoder_seq_lens_tensor = maybe_make_int_tensor(encoder_seq_lens, device)
557 + max_encoder_seq_len = (None if encoder_seq_lens is None else
558 + max(encoder_seq_lens))
559 +
560 + seq_start_loc = None
561 +
562 + return (seq_lens_tensor, context_lens_tensor, max_context_len, max_seq_len,
563 + seq_start_loc, encoder_seq_lens_tensor, max_encoder_seq_len)
564 +
565 +
566 +def make_kv_cache(num_blocks: int,
567 + num_heads: int,
568 + head_size: int,
569 + block_size: int,
570 + device: Union[torch.device, str],
571 + default_val: float = 0.0) -> torch.Tensor:
572 + '''
573 + Create a fake KV cache.
574 +
575 + Arguments:
576 +
577 + * num_blocks: number of blocks in the KV cache
578 + * num_heads: number of attention heads
579 + * head_size: head dimension
580 + * block_size: number of offsets within a block
581 + * device: CPU or CUDA device
582 + * default_val: initialization value for KV cache elements
583 +
584 + Returns:
585 +
586 + * kv_cache: 2 x num_blocks x (block_size * num_heads * head_size)
587 + '''
588 +
589 + kv_cache = torch.rand(
590 + (2, num_blocks, block_size * num_heads * head_size)).to(device)
591 + if default_val is not None:
592 + kv_cache[:, :, :] = default_val
593 + return kv_cache
594 +
595 +
596 +def _num_tokens_to_min_blocks(num_tokens: int, block_size: int) -> int:
597 + '''
598 + Compute the minimum number of blocks required to hold num_tokens tokens,
599 + given block_size
600 + '''
601 + return (num_tokens + block_size) // block_size
602 +
603 +
604 +def make_empty_slot_mapping_tensor(device: Union[torch.device, str]):
605 + return maybe_make_long_tensor([], device)
606 +
607 +
608 +def make_empty_block_tables_tensor(device: Union[torch.device, str]):
609 + return torch.tensor([], device=device)
610 +
611 +
612 +def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: List[int],
613 + device: Union[torch.device, str]):
614 + '''
615 + Split a slot mapping into valid prefill- and decode-phase slot mappings.
616 +
617 + Context:
618 + * Your goal is to test (1) prefill of N prompts, with prompt-lengths
619 + {K_i \\forall i \\in [0,N)}, followed by (2) decoding of a single token
620 + for all N prompts (N tokens total); the resultant sequence lengths
621 + after decode would be {K_i + 1 for i \\in [0,N)}
622 + * The test you want to do requires (1) having the prefill slot mapping
623 + for all tokens present during prefill, the number of which is
624 + M = \\sum_i{K_i}, and (2) having the decode slot mapping for all N
625 + decoded tokens
626 +
627 + This function consumes a single 1D slot mapping, which is the
628 + concatenation of N slot mappings each of length K_i + 1 (corresponding
629 + to the sequence lengths after decode), with a total length of
630 + P = \\sum_i{K_i + 1} = M + N
631 +
632 + The prefill-phase slot mapping results from excising the (K_i + 1)-th entry
633 + from each of the N subsequences in the slot mapping (i.e. omitting the
634 + decoded token's mapping.)
635 +
636 + The N excised entries are appended to obtain the decode-phase slot mapping
637 +
638 + Arguments:
639 +
640 + * slot_mapping_list: Length-P 1D slot mapping (as List) reflecting all N
641 + post-decode sequences
642 + * seq_lens: List of N post-decode sequence lengths (K_i + 1 in the
643 + description above)
644 + * device: cuda, cpu, etc.
645 +
646 + Returns:
647 +
648 + * prefill_slot_mapping: Length-M 1D slot mapping (as Tensor)
649 + reflecting all N prefill prompts
650 + * decode_slot_mapping: Length-N 1D slot mapping (as Tensor) reflecting
651 + all N decoded tokens
652 + '''
653 +
654 + prefill_slot_mapping = []
655 + decode_slot_mapping = []
656 +
657 + base_idx = 0
658 + for seq_len in seq_lens:
659 + prefill_slot_mapping.extend(slot_mapping_list[base_idx:(base_idx +
660 + seq_len - 1)])
661 + decode_slot_mapping.append(slot_mapping_list[base_idx + seq_len - 1])
662 + base_idx += seq_len
663 +
664 + return (maybe_make_long_tensor(prefill_slot_mapping, device),
665 + maybe_make_long_tensor(decode_slot_mapping, device))
666 +
667 +
668 +def make_block_tables_slot_mapping(
669 + block_size: int,
670 + seq_lens: List[int],
671 + device: Union[torch.device, str],
672 + block_base_addr: int = 0) -> Tuple[torch.Tensor, List[int], int]:
673 + '''
674 + Construct fake block tables & slot mappings.
675 +
676 + For a sequence with num_tokens tokens the minimum number
677 + of required KV cache blocks is
678 +
679 + num_blocks = (num_tokens + block_size) // block_size
680 +
681 + Then the minimum KV cache size in blocks is
682 +
683 + total_cache_blocks = sum(num_blocks for all seqs)
684 +
685 + Then, the blocktable mapping counts downward from
686 +
687 + block_base_addr + total_cache_blocks
688 +
689 + to
690 +
691 + block_base_addr
692 +
693 +
694 + The constructed block-tables and slot-mapping are sized to the
695 + lengths of the sequences in their entirety (as reflected by seq_lens),
696 + i.e. the total of prefill prompt tokens + decoded tokens.
697 +
698 + Arguments:
699 +
700 + * block_size: number of offsets per block
701 + * seq_lens: list of token-counts for each sequence
702 + * block_base_addr: the block table base address
703 + * device: CPU or CUDA device
704 +
705 + Return:
706 +
707 + * block_tables_tensor: block table for sequence
708 + * slot_mapping_list: slot mapping for sequence
709 + * max_block_idx: the highest block address within this block table
710 + '''
711 +
712 + # Provision minimum number of KV cache blocks
713 + num_blocks_list = [
714 + _num_tokens_to_min_blocks(num_tokens, block_size)
715 + for num_tokens in seq_lens
716 + ]
717 + max_block_table_len = max(num_blocks_list)
718 + block_table_pad_tokens = 10
719 +
720 + block_tables = []
721 + slot_mapping_list = []
722 + # Compute uppermost address of block table
723 + total_cache_blocks = sum(num_blocks_list)
724 + block_base_idx = block_base_addr + total_cache_blocks
725 + max_block_idx = block_base_idx
726 + for sdx, num_tokens in enumerate(seq_lens):
727 + num_blocks = num_blocks_list[sdx]
728 + block_table = list(
729 + range(block_base_idx, block_base_idx - num_blocks, -1))
730 + for idx in range(num_tokens):
731 + mapping_value = (
732 + idx % block_size) + block_table[idx // block_size] * block_size
733 + slot_mapping_list.append(mapping_value)
734 +
735 + block_base_idx -= num_blocks
736 + block_tables.append(block_table)
737 +
738 + block_tables_tensor = make_tensor_with_pad(
739 + block_tables,
740 + max_len=max_block_table_len + block_table_pad_tokens,
741 + pad=0,
742 + dtype=torch.int,
743 + device=device,
744 + )
745 +
746 + return (block_tables_tensor, slot_mapping_list, max_block_idx)
747 +
748 +
749 +def make_test_metadata(
750 + attn_backend: AttentionBackend,
751 + is_prompt: bool,
752 + seq_lens: Optional[List[int]],
753 + decoder_test_params: Optional[PhaseTestParameters],
754 + device: Union[torch.device, str],
755 + encoder_test_params: Optional[PhaseTestParameters] = None,
756 + cross_test_params: Optional[PhaseTestParameters] = None
757 +) -> AttentionMetadata:
758 + '''
759 + Construct fake attention metadata for a given test phase
760 + (prefill-phase or decode-phase).
761 +
762 + encoder_test_params and cross_test_params arguments allow encoder
763 + attention and enc/dec cross-attention (respectively) to use distinct
764 + metadata values from decoder self-attention (decoder_test_params.)
765 +
766 + if encoder_test_params and cross_test_params are None, the attention
767 + metadata will support decoder-only scenario.
768 +
769 + Assumptions:
770 +
771 + * No chunked prefill -> a batch is 100% prefill or 100% decode, never both
772 +
773 + Arguments:
774 +
775 + * attn_backend: Backend for sourcing attention kernels
776 + * is_prompt: prefill if True, o/w decode
777 + * seq_lens: list of token counts for each sequence
778 + * decoder_test_params: decoder self-attention test params;
779 + this function requires
780 + kv_mmap (memory mapping) field
781 + * device: CPU or CUDA device
782 + * encoder_test_params: encoder attention test params;
783 + this function requires encoder query
784 + sequence lengths field. If None,
785 + encoder query sequence lengths are
786 + treated as None
787 + * cross_test_params: enc/dec cross-attention test params;
788 + this function requires kv_mmap field.
789 + If None, KV cache memory map data
790 + structures are treated as None
791 +
792 + Return:
793 +
794 + * AttentionMetadata structure
795 + '''
796 +
797 + # Decoder self-attention memory mapping
798 + # decoder_test_params is None signals encoder-only
799 + # scenario, so kv_mmap is None
800 + kv_mmap = (None
801 + if decoder_test_params is None else decoder_test_params.kv_mmap)
802 +
803 + # This function constructs metadata assuming no chunked prefill,
804 + # i.e. 100% prefill tokens or 100% decode tokens
805 + #
806 + # - If is_prompt, num_prefills_or_decodes is the number of prefills
807 + # and num_prefill_or_decode_tokens is the number of prefill tokens
808 + # - If not is_prompt, num_prefills_or_decodes is the number of decodes
809 + # and num_prefill_or_decode_tokens is the number of decode tokens
810 + #
811 + # seq_lens is None signals encoder-only
812 + # scenario, in which case num_prefills_or_decodes and
813 + # num_prefill_or_decode_tokens are unused
814 + num_prefills_or_decodes = (None if seq_lens is None else len(seq_lens))
815 +
816 + num_prefill_or_decode_tokens = (None if seq_lens is None else (
817 + sum(seq_lens) if is_prompt else len(seq_lens)))
818 +
819 + # Seems for non-prefix-caching scenarios context_lens
820 + # is never needed
821 + context_lens = None
822 +
823 + if encoder_test_params is None:
824 + encoder_seq_lens = None
825 + num_encoder_tokens = None
826 + else:
827 + # Encoder/decoder or encoder-only models only:
828 + # * Extract encoder input sequence lengths
829 + assert encoder_test_params.packed_qkvo.packed_qkv is not None
830 + encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens
831 + num_encoder_tokens = (None if encoder_seq_lens is None else
832 + (sum(encoder_seq_lens)))
833 +
834 + if cross_test_params is None:
835 + cross_kv_mmap = None
836 + else:
837 + # Encoder/decoder or encoder-only models only:
838 + # * Extract *cross-attention* slot_mapping and block table
839 + # (kv_mmap)
840 + cross_kv_mmap = cross_test_params.kv_mmap
841 +
842 + if is_prompt:
843 + # Prefill-phase scenario
844 +
845 + num_prefills = num_prefills_or_decodes
846 + num_prefill_tokens = num_prefill_or_decode_tokens
847 + num_decode_tokens = 0
848 +
849 + (
850 + seq_lens_tensor,
851 + context_lens_tensor,
852 + _,
853 + _,
854 + _,
855 + encoder_seq_lens_tensor,
856 + max_encoder_seq_len,
857 + ) = _make_metadata_tensors(seq_lens,
858 + context_lens,
859 + encoder_seq_lens,
860 + device=device)
861 +
862 + return attn_backend.make_metadata(
863 + num_prefills=num_prefills,
864 + slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
865 + num_prefill_tokens=num_prefill_tokens,
866 + num_decode_tokens=num_decode_tokens,
867 + seq_lens=seq_lens,
868 + seq_lens_tensor=seq_lens_tensor,
869 + max_prefill_seq_len=None if seq_lens is None else max(seq_lens),
870 + max_decode_seq_len=0,
871 + context_lens_tensor=context_lens_tensor,
872 + block_tables=(None if kv_mmap is None else kv_mmap.block_tables),
873 + use_cuda_graph=False,
874 + num_encoder_tokens=num_encoder_tokens,
875 + encoder_seq_lens=encoder_seq_lens,
876 + encoder_seq_lens_tensor=encoder_seq_lens_tensor,
877 + max_encoder_seq_len=max_encoder_seq_len,
878 + cross_slot_mapping=(None if cross_kv_mmap is None else
879 + cross_kv_mmap.slot_mapping),
880 + cross_block_tables=(None if cross_kv_mmap is None else
881 + cross_kv_mmap.block_tables))
882 +
883 + else: # not is_prompt
884 + # Decode-phase scenario
885 +
886 + assert kv_mmap is not None
887 + assert num_prefill_or_decode_tokens is not None
888 + assert seq_lens is not None
889 +
890 + num_prefills = 0
891 + num_prefill_tokens = 0
892 + num_decode_tokens = num_prefill_or_decode_tokens
893 +
894 + (
895 + seq_lens_tensor,
896 + context_lens_tensor,
897 + _,
898 + _,
899 + _,
900 + encoder_seq_lens_tensor,
901 + max_encoder_seq_len,
902 + ) = _make_metadata_tensors(seq_lens,
903 + context_lens,
904 + encoder_seq_lens,
905 + device=device)
906 +
907 + return attn_backend.make_metadata(
908 + num_prefills=num_prefills,
909 + slot_mapping=kv_mmap.slot_mapping,
910 + num_prefill_tokens=num_prefill_tokens,
911 + num_decode_tokens=num_decode_tokens,
912 + seq_lens=seq_lens,
913 + seq_lens_tensor=seq_lens_tensor,
914 + max_prefill_seq_len=0,
915 + max_decode_seq_len=max(seq_lens),
916 + context_lens_tensor=context_lens_tensor,
917 + block_tables=kv_mmap.block_tables,
918 + use_cuda_graph=False,
919 + num_encoder_tokens=num_encoder_tokens,
920 + encoder_seq_lens=encoder_seq_lens,
921 + encoder_seq_lens_tensor=encoder_seq_lens_tensor,
922 + max_encoder_seq_len=max_encoder_seq_len,
923 + cross_slot_mapping=(None if cross_kv_mmap is None else
924 + cross_kv_mmap.slot_mapping),
925 + cross_block_tables=(None if cross_kv_mmap is None else
926 + cross_kv_mmap.block_tables))
927 +
928 +
929 +def assert_actual_matches_ideal(test_params: PhaseTestParameters,
930 + output_under_test: torch.Tensor) -> None:
931 + '''
932 + Assert that observed output matches the ideal output
933 + contained in the test parameters data structure.
934 +
935 + Arguments:
936 +
937 + * test_params: Test parameters including packed ideal output
938 + * output_under_test: actually observed output value
939 + '''
940 + ideal_output = test_params.packed_qkvo.ideal_output
941 + assert torch.allclose(ideal_output,
942 + output_under_test.view_as(ideal_output))
vllm/attention/backends/abstract.py
+8
@@ -1,11 +1,18 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass, fields
3 +from enum import Enum, auto
4 from typing import (Any, Dict, Generic, List, Optional, Set, Tuple, Type,
5 TypeVar)
6
7 import torch
8
9
10 +class AttentionType(Enum):
11 + DECODER = auto() # Decoder attention between previous layer Q/K/V
12 + ENCODER = auto() # Encoder attention between previous layer Q/K/V
13 + ENCODER_DECODER = auto() # Attention between dec. Q and enc. K/V
14 +
15 +
16 class AttentionBackend(ABC):
17 """Abstract class for attention backends."""
18
@@ -128,5 +135,6 @@ class AttentionImpl(ABC, Generic[T]):
135 kv_cache: torch.Tensor,
136 attn_metadata: T,
137 kv_scale: float = 1.0,
138 + attn_type: AttentionType = AttentionType.DECODER,
139 ) -> torch.Tensor:
140 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+8 -1
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4 import torch
5
6 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 - AttentionMetadata)
7 + AttentionMetadata, AttentionType)
8 from vllm.attention.ops.blocksparse_attention.interface import (
9 LocalStridedBlockSparseAttn, get_head_sliding_step)
10 from vllm.attention.ops.paged_attn import PagedAttention
@@ -328,6 +328,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
328 kv_cache: torch.Tensor,
329 attn_metadata: BlocksparseFlashAttentionMetadata,
330 kv_scale: float = 1.0,
331 + attn_type: AttentionType = AttentionType.DECODER,
332 ) -> torch.Tensor:
333 """Forward pass with FlashAttention and PagedAttention.
334
@@ -340,6 +341,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
341 Returns:
342 shape = [num_tokens, num_heads * head_size]
343 """
344 + if attn_type != AttentionType.DECODER:
345 + raise NotImplementedError("Encoder self-attention and "
346 + "encoder/decoder cross-attention "
347 + "are not implemented for "
348 + "BlocksparseFlashAttentionImpl")
349 +
350 num_tokens, hidden_size = query.shape
351 # Reshape the query, key, and value tensors.
352 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/flash_attn.py
+8 -1
@@ -7,7 +7,7 @@ from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8 from vllm import _custom_ops as ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata)
10 + AttentionMetadata, AttentionType)
11
12
13 class FlashAttentionBackend(AttentionBackend):
@@ -257,6 +257,7 @@ class FlashAttentionImpl(AttentionImpl):
257 kv_cache: torch.Tensor,
258 attn_metadata: FlashAttentionMetadata,
259 kv_scale: float = 1.0,
260 + attn_type: AttentionType = AttentionType.DECODER,
261 ) -> torch.Tensor:
262 """Forward pass with FlashAttention.
263
@@ -269,6 +270,12 @@ class FlashAttentionImpl(AttentionImpl):
270 Returns:
271 shape = [num_tokens, num_heads * head_size]
272 """
273 + if attn_type != AttentionType.DECODER:
274 + raise NotImplementedError("Encoder self-attention and "
275 + "encoder/decoder cross-attention "
276 + "are not implemented for "
277 + "FlashAttentionImpl")
278 +
279 # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
280 assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
281
vllm/attention/backends/flashinfer.py
+7 -1
@@ -14,7 +14,7 @@ import torch
14
15 from vllm import _custom_ops as ops
16 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17 - AttentionMetadata)
17 + AttentionMetadata, AttentionType)
18
19
20 class FlashInferBackend(AttentionBackend):
@@ -224,8 +224,14 @@ class FlashInferImpl(AttentionImpl):
224 kv_cache: Optional[torch.Tensor],
225 attn_metadata: FlashInferMetadata,
226 kv_scale: float = 1.0,
227 + attn_type: AttentionType = AttentionType.DECODER,
228 ) -> torch.Tensor:
229 assert kv_scale == 1.0
230 + if attn_type != AttentionType.DECODER:
231 + raise NotImplementedError("Encoder self-attention and "
232 + "encoder/decoder cross-attention "
233 + "are not implemented for "
234 + "FlashInferImpl")
235 num_tokens, hidden_size = query.shape
236 query = query.view(-1, self.num_heads, self.head_size)
237 key = key.view(-1, self.num_kv_heads, self.head_size)
vllm/attention/backends/ipex_attn.py
+7 -1
@@ -7,7 +7,7 @@ import torch
7
8 from vllm._ipex_ops import ipex_ops
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata)
10 + AttentionMetadata, AttentionType)
11 from vllm.attention.ops.paged_attn import (PagedAttention,
12 PagedAttentionMetadata)
13
@@ -157,6 +157,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
157 kv_cache: Optional[torch.Tensor],
158 attn_metadata: IpexAttnMetadata, # type: ignore
159 kv_scale: float = 1.0,
160 + attn_type: AttentionType = AttentionType.DECODER,
161 ) -> torch.Tensor:
162 """Forward pass with IPEX varlen_attention and PagedAttention.
163
@@ -170,6 +171,11 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
171 shape = [num_tokens, num_heads * head_size]
172 """
173 assert kv_scale == 1.0
174 + if attn_type != AttentionType.DECODER:
175 + raise NotImplementedError("Encoder self-attention and "
176 + "encoder/decoder cross-attention "
177 + "are not implemented for "
178 + "IpexAttnBackendImpl")
179 num_tokens, hidden_size = query.shape
180 # Reshape the query, key, and value tensors.
181 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+7 -1
@@ -6,7 +6,7 @@ import torch_xla.experimental.custom_kernel # Required to register custom ops.
6 import torch_xla.experimental.dynamo_set_buffer_donor
7
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 - AttentionMetadata)
9 + AttentionMetadata, AttentionType)
10
11
12 class PallasAttentionBackend(AttentionBackend):
@@ -132,6 +132,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
132 kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
133 attn_metadata: PallasMetadata,
134 kv_scale: float = 1.0,
135 + attn_type: AttentionType = AttentionType.DECODER,
136 ) -> torch.Tensor:
137 """Forward pass with Pallas attention.
138
@@ -146,6 +147,11 @@ class PallasAttentionBackendImpl(AttentionImpl):
147 shape = [batch_size, seq_len, num_heads * head_size]
148 """
149 assert kv_scale == 1.0
150 + if attn_type != AttentionType.DECODER:
151 + raise NotImplementedError("Encoder self-attention and "
152 + "encoder/decoder cross-attention "
153 + "are not implemented for "
154 + "PallasAttentionBackendImpl")
155 batch_size, seq_len, hidden_size = query.shape
156 query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
157 key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+8 -1
@@ -6,7 +6,7 @@ import torch
6
7 import vllm.envs as envs
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9 - AttentionMetadata)
9 + AttentionMetadata, AttentionType)
10 from vllm.attention.ops.paged_attn import (PagedAttention,
11 PagedAttentionMetadata)
12 from vllm.logger import init_logger
@@ -297,6 +297,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
297 kv_cache: torch.Tensor,
298 attn_metadata: ROCmFlashAttentionMetadata,
299 kv_scale: float = 1.0,
300 + attn_type: AttentionType = AttentionType.DECODER,
301 ) -> torch.Tensor:
302 """Forward pass with FlashAttention and PagedAttention.
303
@@ -309,6 +310,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
310 Returns:
311 shape = [num_tokens, num_heads * head_size]
312 """
313 + if attn_type != AttentionType.DECODER:
314 + raise NotImplementedError("Encoder self-attention and "
315 + "encoder/decoder cross-attention "
316 + "are not implemented for "
317 + "ROCmFlashAttentionImpl")
318 +
319 num_tokens, hidden_size = query.shape
320 # Reshape the query, key, and value tensors.
321 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/torch_sdpa.py
+7 -1
@@ -7,7 +7,7 @@ import torch
7 from torch.nn.functional import scaled_dot_product_attention
8
9 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10 - AttentionMetadata)
10 + AttentionMetadata, AttentionType)
11 from vllm.attention.ops.paged_attn import PagedAttentionMetadata
12 from vllm.utils import is_cpu
13
@@ -145,6 +145,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
145 kv_cache: Optional[torch.Tensor],
146 attn_metadata: TorchSDPAMetadata, # type: ignore
147 kv_scale: float = 1.0,
148 + attn_type: AttentionType = AttentionType.DECODER,
149 ) -> torch.Tensor:
150 """Forward pass with torch SDPA and PagedAttention.
151
@@ -158,6 +159,11 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
159 shape = [num_tokens, num_heads * head_size]
160 """
161 assert kv_scale == 1.0
162 + if attn_type != AttentionType.DECODER:
163 + raise NotImplementedError("Encoder self-attention and "
164 + "encoder/decoder cross-attention "
165 + "are not implemented for "
166 + "TorchSDPABackendImpl")
167 num_tokens, hidden_size = query.shape
168 # Reshape the query, key, and value tensors.
169 query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/utils.py new
+7
@@ -0,0 +1,7 @@
1 +"""Attention backend utils"""
2 +
3 +# Error string(s) for encoder/decoder
4 +# unsupported attention scenarios
5 +
6 +STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
7 + "with encoder/decoder models.")
vllm/attention/backends/xformers.py
+394 -78
@@ -6,10 +6,11 @@ import torch
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import (AttentionBias,
8 BlockDiagonalCausalMask,
9 + BlockDiagonalMask,
10 LowerTriangularMaskWithTensorBias)
11
12 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12 - AttentionMetadata)
13 + AttentionMetadata, AttentionType)
14 from vllm.attention.ops.paged_attn import (PagedAttention,
15 PagedAttentionMetadata)
16 from vllm.logger import init_logger
@@ -66,11 +67,6 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
67 dynamically, it should be stored in tensor. The tensor has to be
68 updated from `CUDAGraphRunner.forward` API.
69 """
69 - # (batch_size,). The sequence length per sequence. Sequence length means
70 - # the computed tokens + new tokens None if it is a decoding.
71 - seq_lens: Optional[List[int]]
72 - # seq_lens stored as a tensor.
73 - seq_lens_tensor: Optional[torch.Tensor]
70
71 # |---------- N-1 iteration --------|
72 # |---------------- N iteration ---------------------|
@@ -79,8 +75,9 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
75 # |-------------------- seq_len ----------------------|
76 # |-- query_len ---|
77
82 - # Maximum query length in the batch. None for decoding.
83 - max_query_len: Optional[int]
78 + # seq_lens stored as a tensor.
79 + seq_lens_tensor: Optional[torch.Tensor]
80 +
81 # FIXME: It is for flash attn.
82 # Maximum sequence length among prefill batch. 0 if there are decoding
83 # requests only.
@@ -88,26 +85,55 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
85 # Maximum sequence length among decode batch. 0 if there are prefill
86 # requests only.
87 max_decode_seq_len: int
91 - # (batch_size + 1,). The cumulative subquery lengths of the sequences in
92 - # the batch, used to index into subquery. E.g., if the subquery length
93 - # is [4, 6], it is [0, 4, 10].
94 - query_start_loc: Optional[torch.Tensor]
88 +
89 + # Whether or not if cuda graph is enabled.
90 + # Cuda-graph is currently enabled for decoding only.
91 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
92 + use_cuda_graph: bool
93 +
94 + # (batch_size,). The sequence length per sequence. Sequence length means
95 + # the computed tokens + new tokens None if it is a decoding.
96 + seq_lens: Optional[List[int]] = None
97 +
98 # FIXME: It is for flash attn.
99 # (batch_size + 1,). The cumulative sequence lengths of the sequences in
100 # the batch, used to index into sequence. E.g., if the sequence length is
101 # [4, 6], it is [0, 4, 10].
99 - seq_start_loc: Optional[torch.Tensor]
102 + seq_start_loc: Optional[torch.Tensor] = None
103 +
104 # (batch_size,) A tensor of context lengths (tokens that are computed
105 # so far).
102 - context_lens_tensor: Optional[torch.Tensor]
106 + context_lens_tensor: Optional[torch.Tensor] = None
107
104 - # Whether or not if cuda graph is enabled.
105 - # Cuda-graph is currently enabled for decoding only.
106 - # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
107 - use_cuda_graph: bool
108 + # Maximum query length in the batch. None for decoding.
109 + max_query_len: Optional[int] = None
110 +
111 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
112 + # the batch, used to index into subquery. E.g., if the subquery length
113 + # is [4, 6], it is [0, 4, 10].
114 + query_start_loc: Optional[torch.Tensor] = None
115 +
116 + # Self-attention prefill/decode metadata cache
117 _cached_prefill_metadata: Optional["XFormersMetadata"] = None
118 _cached_decode_metadata: Optional["XFormersMetadata"] = None
119
120 + # Begin encoder attn & enc/dec cross-attn fields...
121 +
122 + # Encoder sequence lengths representation
123 + encoder_seq_lens: Optional[List[int]] = None
124 + encoder_seq_lens_tensor: Optional[torch.Tensor] = None
125 +
126 + # Maximum sequence length among encoder sequences
127 + max_encoder_seq_len: Optional[int] = None
128 +
129 + # Number of tokens input to encoder
130 + num_encoder_tokens: Optional[int] = None
131 +
132 + # Cross-attention memory-mapping data structures: slot mapping
133 + # and block tables
134 + cross_slot_mapping: Optional[torch.Tensor] = None
135 + cross_block_tables: Optional[torch.Tensor] = None
136 +
137 def __post_init__(self):
138 # Set during the execution of the first attention op.
139 # It is a list because it is needed to set per prompt
@@ -115,6 +141,28 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
141 # from xformer API.
142 # will not appear in the __repr__ and __init__
143 self.attn_bias: Optional[List[AttentionBias]] = None
144 + self.encoder_attn_bias: Optional[List[AttentionBias]] = None
145 + self.cross_attn_bias: Optional[List[AttentionBias]] = None
146 +
147 + @property
148 + def is_all_encoder_attn_metadata_set(self):
149 + '''
150 + All attention metadata required for encoder attention is set.
151 + '''
152 + return ((self.encoder_seq_lens is not None)
153 + and (self.encoder_seq_lens_tensor is not None)
154 + and (self.max_encoder_seq_len is not None))
155 +
156 + @property
157 + def is_all_cross_attn_metadata_set(self):
158 + '''
159 + All attention metadata required for enc/dec cross-attention is set.
160 +
161 + Superset of encoder attention required metadata.
162 + '''
163 + return (self.is_all_encoder_attn_metadata_set
164 + and (self.cross_slot_mapping is not None)
165 + and (self.cross_block_tables is not None))
166
167 @property
168 def prefill_metadata(self) -> Optional["XFormersMetadata"]:
@@ -122,30 +170,50 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
170 return None
171
172 if self._cached_prefill_metadata is not None:
173 + # Recover cached prefill-phase attention
174 + # metadata structure
175 return self._cached_prefill_metadata
176
127 - assert self.seq_lens is not None
128 - assert self.seq_lens_tensor is not None
129 - assert self.query_start_loc is not None
130 - assert self.context_lens_tensor is not None
131 - assert self.block_tables is not None
132 -
177 + assert ((self.seq_lens is not None)
178 + or (self.encoder_seq_lens is not None))
179 + assert ((self.seq_lens_tensor is not None)
180 + or (self.encoder_seq_lens_tensor is not None))
181 +
182 + # Compute some attn_metadata fields which default to None
183 + query_start_loc = (None if self.query_start_loc is None else
184 + self.query_start_loc[:self.num_prefills + 1])
185 + slot_mapping = (None if self.slot_mapping is None else
186 + self.slot_mapping[:self.num_prefill_tokens])
187 + seq_lens = (None if self.seq_lens is None else
188 + self.seq_lens[:self.num_prefills])
189 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
190 + self.seq_lens_tensor[:self.num_prefills])
191 + context_lens_tensor = (None if self.context_lens_tensor is None else
192 + self.context_lens_tensor[:self.num_prefills])
193 + block_tables = (None if self.block_tables is None else
194 + self.block_tables[:self.num_prefills])
195 +
196 + # Construct & cache prefill-phase attention metadata structure
197 self._cached_prefill_metadata = XFormersMetadata(
198 num_prefills=self.num_prefills,
199 num_prefill_tokens=self.num_prefill_tokens,
200 num_decode_tokens=0,
137 - slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
138 - seq_lens=self.seq_lens[:self.num_prefills],
139 - seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
201 + slot_mapping=slot_mapping,
202 + seq_lens=seq_lens,
203 + seq_lens_tensor=seq_lens_tensor,
204 max_query_len=self.max_query_len,
205 max_prefill_seq_len=self.max_prefill_seq_len,
206 max_decode_seq_len=0,
143 - query_start_loc=self.query_start_loc[:self.num_prefills + 1],
144 - seq_start_loc=None,
145 - context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
146 - block_tables=self.block_tables[:self.num_prefills],
207 + query_start_loc=query_start_loc,
208 + context_lens_tensor=context_lens_tensor,
209 + block_tables=block_tables,
210 use_cuda_graph=False,
148 - )
211 + # Begin encoder & cross attn fields below...
212 + encoder_seq_lens=self.encoder_seq_lens,
213 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
214 + max_encoder_seq_len=self.max_encoder_seq_len,
215 + cross_slot_mapping=self.cross_slot_mapping,
216 + cross_block_tables=self.cross_block_tables)
217 return self._cached_prefill_metadata
218
219 @property
@@ -154,29 +222,146 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
222 return None
223
224 if self._cached_decode_metadata is not None:
225 + # Recover cached decode-phase attention
226 + # metadata structure
227 return self._cached_decode_metadata
158 - assert self.block_tables is not None
159 - assert self.seq_lens_tensor is not None
160 -
228 + assert ((self.seq_lens_tensor is not None)
229 + or (self.encoder_seq_lens_tensor is not None))
230 +
231 + # Compute some attn_metadata fields which default to None
232 + slot_mapping = (None if self.slot_mapping is None else
233 + self.slot_mapping[self.num_prefill_tokens:])
234 + seq_lens_tensor = (None if self.seq_lens_tensor is None else
235 + self.seq_lens_tensor[self.num_prefills:])
236 + block_tables = (None if self.block_tables is None else
237 + self.block_tables[self.num_prefills:])
238 +
239 + # Construct & cache decode-phase attention metadata structure
240 self._cached_decode_metadata = XFormersMetadata(
241 num_prefills=0,
242 num_prefill_tokens=0,
243 num_decode_tokens=self.num_decode_tokens,
165 - slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
166 - seq_lens=None,
167 - seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
168 - max_query_len=None,
244 + slot_mapping=slot_mapping,
245 + seq_lens_tensor=seq_lens_tensor,
246 max_prefill_seq_len=0,
247 max_decode_seq_len=self.max_decode_seq_len,
171 - query_start_loc=None,
172 - seq_start_loc=None,
173 - context_lens_tensor=None,
174 - block_tables=self.block_tables[self.num_prefills:],
248 + block_tables=block_tables,
249 use_cuda_graph=self.use_cuda_graph,
176 - )
250 + # Begin encoder & cross attn fields below...
251 + encoder_seq_lens=self.encoder_seq_lens,
252 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
253 + max_encoder_seq_len=self.max_encoder_seq_len,
254 + cross_slot_mapping=self.cross_slot_mapping,
255 + cross_block_tables=self.cross_block_tables)
256 return self._cached_decode_metadata
257
258
259 +def _get_attn_bias(
260 + attn_metadata: XFormersMetadata,
261 + attn_type: AttentionType,
262 +) -> Optional[AttentionBias]:
263 + '''
264 + Extract appropriate attention bias from attention metadata
265 + according to attention type.
266 +
267 + Arguments:
268 +
269 + * attn_metadata: Attention metadata structure associated with attention
270 + * attn_type: encoder attention, decoder self-attention,
271 + encoder/decoder cross-attention
272 +
273 + Returns:
274 + * Appropriate attention bias value given the attention type
275 + '''
276 +
277 + if attn_type == AttentionType.DECODER:
278 + return attn_metadata.attn_bias
279 + elif attn_type == AttentionType.ENCODER:
280 + return attn_metadata.encoder_attn_bias
281 + else:
282 + # attn_type == AttentionType.ENCODER_DECODER
283 + return attn_metadata.cross_attn_bias
284 +
285 +
286 +def _set_attn_bias(
287 + attn_metadata: XFormersMetadata,
288 + attn_bias: List[Optional[AttentionBias]],
289 + attn_type: AttentionType,
290 +) -> None:
291 + '''
292 + Update appropriate attention bias field of attention metadata,
293 + according to attention type.
294 +
295 + Arguments:
296 +
297 + * attn_metadata: Attention metadata structure associated with attention
298 + * attn_bias: The desired attention bias value
299 + * attn_type: encoder attention, decoder self-attention,
300 + encoder/decoder cross-attention
301 + '''
302 +
303 + if attn_type == AttentionType.DECODER:
304 + attn_metadata.attn_bias = attn_bias
305 + elif attn_type == AttentionType.ENCODER:
306 + attn_metadata.encoder_attn_bias = attn_bias
307 + elif attn_type == AttentionType.ENCODER_DECODER:
308 + attn_metadata.cross_attn_bias = attn_bias
309 + else:
310 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
311 +
312 +
313 +def _get_seq_len_block_table_args(
314 + attn_metadata: XFormersMetadata,
315 + is_prompt: bool,
316 + attn_type: AttentionType,
317 +) -> tuple:
318 + '''
319 + The particular choice of sequence-length- and block-table-related
320 + attributes which should be extracted from attn_metadata is dependent
321 + on the type of attention operation.
322 +
323 + Decoder attn -> select entirely decoder self-attention-related fields
324 + Encoder/decoder cross-attn -> select encoder sequence lengths &
325 + cross-attn block-tables fields
326 + Encoder attn -> select encoder sequence lengths fields & no block tables
327 +
328 + Arguments:
329 +
330 + * attn_metadata: Attention metadata structure associated with attention op
331 + * is_prompt: True if prefill, False otherwise
332 + * attn_type: encoder attention, decoder self-attention,
333 + encoder/decoder cross-attention
334 +
335 + Returns:
336 +
337 + * Appropriate sequence-lengths tensor
338 + * Appropriate max sequence-length scalar
339 + * Appropriate block tables (or None)
340 + '''
341 +
342 + if attn_type == AttentionType.DECODER:
343 + # Decoder self-attention
344 + # Choose max_seq_len based on whether we are in prompt_run
345 + if is_prompt:
346 + max_seq_len = attn_metadata.max_prefill_seq_len
347 + else:
348 + max_seq_len = attn_metadata.max_decode_seq_len
349 + return (attn_metadata.seq_lens_tensor, max_seq_len,
350 + attn_metadata.block_tables)
351 + elif attn_type == AttentionType.ENCODER_DECODER:
352 + # Enc/dec cross-attention KVs match encoder sequence length;
353 + # cross-attention utilizes special "cross" block tables
354 + return (attn_metadata.encoder_seq_lens_tensor,
355 + attn_metadata.max_encoder_seq_len,
356 + attn_metadata.cross_block_tables)
357 + elif attn_type == AttentionType.ENCODER:
358 + # No block tables associated with encoder attention
359 + return (attn_metadata.encoder_seq_lens_tensor,
360 + attn_metadata.max_encoder_seq_len, None)
361 + else:
362 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
363 +
364 +
365 class XFormersImpl(AttentionImpl[XFormersMetadata]):
366 """
367 If the input tensors contain prompt tokens, the layout is as follows:
@@ -238,51 +423,144 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
423 def forward(
424 self,
425 query: torch.Tensor,
241 - key: torch.Tensor,
242 - value: torch.Tensor,
426 + key: Optional[torch.Tensor],
427 + value: Optional[torch.Tensor],
428 kv_cache: Optional[torch.Tensor],
429 attn_metadata: "XFormersMetadata",
430 kv_scale: float = 1.0,
431 + attn_type: AttentionType = AttentionType.DECODER,
432 ) -> torch.Tensor:
433 """Forward pass with xFormers and PagedAttention.
434
435 + For decoder-only models: query, key and value must be non-None.
436 +
437 + For encoder/decoder models:
438 + * XFormersImpl.forward() may be invoked for both self- and cross-
439 + attention layers.
440 + * For self-attention: query, key and value must be non-None.
441 + * For cross-attention:
442 + * Query must be non-None
443 + * During prefill, key and value must be non-None; key and value
444 + get cached for use during decode.
445 + * During decode, key and value may be None, since:
446 + (1) key and value tensors were cached during prefill, and
447 + (2) cross-attention key and value tensors do not grow during
448 + decode
449 +
450 + A note on how the attn_type (attention type enum) argument impacts
451 + attention forward() behavior:
452 +
453 + * DECODER: normal decoder-only behavior;
454 + use decoder self-attention block table
455 + * ENCODER: no KV caching; pass encoder sequence
456 + attributes (encoder_seq_lens/encoder_seq_lens_tensor/
457 + max_encoder_seq_len) to kernel, in lieu of decoder
458 + sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
459 + * ENCODER_DECODER: cross-attention behavior;
460 + use cross-attention block table for caching KVs derived
461 + from encoder hidden states; since KV sequence lengths
462 + will match encoder sequence lengths, pass encoder sequence
463 + attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
464 + max_encoder_seq_len)
465 +
466 Args:
467 query: shape = [num_tokens, num_heads * head_size]
468 key: shape = [num_tokens, num_kv_heads * head_size]
469 value: shape = [num_tokens, num_kv_heads * head_size]
470 kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
471 attn_metadata: Metadata for attention.
472 + attn_type: Select attention type, between encoder attention,
473 + decoder self-attention, or encoder/decoder cross-
474 + attention. Defaults to decoder self-attention,
475 + which is the vLLM default generally
476 Returns:
477 shape = [num_tokens, num_heads * head_size]
478 """
258 - query = query.view(-1, self.num_heads, self.head_size)
259 - key = key.view(-1, self.num_kv_heads, self.head_size)
260 - value = value.view(-1, self.num_kv_heads, self.head_size)
479
262 - if kv_cache is not None:
480 + # Check that appropriate attention metadata attributes are
481 + # selected for the desired attention type
482 + if (attn_type == AttentionType.ENCODER
483 + and (not attn_metadata.is_all_encoder_attn_metadata_set)):
484 + raise AttributeError("Encoder attention requires setting "
485 + "encoder metadata attributes.")
486 + elif (attn_type == AttentionType.ENCODER_DECODER
487 + and (not attn_metadata.is_all_cross_attn_metadata_set)):
488 + raise AttributeError("Encoder/decoder cross-attention "
489 + "requires setting cross-attention "
490 + "metadata attributes.")
491 +
492 + query = query.view(-1, self.num_heads, self.head_size)
493 + if key is not None:
494 + assert value is not None
495 + key = key.view(-1, self.num_kv_heads, self.head_size)
496 + value = value.view(-1, self.num_kv_heads, self.head_size)
497 + else:
498 + assert value is None
499 +
500 + # Self-attention vs. cross-attention will impact
501 + # which KV cache memory-mapping & which
502 + # seqlen datastructures we utilize
503 +
504 + if (attn_type != AttentionType.ENCODER and kv_cache is not None):
505 + # KV-cache during decoder-self- or
506 + # encoder-decoder-cross-attention, but not
507 + # during encoder attention.
508 + #
509 + # Even if there are no new key/value pairs to cache,
510 + # we still need to break out key_cache and value_cache
511 + # i.e. for later use by paged attention
512 key_cache, value_cache = PagedAttention.split_kv_cache(
513 kv_cache, self.num_kv_heads, self.head_size)
514
266 - # Reshape the input keys and values and store them in the cache.
267 - # If kv_cache is not provided, the new key and value tensors are
268 - # not cached. This happens during the initial memory profiling run.
269 - PagedAttention.write_to_paged_cache(key, value, key_cache,
270 - value_cache,
271 - attn_metadata.slot_mapping,
272 - self.kv_cache_dtype, kv_scale)
273 -
274 - num_prefill_tokens = attn_metadata.num_prefill_tokens
275 - num_decode_tokens = attn_metadata.num_decode_tokens
276 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens
277 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens
515 + if (key is not None) and (value is not None):
516 +
517 + if attn_type == AttentionType.ENCODER_DECODER:
518 + # Update cross-attention KV cache (prefill-only)
519 + # During cross-attention decode, key & value will be None,
520 + # preventing this IF-statement branch from running
521 + updated_slot_mapping = attn_metadata.cross_slot_mapping
522 + else:
523 + # Update self-attention KV cache (prefill/decode)
524 + updated_slot_mapping = attn_metadata.slot_mapping
525 +
526 + # Reshape the input keys and values and store them in the cache.
527 + # If kv_cache is not provided, the new key and value tensors are
528 + # not cached. This happens during the initial memory
529 + # profiling run.
530 + PagedAttention.write_to_paged_cache(key, value, key_cache,
531 + value_cache,
532 + updated_slot_mapping,
533 + self.kv_cache_dtype,
534 + kv_scale)
535 +
536 + if attn_type != AttentionType.ENCODER:
537 + # Decoder self-attention supports chunked prefill.
538 + # Encoder/decoder cross-attention requires no chunked
539 + # prefill (100% prefill or 100% decode tokens, no mix)
540 + num_prefill_tokens = attn_metadata.num_prefill_tokens
541 + num_decode_tokens = attn_metadata.num_decode_tokens
542 + else:
543 + # Encoder attention - chunked prefill is not applicable;
544 + # derive token-count from query shape & and treat them
545 + # as 100% prefill tokens
546 + assert attn_metadata.num_encoder_tokens is not None
547 + num_prefill_tokens = attn_metadata.num_encoder_tokens
548 + num_decode_tokens = 0
549 +
550 + if attn_type == AttentionType.DECODER:
551 + # Only enforce this shape-constraint for decoder
552 + # self-attention
553 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens
554 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens
555
556 output = torch.empty_like(query)
557 # Query for decode. KV is not needed because it is already cached.
558 decode_query = query[num_prefill_tokens:]
559 # QKV for prefill.
560 query = query[:num_prefill_tokens]
284 - key = key[:num_prefill_tokens]
285 - value = value[:num_prefill_tokens]
561 + if key is not None and value is not None:
562 + key = key[:num_prefill_tokens]
563 + value = value[:num_prefill_tokens]
564
565 assert query.shape[0] == num_prefill_tokens
566 assert decode_query.shape[0] == num_decode_tokens
@@ -294,10 +572,14 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
572 # block tables are empty if the prompt does not have a cached
573 # prefix.
574 out = self._run_memory_efficient_xformers_forward(
297 - query, key, value, prefill_meta)
575 + query, key, value, prefill_meta, attn_type=attn_type)
576 assert out.shape == output[:num_prefill_tokens].shape
577 output[:num_prefill_tokens] = out
578 else:
579 +
580 + assert prefill_meta.query_start_loc is not None
581 + assert prefill_meta.max_query_len is not None
582 +
583 # prefix-enabled attention
584 # TODO(Hai) this triton kernel has regression issue (broke) to
585 # deal with different data types between KV and FP8 KV cache,
@@ -320,13 +602,20 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
602 output[:num_prefill_tokens] = out
603
604 if decode_meta := attn_metadata.decode_metadata:
605 +
606 + (
607 + seq_lens_arg,
608 + max_seq_len_arg,
609 + block_tables_arg,
610 + ) = _get_seq_len_block_table_args(decode_meta, False, attn_type)
611 +
612 output[num_prefill_tokens:] = PagedAttention.forward_decode(
613 decode_query,
614 key_cache,
615 value_cache,
327 - decode_meta.block_tables,
328 - decode_meta.seq_lens_tensor,
329 - decode_meta.max_decode_seq_len,
616 + block_tables_arg,
617 + seq_lens_arg,
618 + max_seq_len_arg,
619 self.kv_cache_dtype,
620 self.num_kv_heads,
621 self.scale,
@@ -343,6 +632,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
632 key: torch.Tensor,
633 value: torch.Tensor,
634 attn_metadata: XFormersMetadata,
635 + attn_type: AttentionType = AttentionType.DECODER,
636 ) -> torch.Tensor:
637 """Attention for 1D query of multiple prompts. Multiple prompt
638 tokens are flattened in to `query` input.
@@ -356,8 +646,12 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
646 key: shape = [num_prefill_tokens, num_kv_heads, head_size]
647 value: shape = [num_prefill_tokens, num_kv_heads, head_size]
648 attn_metadata: Metadata for attention.
649 + attn_type: Select attention type, between encoder attention,
650 + decoder self-attention, or encoder/decoder cross-
651 + attention. Defaults to decoder self-attention,
652 + which is the vLLM default generally
653 """
360 - assert attn_metadata.seq_lens is not None
654 +
655 original_query = query
656 if self.num_kv_heads != self.num_heads:
657 # GQA/MQA requires the shape [B, M, G, H, K].
@@ -375,18 +669,39 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
669 # Set attention bias if not provided. This typically happens at
670 # the very attention layer of every iteration.
671 # FIXME(woosuk): This is a hack.
378 - if attn_metadata.attn_bias is None:
672 + attn_bias = _get_attn_bias(attn_metadata, attn_type)
673 + if attn_bias is None:
674 if self.alibi_slopes is None:
380 - attn_bias = BlockDiagonalCausalMask.from_seqlens(
381 - attn_metadata.seq_lens)
675 + if (attn_type == AttentionType.ENCODER_DECODER):
676 + assert attn_metadata.seq_lens is not None
677 + assert attn_metadata.encoder_seq_lens is not None
678 +
679 + # Default enc/dec cross-attention mask is non-causal
680 + attn_bias = BlockDiagonalMask.from_seqlens(
681 + attn_metadata.seq_lens, attn_metadata.encoder_seq_lens)
682 + elif attn_type == AttentionType.ENCODER:
683 + assert attn_metadata.encoder_seq_lens is not None
684 +
685 + # Default encoder self-attention mask is non-causal
686 + attn_bias = BlockDiagonalMask.from_seqlens(
687 + attn_metadata.encoder_seq_lens)
688 + else:
689 + assert attn_metadata.seq_lens is not None
690 +
691 + # Default decoder self-attention mask is causal
692 + attn_bias = BlockDiagonalCausalMask.from_seqlens(
693 + attn_metadata.seq_lens)
694 if self.sliding_window is not None:
695 attn_bias = attn_bias.make_local_attention(
696 self.sliding_window)
385 - attn_metadata.attn_bias = [attn_bias]
697 + attn_bias = [attn_bias]
698 else:
387 - attn_metadata.attn_bias = _make_alibi_bias(
388 - self.alibi_slopes, self.num_kv_heads, query.dtype,
389 - attn_metadata.seq_lens)
699 + assert attn_metadata.seq_lens is not None
700 + attn_bias = _make_alibi_bias(self.alibi_slopes,
701 + self.num_kv_heads, query.dtype,
702 + attn_metadata.seq_lens)
703 +
704 + _set_attn_bias(attn_metadata, attn_bias, attn_type)
705
706 # No alibi slopes.
707 # TODO(woosuk): Too many view operations. Let's try to reduce
@@ -400,7 +715,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
715 query,
716 key,
717 value,
403 - attn_bias=attn_metadata.attn_bias[0],
718 + attn_bias=attn_bias[0],
719 p=0.0,
720 scale=self.scale)
721 return out.view_as(original_query)
@@ -409,6 +724,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
724 # FIXME(woosuk): Because xformers does not support dynamic sequence
725 # lengths with custom attention bias, we process each prompt one by
726 # one. This is inefficient, especially when we have many short prompts.
727 + assert attn_metadata.seq_lens is not None
728 output = torch.empty_like(original_query)
729 start = 0
730 for i, seq_len in enumerate(attn_metadata.seq_lens):
@@ -417,7 +733,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
733 query[None, start:end],
734 key[None, start:end],
735 value[None, start:end],
420 - attn_bias=attn_metadata.attn_bias[i],
736 + attn_bias=attn_bias[i],
737 p=0.0,
738 scale=self.scale)
739 # TODO(woosuk): Unnecessary copy. Optimize.
vllm/attention/layer.py
+10 -3
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional
4 import torch
5 import torch.nn as nn
6
7 -from vllm.attention.backends.abstract import AttentionMetadata
7 +from vllm.attention.backends.abstract import AttentionMetadata, AttentionType
8 from vllm.attention.selector import get_attn_backend
9 from vllm.config import CacheConfig
10 from vllm.model_executor.layers.quantization.base_config import (
@@ -90,9 +90,16 @@ class Attention(nn.Module):
90 value: torch.Tensor,
91 kv_cache: Optional[torch.Tensor],
92 attn_metadata: AttentionMetadata,
93 + attn_type: AttentionType = AttentionType.DECODER,
94 ) -> torch.Tensor:
94 - return self.impl.forward(query, key, value, kv_cache, attn_metadata,
95 - self._kv_scale)
95 +
96 + return self.impl.forward(query,
97 + key,
98 + value,
99 + kv_cache,
100 + attn_metadata,
101 + self._kv_scale,
102 + attn_type=attn_type)
103
104 def extra_repr(self) -> str:
105 s = f"head_size={self.impl.head_size}" # type: ignore