A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Kernel] Correctly invoke prefill & decode kernels for cross-attention (towards eventual encoder/decoder model support) (#4888)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
afeldman-nm committed
Jul 8, 2024 at 13:12 UTC
543aa4857362ff385af8a9f496392b8831c42833
14 files changed
+2351
-95
tests/kernels/test_attention_selector.py
+7
-7
@@ -47,32 +47,32 @@ def test_flash_attn(monkeypatch):
47
# Unsupported CUDA arch
48
with patch("torch.cuda.get_device_capability", return_value=[7, 5]):
49
backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
50
- assert backend.name != "FLASH_ATTN"
50
+ assert backend.name != STR_FLASH_ATTN_VAL
51
52
# Unsupported data type
53
backend = which_attn_to_use(8, 16, 8, None, torch.float8_e4m3fn, None, 16)
54
- assert backend.name != "FLASH_ATTN"
54
+ assert backend.name != STR_FLASH_ATTN_VAL
55
56
# Unsupported kv cache data type
57
backend = which_attn_to_use(8, 16, 8, None, torch.float16, "fp8", 16)
58
- assert backend.name != "FLASH_ATTN"
58
+ assert backend.name != STR_FLASH_ATTN_VAL
59
60
# Unsupported block size
61
backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 8)
62
- assert backend.name != "FLASH_ATTN"
62
+ assert backend.name != STR_FLASH_ATTN_VAL
63
64
# Unsupported sliding window
65
backend = which_attn_to_use(8, 16, 8, 1, torch.float16, None, 16)
66
- assert backend.name != "FLASH_ATTN"
66
+ assert backend.name != STR_FLASH_ATTN_VAL
67
68
# flash-attn is not installed
69
with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70
backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
71
- assert backend.name != "FLASH_ATTN"
71
+ assert backend.name != STR_FLASH_ATTN_VAL
72
73
# Unsupported head size
74
backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
75
- assert backend.name != "FLASH_ATTN"
75
+ assert backend.name != STR_FLASH_ATTN_VAL
76
77
78
def test_invalid_env(monkeypatch):
tests/kernels/test_encoder_decoder_attn.py
new
+953
@@ -0,0 +1,953 @@
1
+"""
2
+Tests:
3
+
4
+* E2E test of Encoder attention + Decoder self-attention +
5
+ Encoder/decoder cross-attention (collectively
6
+ "encoder/decoder attention")
7
+* Confirm enc/dec models will fail for chunked prefill
8
+* Confirm enc/dec models will fail for prefix caching
9
+
10
+"""
11
+
12
+from typing import NamedTuple, Optional
13
+
14
+import pytest
15
+import torch
16
+
17
+from tests.kernels.utils import *
18
+from tests.kernels.utils import make_causal_mask, maybe_make_long_tensor
19
+from vllm.attention import Attention, AttentionMetadata
20
+from vllm.attention.backends.abstract import AttentionBackend, AttentionType
21
+from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
22
+from vllm.utils import is_hip
23
+
24
+HEAD_SIZES = [64, 256]
25
+
26
+NUM_HEADS = [1, 16]
27
+
28
+BATCH_SIZES = [1, 16]
29
+BLOCK_SIZES = [16]
30
+BACKEND_NAMES = [STR_XFORMERS_ATTN_VAL]
31
+CUDA_DEVICE = "cuda:0"
32
+
33
+MAX_DEC_SEQ_LENS = [128]
34
+MAX_ENC_SEQ_LENS = [128]
35
+
36
+# Narrow teest-cases for unsupported-scenario
37
+# tests
38
+HEAD_SIZES_FOR_UNSUPP = [HEAD_SIZES[0]]
39
+
40
+
41
+class TestPoint(NamedTuple):
42
+ """
43
+ Encapsulates the attributes which define a single invocation
44
+ of the test_e2e_enc_dec_attn() test
45
+
46
+ Attributes:
47
+ num_heads: The number of heads in the model.
48
+ head_size: Head dimension
49
+ backend_name: Name of the backend framework used.
50
+ batch_size: Number of samples per batch.
51
+ block_size: Size of each block of data processed.
52
+ max_dec_seq_len: Maximum sequence length for the decoder.
53
+ max_enc_seq_len: Maximum sequence length for the encoder.
54
+ num_blocks: Number of blocks in the model.
55
+ """
56
+
57
+ num_heads: int
58
+ head_size: int
59
+ backend_name: str
60
+ batch_size: int
61
+ block_size: int
62
+ max_dec_seq_len: int
63
+ max_enc_seq_len: int
64
+ num_blocks: int
65
+
66
+
67
+class TestResources(NamedTuple):
68
+ '''
69
+ Encapsulates key components for performing an
70
+ encoder/decoder attention test
71
+
72
+ Note that
73
+ (1) attn automatically selects an attention backend
74
+ based on platform info & a set of canned
75
+ heuristics
76
+ (2) attn_backend is thus *not the same backend
77
+ instance* used by attn, but rather it is
78
+ intended to be a
79
+ *different instance* of the *same backend class*;
80
+ it is assumed that the user of TestResources
81
+ will leverage attn_backend for the purpose of
82
+ constructing backend-compatible attention
83
+ metadata instances
84
+
85
+ Attributes:
86
+
87
+ * scale: 1/sqrt(d) scale factor for attn
88
+ * attn_backend: implementatino of abstraction
89
+ attention interface using
90
+ a particular kernel library
91
+ i.e. XFormers
92
+ * attn: Attention layer instance
93
+ * kv_cache: shared key/value cache for all attention
94
+ '''
95
+
96
+ scale: float
97
+ attn_backend: AttentionBackend
98
+ attn: Attention
99
+ kv_cache: torch.Tensor
100
+
101
+
102
+def _make_test_resources(test_pt: TestPoint, ) -> TestResources:
103
+ '''
104
+ Build key components for performing encoder/decoder attention test.
105
+
106
+ Note that
107
+ (1) The Attention instance constructed here, automatically selects
108
+ an attention backend class based on platform info & a set of canned
109
+ heuristics, so
110
+ (2) The attention backend instance constructed here is thus *not
111
+ the same backend instance* used by attn, but rather it is
112
+ intended to be a *different instance* of the *same backend class*;
113
+ therefore,
114
+ (3) This function requires that test_pt.backend_name matches the backend
115
+ class that Attention will automatically select when it is constructed.
116
+
117
+
118
+ Arguments:
119
+
120
+ * test_pt: TestPoint data structure; this function relies on the
121
+ following fields: num_heads, head_size, num_blocks,
122
+ block_size, backend_name
123
+
124
+ Returns:
125
+
126
+ * TestResources data structure.
127
+ '''
128
+
129
+ scale = float(1.0 / (test_pt.head_size**0.5))
130
+ attn_backend = make_backend(test_pt.backend_name)
131
+ attn = Attention(
132
+ test_pt.num_heads,
133
+ test_pt.head_size,
134
+ scale=scale,
135
+ )
136
+ if test_pt.num_blocks is None or test_pt.num_heads is None:
137
+ # Caller does not require a KV cache
138
+ return TestResources(scale, attn_backend, attn, None)
139
+
140
+ # Construct KV cache
141
+ kv_cache = make_kv_cache(test_pt.num_blocks,
142
+ test_pt.num_heads,
143
+ test_pt.head_size,
144
+ test_pt.block_size,
145
+ device=CUDA_DEVICE)
146
+ return TestResources(scale, attn_backend, attn, kv_cache)
147
+
148
+
149
+def _encoder_attn_setup(
150
+ test_pt: TestPoint,
151
+ test_rsrcs: TestResources,
152
+) -> PhaseTestParameters:
153
+ '''
154
+ Set up test vectors & data structures for encoder attention test.
155
+
156
+ A triplet of synthetic query/key/value tensors are constructed.
157
+ Given this is an encoder attention test, the key & value
158
+ sequences will have the same length as the corresponding queries.
159
+
160
+ The query/key/value tensors are passed to an ideal reference
161
+ self-attention implementation to generate an ideal output tensor.
162
+
163
+ Encoder inference does not populate the KV cache, therefore
164
+ no KV cache memory mapping is constructed
165
+
166
+ Arguments:
167
+
168
+ * test_pt: TestPoint data structure; this function relies on the
169
+ following fields: batch_size, num_heads, head_size,
170
+ block_size, max_q_seq_len
171
+ * test_rsrcs: TestResources data structure; this function relies on the
172
+ scale field
173
+
174
+
175
+ Returns:
176
+
177
+ * PhaseTestParameters data structure comprising (1) packed query/key/value
178
+ tensors, (2) the ideal output of attention computed using a naive
179
+ implementation, and (3) KVCache field set to None
180
+ '''
181
+
182
+ (
183
+ num_heads,
184
+ head_size,
185
+ _,
186
+ batch_size,
187
+ _,
188
+ _,
189
+ max_q_seq_len,
190
+ _,
191
+ ) = test_pt
192
+
193
+ scale = test_rsrcs.scale
194
+
195
+ max_kv_seq_len = max_q_seq_len
196
+
197
+ # Make test tensors
198
+
199
+ qkv_in, _, _ = make_qkv(batch_size,
200
+ max_q_seq_len,
201
+ max_kv_seq_len,
202
+ num_heads,
203
+ head_size,
204
+ attn_type=AttentionType.ENCODER,
205
+ device=CUDA_DEVICE)
206
+
207
+ # Compute correct answer using naive non-causal attention
208
+ # implementation
209
+
210
+ ideal_output = ref_masked_attention(qkv_in.query,
211
+ qkv_in.key,
212
+ qkv_in.value,
213
+ scale=scale,
214
+ q_seq_lens=qkv_in.q_seq_lens,
215
+ kv_seq_lens=qkv_in.kv_seq_lens)
216
+
217
+ packed_ideal_output, _ = pack_tensor(ideal_output,
218
+ qkv_in.q_seq_lens,
219
+ device=CUDA_DEVICE)
220
+
221
+ packed_qkv = pack_qkv(qkv_in, device=CUDA_DEVICE)
222
+
223
+ return PhaseTestParameters(
224
+ PackedQKVO(packed_qkv, packed_ideal_output),
225
+ None # No KV cache
226
+ )
227
+
228
+
229
+def _decoder_attn_setup(
230
+ test_pt: TestPoint,
231
+ test_rsrcs: TestResources,
232
+ block_base_addr: int = 0,
233
+) -> Tuple[QKVInputs, PhaseTestParameters, PhaseTestParameters, int]:
234
+ '''
235
+ Set up test vectors & data structures for self-attention test.
236
+
237
+ A triplet of synthetic query/key/value tensors are constructed ("baseline"
238
+ query/key/value). Given this is a self-attention test, the key & value
239
+ sequences will have the same length as the corresponding queries.
240
+
241
+ "Prefill" query/key/value tensors are derived by masking out the last value
242
+ in each baseline query/key/value. These tensors are used to test prefill &
243
+ populate KV cache for a subsequent decode test.
244
+
245
+ "Decode" query/key/value tensors are derived by extracting *only* the last
246
+ value from each baseline query/key/value (i.e. complement of the prefill
247
+ tensors.) These tensors are used to test decode, conditional on the kv cache
248
+ being populated during the prefill test.
249
+
250
+ The baseline query/key/value tensors are passed to an ideal reference
251
+ self-attention implementation to generate a "Baseline" ideal output tensor.
252
+ This tensor is split into the "Prefill" ideal output tensor (all but the
253
+ last element of each output sequence) and the "Decode" ideal output tensor
254
+ (*only* the last element of each output sequence); the "Prefill" and
255
+ "Decode" ideal output tensors can be used to validate the prefill and decode
256
+ test results, respectively.
257
+
258
+ This function also constructs the self-attention KV cache memory mapping
259
+ (slot mapping and block table), ensuring that the block table starts at
260
+ block_base_addr
261
+
262
+ Arguments:
263
+
264
+ * test_pt: TestPoint data structure; this function relies on the
265
+ following fields: batch_size, num_heads, head_size,
266
+ block_size, max_q_seq_len
267
+ * test_rsrcs: TestResources data structure; this function relies on the
268
+ scale field
269
+ * block_base_addr: decoder self-attention block-table base address
270
+
271
+ Returns:
272
+ * qkv: Unpacked (batch_size x padded_seq_len x num_heads x
273
+ head_size) query/key/value tensors
274
+ * Prefill-phase decoder self-attention PhaseTestParameters data structure,
275
+ including (1) packed (number_of_tokens x num_heads x head_size)
276
+ query/key/value tensors along with (2) ideal attention output
277
+ computed using a naive implementation, and (3) memory-mapping data
278
+ structures appropriate for prefill phase.
279
+ * Decode-phase decoder self-attention PhaseTestParameters data structure,
280
+ including (1) packed (number_of_tokens x num_heads x head_size)
281
+ query/key/value tensors along with (2) ideal attention output
282
+ computed using a naive implementation, and (3) memory-mapping data
283
+ structures appropriate for decode phase.
284
+ * max_block_idx: max physical address in decoder self-attention block-table
285
+ (intended to be used as the base address for the encoder/
286
+ decoder cross-attention block-table, which is not
287
+ constructed in this function)
288
+ '''
289
+
290
+ (
291
+ num_heads,
292
+ head_size,
293
+ _,
294
+ batch_size,
295
+ block_size,
296
+ max_q_seq_len,
297
+ _,
298
+ _,
299
+ ) = test_pt
300
+
301
+ scale = test_rsrcs.scale
302
+
303
+ max_kv_seq_len = max_q_seq_len
304
+
305
+ # Build test tensors
306
+
307
+ (
308
+ qkv,
309
+ prefill_qkv,
310
+ decode_qkv,
311
+ ) = make_qkv(batch_size,
312
+ max_q_seq_len,
313
+ max_kv_seq_len,
314
+ num_heads,
315
+ head_size,
316
+ attn_type=AttentionType.DECODER,
317
+ device=CUDA_DEVICE)
318
+
319
+ # Compute correct answer using naive attention implementation
320
+ # with causal attention mask
321
+
322
+ causal_mask = make_causal_mask(max_q_seq_len,
323
+ max_kv_seq_len).to(CUDA_DEVICE)
324
+
325
+ ideal_output = ref_masked_attention(qkv.query,
326
+ qkv.key,
327
+ qkv.value,
328
+ scale=scale,
329
+ custom_mask=causal_mask,
330
+ q_seq_lens=qkv.q_seq_lens,
331
+ kv_seq_lens=qkv.kv_seq_lens)
332
+
333
+ # Split out the prefill- & decode-phase ideal answers & pack them
334
+
335
+ prefill_ideal_output = torch.zeros_like(ideal_output)
336
+ decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
337
+ for bdx, prefill_q_seq_len in enumerate(prefill_qkv.q_seq_lens):
338
+ prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
339
+ bdx, :prefill_q_seq_len]
340
+ decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
341
+ prefill_q_seq_len + 1)]
342
+
343
+ prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
344
+ prefill_qkv.q_seq_lens,
345
+ device=CUDA_DEVICE)
346
+ decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
347
+ [1 for _ in range(batch_size)],
348
+ device=CUDA_DEVICE)
349
+
350
+ # Build prefill- & decode-phase data structures
351
+ # for decoder self-attention. Block tables and
352
+ # slot mapping must be in a format compatible
353
+ # with KV caching & attention kernels
354
+ #
355
+ # Prefill-phase:
356
+ #
357
+ # * Empty block-tables tensor
358
+ # * Slot-mapping with entries for prompt tokens
359
+ #
360
+ # Decode-phase:
361
+ # * Block-tables tensor with minimum number of blocks
362
+ # required by total num. tokens in the entirety of all sequences
363
+ # (including both prefill & decode)
364
+ # * Slot-mapping with entries for tokens that will be decoded in the
365
+ # current decode iteration
366
+ #
367
+ # Note: the format described above is simply mirroring what ModelRunner
368
+ # produces
369
+
370
+ prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
371
+
372
+ (
373
+ decode_block_tables,
374
+ slot_mapping_list,
375
+ max_block_idx,
376
+ ) = make_block_tables_slot_mapping(block_size,
377
+ qkv.q_seq_lens,
378
+ device=CUDA_DEVICE,
379
+ block_base_addr=block_base_addr)
380
+
381
+ (
382
+ prefill_slot_mapping,
383
+ decode_slot_mapping,
384
+ ) = split_slot_mapping(slot_mapping_list,
385
+ qkv.q_seq_lens,
386
+ device=CUDA_DEVICE)
387
+
388
+ prefill_pckd_qkv = pack_qkv(prefill_qkv, device=CUDA_DEVICE)
389
+
390
+ decode_pckd_qkv = pack_qkv(decode_qkv, device=CUDA_DEVICE)
391
+
392
+ return (
393
+ qkv,
394
+ PhaseTestParameters( # Prefill test params
395
+ PackedQKVO(prefill_pckd_qkv, prefill_packed_ideal_output),
396
+ KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
397
+ PhaseTestParameters( # Decode test params
398
+ PackedQKVO(decode_pckd_qkv, decode_packed_ideal_output),
399
+ KVMemoryMap(decode_block_tables, decode_slot_mapping)),
400
+ max_block_idx)
401
+
402
+
403
+def _enc_dec_cross_attn_setup_reuses_query(
404
+ decoder_qkv: QKVInputs,
405
+ encoder_test_params: PhaseTestParameters,
406
+ prefill_decoder_phase_test_params: PhaseTestParameters,
407
+ test_pt: TestPoint,
408
+ test_rsrcs: TestResources,
409
+ block_base_addr: int = 0,
410
+) -> Tuple[PhaseTestParameters, PhaseTestParameters]:
411
+ '''
412
+ Set up test vectors & data structures for cross-attention test.
413
+
414
+ A triplet of synthetic cross-attention key/value tensors are constructed
415
+ ("baseline" key/value). Given this is a cross-attention test, we assume
416
+ query tensors were already synthesized for a prior self-attention test and
417
+ will be reused for cross-attention. The key & value sequences generated here
418
+ may have a different length than the corresponding queries (as is often
419
+ the case for cross-attention between decoder and encoder sequences.)
420
+
421
+ Cross attention key & value tensors do not grow during autoregressive
422
+ inference; thus this function obtains a single key/value pair suitable for
423
+ both prefill and decode.
424
+
425
+ The "baseline" query tensor is received as an argument. The "baseline"
426
+ query/key/value tensors are passed to an ideal reference cross-attention
427
+ implementation to generate a "baseline" ideal output tensor. This tensor is
428
+ split into the "Prefill" ideal output tensor (all but the last element of
429
+ each output sequence) and the "Decode" ideal output tensor (*only* the last
430
+ element of each output sequence); the "Prefill" and "Decode" ideal output
431
+ tensors can be used to validate the prefill and decode test results,
432
+ respectively.
433
+
434
+ This function also constructs the cross-attention KV cache memory mapping
435
+ (slot mapping and block table), ensuring that the block table starts at
436
+ block_base_addr.
437
+
438
+ Arguments:
439
+
440
+ * decoder_qkv: pre-existing unpacked (batch_size x padded_seq_len x
441
+ num_heads x head_size) decoder self-attention inputs;
442
+ this function relies on the query and q_seq_lens
443
+ fields
444
+ * encoder_test_params: PhaseTestParameters data structure which was
445
+ used for encoder inference; KV cache field
446
+ is not used by this function
447
+ * prefill_decoder_phase_test_params: PhaseTestParameters data structure
448
+ used for prefill-phase decoder
449
+ self-attention; all fields
450
+ including KV cache required
451
+ * test_pt: TestPoint data structure; this function relies on the
452
+ following fields: batch_size, num_heads, head_size,
453
+ block_size, max_q_seq_len
454
+ * test_rsrcs: TestResources data structure; this function relies on the
455
+ scale field
456
+ * block_base_addr: decoder self-attention block-table base address
457
+
458
+ Returns:
459
+
460
+ * Prefill-phase encoder/decoder cross-attention PhaseTestParameters data
461
+ structure, including (1) packed
462
+ (number_of_tokens x num_heads x head_size) query/key/value tensors
463
+ along with (2) ideal attention output computed using a
464
+ naive implementation, and (3) memory-mapping data structures appropriate
465
+ for prefill phase.
466
+ * Decode-phase encoder/decoder cross-attention PhaseTestParameters data
467
+ structure, including (1) packed
468
+ (number_of_tokens x num_heads x head_size) query/key/value tensors
469
+ along with (2) ideal attention output computed using a
470
+ naive implementation, and (3) memory-mapping data structures appropriate
471
+ for decode phase.
472
+ '''
473
+
474
+ assert encoder_test_params.packed_qkvo.packed_qkv is not None
475
+ assert prefill_decoder_phase_test_params.packed_qkvo.packed_qkv is not None
476
+
477
+ (
478
+ num_heads,
479
+ head_size,
480
+ _,
481
+ batch_size,
482
+ block_size,
483
+ max_decoder_seq_len,
484
+ max_encoder_seq_len,
485
+ _,
486
+ ) = test_pt
487
+
488
+ scale = test_rsrcs.scale
489
+
490
+ decoder_query = decoder_qkv.query
491
+ decoder_seq_lens = decoder_qkv.q_seq_lens
492
+ encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens
493
+ prefill_q_seq_lens = (
494
+ prefill_decoder_phase_test_params.packed_qkvo.packed_qkv.q_seq_lens)
495
+
496
+ assert prefill_q_seq_lens is not None
497
+
498
+ (
499
+ cross_kv,
500
+ _,
501
+ _,
502
+ ) = make_qkv(batch_size,
503
+ max_decoder_seq_len,
504
+ max_encoder_seq_len,
505
+ num_heads,
506
+ head_size,
507
+ force_kv_seq_lens=encoder_seq_lens,
508
+ attn_type=AttentionType.ENCODER_DECODER,
509
+ device=CUDA_DEVICE)
510
+
511
+ ideal_output = ref_masked_attention(decoder_query,
512
+ cross_kv.key,
513
+ cross_kv.value,
514
+ scale=scale,
515
+ q_seq_lens=decoder_seq_lens,
516
+ kv_seq_lens=cross_kv.kv_seq_lens)
517
+
518
+ prefill_ideal_output = torch.zeros_like(ideal_output)
519
+ decode_ideal_output = torch.zeros_like(ideal_output[:, 0:1])
520
+ for bdx, prefill_q_seq_len in enumerate(prefill_q_seq_lens):
521
+ prefill_ideal_output[bdx, :prefill_q_seq_len] = ideal_output[
522
+ bdx, :prefill_q_seq_len]
523
+ decode_ideal_output[bdx, :] = ideal_output[bdx, prefill_q_seq_len:(
524
+ prefill_q_seq_len + 1)]
525
+
526
+ prefill_packed_ideal_output, _ = pack_tensor(prefill_ideal_output,
527
+ prefill_q_seq_lens,
528
+ device=CUDA_DEVICE)
529
+ decode_packed_ideal_output, _ = pack_tensor(decode_ideal_output,
530
+ [1 for _ in range(batch_size)],
531
+ device=CUDA_DEVICE)
532
+
533
+ # Build prefill- & decode-phase data structures
534
+ # for encoder/decoder cross-attention. Block tables and
535
+ # slot mapping must be in a format compatible
536
+ # with KV caching & attention kernels
537
+ #
538
+ # Whereas decoder self-attention extracts relationships between
539
+ # equal-length Q/K/V sequences, which mutually grow in length
540
+ # with each decoded token, cross-attention relates the Q sequence
541
+ # - which grows with each new decoded token - to fixed-length
542
+ # K and V sequences derived from the encoder hidden states.
543
+ #
544
+ # Prefill-phase:
545
+ #
546
+ # * Empty block-tables tensor
547
+ # * Slot-mapping with as many entries as there are tokens in the encoder
548
+ # prompt.
549
+ #
550
+ # Decode-phase:
551
+ # * Block-tables tensor with minimum number of blocks to
552
+ # accommodate K & V tensors which are equal in lnegth
553
+ # to the encoder prompt length
554
+ # * Empty slot-mapping tensor (since K & V are fixed in size,
555
+ # new decoded tokens are not KV-cached and require no slot-
556
+ # mapping)
557
+ #
558
+ # Note: the format above is simply an extension of what ModelRunner
559
+ # produces for decoder-only models
560
+
561
+ prefill_block_tables = make_empty_block_tables_tensor(device=CUDA_DEVICE)
562
+ decode_slot_mapping = make_empty_slot_mapping_tensor(device=CUDA_DEVICE)
563
+
564
+ (
565
+ decode_block_tables,
566
+ prefill_slot_mapping_list,
567
+ _,
568
+ ) = make_block_tables_slot_mapping(block_size,
569
+ cross_kv.kv_seq_lens,
570
+ block_base_addr=block_base_addr,
571
+ device=CUDA_DEVICE)
572
+
573
+ prefill_slot_mapping = maybe_make_long_tensor(prefill_slot_mapping_list,
574
+ device=CUDA_DEVICE)
575
+
576
+ # Packed key/value (query is already provided)
577
+ packed_cross_kv = pack_qkv(cross_kv, device=CUDA_DEVICE)
578
+
579
+ return (
580
+ PhaseTestParameters( # Prefill-phase test params
581
+ PackedQKVO(packed_cross_kv, prefill_packed_ideal_output),
582
+ KVMemoryMap(prefill_block_tables, prefill_slot_mapping)),
583
+ PhaseTestParameters( # Decode-phase test params
584
+ PackedQKVO(None, decode_packed_ideal_output),
585
+ KVMemoryMap(decode_block_tables, decode_slot_mapping)))
586
+
587
+
588
+def _run_encoder_attention_test(
589
+ attn: Attention,
590
+ encoder_test_params: PhaseTestParameters,
591
+ attn_metadata: AttentionMetadata,
592
+) -> torch.Tensor:
593
+ '''
594
+ Run encoder attention.
595
+
596
+ attn.forward() is passed attn_type=AttentionType.ENCODER in order
597
+ to configure the kernel invocation for encoder attention
598
+
599
+ Requires attn_metadata.num_decode_tokens == 0
600
+ (There is no encoder execution in the decode-phase)
601
+
602
+ Arguments:
603
+
604
+ * attn: Attention wrapper instance
605
+ * encoder_test_params: encoder PhaseTestParameters data structure;
606
+ this function relies on the packed
607
+ (number_of_tokens x num_heads x head_size)
608
+ query/key/value fields
609
+ * attn_metadata: attention metadata for encoder/decoder-self attention
610
+
611
+ Returns:
612
+ * Attention.forward() applied to packed {query,key,value} and
613
+ & attn_metadata
614
+ '''
615
+ assert attn_metadata.num_decode_tokens == 0
616
+ attn_type = AttentionType.ENCODER
617
+ packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
618
+ assert packed_qkv is not None
619
+ return attn.forward(packed_qkv.query,
620
+ packed_qkv.key,
621
+ packed_qkv.value,
622
+ None,
623
+ attn_metadata,
624
+ attn_type=attn_type)
625
+
626
+
627
+def _run_decoder_self_attention_test(
628
+ test_rsrcs: TestResources,
629
+ decoder_test_params: PhaseTestParameters,
630
+ attn_metadata: AttentionMetadata,
631
+) -> torch.Tensor:
632
+ '''
633
+ Run decoder self-attention test.
634
+
635
+ attn.forward() is passed attn_type=AttentionType.DECODER
636
+ in order to configure the kernel invocation for decoder self-attention.
637
+
638
+ Arguments:
639
+
640
+ * test_rsrcs: TestResources instance; this function relies on the kv_cache
641
+ and attn (Attention wrapper instance) fields
642
+ * decoder_test_params: decoder PhaseTestParameters data structure;
643
+ this function relies on the packed
644
+ (number_of_tokens x num_heads x head_size)
645
+ query/key/value fields
646
+ * attn_metadata: attention metadata for decoder-self attention
647
+ (contains KV cache memory-mapping)
648
+
649
+ Returns:
650
+ * Attention.forward() applied to packed_{query,key,value}, kv_cache
651
+ & attn_metadata
652
+ '''
653
+ attn_type = AttentionType.DECODER
654
+ attn = test_rsrcs.attn
655
+ kv_cache = test_rsrcs.kv_cache
656
+ packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
657
+ assert packed_qkv is not None
658
+ return attn.forward(packed_qkv.query,
659
+ packed_qkv.key,
660
+ packed_qkv.value,
661
+ kv_cache,
662
+ attn_metadata,
663
+ attn_type=attn_type)
664
+
665
+
666
+def _run_encoder_decoder_cross_attention_test(
667
+ test_rsrcs: TestResources,
668
+ decoder_test_params: PhaseTestParameters,
669
+ cross_test_params: Optional[PhaseTestParameters],
670
+ attn_metadata: AttentionMetadata,
671
+) -> torch.Tensor:
672
+ '''
673
+ Run encoder/decoder cross-attention test.
674
+
675
+ Via PhaseTestParameters data structures, consumes the same query utilized
676
+ for decoder self-attention, plus a key/value specific to cross-attention.
677
+
678
+ if cross_test_params is None or cross_test_params.packed_qkvo.packed_qkv
679
+ is None, this reflects that in decode-phase cross attention there
680
+ is no growth in the key and value tensors.
681
+
682
+ attn.forward() is passed attn_type=AttentionType.ENCODER_DECODER
683
+ in order to configure the kernel invocation for encoder/decoder cross-
684
+ attention.
685
+
686
+ Arguments:
687
+
688
+ * test_rsrcs: TestResources instance; this function relies on the kv_cache
689
+ and attn (Attention wrapper instance) fields
690
+ * decoder_test_params: decoder PhaseTestParameters data structure;
691
+ this function relies on the packed
692
+ (number_of_tokens x num_heads x head_size)
693
+ query field
694
+ * cross_test_params: encoder/decoder PhaseTestParameters data structure;
695
+ this function relies on the packed
696
+ (number_of_tokens x num_heads x head_size)
697
+ key/value fields
698
+ * attn_metadata: attention metadata for encoder/decoder-self attention
699
+
700
+ Returns:
701
+ * Attention.forward() applied to packed_{query,key,value}, kv_cache
702
+ & attn_metadata
703
+ '''
704
+ assert decoder_test_params.packed_qkvo.packed_qkv is not None
705
+
706
+ attn_type = AttentionType.ENCODER_DECODER
707
+ attn = test_rsrcs.attn
708
+ kv_cache = test_rsrcs.kv_cache
709
+ if cross_test_params is None:
710
+ key = None
711
+ value = None
712
+ else:
713
+ cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
714
+ key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
715
+ value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
716
+ return attn.forward(decoder_test_params.packed_qkvo.packed_qkv.query,
717
+ key,
718
+ value,
719
+ kv_cache,
720
+ attn_metadata,
721
+ attn_type=attn_type)
722
+
723
+
724
+@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
725
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
726
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
727
+@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
728
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
729
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
730
+@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
731
+@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
732
+def test_encoder_only(num_heads: int, head_size: int, backend_name: str,
733
+ batch_size: int, block_size: int, max_dec_seq_len: int,
734
+ max_enc_seq_len: int, monkeypatch):
735
+
736
+ # Force Attention wrapper backend
737
+ override_backend_env_variable(monkeypatch, backend_name)
738
+
739
+ # Note: KV cache size of 4096 is arbitrary & chosen intentionally
740
+ # to be more than necessary, since exceeding the kv cache size
741
+ # is not part of this test
742
+ test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
743
+ block_size, max_dec_seq_len, max_enc_seq_len, 4096)
744
+
745
+ # Attention scale factor, attention backend instance, attention wrapper
746
+ # instance, KV cache init
747
+ test_rsrcs = _make_test_resources(test_pt)
748
+
749
+ # Construct encoder attention test params (only used
750
+ # during prefill)
751
+
752
+ enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
753
+
754
+ # Shared prefill metadata structure
755
+
756
+ prephase_attn_metadata: AttentionMetadata = make_test_metadata(
757
+ test_rsrcs.attn_backend,
758
+ True,
759
+ None,
760
+ decoder_test_params=None,
761
+ encoder_test_params=enc_test_params,
762
+ cross_test_params=None,
763
+ device=CUDA_DEVICE)
764
+
765
+ # PREFILL: encoder attention
766
+
767
+ enc_pckd_act_out: torch.Tensor = (_run_encoder_attention_test(
768
+ test_rsrcs.attn, enc_test_params, prephase_attn_metadata))
769
+
770
+ # - Is encoder attention result correct?
771
+ assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
772
+
773
+
774
+@pytest.mark.skipif(is_hip(), reason=STR_NOT_IMPL_ENC_DEC_ROCM_HIP)
775
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
776
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
777
+@pytest.mark.parametrize("backend_name", BACKEND_NAMES)
778
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
779
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
780
+@pytest.mark.parametrize("max_dec_seq_len", MAX_DEC_SEQ_LENS)
781
+@pytest.mark.parametrize("max_enc_seq_len", MAX_ENC_SEQ_LENS)
782
+def test_e2e_enc_dec_attn(
783
+ num_heads: int,
784
+ head_size: int,
785
+ backend_name: str,
786
+ batch_size: int,
787
+ block_size: int,
788
+ max_dec_seq_len: int,
789
+ max_enc_seq_len: int,
790
+ monkeypatch,
791
+) -> None:
792
+ '''
793
+ End-to-end encoder/decoder test:
794
+
795
+ * Construct fake test vectors for (1) encoder attention,
796
+ (2) decoder self-attention, and (3) encoder/decoder cross-attention
797
+ * Construct (1) attention metadata structure with self- and cross-attention
798
+ attributes for prefill-phase, and (2) an analogous attention metadata
799
+ structure but for decode-phase
800
+ * Test attention steps in the following order
801
+
802
+ * Encoder attention
803
+ * Prefill self-attention
804
+ * Prefill cross-attention
805
+ * Decode self-attention
806
+ * Decode cross-attention
807
+ * Besides being reflective of realistic use-cases, this order would
808
+ exacerbate any accidental overlap in the self-/cross-attention
809
+ block tables, which one hopes to avoid
810
+
811
+
812
+ * Validate output correctness against ideal reference attention
813
+ implementation
814
+
815
+ Block tables are constructed such that cross-attention KV cache is in a
816
+ higher, non-intersecting address-space than self-attention KV cache.
817
+
818
+ Self- and cross-attention share the same query tensor but not the K/V
819
+ tensors. Self-attention K/Vs must have the same seq len as Q while
820
+ cross-attention K/Vs are allowed to differ in seq len, as is often the case
821
+ for cross-attention.
822
+
823
+ This test utilizes PyTest monkey patching to force the attention backend
824
+ via an environment variable.
825
+
826
+ Note on ROCm/HIP: currently encoder/decoder models are not supported on
827
+ AMD GPUs, therefore this test simply is skipped if is_hip().
828
+
829
+ Note on metadata: there is a single attention metadata structure shared by
830
+ all prefill-phase attention operations (encoder, decoder, enc/dec cross),
831
+ and a single one shared by all decode-phase attention operations
832
+ (decoder & enc/dec cross.) This is intended to reflect the behavior
833
+ of ModelRunner, which constructs a single attention metadata structure for
834
+ each prefill or decode run. A realistic scenario would rely on the
835
+ attention backend to utilize the appropriate attention metadata fields
836
+ according to the value of attn_metadata.attention_type. Thus, this test is
837
+ organized so as to confirm that the backend-under-test can handle a
838
+ shared prefill attention metadata structure & a shared decode attention
839
+ metadata structure.
840
+ '''
841
+
842
+ # Force Attention wrapper backend
843
+ override_backend_env_variable(monkeypatch, backend_name)
844
+
845
+ # Note: KV cache size of 4096 is arbitrary & chosen intentionally
846
+ # to be more than necessary, since exceeding the kv cache size
847
+ # is not part of this test
848
+ test_pt = TestPoint(num_heads, head_size, backend_name, batch_size,
849
+ block_size, max_dec_seq_len, max_enc_seq_len, 4096)
850
+
851
+ # Attention scale factor, attention backend instance, attention wrapper
852
+ # instance, KV cache init
853
+ test_rsrcs = _make_test_resources(test_pt)
854
+
855
+ # Construct encoder attention test params (only used
856
+ # during prefill)
857
+
858
+ enc_test_params = _encoder_attn_setup(test_pt, test_rsrcs)
859
+
860
+ # Construct Decoder self-attention prefill-phase & decode-phase
861
+ # test params, including query/key/value tensors, decoder self-attention
862
+ # memory-mapping. cross_block_base_addr is the uppermost address in the
863
+ # decoder self-attention block-table, i.e. a base address which the
864
+ # encoder/decoder cross-attention block-table may build downward toward.
865
+
866
+ (
867
+ dec_qkv,
868
+ prephase_dec_test_params,
869
+ decphase_dec_test_params,
870
+ cross_block_base_addr,
871
+ ) = _decoder_attn_setup(test_pt, test_rsrcs)
872
+
873
+ # Construct encoder/decoder cross-attention prefill-phase & decode-phase
874
+ # test params, including key/value tensors, cross-attention memory-mapping
875
+
876
+ (
877
+ prephase_cross_test_params,
878
+ decphase_cross_test_params,
879
+ ) = _enc_dec_cross_attn_setup_reuses_query(
880
+ dec_qkv,
881
+ enc_test_params,
882
+ prephase_dec_test_params,
883
+ test_pt,
884
+ test_rsrcs,
885
+ block_base_addr=cross_block_base_addr)
886
+
887
+ # Shared prefill metadata structure
888
+ assert prephase_dec_test_params.packed_qkvo.packed_qkv is not None
889
+ prephase_attn_metadata: AttentionMetadata = make_test_metadata(
890
+ test_rsrcs.attn_backend,
891
+ True,
892
+ prephase_dec_test_params.packed_qkvo.packed_qkv.q_seq_lens,
893
+ decoder_test_params=prephase_dec_test_params,
894
+ encoder_test_params=enc_test_params,
895
+ cross_test_params=prephase_cross_test_params,
896
+ device=CUDA_DEVICE)
897
+
898
+ # PREFILL: encoder attention
899
+
900
+ enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
901
+ enc_test_params,
902
+ prephase_attn_metadata)
903
+
904
+ # - Is encoder attention result correct?
905
+ assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out)
906
+
907
+ # PREFILL: decoder self-attention test
908
+
909
+ prephase_dec_pckd_act_out = _run_decoder_self_attention_test(
910
+ test_rsrcs, prephase_dec_test_params, prephase_attn_metadata)
911
+
912
+ # - Is prefill decoder self-attention correct?
913
+ assert_actual_matches_ideal(prephase_dec_test_params,
914
+ prephase_dec_pckd_act_out)
915
+
916
+ # PREFILL: encoder/decoder cross-attention test
917
+
918
+ prephase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
919
+ test_rsrcs, prephase_dec_test_params, prephase_cross_test_params,
920
+ prephase_attn_metadata)
921
+
922
+ # - Is prefill encoder/decoder cross-attention correct?
923
+ assert_actual_matches_ideal(prephase_cross_test_params,
924
+ prephase_cross_pckd_act_out)
925
+
926
+ # DECODE: build decode-phase attention metadata
927
+
928
+ decphase_attn_metadata: AttentionMetadata = make_test_metadata(
929
+ test_rsrcs.attn_backend,
930
+ False,
931
+ dec_qkv.q_seq_lens,
932
+ decoder_test_params=decphase_dec_test_params,
933
+ encoder_test_params=enc_test_params,
934
+ cross_test_params=decphase_cross_test_params,
935
+ device=CUDA_DEVICE)
936
+
937
+ # DECODE: decoder self-attention test
938
+
939
+ decphase_dec_pckd_act_out = _run_decoder_self_attention_test(
940
+ test_rsrcs, decphase_dec_test_params, decphase_attn_metadata)
941
+
942
+ # - Is decode-phase decoder self-attention correct?
943
+ assert_actual_matches_ideal(decphase_dec_test_params,
944
+ decphase_dec_pckd_act_out)
945
+
946
+ # DECODE: encoder/decoder cross-attention test
947
+
948
+ decphase_cross_pckd_act_out = _run_encoder_decoder_cross_attention_test(
949
+ test_rsrcs, decphase_dec_test_params, None, decphase_attn_metadata)
950
+
951
+ # - Is decode-phase encoder/decoder cross-attention correct?
952
+ assert_actual_matches_ideal(decphase_cross_test_params,
953
+ decphase_cross_pckd_act_out)
tests/kernels/utils.py
+920
@@ -1,12 +1,211 @@
1
"""Kernel test utils"""
2
3
+import itertools
4
+import random
5
+from numbers import Number
6
+from typing import Any, List, NamedTuple, Optional, Tuple, Union
7
+
8
import pytest
9
+import torch
10
+
11
+from vllm.attention.backends.abstract import (AttentionBackend,
12
+ AttentionMetadata, AttentionType)
13
+from vllm.attention.backends.xformers import XFormersBackend
14
+from vllm.utils import make_tensor_with_pad
15
16
+# String name of register which may be set in order to
17
+# force auto-selection of attention backend by Attention
18
+# wrapper
19
STR_BACKEND_ENV_VAR: str = "VLLM_ATTENTION_BACKEND"
20
+
21
+# Possible string values of STR_BACKEND_ENV_VAR
22
+# register, corresponding to possible backends
23
+STR_FLASHINFER_ATTN_VAL: str = "FLASHINFER"
24
+STR_TORCH_SDPA_ATTN_VAL: str = "TORCH_SDPA"
25
+STR_ROCM_FLASH_ATTN_VAL: str = "ROCM_FLASH"
26
+STR_XFORMERS_ATTN_VAL: str = "XFORMERS"
27
STR_FLASH_ATTN_VAL: str = "FLASH_ATTN"
28
STR_INVALID_VAL: str = "INVALID"
29
30
31
+class QKVInputs(NamedTuple):
32
+ '''
33
+ Data structure for representing unpacked attention inputs,
34
+ query/key/values and their sequence lengths.
35
+
36
+ Attributes:
37
+
38
+ * {query,key,value}: unpacked (batch_size x padded_seq_len x
39
+ num_heads x head_size) attention inputs
40
+ * q_seq_lens: query sequence lengths list
41
+ * kv_seq_lens: shared key/value sequence lengths list
42
+ '''
43
+
44
+ query: torch.Tensor
45
+ key: torch.Tensor
46
+ value: torch.Tensor
47
+ q_seq_lens: List[int]
48
+ kv_seq_lens: List[int]
49
+
50
+
51
+class QKVO(NamedTuple):
52
+ '''
53
+ Data structure for representing unpacked attention inputs,
54
+ alongside unpacked known-correct attention output
55
+
56
+ Attributes:
57
+
58
+ * qkv: unpacked (batch_size x padded_seq_len x
59
+ num_heads x head_size) attention inputs
60
+ * ideal_output: unpacked (batch_size x padded_seq_len x
61
+ num_heads x head_size) known-correct attention output
62
+ '''
63
+
64
+ qkv: QKVInputs
65
+ ideal_output: torch.Tensor
66
+
67
+
68
+class PackedQKVInputs(NamedTuple):
69
+ '''
70
+ Data structure for representing packed attention inputs
71
+
72
+ Attributes:
73
+
74
+ * {query,key,value}: packed (number_of_tokens x num_heads
75
+ x head_size) attention inputs
76
+ * q_start_loc_list: list of query start locations within packed tensor
77
+ * kv_start_loc_list: shared list of key/value start locations within
78
+ packed tensor
79
+ * q_seq_lens: query sequence lengths list
80
+ * kv_seq_lens: shared key/value sequence lengths list
81
+ '''
82
+
83
+ query: torch.Tensor
84
+ key: torch.Tensor
85
+ value: torch.Tensor
86
+ q_start_loc_list: Optional[List[int]]
87
+ kv_start_loc_list: Optional[List[int]]
88
+ q_seq_lens: Optional[List[int]]
89
+ kv_seq_lens: Optional[List[int]]
90
+
91
+
92
+class PackedQKVO(NamedTuple):
93
+ '''
94
+ Data structure for representing packed attention inputs,
95
+ alongside packed known-correct attention output
96
+
97
+ Attributes:
98
+
99
+ * packed_qkv: packed (number_of_tokens x num_heads
100
+ x head_size) attention inputs
101
+ * ideal_output: packed (number_of_tokens x num_heads
102
+ x head_size) known-correct attention output
103
+ '''
104
+
105
+ packed_qkv: Optional[PackedQKVInputs]
106
+ ideal_output: torch.Tensor
107
+
108
+
109
+class KVMemoryMap(NamedTuple):
110
+ '''
111
+ Data structure for encapsulating KV cache memory mapping.
112
+
113
+ Attributes:
114
+
115
+ * block_tables: KV cache block tables
116
+ * slot_mapping: mapping of sequence offset to physical address
117
+ '''
118
+
119
+ block_tables: torch.Tensor
120
+ slot_mapping: torch.Tensor
121
+
122
+
123
+class PhaseTestParameters(NamedTuple):
124
+ '''
125
+ Data structure for encapsulating the test parameters
126
+ for a given test "phase" (prefill or decode phase) and attention
127
+ scenario (encoder, decoder-self, encoder/decoder-cross)
128
+
129
+ Attributes:
130
+
131
+ * packed_qkvo: packed (number_of_tokens x num_heads
132
+ x head_size) attention inputs & known-correct
133
+ output
134
+ * kv_mmap: KV cache memory mapping, specific to this test phase &
135
+ attention scenario
136
+ '''
137
+
138
+ packed_qkvo: PackedQKVO
139
+ kv_mmap: Optional[KVMemoryMap]
140
+
141
+
142
+def maybe_make_int_tensor(
143
+ _list: Optional[List[int]],
144
+ device: Union[torch.device, str],
145
+) -> torch.Tensor:
146
+ '''
147
+ Convert Python int list to a 1D int torch.Tensor on `device`
148
+
149
+ Returns:
150
+
151
+ * If _list is not None: 1D int torch.Tensor on `device`
152
+ * None otherwise
153
+ '''
154
+ return None if _list is None else torch.tensor(
155
+ _list, dtype=torch.int, device=device)
156
+
157
+
158
+def maybe_make_long_tensor(
159
+ _list: Optional[List[int]],
160
+ device: Union[torch.device, str],
161
+) -> torch.Tensor:
162
+ '''
163
+ Convert Python int list to a 1D long torch.Tensor on `device`
164
+
165
+ Returns:
166
+
167
+ * If _list is not None: 1D long torch.Tensor on `device`
168
+ * None otherwise
169
+ '''
170
+ return None if _list is None else torch.tensor(
171
+ _list, dtype=torch.long, device=device)
172
+
173
+
174
+def maybe_max(_list: Optional[List]) -> Optional[Number]:
175
+ '''
176
+ Returns:
177
+
178
+ * If _list is not None: max(_list)
179
+ * None otherwise
180
+ '''
181
+ return None if _list is None else max(_list)
182
+
183
+
184
+def make_causal_mask(
185
+ q_max_seq_len: int,
186
+ kv_max_seq_len: int,
187
+) -> torch.Tensor:
188
+ '''
189
+ Create a q_max_seq_len x kv_max_seq_len causal mask
190
+
191
+ Arguments:
192
+
193
+ * q_max_seq_len: query max seq len
194
+ * kv_max_seq_len: key/value max seq len
195
+
196
+ Returns:
197
+
198
+ * 2D tensor, q_max_seq_len x kv_max_seq_len
199
+ '''
200
+
201
+ # Create a matrix where entry (i, j) is True if i >= j
202
+ mask = torch.triu(torch.ones(q_max_seq_len, kv_max_seq_len), diagonal=1)
203
+ # Replace True with float('-inf') and False with 0
204
+ mask = mask.masked_fill(mask == 1,
205
+ float('-inf')).masked_fill(mask == 0, 0.0)
206
+ return mask
207
+
208
+
209
def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
210
backend_name: str) -> None:
211
'''
@@ -20,3 +219,724 @@ def override_backend_env_variable(mpatch: pytest.MonkeyPatch,
219
* backend_name: attention backend name to force
220
'''
221
mpatch.setenv(STR_BACKEND_ENV_VAR, backend_name)
222
+
223
+
224
+def ref_masked_attention(query: torch.Tensor,
225
+ key: torch.Tensor,
226
+ value: torch.Tensor,
227
+ scale: float,
228
+ custom_mask: Optional[torch.Tensor] = None,
229
+ q_seq_lens: Optional[List] = None,
230
+ kv_seq_lens: Optional[List] = None) -> torch.Tensor:
231
+ '''
232
+ "Golden" masked attention reference. Supports two types of masking:
233
+
234
+ * Basic attention mask, utilizing {q,kv}_seq_lens args to mask out
235
+ padding elements
236
+ * Custom attention mask, which can force an arbitrary mask tensor, i.e.
237
+ causal
238
+
239
+ Arguments:
240
+
241
+ * query: batch_size x q_padded_seq_len x num_heads x head_size
242
+ * key: batch_size x kv_padded_seq_len x num_heads x head_size
243
+ * value: batch_size x kv_padded_seq_len x num_heads x head_size
244
+ * scale: Attention scale factor
245
+ * custom_mask: custom attention mask; good place to inject a causal
246
+ attention mask
247
+ * q_seq_lens: list of unpadded query seq_lens for each batch index
248
+ * kv_seq_lens: list of unpadded key/value seq_lens for each batch index
249
+
250
+ Returns:
251
+
252
+ * Attention result, batch_size x q_padded_seq_len x num_heads x head_size
253
+ '''
254
+
255
+ assert q_seq_lens is not None
256
+ assert kv_seq_lens is not None
257
+
258
+ batch_size = query.shape[0]
259
+ assert (len(q_seq_lens) == batch_size)
260
+ assert (len(kv_seq_lens) == batch_size)
261
+
262
+ attn_weights = scale * torch.einsum("bqhd,bkhd->bhqk", query, key).float()
263
+
264
+ # Basic attention mask, derived from seq lens
265
+ if (q_seq_lens is not None) or (kv_seq_lens is not None):
266
+ attn_mask = torch.zeros_like(attn_weights)
267
+ if q_seq_lens is not None:
268
+ for bdx, plen in enumerate(q_seq_lens):
269
+ attn_mask[bdx, :, plen:, :] = -torch.inf
270
+ if kv_seq_lens is not None:
271
+ for bdx, plen in enumerate(kv_seq_lens):
272
+ attn_mask[bdx, :, :, plen:] = -torch.inf
273
+
274
+ attn_weights = attn_weights + attn_mask.float()
275
+
276
+ # Custom attention mask
277
+ if custom_mask is not None:
278
+ attn_weights = attn_weights + custom_mask.float()
279
+
280
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
281
+ out = torch.einsum("bhqk,bkhd->bqhd", attn_weights, value)
282
+ return out
283
+
284
+
285
+def make_qkv(
286
+ batch_size: int,
287
+ max_q_seq_len: int,
288
+ max_kv_seq_len: Optional[int],
289
+ num_heads: int,
290
+ head_size: int,
291
+ device: Union[torch.device, str],
292
+ force_kv_seq_lens: Optional[List[int]] = None,
293
+ attn_type: AttentionType = AttentionType.ENCODER_DECODER,
294
+ force_max_len: bool = False,
295
+) -> Tuple[QKVInputs, QKVInputs, QKVInputs]:
296
+ '''
297
+ Construct QKV test tensors for self- and cross-attention.
298
+
299
+ Generates three query/key/value triplets:
300
+
301
+ * "Baseline" query/key/value (for input to reference attention function)
302
+ * "Prefill" query/key/value (last sequence offset zero'd out, for use as
303
+ input to prefill kernel)
304
+ * "Decode" query/key/value (only the last sequence offset from baseline,
305
+ for use as input to decode kernel)
306
+
307
+ Each Q/K/V triplet is associated with a list of q seqlens and a list of k/v
308
+ seqlens
309
+
310
+ Arguments:
311
+
312
+ * batch_size
313
+ * max_q_seq_len: max query seq len
314
+ * max_kv_seq_len: max key/value seq len
315
+ * num_heads
316
+ * head_size
317
+ * is_encoder_decoder_attn: if True, query seqlen may differ from
318
+ key/value seqlen (as is often the case for cross-attention);
319
+ o/w, query/key/value seqlens match at each batch index
320
+ (max_kv_seq_len is unused)
321
+ * force_kv_seq_lens: if not None, overrides kv sequence lengths
322
+ * attn_type: encoder, decoder self, or enc/dec cross attention
323
+ * force_max_len: if True, all query seqlens are max_q_seq_len; o/w query
324
+ seqlens are random in [2,max_q_seq_lens]. Same for key/value seqlens
325
+ and max_kv_seq_len, unless forced by is_encoder_decoder_attn=False
326
+ * device: CPU or CUDA device
327
+
328
+ Returns:
329
+
330
+ * Overall QKVInputs structure (containing full unpacked Q/K/V tensors)
331
+ * Prefill QKVInputs structure (containing all but the last sequence offset)
332
+ * Decode QKVInputs structure (containing all only the last sequence offset)
333
+ '''
334
+
335
+ if force_max_len:
336
+ q_seq_lens = [max_q_seq_len for _ in range(batch_size)]
337
+ else:
338
+ q_seq_lens = [
339
+ random.randint(2, max_q_seq_len) for _ in range(batch_size)
340
+ ]
341
+ kv_seq_lens = None
342
+ if force_kv_seq_lens is not None:
343
+ kv_seq_lens = force_kv_seq_lens
344
+ elif attn_type != AttentionType.ENCODER_DECODER:
345
+ # K,V seq lens match Q for self-attention
346
+ kv_seq_lens = q_seq_lens
347
+ else:
348
+ # K,V seq lens are distinct from Q seq lens & random
349
+ assert max_kv_seq_len is not None
350
+ if force_max_len:
351
+ kv_seq_lens = [max_kv_seq_len] * batch_size
352
+ else:
353
+ kv_seq_lens = [
354
+ random.randint(2, max_kv_seq_len) for _ in range(batch_size)
355
+ ]
356
+
357
+ query = torch.rand(
358
+ (batch_size, max_q_seq_len, num_heads, head_size)).to(device)
359
+ key = torch.rand(
360
+ (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
361
+ value = torch.rand(
362
+ (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
363
+
364
+ prefill_query = torch.zeros(
365
+ (batch_size, max_q_seq_len, num_heads, head_size)).to(device)
366
+ prefill_key = torch.zeros(
367
+ (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
368
+ prefill_value = torch.zeros(
369
+ (batch_size, max_kv_seq_len, num_heads, head_size)).to(device)
370
+
371
+ decode_query = torch.zeros(
372
+ (batch_size, 1, num_heads, head_size)).to(device)
373
+ decode_key = torch.zeros((batch_size, 1, num_heads, head_size)).to(device)
374
+ decode_value = torch.zeros(
375
+ (batch_size, 1, num_heads, head_size)).to(device)
376
+
377
+ for bdx, (q_seq_len, kv_seq_len) in enumerate(zip(q_seq_lens,
378
+ kv_seq_lens)):
379
+ query[bdx, q_seq_len:, :, :] = 0
380
+ key[bdx, kv_seq_len:, :, :] = 0
381
+ value[bdx, kv_seq_len:, :, :] = 0
382
+
383
+ prefill_query[bdx,
384
+ 0:(q_seq_len - 1), :, :] = query[bdx,
385
+ 0:(q_seq_len - 1), :, :]
386
+ prefill_key[bdx,
387
+ 0:(kv_seq_len - 1), :, :] = key[bdx,
388
+ 0:(kv_seq_len - 1), :, :]
389
+ prefill_value[bdx, 0:(kv_seq_len -
390
+ 1), :, :] = value[bdx, 0:(kv_seq_len - 1), :, :]
391
+
392
+ decode_query[bdx, :, :, :] = query[bdx,
393
+ (q_seq_len - 1):q_seq_len, :, :]
394
+ decode_key[bdx, :, :, :] = key[bdx, (kv_seq_len - 1):kv_seq_len, :, :]
395
+ decode_value[bdx, :, :, :] = value[bdx,
396
+ (kv_seq_len - 1):kv_seq_len, :, :]
397
+
398
+ prefill_q_seq_lens = [plen - 1 for plen in q_seq_lens]
399
+ prefill_kv_seq_lens = [plen - 1 for plen in kv_seq_lens]
400
+
401
+ decode_q_seq_lens = [1 for _ in q_seq_lens]
402
+ decode_kv_seq_lens = [1 for _ in kv_seq_lens]
403
+
404
+ return (
405
+ QKVInputs(
406
+ query, # Overall QKV inputs
407
+ key,
408
+ value,
409
+ q_seq_lens,
410
+ kv_seq_lens),
411
+ QKVInputs(
412
+ prefill_query, # Prefill subset of QKV sequences
413
+ prefill_key,
414
+ prefill_value,
415
+ prefill_q_seq_lens,
416
+ prefill_kv_seq_lens),
417
+ QKVInputs(
418
+ decode_query, # Decode subset of KV sequences
419
+ decode_key,
420
+ decode_value,
421
+ decode_q_seq_lens,
422
+ decode_kv_seq_lens))
423
+
424
+
425
+def pack_tensor(
426
+ unpacked_tensor: torch.Tensor, seq_lens: List[int],
427
+ device: Union[torch.device, str]) -> Tuple[torch.Tensor, List[int]]:
428
+ '''
429
+ Pack a batch_size x padded_seq_len x num_heads x head_size tensor into an
430
+ unpadded number_of_tokens x num_heads x head_size tensor, where
431
+ number_of_tokens = sum(seq_lens)
432
+
433
+ Arguments:
434
+
435
+ * unpacked_tensor: batch_size x padded_seq_len x num_heads x head_size
436
+ * seq_lens: list of token counts for each seq
437
+ * device: CPU or CUDA device
438
+
439
+ Returns
440
+
441
+ * packed_tensor: number_of_tokens x num_heads x head_size
442
+ * start_loc_list: start idx of each batch elt in packed_tensor; [0] +
443
+ list(itertools.accumulate(seq_lens))
444
+ '''
445
+
446
+ num_tok = sum(seq_lens)
447
+ num_heads = unpacked_tensor.shape[-2]
448
+ head_size = unpacked_tensor.shape[-1]
449
+ start_loc_list = [0] + list(itertools.accumulate(seq_lens))
450
+ packed_tensor = torch.zeros((num_tok, num_heads, head_size), device=device)
451
+
452
+ for bdx, (seq_len, start_loc) in enumerate(zip(seq_lens, start_loc_list)):
453
+
454
+ packed_tensor[start_loc:(
455
+ start_loc + seq_len), :, :] = unpacked_tensor[bdx, :seq_len, :, :]
456
+
457
+ return packed_tensor, start_loc_list
458
+
459
+
460
+def pack_qkv(qkv: QKVInputs, device: Union[torch.device,
461
+ str]) -> PackedQKVInputs:
462
+ '''
463
+ Individually pack each of Q, K and V, each with dimensions batch_size x
464
+ padded_seq_len x num_heads x head_size, into respective number_of_tokens x
465
+ num_heads x head_size tensors.
466
+
467
+ For Q, number_of_tokens = sum(q_seq_lens).
468
+
469
+ For K and V, number_of_tokens = sum(kv_seq_lens)
470
+
471
+ Arguments:
472
+
473
+ * qkv: Unpacked (batch_size x padded_seq_len x num_heads x head_size)
474
+ attention inputs
475
+ * device: CPU or CUDA device
476
+
477
+ Returns
478
+
479
+ * Packed (number_of_tokens x num_heads x head_size) QKV inputs
480
+ derived from unpacked inputs
481
+ '''
482
+
483
+ if qkv.query is None:
484
+ packed_query = None
485
+ q_start_loc_list = None
486
+ else:
487
+ packed_query, q_start_loc_list = pack_tensor(qkv.query,
488
+ qkv.q_seq_lens,
489
+ device=device)
490
+ packed_key, kv_start_loc_list = pack_tensor(qkv.key,
491
+ qkv.kv_seq_lens,
492
+ device=device)
493
+ packed_value, _ = pack_tensor(qkv.value, qkv.kv_seq_lens, device=device)
494
+ return PackedQKVInputs(
495
+ packed_query, packed_key, packed_value, q_start_loc_list,
496
+ kv_start_loc_list,
497
+ (None if q_start_loc_list is None else qkv.q_seq_lens),
498
+ qkv.kv_seq_lens)
499
+
500
+
501
+def make_backend(backend_name: str) -> AttentionBackend:
502
+ '''
503
+ Construct the backend instance determined by the backend_name string
504
+ argument.
505
+
506
+ "XFORMERS" -> construct xformers backend
507
+
508
+ TODO: other backends
509
+
510
+ Note: at time of writing the Attention wrapper automatically selects
511
+ its own backend for Attention.forward(); so the backend instance which
512
+ you generate with this function is not meant to be used for *running*
513
+ inference, but rather for generating compatible metadata structures
514
+ using backend.make_metadata()
515
+
516
+
517
+ Returns:
518
+
519
+ * Backend instance
520
+ '''
521
+ if backend_name == STR_XFORMERS_ATTN_VAL:
522
+ return XFormersBackend()
523
+ raise AssertionError(
524
+ f"Unrecognized backend_name {backend_name} for unit test")
525
+
526
+
527
+def _make_metadata_tensors(
528
+ seq_lens: Optional[List[int]], context_lens: Optional[List[int]],
529
+ encoder_seq_lens: Optional[List[int]], device: Union[torch.device, str]
530
+) -> Tuple[torch.Tensor, torch.Tensor, Any, Any, Optional[List[int]],
531
+ torch.Tensor, Optional[int]]:
532
+ '''
533
+ Build scalar & tensor values required to build attention metadata structure.
534
+
535
+ Arguments:
536
+
537
+ * seq_lens: list of token-counts for each decoder input seq
538
+ * context_lens: list of context length values for each seq
539
+ * encoder_seq_lens: list of token-counts for each encoder input seq
540
+ * device: CPU or CUDA device
541
+
542
+ Returns:
543
+
544
+ * seq_lens_tensor: decoder seq_lens list, as tensor
545
+ * context_lens_tensor: context_lens list, as tensor
546
+ * max_context_len: max(context_lens)
547
+ * max_seq_len: max(seq_lens)
548
+ * seq_start_loc: start idx of each sequence
549
+ * max_encoder_seq_len: encoder seq_lens list, as tensor
550
+ '''
551
+ seq_lens_tensor = maybe_make_int_tensor(seq_lens, device)
552
+ context_lens_tensor = maybe_make_int_tensor(context_lens, device)
553
+ max_context_len = maybe_max(context_lens)
554
+ max_seq_len = maybe_max(seq_lens)
555
+
556
+ encoder_seq_lens_tensor = maybe_make_int_tensor(encoder_seq_lens, device)
557
+ max_encoder_seq_len = (None if encoder_seq_lens is None else
558
+ max(encoder_seq_lens))
559
+
560
+ seq_start_loc = None
561
+
562
+ return (seq_lens_tensor, context_lens_tensor, max_context_len, max_seq_len,
563
+ seq_start_loc, encoder_seq_lens_tensor, max_encoder_seq_len)
564
+
565
+
566
+def make_kv_cache(num_blocks: int,
567
+ num_heads: int,
568
+ head_size: int,
569
+ block_size: int,
570
+ device: Union[torch.device, str],
571
+ default_val: float = 0.0) -> torch.Tensor:
572
+ '''
573
+ Create a fake KV cache.
574
+
575
+ Arguments:
576
+
577
+ * num_blocks: number of blocks in the KV cache
578
+ * num_heads: number of attention heads
579
+ * head_size: head dimension
580
+ * block_size: number of offsets within a block
581
+ * device: CPU or CUDA device
582
+ * default_val: initialization value for KV cache elements
583
+
584
+ Returns:
585
+
586
+ * kv_cache: 2 x num_blocks x (block_size * num_heads * head_size)
587
+ '''
588
+
589
+ kv_cache = torch.rand(
590
+ (2, num_blocks, block_size * num_heads * head_size)).to(device)
591
+ if default_val is not None:
592
+ kv_cache[:, :, :] = default_val
593
+ return kv_cache
594
+
595
+
596
+def _num_tokens_to_min_blocks(num_tokens: int, block_size: int) -> int:
597
+ '''
598
+ Compute the minimum number of blocks required to hold num_tokens tokens,
599
+ given block_size
600
+ '''
601
+ return (num_tokens + block_size) // block_size
602
+
603
+
604
+def make_empty_slot_mapping_tensor(device: Union[torch.device, str]):
605
+ return maybe_make_long_tensor([], device)
606
+
607
+
608
+def make_empty_block_tables_tensor(device: Union[torch.device, str]):
609
+ return torch.tensor([], device=device)
610
+
611
+
612
+def split_slot_mapping(slot_mapping_list: torch.Tensor, seq_lens: List[int],
613
+ device: Union[torch.device, str]):
614
+ '''
615
+ Split a slot mapping into valid prefill- and decode-phase slot mappings.
616
+
617
+ Context:
618
+ * Your goal is to test (1) prefill of N prompts, with prompt-lengths
619
+ {K_i \\forall i \\in [0,N)}, followed by (2) decoding of a single token
620
+ for all N prompts (N tokens total); the resultant sequence lengths
621
+ after decode would be {K_i + 1 for i \\in [0,N)}
622
+ * The test you want to do requires (1) having the prefill slot mapping
623
+ for all tokens present during prefill, the number of which is
624
+ M = \\sum_i{K_i}, and (2) having the decode slot mapping for all N
625
+ decoded tokens
626
+
627
+ This function consumes a single 1D slot mapping, which is the
628
+ concatenation of N slot mappings each of length K_i + 1 (corresponding
629
+ to the sequence lengths after decode), with a total length of
630
+ P = \\sum_i{K_i + 1} = M + N
631
+
632
+ The prefill-phase slot mapping results from excising the (K_i + 1)-th entry
633
+ from each of the N subsequences in the slot mapping (i.e. omitting the
634
+ decoded token's mapping.)
635
+
636
+ The N excised entries are appended to obtain the decode-phase slot mapping
637
+
638
+ Arguments:
639
+
640
+ * slot_mapping_list: Length-P 1D slot mapping (as List) reflecting all N
641
+ post-decode sequences
642
+ * seq_lens: List of N post-decode sequence lengths (K_i + 1 in the
643
+ description above)
644
+ * device: cuda, cpu, etc.
645
+
646
+ Returns:
647
+
648
+ * prefill_slot_mapping: Length-M 1D slot mapping (as Tensor)
649
+ reflecting all N prefill prompts
650
+ * decode_slot_mapping: Length-N 1D slot mapping (as Tensor) reflecting
651
+ all N decoded tokens
652
+ '''
653
+
654
+ prefill_slot_mapping = []
655
+ decode_slot_mapping = []
656
+
657
+ base_idx = 0
658
+ for seq_len in seq_lens:
659
+ prefill_slot_mapping.extend(slot_mapping_list[base_idx:(base_idx +
660
+ seq_len - 1)])
661
+ decode_slot_mapping.append(slot_mapping_list[base_idx + seq_len - 1])
662
+ base_idx += seq_len
663
+
664
+ return (maybe_make_long_tensor(prefill_slot_mapping, device),
665
+ maybe_make_long_tensor(decode_slot_mapping, device))
666
+
667
+
668
+def make_block_tables_slot_mapping(
669
+ block_size: int,
670
+ seq_lens: List[int],
671
+ device: Union[torch.device, str],
672
+ block_base_addr: int = 0) -> Tuple[torch.Tensor, List[int], int]:
673
+ '''
674
+ Construct fake block tables & slot mappings.
675
+
676
+ For a sequence with num_tokens tokens the minimum number
677
+ of required KV cache blocks is
678
+
679
+ num_blocks = (num_tokens + block_size) // block_size
680
+
681
+ Then the minimum KV cache size in blocks is
682
+
683
+ total_cache_blocks = sum(num_blocks for all seqs)
684
+
685
+ Then, the blocktable mapping counts downward from
686
+
687
+ block_base_addr + total_cache_blocks
688
+
689
+ to
690
+
691
+ block_base_addr
692
+
693
+
694
+ The constructed block-tables and slot-mapping are sized to the
695
+ lengths of the sequences in their entirety (as reflected by seq_lens),
696
+ i.e. the total of prefill prompt tokens + decoded tokens.
697
+
698
+ Arguments:
699
+
700
+ * block_size: number of offsets per block
701
+ * seq_lens: list of token-counts for each sequence
702
+ * block_base_addr: the block table base address
703
+ * device: CPU or CUDA device
704
+
705
+ Return:
706
+
707
+ * block_tables_tensor: block table for sequence
708
+ * slot_mapping_list: slot mapping for sequence
709
+ * max_block_idx: the highest block address within this block table
710
+ '''
711
+
712
+ # Provision minimum number of KV cache blocks
713
+ num_blocks_list = [
714
+ _num_tokens_to_min_blocks(num_tokens, block_size)
715
+ for num_tokens in seq_lens
716
+ ]
717
+ max_block_table_len = max(num_blocks_list)
718
+ block_table_pad_tokens = 10
719
+
720
+ block_tables = []
721
+ slot_mapping_list = []
722
+ # Compute uppermost address of block table
723
+ total_cache_blocks = sum(num_blocks_list)
724
+ block_base_idx = block_base_addr + total_cache_blocks
725
+ max_block_idx = block_base_idx
726
+ for sdx, num_tokens in enumerate(seq_lens):
727
+ num_blocks = num_blocks_list[sdx]
728
+ block_table = list(
729
+ range(block_base_idx, block_base_idx - num_blocks, -1))
730
+ for idx in range(num_tokens):
731
+ mapping_value = (
732
+ idx % block_size) + block_table[idx // block_size] * block_size
733
+ slot_mapping_list.append(mapping_value)
734
+
735
+ block_base_idx -= num_blocks
736
+ block_tables.append(block_table)
737
+
738
+ block_tables_tensor = make_tensor_with_pad(
739
+ block_tables,
740
+ max_len=max_block_table_len + block_table_pad_tokens,
741
+ pad=0,
742
+ dtype=torch.int,
743
+ device=device,
744
+ )
745
+
746
+ return (block_tables_tensor, slot_mapping_list, max_block_idx)
747
+
748
+
749
+def make_test_metadata(
750
+ attn_backend: AttentionBackend,
751
+ is_prompt: bool,
752
+ seq_lens: Optional[List[int]],
753
+ decoder_test_params: Optional[PhaseTestParameters],
754
+ device: Union[torch.device, str],
755
+ encoder_test_params: Optional[PhaseTestParameters] = None,
756
+ cross_test_params: Optional[PhaseTestParameters] = None
757
+) -> AttentionMetadata:
758
+ '''
759
+ Construct fake attention metadata for a given test phase
760
+ (prefill-phase or decode-phase).
761
+
762
+ encoder_test_params and cross_test_params arguments allow encoder
763
+ attention and enc/dec cross-attention (respectively) to use distinct
764
+ metadata values from decoder self-attention (decoder_test_params.)
765
+
766
+ if encoder_test_params and cross_test_params are None, the attention
767
+ metadata will support decoder-only scenario.
768
+
769
+ Assumptions:
770
+
771
+ * No chunked prefill -> a batch is 100% prefill or 100% decode, never both
772
+
773
+ Arguments:
774
+
775
+ * attn_backend: Backend for sourcing attention kernels
776
+ * is_prompt: prefill if True, o/w decode
777
+ * seq_lens: list of token counts for each sequence
778
+ * decoder_test_params: decoder self-attention test params;
779
+ this function requires
780
+ kv_mmap (memory mapping) field
781
+ * device: CPU or CUDA device
782
+ * encoder_test_params: encoder attention test params;
783
+ this function requires encoder query
784
+ sequence lengths field. If None,
785
+ encoder query sequence lengths are
786
+ treated as None
787
+ * cross_test_params: enc/dec cross-attention test params;
788
+ this function requires kv_mmap field.
789
+ If None, KV cache memory map data
790
+ structures are treated as None
791
+
792
+ Return:
793
+
794
+ * AttentionMetadata structure
795
+ '''
796
+
797
+ # Decoder self-attention memory mapping
798
+ # decoder_test_params is None signals encoder-only
799
+ # scenario, so kv_mmap is None
800
+ kv_mmap = (None
801
+ if decoder_test_params is None else decoder_test_params.kv_mmap)
802
+
803
+ # This function constructs metadata assuming no chunked prefill,
804
+ # i.e. 100% prefill tokens or 100% decode tokens
805
+ #
806
+ # - If is_prompt, num_prefills_or_decodes is the number of prefills
807
+ # and num_prefill_or_decode_tokens is the number of prefill tokens
808
+ # - If not is_prompt, num_prefills_or_decodes is the number of decodes
809
+ # and num_prefill_or_decode_tokens is the number of decode tokens
810
+ #
811
+ # seq_lens is None signals encoder-only
812
+ # scenario, in which case num_prefills_or_decodes and
813
+ # num_prefill_or_decode_tokens are unused
814
+ num_prefills_or_decodes = (None if seq_lens is None else len(seq_lens))
815
+
816
+ num_prefill_or_decode_tokens = (None if seq_lens is None else (
817
+ sum(seq_lens) if is_prompt else len(seq_lens)))
818
+
819
+ # Seems for non-prefix-caching scenarios context_lens
820
+ # is never needed
821
+ context_lens = None
822
+
823
+ if encoder_test_params is None:
824
+ encoder_seq_lens = None
825
+ num_encoder_tokens = None
826
+ else:
827
+ # Encoder/decoder or encoder-only models only:
828
+ # * Extract encoder input sequence lengths
829
+ assert encoder_test_params.packed_qkvo.packed_qkv is not None
830
+ encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens
831
+ num_encoder_tokens = (None if encoder_seq_lens is None else
832
+ (sum(encoder_seq_lens)))
833
+
834
+ if cross_test_params is None:
835
+ cross_kv_mmap = None
836
+ else:
837
+ # Encoder/decoder or encoder-only models only:
838
+ # * Extract *cross-attention* slot_mapping and block table
839
+ # (kv_mmap)
840
+ cross_kv_mmap = cross_test_params.kv_mmap
841
+
842
+ if is_prompt:
843
+ # Prefill-phase scenario
844
+
845
+ num_prefills = num_prefills_or_decodes
846
+ num_prefill_tokens = num_prefill_or_decode_tokens
847
+ num_decode_tokens = 0
848
+
849
+ (
850
+ seq_lens_tensor,
851
+ context_lens_tensor,
852
+ _,
853
+ _,
854
+ _,
855
+ encoder_seq_lens_tensor,
856
+ max_encoder_seq_len,
857
+ ) = _make_metadata_tensors(seq_lens,
858
+ context_lens,
859
+ encoder_seq_lens,
860
+ device=device)
861
+
862
+ return attn_backend.make_metadata(
863
+ num_prefills=num_prefills,
864
+ slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),
865
+ num_prefill_tokens=num_prefill_tokens,
866
+ num_decode_tokens=num_decode_tokens,
867
+ seq_lens=seq_lens,
868
+ seq_lens_tensor=seq_lens_tensor,
869
+ max_prefill_seq_len=None if seq_lens is None else max(seq_lens),
870
+ max_decode_seq_len=0,
871
+ context_lens_tensor=context_lens_tensor,
872
+ block_tables=(None if kv_mmap is None else kv_mmap.block_tables),
873
+ use_cuda_graph=False,
874
+ num_encoder_tokens=num_encoder_tokens,
875
+ encoder_seq_lens=encoder_seq_lens,
876
+ encoder_seq_lens_tensor=encoder_seq_lens_tensor,
877
+ max_encoder_seq_len=max_encoder_seq_len,
878
+ cross_slot_mapping=(None if cross_kv_mmap is None else
879
+ cross_kv_mmap.slot_mapping),
880
+ cross_block_tables=(None if cross_kv_mmap is None else
881
+ cross_kv_mmap.block_tables))
882
+
883
+ else: # not is_prompt
884
+ # Decode-phase scenario
885
+
886
+ assert kv_mmap is not None
887
+ assert num_prefill_or_decode_tokens is not None
888
+ assert seq_lens is not None
889
+
890
+ num_prefills = 0
891
+ num_prefill_tokens = 0
892
+ num_decode_tokens = num_prefill_or_decode_tokens
893
+
894
+ (
895
+ seq_lens_tensor,
896
+ context_lens_tensor,
897
+ _,
898
+ _,
899
+ _,
900
+ encoder_seq_lens_tensor,
901
+ max_encoder_seq_len,
902
+ ) = _make_metadata_tensors(seq_lens,
903
+ context_lens,
904
+ encoder_seq_lens,
905
+ device=device)
906
+
907
+ return attn_backend.make_metadata(
908
+ num_prefills=num_prefills,
909
+ slot_mapping=kv_mmap.slot_mapping,
910
+ num_prefill_tokens=num_prefill_tokens,
911
+ num_decode_tokens=num_decode_tokens,
912
+ seq_lens=seq_lens,
913
+ seq_lens_tensor=seq_lens_tensor,
914
+ max_prefill_seq_len=0,
915
+ max_decode_seq_len=max(seq_lens),
916
+ context_lens_tensor=context_lens_tensor,
917
+ block_tables=kv_mmap.block_tables,
918
+ use_cuda_graph=False,
919
+ num_encoder_tokens=num_encoder_tokens,
920
+ encoder_seq_lens=encoder_seq_lens,
921
+ encoder_seq_lens_tensor=encoder_seq_lens_tensor,
922
+ max_encoder_seq_len=max_encoder_seq_len,
923
+ cross_slot_mapping=(None if cross_kv_mmap is None else
924
+ cross_kv_mmap.slot_mapping),
925
+ cross_block_tables=(None if cross_kv_mmap is None else
926
+ cross_kv_mmap.block_tables))
927
+
928
+
929
+def assert_actual_matches_ideal(test_params: PhaseTestParameters,
930
+ output_under_test: torch.Tensor) -> None:
931
+ '''
932
+ Assert that observed output matches the ideal output
933
+ contained in the test parameters data structure.
934
+
935
+ Arguments:
936
+
937
+ * test_params: Test parameters including packed ideal output
938
+ * output_under_test: actually observed output value
939
+ '''
940
+ ideal_output = test_params.packed_qkvo.ideal_output
941
+ assert torch.allclose(ideal_output,
942
+ output_under_test.view_as(ideal_output))
vllm/attention/backends/abstract.py
+8
@@ -1,11 +1,18 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass, fields
3
+from enum import Enum, auto
4
from typing import (Any, Dict, Generic, List, Optional, Set, Tuple, Type,
5
TypeVar)
6
7
import torch
8
9
10
+class AttentionType(Enum):
11
+ DECODER = auto() # Decoder attention between previous layer Q/K/V
12
+ ENCODER = auto() # Encoder attention between previous layer Q/K/V
13
+ ENCODER_DECODER = auto() # Attention between dec. Q and enc. K/V
14
+
15
+
16
class AttentionBackend(ABC):
17
"""Abstract class for attention backends."""
18
@@ -128,5 +135,6 @@ class AttentionImpl(ABC, Generic[T]):
135
kv_cache: torch.Tensor,
136
attn_metadata: T,
137
kv_scale: float = 1.0,
138
+ attn_type: AttentionType = AttentionType.DECODER,
139
) -> torch.Tensor:
140
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+8
-1
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4
import torch
5
6
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7
- AttentionMetadata)
7
+ AttentionMetadata, AttentionType)
8
from vllm.attention.ops.blocksparse_attention.interface import (
9
LocalStridedBlockSparseAttn, get_head_sliding_step)
10
from vllm.attention.ops.paged_attn import PagedAttention
@@ -328,6 +328,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
328
kv_cache: torch.Tensor,
329
attn_metadata: BlocksparseFlashAttentionMetadata,
330
kv_scale: float = 1.0,
331
+ attn_type: AttentionType = AttentionType.DECODER,
332
) -> torch.Tensor:
333
"""Forward pass with FlashAttention and PagedAttention.
334
@@ -340,6 +341,12 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
341
Returns:
342
shape = [num_tokens, num_heads * head_size]
343
"""
344
+ if attn_type != AttentionType.DECODER:
345
+ raise NotImplementedError("Encoder self-attention and "
346
+ "encoder/decoder cross-attention "
347
+ "are not implemented for "
348
+ "BlocksparseFlashAttentionImpl")
349
+
350
num_tokens, hidden_size = query.shape
351
# Reshape the query, key, and value tensors.
352
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/flash_attn.py
+8
-1
@@ -7,7 +7,7 @@ from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
7
8
from vllm import _custom_ops as ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata)
10
+ AttentionMetadata, AttentionType)
11
12
13
class FlashAttentionBackend(AttentionBackend):
@@ -257,6 +257,7 @@ class FlashAttentionImpl(AttentionImpl):
257
kv_cache: torch.Tensor,
258
attn_metadata: FlashAttentionMetadata,
259
kv_scale: float = 1.0,
260
+ attn_type: AttentionType = AttentionType.DECODER,
261
) -> torch.Tensor:
262
"""Forward pass with FlashAttention.
263
@@ -269,6 +270,12 @@ class FlashAttentionImpl(AttentionImpl):
270
Returns:
271
shape = [num_tokens, num_heads * head_size]
272
"""
273
+ if attn_type != AttentionType.DECODER:
274
+ raise NotImplementedError("Encoder self-attention and "
275
+ "encoder/decoder cross-attention "
276
+ "are not implemented for "
277
+ "FlashAttentionImpl")
278
+
279
# NOTE(woosuk): FlashAttention does not support FP8 KV cache.
280
assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
281
vllm/attention/backends/flashinfer.py
+7
-1
@@ -14,7 +14,7 @@ import torch
14
15
from vllm import _custom_ops as ops
16
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17
- AttentionMetadata)
17
+ AttentionMetadata, AttentionType)
18
19
20
class FlashInferBackend(AttentionBackend):
@@ -224,8 +224,14 @@ class FlashInferImpl(AttentionImpl):
224
kv_cache: Optional[torch.Tensor],
225
attn_metadata: FlashInferMetadata,
226
kv_scale: float = 1.0,
227
+ attn_type: AttentionType = AttentionType.DECODER,
228
) -> torch.Tensor:
229
assert kv_scale == 1.0
230
+ if attn_type != AttentionType.DECODER:
231
+ raise NotImplementedError("Encoder self-attention and "
232
+ "encoder/decoder cross-attention "
233
+ "are not implemented for "
234
+ "FlashInferImpl")
235
num_tokens, hidden_size = query.shape
236
query = query.view(-1, self.num_heads, self.head_size)
237
key = key.view(-1, self.num_kv_heads, self.head_size)
vllm/attention/backends/ipex_attn.py
+7
-1
@@ -7,7 +7,7 @@ import torch
7
8
from vllm._ipex_ops import ipex_ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata)
10
+ AttentionMetadata, AttentionType)
11
from vllm.attention.ops.paged_attn import (PagedAttention,
12
PagedAttentionMetadata)
13
@@ -157,6 +157,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
157
kv_cache: Optional[torch.Tensor],
158
attn_metadata: IpexAttnMetadata, # type: ignore
159
kv_scale: float = 1.0,
160
+ attn_type: AttentionType = AttentionType.DECODER,
161
) -> torch.Tensor:
162
"""Forward pass with IPEX varlen_attention and PagedAttention.
163
@@ -170,6 +171,11 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
171
shape = [num_tokens, num_heads * head_size]
172
"""
173
assert kv_scale == 1.0
174
+ if attn_type != AttentionType.DECODER:
175
+ raise NotImplementedError("Encoder self-attention and "
176
+ "encoder/decoder cross-attention "
177
+ "are not implemented for "
178
+ "IpexAttnBackendImpl")
179
num_tokens, hidden_size = query.shape
180
# Reshape the query, key, and value tensors.
181
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/pallas.py
+7
-1
@@ -6,7 +6,7 @@ import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
import torch_xla.experimental.dynamo_set_buffer_donor
7
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
- AttentionMetadata)
9
+ AttentionMetadata, AttentionType)
10
11
12
class PallasAttentionBackend(AttentionBackend):
@@ -132,6 +132,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
132
kv_cache: Tuple[Optional[torch.Tensor], Optional[torch.Tensor]],
133
attn_metadata: PallasMetadata,
134
kv_scale: float = 1.0,
135
+ attn_type: AttentionType = AttentionType.DECODER,
136
) -> torch.Tensor:
137
"""Forward pass with Pallas attention.
138
@@ -146,6 +147,11 @@ class PallasAttentionBackendImpl(AttentionImpl):
147
shape = [batch_size, seq_len, num_heads * head_size]
148
"""
149
assert kv_scale == 1.0
150
+ if attn_type != AttentionType.DECODER:
151
+ raise NotImplementedError("Encoder self-attention and "
152
+ "encoder/decoder cross-attention "
153
+ "are not implemented for "
154
+ "PallasAttentionBackendImpl")
155
batch_size, seq_len, hidden_size = query.shape
156
query = query.view(batch_size, seq_len, self.num_heads, self.head_size)
157
key = key.view(batch_size, seq_len, self.num_kv_heads, self.head_size)
vllm/attention/backends/rocm_flash_attn.py
+8
-1
@@ -6,7 +6,7 @@ import torch
6
7
import vllm.envs as envs
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
- AttentionMetadata)
9
+ AttentionMetadata, AttentionType)
10
from vllm.attention.ops.paged_attn import (PagedAttention,
11
PagedAttentionMetadata)
12
from vllm.logger import init_logger
@@ -297,6 +297,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
297
kv_cache: torch.Tensor,
298
attn_metadata: ROCmFlashAttentionMetadata,
299
kv_scale: float = 1.0,
300
+ attn_type: AttentionType = AttentionType.DECODER,
301
) -> torch.Tensor:
302
"""Forward pass with FlashAttention and PagedAttention.
303
@@ -309,6 +310,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
310
Returns:
311
shape = [num_tokens, num_heads * head_size]
312
"""
313
+ if attn_type != AttentionType.DECODER:
314
+ raise NotImplementedError("Encoder self-attention and "
315
+ "encoder/decoder cross-attention "
316
+ "are not implemented for "
317
+ "ROCmFlashAttentionImpl")
318
+
319
num_tokens, hidden_size = query.shape
320
# Reshape the query, key, and value tensors.
321
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/torch_sdpa.py
+7
-1
@@ -7,7 +7,7 @@ import torch
7
from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
- AttentionMetadata)
10
+ AttentionMetadata, AttentionType)
11
from vllm.attention.ops.paged_attn import PagedAttentionMetadata
12
from vllm.utils import is_cpu
13
@@ -145,6 +145,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
145
kv_cache: Optional[torch.Tensor],
146
attn_metadata: TorchSDPAMetadata, # type: ignore
147
kv_scale: float = 1.0,
148
+ attn_type: AttentionType = AttentionType.DECODER,
149
) -> torch.Tensor:
150
"""Forward pass with torch SDPA and PagedAttention.
151
@@ -158,6 +159,11 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
159
shape = [num_tokens, num_heads * head_size]
160
"""
161
assert kv_scale == 1.0
162
+ if attn_type != AttentionType.DECODER:
163
+ raise NotImplementedError("Encoder self-attention and "
164
+ "encoder/decoder cross-attention "
165
+ "are not implemented for "
166
+ "TorchSDPABackendImpl")
167
num_tokens, hidden_size = query.shape
168
# Reshape the query, key, and value tensors.
169
query = query.view(-1, self.num_heads, self.head_size)
vllm/attention/backends/utils.py
new
+7
@@ -0,0 +1,7 @@
1
+"""Attention backend utils"""
2
+
3
+# Error string(s) for encoder/decoder
4
+# unsupported attention scenarios
5
+
6
+STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
7
+ "with encoder/decoder models.")
vllm/attention/backends/xformers.py
+394
-78
@@ -6,10 +6,11 @@ import torch
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import (AttentionBias,
8
BlockDiagonalCausalMask,
9
+ BlockDiagonalMask,
10
LowerTriangularMaskWithTensorBias)
11
12
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
12
- AttentionMetadata)
13
+ AttentionMetadata, AttentionType)
14
from vllm.attention.ops.paged_attn import (PagedAttention,
15
PagedAttentionMetadata)
16
from vllm.logger import init_logger
@@ -66,11 +67,6 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
67
dynamically, it should be stored in tensor. The tensor has to be
68
updated from `CUDAGraphRunner.forward` API.
69
"""
69
- # (batch_size,). The sequence length per sequence. Sequence length means
70
- # the computed tokens + new tokens None if it is a decoding.
71
- seq_lens: Optional[List[int]]
72
- # seq_lens stored as a tensor.
73
- seq_lens_tensor: Optional[torch.Tensor]
70
71
# |---------- N-1 iteration --------|
72
# |---------------- N iteration ---------------------|
@@ -79,8 +75,9 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
75
# |-------------------- seq_len ----------------------|
76
# |-- query_len ---|
77
82
- # Maximum query length in the batch. None for decoding.
83
- max_query_len: Optional[int]
78
+ # seq_lens stored as a tensor.
79
+ seq_lens_tensor: Optional[torch.Tensor]
80
+
81
# FIXME: It is for flash attn.
82
# Maximum sequence length among prefill batch. 0 if there are decoding
83
# requests only.
@@ -88,26 +85,55 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
85
# Maximum sequence length among decode batch. 0 if there are prefill
86
# requests only.
87
max_decode_seq_len: int
91
- # (batch_size + 1,). The cumulative subquery lengths of the sequences in
92
- # the batch, used to index into subquery. E.g., if the subquery length
93
- # is [4, 6], it is [0, 4, 10].
94
- query_start_loc: Optional[torch.Tensor]
88
+
89
+ # Whether or not if cuda graph is enabled.
90
+ # Cuda-graph is currently enabled for decoding only.
91
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
92
+ use_cuda_graph: bool
93
+
94
+ # (batch_size,). The sequence length per sequence. Sequence length means
95
+ # the computed tokens + new tokens None if it is a decoding.
96
+ seq_lens: Optional[List[int]] = None
97
+
98
# FIXME: It is for flash attn.
99
# (batch_size + 1,). The cumulative sequence lengths of the sequences in
100
# the batch, used to index into sequence. E.g., if the sequence length is
101
# [4, 6], it is [0, 4, 10].
99
- seq_start_loc: Optional[torch.Tensor]
102
+ seq_start_loc: Optional[torch.Tensor] = None
103
+
104
# (batch_size,) A tensor of context lengths (tokens that are computed
105
# so far).
102
- context_lens_tensor: Optional[torch.Tensor]
106
+ context_lens_tensor: Optional[torch.Tensor] = None
107
104
- # Whether or not if cuda graph is enabled.
105
- # Cuda-graph is currently enabled for decoding only.
106
- # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
107
- use_cuda_graph: bool
108
+ # Maximum query length in the batch. None for decoding.
109
+ max_query_len: Optional[int] = None
110
+
111
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
112
+ # the batch, used to index into subquery. E.g., if the subquery length
113
+ # is [4, 6], it is [0, 4, 10].
114
+ query_start_loc: Optional[torch.Tensor] = None
115
+
116
+ # Self-attention prefill/decode metadata cache
117
_cached_prefill_metadata: Optional["XFormersMetadata"] = None
118
_cached_decode_metadata: Optional["XFormersMetadata"] = None
119
120
+ # Begin encoder attn & enc/dec cross-attn fields...
121
+
122
+ # Encoder sequence lengths representation
123
+ encoder_seq_lens: Optional[List[int]] = None
124
+ encoder_seq_lens_tensor: Optional[torch.Tensor] = None
125
+
126
+ # Maximum sequence length among encoder sequences
127
+ max_encoder_seq_len: Optional[int] = None
128
+
129
+ # Number of tokens input to encoder
130
+ num_encoder_tokens: Optional[int] = None
131
+
132
+ # Cross-attention memory-mapping data structures: slot mapping
133
+ # and block tables
134
+ cross_slot_mapping: Optional[torch.Tensor] = None
135
+ cross_block_tables: Optional[torch.Tensor] = None
136
+
137
def __post_init__(self):
138
# Set during the execution of the first attention op.
139
# It is a list because it is needed to set per prompt
@@ -115,6 +141,28 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
141
# from xformer API.
142
# will not appear in the __repr__ and __init__
143
self.attn_bias: Optional[List[AttentionBias]] = None
144
+ self.encoder_attn_bias: Optional[List[AttentionBias]] = None
145
+ self.cross_attn_bias: Optional[List[AttentionBias]] = None
146
+
147
+ @property
148
+ def is_all_encoder_attn_metadata_set(self):
149
+ '''
150
+ All attention metadata required for encoder attention is set.
151
+ '''
152
+ return ((self.encoder_seq_lens is not None)
153
+ and (self.encoder_seq_lens_tensor is not None)
154
+ and (self.max_encoder_seq_len is not None))
155
+
156
+ @property
157
+ def is_all_cross_attn_metadata_set(self):
158
+ '''
159
+ All attention metadata required for enc/dec cross-attention is set.
160
+
161
+ Superset of encoder attention required metadata.
162
+ '''
163
+ return (self.is_all_encoder_attn_metadata_set
164
+ and (self.cross_slot_mapping is not None)
165
+ and (self.cross_block_tables is not None))
166
167
@property
168
def prefill_metadata(self) -> Optional["XFormersMetadata"]:
@@ -122,30 +170,50 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
170
return None
171
172
if self._cached_prefill_metadata is not None:
173
+ # Recover cached prefill-phase attention
174
+ # metadata structure
175
return self._cached_prefill_metadata
176
127
- assert self.seq_lens is not None
128
- assert self.seq_lens_tensor is not None
129
- assert self.query_start_loc is not None
130
- assert self.context_lens_tensor is not None
131
- assert self.block_tables is not None
132
-
177
+ assert ((self.seq_lens is not None)
178
+ or (self.encoder_seq_lens is not None))
179
+ assert ((self.seq_lens_tensor is not None)
180
+ or (self.encoder_seq_lens_tensor is not None))
181
+
182
+ # Compute some attn_metadata fields which default to None
183
+ query_start_loc = (None if self.query_start_loc is None else
184
+ self.query_start_loc[:self.num_prefills + 1])
185
+ slot_mapping = (None if self.slot_mapping is None else
186
+ self.slot_mapping[:self.num_prefill_tokens])
187
+ seq_lens = (None if self.seq_lens is None else
188
+ self.seq_lens[:self.num_prefills])
189
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
190
+ self.seq_lens_tensor[:self.num_prefills])
191
+ context_lens_tensor = (None if self.context_lens_tensor is None else
192
+ self.context_lens_tensor[:self.num_prefills])
193
+ block_tables = (None if self.block_tables is None else
194
+ self.block_tables[:self.num_prefills])
195
+
196
+ # Construct & cache prefill-phase attention metadata structure
197
self._cached_prefill_metadata = XFormersMetadata(
198
num_prefills=self.num_prefills,
199
num_prefill_tokens=self.num_prefill_tokens,
200
num_decode_tokens=0,
137
- slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
138
- seq_lens=self.seq_lens[:self.num_prefills],
139
- seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
201
+ slot_mapping=slot_mapping,
202
+ seq_lens=seq_lens,
203
+ seq_lens_tensor=seq_lens_tensor,
204
max_query_len=self.max_query_len,
205
max_prefill_seq_len=self.max_prefill_seq_len,
206
max_decode_seq_len=0,
143
- query_start_loc=self.query_start_loc[:self.num_prefills + 1],
144
- seq_start_loc=None,
145
- context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
146
- block_tables=self.block_tables[:self.num_prefills],
207
+ query_start_loc=query_start_loc,
208
+ context_lens_tensor=context_lens_tensor,
209
+ block_tables=block_tables,
210
use_cuda_graph=False,
148
- )
211
+ # Begin encoder & cross attn fields below...
212
+ encoder_seq_lens=self.encoder_seq_lens,
213
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
214
+ max_encoder_seq_len=self.max_encoder_seq_len,
215
+ cross_slot_mapping=self.cross_slot_mapping,
216
+ cross_block_tables=self.cross_block_tables)
217
return self._cached_prefill_metadata
218
219
@property
@@ -154,29 +222,146 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
222
return None
223
224
if self._cached_decode_metadata is not None:
225
+ # Recover cached decode-phase attention
226
+ # metadata structure
227
return self._cached_decode_metadata
158
- assert self.block_tables is not None
159
- assert self.seq_lens_tensor is not None
160
-
228
+ assert ((self.seq_lens_tensor is not None)
229
+ or (self.encoder_seq_lens_tensor is not None))
230
+
231
+ # Compute some attn_metadata fields which default to None
232
+ slot_mapping = (None if self.slot_mapping is None else
233
+ self.slot_mapping[self.num_prefill_tokens:])
234
+ seq_lens_tensor = (None if self.seq_lens_tensor is None else
235
+ self.seq_lens_tensor[self.num_prefills:])
236
+ block_tables = (None if self.block_tables is None else
237
+ self.block_tables[self.num_prefills:])
238
+
239
+ # Construct & cache decode-phase attention metadata structure
240
self._cached_decode_metadata = XFormersMetadata(
241
num_prefills=0,
242
num_prefill_tokens=0,
243
num_decode_tokens=self.num_decode_tokens,
165
- slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
166
- seq_lens=None,
167
- seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
168
- max_query_len=None,
244
+ slot_mapping=slot_mapping,
245
+ seq_lens_tensor=seq_lens_tensor,
246
max_prefill_seq_len=0,
247
max_decode_seq_len=self.max_decode_seq_len,
171
- query_start_loc=None,
172
- seq_start_loc=None,
173
- context_lens_tensor=None,
174
- block_tables=self.block_tables[self.num_prefills:],
248
+ block_tables=block_tables,
249
use_cuda_graph=self.use_cuda_graph,
176
- )
250
+ # Begin encoder & cross attn fields below...
251
+ encoder_seq_lens=self.encoder_seq_lens,
252
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
253
+ max_encoder_seq_len=self.max_encoder_seq_len,
254
+ cross_slot_mapping=self.cross_slot_mapping,
255
+ cross_block_tables=self.cross_block_tables)
256
return self._cached_decode_metadata
257
258
259
+def _get_attn_bias(
260
+ attn_metadata: XFormersMetadata,
261
+ attn_type: AttentionType,
262
+) -> Optional[AttentionBias]:
263
+ '''
264
+ Extract appropriate attention bias from attention metadata
265
+ according to attention type.
266
+
267
+ Arguments:
268
+
269
+ * attn_metadata: Attention metadata structure associated with attention
270
+ * attn_type: encoder attention, decoder self-attention,
271
+ encoder/decoder cross-attention
272
+
273
+ Returns:
274
+ * Appropriate attention bias value given the attention type
275
+ '''
276
+
277
+ if attn_type == AttentionType.DECODER:
278
+ return attn_metadata.attn_bias
279
+ elif attn_type == AttentionType.ENCODER:
280
+ return attn_metadata.encoder_attn_bias
281
+ else:
282
+ # attn_type == AttentionType.ENCODER_DECODER
283
+ return attn_metadata.cross_attn_bias
284
+
285
+
286
+def _set_attn_bias(
287
+ attn_metadata: XFormersMetadata,
288
+ attn_bias: List[Optional[AttentionBias]],
289
+ attn_type: AttentionType,
290
+) -> None:
291
+ '''
292
+ Update appropriate attention bias field of attention metadata,
293
+ according to attention type.
294
+
295
+ Arguments:
296
+
297
+ * attn_metadata: Attention metadata structure associated with attention
298
+ * attn_bias: The desired attention bias value
299
+ * attn_type: encoder attention, decoder self-attention,
300
+ encoder/decoder cross-attention
301
+ '''
302
+
303
+ if attn_type == AttentionType.DECODER:
304
+ attn_metadata.attn_bias = attn_bias
305
+ elif attn_type == AttentionType.ENCODER:
306
+ attn_metadata.encoder_attn_bias = attn_bias
307
+ elif attn_type == AttentionType.ENCODER_DECODER:
308
+ attn_metadata.cross_attn_bias = attn_bias
309
+ else:
310
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
311
+
312
+
313
+def _get_seq_len_block_table_args(
314
+ attn_metadata: XFormersMetadata,
315
+ is_prompt: bool,
316
+ attn_type: AttentionType,
317
+) -> tuple:
318
+ '''
319
+ The particular choice of sequence-length- and block-table-related
320
+ attributes which should be extracted from attn_metadata is dependent
321
+ on the type of attention operation.
322
+
323
+ Decoder attn -> select entirely decoder self-attention-related fields
324
+ Encoder/decoder cross-attn -> select encoder sequence lengths &
325
+ cross-attn block-tables fields
326
+ Encoder attn -> select encoder sequence lengths fields & no block tables
327
+
328
+ Arguments:
329
+
330
+ * attn_metadata: Attention metadata structure associated with attention op
331
+ * is_prompt: True if prefill, False otherwise
332
+ * attn_type: encoder attention, decoder self-attention,
333
+ encoder/decoder cross-attention
334
+
335
+ Returns:
336
+
337
+ * Appropriate sequence-lengths tensor
338
+ * Appropriate max sequence-length scalar
339
+ * Appropriate block tables (or None)
340
+ '''
341
+
342
+ if attn_type == AttentionType.DECODER:
343
+ # Decoder self-attention
344
+ # Choose max_seq_len based on whether we are in prompt_run
345
+ if is_prompt:
346
+ max_seq_len = attn_metadata.max_prefill_seq_len
347
+ else:
348
+ max_seq_len = attn_metadata.max_decode_seq_len
349
+ return (attn_metadata.seq_lens_tensor, max_seq_len,
350
+ attn_metadata.block_tables)
351
+ elif attn_type == AttentionType.ENCODER_DECODER:
352
+ # Enc/dec cross-attention KVs match encoder sequence length;
353
+ # cross-attention utilizes special "cross" block tables
354
+ return (attn_metadata.encoder_seq_lens_tensor,
355
+ attn_metadata.max_encoder_seq_len,
356
+ attn_metadata.cross_block_tables)
357
+ elif attn_type == AttentionType.ENCODER:
358
+ # No block tables associated with encoder attention
359
+ return (attn_metadata.encoder_seq_lens_tensor,
360
+ attn_metadata.max_encoder_seq_len, None)
361
+ else:
362
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
363
+
364
+
365
class XFormersImpl(AttentionImpl[XFormersMetadata]):
366
"""
367
If the input tensors contain prompt tokens, the layout is as follows:
@@ -238,51 +423,144 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
423
def forward(
424
self,
425
query: torch.Tensor,
241
- key: torch.Tensor,
242
- value: torch.Tensor,
426
+ key: Optional[torch.Tensor],
427
+ value: Optional[torch.Tensor],
428
kv_cache: Optional[torch.Tensor],
429
attn_metadata: "XFormersMetadata",
430
kv_scale: float = 1.0,
431
+ attn_type: AttentionType = AttentionType.DECODER,
432
) -> torch.Tensor:
433
"""Forward pass with xFormers and PagedAttention.
434
435
+ For decoder-only models: query, key and value must be non-None.
436
+
437
+ For encoder/decoder models:
438
+ * XFormersImpl.forward() may be invoked for both self- and cross-
439
+ attention layers.
440
+ * For self-attention: query, key and value must be non-None.
441
+ * For cross-attention:
442
+ * Query must be non-None
443
+ * During prefill, key and value must be non-None; key and value
444
+ get cached for use during decode.
445
+ * During decode, key and value may be None, since:
446
+ (1) key and value tensors were cached during prefill, and
447
+ (2) cross-attention key and value tensors do not grow during
448
+ decode
449
+
450
+ A note on how the attn_type (attention type enum) argument impacts
451
+ attention forward() behavior:
452
+
453
+ * DECODER: normal decoder-only behavior;
454
+ use decoder self-attention block table
455
+ * ENCODER: no KV caching; pass encoder sequence
456
+ attributes (encoder_seq_lens/encoder_seq_lens_tensor/
457
+ max_encoder_seq_len) to kernel, in lieu of decoder
458
+ sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
459
+ * ENCODER_DECODER: cross-attention behavior;
460
+ use cross-attention block table for caching KVs derived
461
+ from encoder hidden states; since KV sequence lengths
462
+ will match encoder sequence lengths, pass encoder sequence
463
+ attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
464
+ max_encoder_seq_len)
465
+
466
Args:
467
query: shape = [num_tokens, num_heads * head_size]
468
key: shape = [num_tokens, num_kv_heads * head_size]
469
value: shape = [num_tokens, num_kv_heads * head_size]
470
kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
471
attn_metadata: Metadata for attention.
472
+ attn_type: Select attention type, between encoder attention,
473
+ decoder self-attention, or encoder/decoder cross-
474
+ attention. Defaults to decoder self-attention,
475
+ which is the vLLM default generally
476
Returns:
477
shape = [num_tokens, num_heads * head_size]
478
"""
258
- query = query.view(-1, self.num_heads, self.head_size)
259
- key = key.view(-1, self.num_kv_heads, self.head_size)
260
- value = value.view(-1, self.num_kv_heads, self.head_size)
479
262
- if kv_cache is not None:
480
+ # Check that appropriate attention metadata attributes are
481
+ # selected for the desired attention type
482
+ if (attn_type == AttentionType.ENCODER
483
+ and (not attn_metadata.is_all_encoder_attn_metadata_set)):
484
+ raise AttributeError("Encoder attention requires setting "
485
+ "encoder metadata attributes.")
486
+ elif (attn_type == AttentionType.ENCODER_DECODER
487
+ and (not attn_metadata.is_all_cross_attn_metadata_set)):
488
+ raise AttributeError("Encoder/decoder cross-attention "
489
+ "requires setting cross-attention "
490
+ "metadata attributes.")
491
+
492
+ query = query.view(-1, self.num_heads, self.head_size)
493
+ if key is not None:
494
+ assert value is not None
495
+ key = key.view(-1, self.num_kv_heads, self.head_size)
496
+ value = value.view(-1, self.num_kv_heads, self.head_size)
497
+ else:
498
+ assert value is None
499
+
500
+ # Self-attention vs. cross-attention will impact
501
+ # which KV cache memory-mapping & which
502
+ # seqlen datastructures we utilize
503
+
504
+ if (attn_type != AttentionType.ENCODER and kv_cache is not None):
505
+ # KV-cache during decoder-self- or
506
+ # encoder-decoder-cross-attention, but not
507
+ # during encoder attention.
508
+ #
509
+ # Even if there are no new key/value pairs to cache,
510
+ # we still need to break out key_cache and value_cache
511
+ # i.e. for later use by paged attention
512
key_cache, value_cache = PagedAttention.split_kv_cache(
513
kv_cache, self.num_kv_heads, self.head_size)
514
266
- # Reshape the input keys and values and store them in the cache.
267
- # If kv_cache is not provided, the new key and value tensors are
268
- # not cached. This happens during the initial memory profiling run.
269
- PagedAttention.write_to_paged_cache(key, value, key_cache,
270
- value_cache,
271
- attn_metadata.slot_mapping,
272
- self.kv_cache_dtype, kv_scale)
273
-
274
- num_prefill_tokens = attn_metadata.num_prefill_tokens
275
- num_decode_tokens = attn_metadata.num_decode_tokens
276
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens
277
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens
515
+ if (key is not None) and (value is not None):
516
+
517
+ if attn_type == AttentionType.ENCODER_DECODER:
518
+ # Update cross-attention KV cache (prefill-only)
519
+ # During cross-attention decode, key & value will be None,
520
+ # preventing this IF-statement branch from running
521
+ updated_slot_mapping = attn_metadata.cross_slot_mapping
522
+ else:
523
+ # Update self-attention KV cache (prefill/decode)
524
+ updated_slot_mapping = attn_metadata.slot_mapping
525
+
526
+ # Reshape the input keys and values and store them in the cache.
527
+ # If kv_cache is not provided, the new key and value tensors are
528
+ # not cached. This happens during the initial memory
529
+ # profiling run.
530
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
531
+ value_cache,
532
+ updated_slot_mapping,
533
+ self.kv_cache_dtype,
534
+ kv_scale)
535
+
536
+ if attn_type != AttentionType.ENCODER:
537
+ # Decoder self-attention supports chunked prefill.
538
+ # Encoder/decoder cross-attention requires no chunked
539
+ # prefill (100% prefill or 100% decode tokens, no mix)
540
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
541
+ num_decode_tokens = attn_metadata.num_decode_tokens
542
+ else:
543
+ # Encoder attention - chunked prefill is not applicable;
544
+ # derive token-count from query shape & and treat them
545
+ # as 100% prefill tokens
546
+ assert attn_metadata.num_encoder_tokens is not None
547
+ num_prefill_tokens = attn_metadata.num_encoder_tokens
548
+ num_decode_tokens = 0
549
+
550
+ if attn_type == AttentionType.DECODER:
551
+ # Only enforce this shape-constraint for decoder
552
+ # self-attention
553
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens
554
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens
555
556
output = torch.empty_like(query)
557
# Query for decode. KV is not needed because it is already cached.
558
decode_query = query[num_prefill_tokens:]
559
# QKV for prefill.
560
query = query[:num_prefill_tokens]
284
- key = key[:num_prefill_tokens]
285
- value = value[:num_prefill_tokens]
561
+ if key is not None and value is not None:
562
+ key = key[:num_prefill_tokens]
563
+ value = value[:num_prefill_tokens]
564
565
assert query.shape[0] == num_prefill_tokens
566
assert decode_query.shape[0] == num_decode_tokens
@@ -294,10 +572,14 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
572
# block tables are empty if the prompt does not have a cached
573
# prefix.
574
out = self._run_memory_efficient_xformers_forward(
297
- query, key, value, prefill_meta)
575
+ query, key, value, prefill_meta, attn_type=attn_type)
576
assert out.shape == output[:num_prefill_tokens].shape
577
output[:num_prefill_tokens] = out
578
else:
579
+
580
+ assert prefill_meta.query_start_loc is not None
581
+ assert prefill_meta.max_query_len is not None
582
+
583
# prefix-enabled attention
584
# TODO(Hai) this triton kernel has regression issue (broke) to
585
# deal with different data types between KV and FP8 KV cache,
@@ -320,13 +602,20 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
602
output[:num_prefill_tokens] = out
603
604
if decode_meta := attn_metadata.decode_metadata:
605
+
606
+ (
607
+ seq_lens_arg,
608
+ max_seq_len_arg,
609
+ block_tables_arg,
610
+ ) = _get_seq_len_block_table_args(decode_meta, False, attn_type)
611
+
612
output[num_prefill_tokens:] = PagedAttention.forward_decode(
613
decode_query,
614
key_cache,
615
value_cache,
327
- decode_meta.block_tables,
328
- decode_meta.seq_lens_tensor,
329
- decode_meta.max_decode_seq_len,
616
+ block_tables_arg,
617
+ seq_lens_arg,
618
+ max_seq_len_arg,
619
self.kv_cache_dtype,
620
self.num_kv_heads,
621
self.scale,
@@ -343,6 +632,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
632
key: torch.Tensor,
633
value: torch.Tensor,
634
attn_metadata: XFormersMetadata,
635
+ attn_type: AttentionType = AttentionType.DECODER,
636
) -> torch.Tensor:
637
"""Attention for 1D query of multiple prompts. Multiple prompt
638
tokens are flattened in to `query` input.
@@ -356,8 +646,12 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
646
key: shape = [num_prefill_tokens, num_kv_heads, head_size]
647
value: shape = [num_prefill_tokens, num_kv_heads, head_size]
648
attn_metadata: Metadata for attention.
649
+ attn_type: Select attention type, between encoder attention,
650
+ decoder self-attention, or encoder/decoder cross-
651
+ attention. Defaults to decoder self-attention,
652
+ which is the vLLM default generally
653
"""
360
- assert attn_metadata.seq_lens is not None
654
+
655
original_query = query
656
if self.num_kv_heads != self.num_heads:
657
# GQA/MQA requires the shape [B, M, G, H, K].
@@ -375,18 +669,39 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
669
# Set attention bias if not provided. This typically happens at
670
# the very attention layer of every iteration.
671
# FIXME(woosuk): This is a hack.
378
- if attn_metadata.attn_bias is None:
672
+ attn_bias = _get_attn_bias(attn_metadata, attn_type)
673
+ if attn_bias is None:
674
if self.alibi_slopes is None:
380
- attn_bias = BlockDiagonalCausalMask.from_seqlens(
381
- attn_metadata.seq_lens)
675
+ if (attn_type == AttentionType.ENCODER_DECODER):
676
+ assert attn_metadata.seq_lens is not None
677
+ assert attn_metadata.encoder_seq_lens is not None
678
+
679
+ # Default enc/dec cross-attention mask is non-causal
680
+ attn_bias = BlockDiagonalMask.from_seqlens(
681
+ attn_metadata.seq_lens, attn_metadata.encoder_seq_lens)
682
+ elif attn_type == AttentionType.ENCODER:
683
+ assert attn_metadata.encoder_seq_lens is not None
684
+
685
+ # Default encoder self-attention mask is non-causal
686
+ attn_bias = BlockDiagonalMask.from_seqlens(
687
+ attn_metadata.encoder_seq_lens)
688
+ else:
689
+ assert attn_metadata.seq_lens is not None
690
+
691
+ # Default decoder self-attention mask is causal
692
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(
693
+ attn_metadata.seq_lens)
694
if self.sliding_window is not None:
695
attn_bias = attn_bias.make_local_attention(
696
self.sliding_window)
385
- attn_metadata.attn_bias = [attn_bias]
697
+ attn_bias = [attn_bias]
698
else:
387
- attn_metadata.attn_bias = _make_alibi_bias(
388
- self.alibi_slopes, self.num_kv_heads, query.dtype,
389
- attn_metadata.seq_lens)
699
+ assert attn_metadata.seq_lens is not None
700
+ attn_bias = _make_alibi_bias(self.alibi_slopes,
701
+ self.num_kv_heads, query.dtype,
702
+ attn_metadata.seq_lens)
703
+
704
+ _set_attn_bias(attn_metadata, attn_bias, attn_type)
705
706
# No alibi slopes.
707
# TODO(woosuk): Too many view operations. Let's try to reduce
@@ -400,7 +715,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
715
query,
716
key,
717
value,
403
- attn_bias=attn_metadata.attn_bias[0],
718
+ attn_bias=attn_bias[0],
719
p=0.0,
720
scale=self.scale)
721
return out.view_as(original_query)
@@ -409,6 +724,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
724
# FIXME(woosuk): Because xformers does not support dynamic sequence
725
# lengths with custom attention bias, we process each prompt one by
726
# one. This is inefficient, especially when we have many short prompts.
727
+ assert attn_metadata.seq_lens is not None
728
output = torch.empty_like(original_query)
729
start = 0
730
for i, seq_len in enumerate(attn_metadata.seq_lens):
@@ -417,7 +733,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
733
query[None, start:end],
734
key[None, start:end],
735
value[None, start:end],
420
- attn_bias=attn_metadata.attn_bias[i],
736
+ attn_bias=attn_bias[i],
737
p=0.0,
738
scale=self.scale)
739
# TODO(woosuk): Unnecessary copy. Optimize.
vllm/attention/layer.py
+10
-3
@@ -4,7 +4,7 @@ from typing import Any, Dict, List, Optional
4
import torch
5
import torch.nn as nn
6
7
-from vllm.attention.backends.abstract import AttentionMetadata
7
+from vllm.attention.backends.abstract import AttentionMetadata, AttentionType
8
from vllm.attention.selector import get_attn_backend
9
from vllm.config import CacheConfig
10
from vllm.model_executor.layers.quantization.base_config import (
@@ -90,9 +90,16 @@ class Attention(nn.Module):
90
value: torch.Tensor,
91
kv_cache: Optional[torch.Tensor],
92
attn_metadata: AttentionMetadata,
93
+ attn_type: AttentionType = AttentionType.DECODER,
94
) -> torch.Tensor:
94
- return self.impl.forward(query, key, value, kv_cache, attn_metadata,
95
- self._kv_scale)
95
+
96
+ return self.impl.forward(query,
97
+ key,
98
+ value,
99
+ kv_cache,
100
+ attn_metadata,
101
+ self._kv_scale,
102
+ attn_type=attn_type)
103
104
def extra_repr(self) -> str:
105
s = f"head_size={self.impl.head_size}" # type: ignore