A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[ROCm][AMD][Model] llama 3.2 support upstreaming (#12421)
Signed-off-by: Aleksandr Malyshev <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com>
Aleksandr Malyshev committed
Jan 30, 2025 at 20:24 UTC
a1fc18c030e4d0466f2b23cb7dd4d11ce4b85603
2 files changed
+304
-88
vllm/attention/backends/rocm_flash_attn.py
+291
-85
@@ -90,6 +90,17 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
90
seq_lens: Optional[List[int]]
91
# seq_lens stored as a tensor.
92
seq_lens_tensor: Optional[torch.Tensor]
93
+ # Maximum sequence length among prefill batch. 0 if there are decoding
94
+ # requests only.
95
+ max_prefill_seq_len: int
96
+ # Maximum sequence length among decode batch. 0 if there are prefill
97
+ # requests only.
98
+ max_decode_seq_len: int
99
+
100
+ # Whether or not if cuda graph is enabled.
101
+ # Cuda-graph is currently enabled for decoding only.
102
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
103
+ use_cuda_graph: bool
104
105
# NOTE(sang): Definition of context_len, query_len, and seq_len.
106
# |---------- N-1 iteration --------|
@@ -100,30 +111,18 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
111
# |-- query_len ---|
112
113
# Maximum query length in the batch. None for decoding.
103
- max_query_len: Optional[int]
104
- # Maximum sequence length among prefill batch. 0 if there are decoding
105
- # requests only.
106
- max_prefill_seq_len: int
107
- # Maximum sequence length among decode batch. 0 if there are prefill
108
- # requests only.
109
- max_decode_seq_len: int
114
+ max_query_len: Optional[int] = None
115
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
116
# the batch, used to index into subquery. E.g., if the subquery length
117
# is [4, 6], it is [0, 4, 10].
113
- query_start_loc: Optional[torch.Tensor]
118
+ query_start_loc: Optional[torch.Tensor] = None
119
# (batch_size + 1,). The cumulative sequence lengths of the sequences in
120
# the batch, used to index into sequence. E.g., if the sequence length is
121
# [4, 6], it is [0, 4, 10].
117
- seq_start_loc: Optional[torch.Tensor]
118
-
119
- # Whether or not if cuda graph is enabled.
120
- # Cuda-graph is currently enabled for decoding only.
121
- # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
122
- use_cuda_graph: bool
123
-
122
+ seq_start_loc: Optional[torch.Tensor] = None
123
# (batch_size,) A tensor of context lengths (tokens that are computed
124
# so far).
126
- context_lens_tensor: Optional[torch.Tensor]
125
+ context_lens_tensor: Optional[torch.Tensor] = None
126
127
# Max number of query tokens among request in the batch.
128
max_decode_query_len: Optional[int] = None
@@ -131,6 +130,23 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
130
_cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
131
_cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
132
133
+ # Begin encoder attn & enc/dec cross-attn fields...
134
+
135
+ # Encoder sequence lengths representation
136
+ encoder_seq_lens: Optional[List[int]] = None
137
+ encoder_seq_lens_tensor: Optional[torch.Tensor] = None
138
+
139
+ # Maximum sequence length among encoder sequences
140
+ max_encoder_seq_len: Optional[int] = None
141
+
142
+ # Number of tokens input to encoder
143
+ num_encoder_tokens: Optional[int] = None
144
+
145
+ # Cross-attention memory-mapping data structures: slot mapping
146
+ # and block tables
147
+ cross_slot_mapping: Optional[torch.Tensor] = None
148
+ cross_block_tables: Optional[torch.Tensor] = None
149
+
150
@property
151
def prefill_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
152
if self.num_prefills == 0:
@@ -141,10 +157,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
157
158
assert self.seq_lens is not None
159
assert self.seq_lens_tensor is not None
144
- assert self.query_start_loc is not None
145
- assert self.context_lens_tensor is not None
160
assert self.block_tables is not None
147
- assert self.seq_start_loc is not None
161
162
self._cached_prefill_metadata = ROCmFlashAttentionMetadata(
163
num_prefills=self.num_prefills,
@@ -159,12 +172,20 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
172
max_query_len=self.max_query_len,
173
max_prefill_seq_len=self.max_prefill_seq_len,
174
max_decode_seq_len=0,
162
- query_start_loc=self.query_start_loc[:self.num_prefills + 1],
163
- seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
164
- context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
175
+ query_start_loc=None if self.query_start_loc is None else
176
+ self.query_start_loc[:self.num_prefills + 1],
177
+ seq_start_loc=None if self.seq_start_loc is None else
178
+ self.seq_start_loc[:self.num_prefills + 1],
179
+ context_lens_tensor=None if self.context_lens_tensor is None else
180
+ self.context_lens_tensor[:self.num_prefills],
181
block_tables=self.block_tables[:self.num_prefills],
182
use_cuda_graph=False,
167
- )
183
+ # Begin encoder & cross attn fields below...
184
+ encoder_seq_lens=self.encoder_seq_lens,
185
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
186
+ max_encoder_seq_len=self.max_encoder_seq_len,
187
+ cross_slot_mapping=self.cross_slot_mapping,
188
+ cross_block_tables=self.cross_block_tables)
189
return self._cached_prefill_metadata
190
191
@property
@@ -194,7 +215,12 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
215
context_lens_tensor=None,
216
block_tables=self.block_tables[self.num_prefills:],
217
use_cuda_graph=self.use_cuda_graph,
197
- )
218
+ # Begin encoder & cross attn fields below...
219
+ encoder_seq_lens=self.encoder_seq_lens,
220
+ encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
221
+ max_encoder_seq_len=self.max_encoder_seq_len,
222
+ cross_slot_mapping=self.cross_slot_mapping,
223
+ cross_block_tables=self.cross_block_tables)
224
# Batch may be composed of prefill|decodes, adjust query start indices
225
# to refer to the start of decodes when the two are split apart.
226
# E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
@@ -304,6 +330,97 @@ def _make_alibi_bias(alibi_slopes: torch.Tensor,
330
return attn_biases
331
332
333
+def _get_seq_len_block_table_args(
334
+ attn_metadata: ROCmFlashAttentionMetadata,
335
+ attn_type: str,
336
+) -> tuple:
337
+ '''
338
+ The particular choice of sequence-length
339
+ attributes which should be extracted from attn_metadata is dependent
340
+ on the type of attention operation.
341
+
342
+ Decoder attn -> select entirely decoder self-attention-related fields
343
+ Encoder/decoder cross-attn -> select encoder sequence lengths
344
+ Encoder attn -> select encoder sequence lengths fields
345
+
346
+ Arguments:
347
+
348
+ * attn_metadata: Attention metadata structure associated with attention op
349
+ * attn_type: encoder attention, decoder self-attention,
350
+ encoder/decoder cross-attention
351
+
352
+ Returns:
353
+
354
+ * Appropriate sequence-lengths tensors for query and key
355
+ * Appropriate max sequence-length scalar
356
+ '''
357
+
358
+ partial_prefix_sum = 0
359
+ if attn_type == AttentionType.ENCODER:
360
+ assert attn_metadata.encoder_seq_lens is not None
361
+ assert attn_metadata.encoder_seq_lens_tensor is not None
362
+ query_seq_start_loc = torch.tensor(
363
+ [0] + [
364
+ partial_prefix_sum := partial_prefix_sum + i
365
+ for i in attn_metadata.encoder_seq_lens
366
+ ],
367
+ device=attn_metadata.encoder_seq_lens_tensor.device,
368
+ dtype=attn_metadata.encoder_seq_lens_tensor.dtype)
369
+ causal_mask = False
370
+
371
+ # No block tables associated with encoder attention
372
+ return (query_seq_start_loc, attn_metadata.max_encoder_seq_len,
373
+ query_seq_start_loc, attn_metadata.max_encoder_seq_len,
374
+ attn_metadata.encoder_seq_lens, causal_mask)
375
+ elif attn_type == AttentionType.DECODER:
376
+ # Decoder self-attention
377
+ # Choose max_seq_len based on whether we are in prompt_run
378
+ assert attn_metadata.seq_lens is not None
379
+ assert attn_metadata.seq_lens_tensor is not None
380
+ query_seq_start_loc = torch.tensor(
381
+ [0] + [
382
+ partial_prefix_sum := partial_prefix_sum + i
383
+ for i in attn_metadata.seq_lens
384
+ ],
385
+ device=attn_metadata.seq_lens_tensor.device,
386
+ dtype=attn_metadata.seq_lens_tensor.dtype)
387
+ max_seq_len = attn_metadata.max_prefill_seq_len
388
+ causal_mask = True
389
+
390
+ return (query_seq_start_loc, max_seq_len, query_seq_start_loc,
391
+ max_seq_len, attn_metadata.seq_lens, causal_mask)
392
+ elif attn_type == AttentionType.ENCODER_DECODER:
393
+ assert attn_metadata.seq_lens is not None
394
+ assert attn_metadata.encoder_seq_lens_tensor is not None
395
+ query_start_loc = torch.tensor(
396
+ [0] + [
397
+ partial_prefix_sum := partial_prefix_sum + i
398
+ for i in attn_metadata.seq_lens
399
+ ],
400
+ device=attn_metadata.encoder_seq_lens_tensor.device,
401
+ dtype=attn_metadata.encoder_seq_lens_tensor.dtype)
402
+
403
+ partial_prefix_sum = 0
404
+ assert attn_metadata.encoder_seq_lens is not None
405
+ assert attn_metadata.seq_lens_tensor is not None
406
+ key_seq_start_loc = torch.tensor(
407
+ [0] + [
408
+ partial_prefix_sum := partial_prefix_sum + i
409
+ for i in attn_metadata.encoder_seq_lens
410
+ ],
411
+ device=attn_metadata.seq_lens_tensor.device,
412
+ dtype=attn_metadata.seq_lens_tensor.dtype)
413
+ causal_mask = False
414
+
415
+ # Enc/dec cross-attention KVs match encoder sequence length;
416
+ # cross-attention utilizes special "cross" block tables
417
+ return (query_start_loc, attn_metadata.max_prefill_seq_len,
418
+ key_seq_start_loc, attn_metadata.max_encoder_seq_len,
419
+ attn_metadata.seq_lens, causal_mask)
420
+ else:
421
+ raise AttributeError(f"Invalid attention type {str(attn_type)}")
422
+
423
+
424
class ROCmFlashAttentionImpl(AttentionImpl):
425
"""
426
If the input tensors contain prompt tokens, the layout is as follows:
@@ -346,10 +463,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
463
if blocksparse_params is not None:
464
raise ValueError(
465
"ROCmFlashAttention does not support blocksparse attention.")
349
- if logits_soft_cap is not None:
350
- raise ValueError(
351
- "ROCmFlashAttention does not support attention logits soft "
352
- "capping.")
466
+
467
+ if logits_soft_cap is None:
468
+ # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
469
+ self.logits_soft_cap = 0.0
470
+ else:
471
+ self.logits_soft_cap = logits_soft_cap
472
+ self.attn_type = attn_type
473
self.num_heads = num_heads
474
self.head_size = head_size
475
self.scale = float(scale)
@@ -374,6 +494,14 @@ class ROCmFlashAttentionImpl(AttentionImpl):
494
# NOTE: Allow for switching between Triton and CK. Defaulting to triton.
495
self.use_triton_flash_attn = envs.VLLM_USE_TRITON_FLASH_ATTN
496
if self.use_triton_flash_attn:
497
+ if logits_soft_cap is not None:
498
+ raise ValueError(
499
+ "ROCm Triton FlashAttention does not support attention"
500
+ "logits soft capping."
501
+ " please try using the ROCm CK "
502
+ "FA backend instead by setting the env var "
503
+ "`VLLM_USE_TRITON_FLASH_ATTN=0`")
504
+
505
from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
506
triton_attention)
507
self.attn_func = triton_attention
@@ -398,14 +526,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
526
self.use_naive_attn = True
527
528
if self.use_naive_attn:
401
- self.attn_func = _sdpa_attention
402
- logger.debug("Using naive attention in ROCmBackend")
529
+ if logits_soft_cap is not None:
530
+ raise ValueError(
531
+ "ROCm Naive FlashAttention does not support"
532
+ "attention logits soft capping.")
533
404
- if attn_type != AttentionType.DECODER:
405
- raise NotImplementedError("Encoder self-attention and "
406
- "encoder/decoder cross-attention "
407
- "are not implemented for "
408
- "ROCmFlashAttentionImpl")
534
+ self.attn_func = _sdpa_attention
535
+ logger.debug("Using naive (SDPA) attention in ROCmBackend")
536
537
def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
538
"""torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
@@ -427,6 +554,37 @@ class ROCmFlashAttentionImpl(AttentionImpl):
554
) -> torch.Tensor:
555
"""Forward pass with FlashAttention and PagedAttention.
556
557
+ For decoder-only models: query, key and value must be non-None.
558
+
559
+ For encoder/decoder models:
560
+ * ROCmFlashAttentionImpl.forward() may be invoked for both self- and
561
+ cross-attention layers.
562
+ * For self-attention: query, key and value must be non-None.
563
+ * For cross-attention:
564
+ * Query must be non-None
565
+ * During prefill, key and value must be non-None; key and value
566
+ get cached for use during decode.
567
+ * During decode, key and value may be None, since:
568
+ (1) key and value tensors were cached during prefill, and
569
+ (2) cross-attention key and value tensors do not grow during
570
+ decode
571
+
572
+ A note on how the attn_type (attention type enum) argument impacts
573
+ attention forward() behavior:
574
+
575
+ * DECODER: normal decoder-only behavior;
576
+ use decoder self-attention block table
577
+ * ENCODER: no KV caching; pass encoder sequence
578
+ attributes (encoder_seq_lens/encoder_seq_lens_tensor/
579
+ max_encoder_seq_len) to kernel, in lieu of decoder
580
+ sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
581
+ * ENCODER_DECODER: cross-attention behavior;
582
+ use cross-attention block table for caching KVs derived
583
+ from encoder hidden states; since KV sequence lengths
584
+ will match encoder sequence lengths, pass encoder sequence
585
+ attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
586
+ max_encoder_seq_len)
587
+
588
Args:
589
query: shape = [num_tokens, num_heads * head_size]
590
key: shape = [num_tokens, num_kv_heads * head_size]
@@ -435,54 +593,80 @@ class ROCmFlashAttentionImpl(AttentionImpl):
593
NOTE: kv_cache will be an empty tensor with shape [0]
594
for profiling run.
595
attn_metadata: Metadata for attention.
596
+ attn_type: Select attention type, between encoder attention,
597
+ decoder self-attention, or encoder/decoder cross-
598
+ attention. Defaults to decoder self-attention,
599
+ which is the vLLM default generally
600
Returns:
601
shape = [num_tokens, num_heads * head_size]
602
"""
441
- # Reminder: Please update docs/source/features/compatibility_matrix.md
442
- # If the feature combo become valid
443
- num_tokens, hidden_size = query.shape
444
- # Reshape the query, key, and value tensors.
603
query = query.view(-1, self.num_heads, self.head_size)
446
- key = key.view(-1, self.num_kv_heads, self.head_size)
447
- value = value.view(-1, self.num_kv_heads, self.head_size)
604
+ if key is not None:
605
+ assert value is not None
606
+ key = key.view(-1, self.num_kv_heads, self.head_size)
607
+ value = value.view(-1, self.num_kv_heads, self.head_size)
608
+ else:
609
+ assert value is None
610
449
- if kv_cache.numel() > 0:
611
+ if self.attn_type != AttentionType.ENCODER and kv_cache.numel() > 0:
612
key_cache, value_cache = PagedAttention.split_kv_cache(
613
kv_cache, self.num_kv_heads, self.head_size)
614
453
- # Reshape the input keys and values and store them in the cache.
454
- # If kv_cache is not provided, the new key and value tensors are
455
- # not cached. This happens during the initial memory profiling run.
456
- PagedAttention.write_to_paged_cache(
457
- key,
458
- value,
459
- key_cache,
460
- value_cache,
461
- attn_metadata.slot_mapping,
462
- self.kv_cache_dtype,
463
- layer._k_scale,
464
- layer._v_scale,
465
- )
466
-
467
- num_prefill_tokens = attn_metadata.num_prefill_tokens
468
- num_decode_tokens = attn_metadata.num_decode_tokens
469
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens
470
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens
615
+ if key is not None and value is not None:
616
+ # Reshape the input keys and values and store them in the
617
+ # cache. If kv_cache is not provided, the new key and value
618
+ # tensors are not cached. This happens during the initial
619
+ # memory profiling run.
620
+ PagedAttention.write_to_paged_cache(
621
+ key,
622
+ value,
623
+ key_cache,
624
+ value_cache,
625
+ attn_metadata.slot_mapping
626
+ if self.attn_type != AttentionType.ENCODER_DECODER else
627
+ attn_metadata.cross_slot_mapping,
628
+ self.kv_cache_dtype,
629
+ layer._k_scale,
630
+ layer._v_scale,
631
+ )
632
+
633
+ if self.attn_type != AttentionType.ENCODER:
634
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
635
+ else:
636
+ assert attn_metadata.num_encoder_tokens is not None
637
+ num_prefill_tokens = attn_metadata.num_encoder_tokens
638
639
output = torch.empty_like(query)
640
# Query for decode. KV is not needed because it is already cached.
641
decode_query = query[num_prefill_tokens:]
642
# QKV for prefill.
643
query = query[:num_prefill_tokens]
477
- key = key[:num_prefill_tokens]
478
- value = value[:num_prefill_tokens]
644
480
- assert query.shape[0] == num_prefill_tokens
481
- assert decode_query.shape[0] == num_decode_tokens
645
+ if key is not None and value is not None \
646
+ and self.attn_type != AttentionType.ENCODER_DECODER:
647
+ key = key[:num_prefill_tokens]
648
+ value = value[:num_prefill_tokens]
649
650
if prefill_meta := attn_metadata.prefill_metadata:
651
# Prompt run.
485
- assert prefill_meta.seq_lens is not None
652
+ # normal attention and DECODER
653
+ if self.attn_type == AttentionType.DECODER and (
654
+ kv_cache.numel() == 0 or prefill_meta.block_tables is None
655
+ or prefill_meta.block_tables.numel() == 0):
656
+ (query_seq_start_loc, query_max_seq_len, key_seq_start_loc,
657
+ key_max_seq_len, seq_lens,
658
+ causal_mask) = (prefill_meta.seq_start_loc,
659
+ prefill_meta.max_prefill_seq_len,
660
+ prefill_meta.seq_start_loc,
661
+ prefill_meta.max_prefill_seq_len,
662
+ attn_metadata.seq_lens, True)
663
+ # prefix-enabled attention and ENCODER/ENCODER_DECODER
664
+ else:
665
+ (query_seq_start_loc, query_max_seq_len, key_seq_start_loc,
666
+ key_max_seq_len, seq_lens,
667
+ causal_mask) = _get_seq_len_block_table_args(
668
+ prefill_meta, self.attn_type)
669
+ # Prompt run.
670
if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
671
# triton attention
672
# When block_tables are not filled, it means q and k are the
@@ -493,18 +677,18 @@ class ROCmFlashAttentionImpl(AttentionImpl):
677
attn_masks = _make_alibi_bias(
678
self.alibi_slopes,
679
query.dtype,
496
- attn_metadata.seq_lens,
680
+ seq_lens,
681
make_attn_mask=False) # type: ignore
682
out, _ = self.attn_func(
683
query,
684
key,
685
value,
686
None,
503
- prefill_meta.seq_start_loc,
504
- prefill_meta.seq_start_loc,
505
- prefill_meta.max_prefill_seq_len,
506
- prefill_meta.max_prefill_seq_len,
507
- True,
687
+ query_seq_start_loc,
688
+ key_seq_start_loc,
689
+ query_max_seq_len,
690
+ key_max_seq_len,
691
+ causal_mask,
692
self.scale,
693
attn_masks[0][None]
694
if attn_masks is not None else None,
@@ -528,11 +712,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
712
query,
713
key,
714
value,
531
- prefill_meta.seq_lens,
532
- num_tokens,
715
+ query_seq_start_loc,
716
+ num_prefill_tokens,
717
self.num_heads,
718
self.head_size,
719
self.scale,
720
+ causal_mask,
721
attn_masks,
722
)
723
else:
@@ -540,19 +725,23 @@ class ROCmFlashAttentionImpl(AttentionImpl):
725
q=query,
726
k=key,
727
v=value,
543
- cu_seqlens_q=prefill_meta.seq_start_loc,
544
- cu_seqlens_k=prefill_meta.seq_start_loc,
728
+ cu_seqlens_q=query_seq_start_loc,
729
+ cu_seqlens_k=key_seq_start_loc,
730
max_seqlen_q=prefill_meta.max_prefill_seq_len,
546
- max_seqlen_k=prefill_meta.max_prefill_seq_len,
731
+ max_seqlen_k=key_max_seq_len,
732
softmax_scale=self.scale,
733
causal=True,
734
window_size=self.sliding_window,
735
alibi_slopes=self.alibi_slopes,
736
+ softcap=self.logits_soft_cap,
737
)
738
739
# common code for prefill
740
assert output[:num_prefill_tokens].shape == out.shape
555
- output[:num_prefill_tokens] = out
741
+ if output.shape[0] > num_prefill_tokens:
742
+ output[:num_prefill_tokens] = out
743
+ else:
744
+ output = out
745
else:
746
# prefix-enabled attention
747
output[:num_prefill_tokens] = PagedAttention.forward_prefix(
@@ -583,7 +772,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
772
decode_query.dtype, head_size, block_size, gqa_ratio,
773
decode_meta.max_decode_seq_len)
774
if use_custom:
586
- max_seq_len = decode_meta.max_decode_seq_len
775
+ max_seq_len = (decode_meta.max_decode_seq_len if self.attn_type
776
+ != AttentionType.ENCODER_DECODER else
777
+ decode_meta.max_encoder_seq_len)
778
+ assert max_seq_len is not None
779
max_num_partitions = (
780
(max_seq_len + _PARTITION_SIZE_ROCM - 1) //
781
_PARTITION_SIZE_ROCM)
@@ -599,8 +791,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
791
device=output.device,
792
)
793
max_logits = torch.empty_like(exp_sums)
794
+ if num_prefill_tokens > 0:
795
+ out = output[num_prefill_tokens:]
796
+ else:
797
+ out = output
798
ops.paged_attention_rocm(
603
- output[num_prefill_tokens:],
799
+ out,
800
exp_sums,
801
max_logits,
802
tmp_output,
@@ -609,8 +805,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
805
value_cache,
806
self.num_kv_heads,
807
self.scale,
612
- decode_meta.block_tables,
613
- decode_meta.seq_lens_tensor,
808
+ decode_meta.block_tables
809
+ if self.attn_type != AttentionType.ENCODER_DECODER else
810
+ decode_meta.cross_block_tables,
811
+ decode_meta.seq_lens_tensor
812
+ if self.attn_type != AttentionType.ENCODER_DECODER else
813
+ decode_meta.encoder_seq_lens_tensor,
814
block_size,
815
max_seq_len,
816
self.alibi_slopes,
@@ -623,9 +823,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
823
decode_query,
824
key_cache,
825
value_cache,
626
- decode_meta.block_tables,
627
- decode_meta.seq_lens_tensor,
628
- decode_meta.max_decode_seq_len,
826
+ decode_meta.block_tables
827
+ if self.attn_type != AttentionType.ENCODER_DECODER else
828
+ decode_meta.cross_block_tables,
829
+ decode_meta.seq_lens_tensor
830
+ if self.attn_type != AttentionType.ENCODER_DECODER else
831
+ decode_meta.encoder_seq_lens_tensor,
832
+ decode_meta.max_decode_seq_len
833
+ if self.attn_type != AttentionType.ENCODER_DECODER else
834
+ decode_meta.max_encoder_seq_len,
835
self.kv_cache_dtype,
836
self.num_kv_heads,
837
self.scale,
@@ -635,7 +841,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
841
)
842
843
# Reshape the output tensor.
638
- return output.view(num_tokens, hidden_size)
844
+ return output.view(-1, self.num_heads * self.head_size)
845
846
847
def _sdpa_attention(
vllm/model_executor/models/mllama.py
+13
-3
@@ -48,7 +48,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
48
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
49
from vllm.model_executor.layers.vocab_parallel_embedding import (
50
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
51
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51
+from vllm.model_executor.model_loader.weight_utils import (
52
+ default_weight_loader, maybe_remap_kv_scale_name)
53
from vllm.model_executor.sampling_metadata import SamplingMetadata
54
from vllm.multimodal import MULTIMODAL_REGISTRY
55
from vllm.sequence import SequenceData
@@ -847,7 +848,8 @@ class MllamaTextCrossAttention(nn.Module):
848
i,
849
i,
850
)
850
- elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
851
+ elif self.attn.backend in (_Backend.XFORMERS, _Backend.ROCM_FLASH,
852
+ _Backend.TORCH_SDPA):
853
key_cache, value_cache = PagedAttention.split_kv_cache(
854
kv_cache, self.num_local_key_value_heads, self.head_dim)
855
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -859,7 +861,8 @@ class MllamaTextCrossAttention(nn.Module):
861
raise ValueError(
862
f"Unsupported Attention backend {self.attn.backend} "
863
"enum found. Expected the Attention backend to be "
862
- "FLASH_ATTN, FLASH_ATTN_VLLM_V1, XFORMERS or TORCH_SDPA.")
864
+ "FLASH_ATTN, FLASH_ATTN_VLLM_V1, "
865
+ "XFORMERS or TORCH_SDPA.")
866
867
# We have to call torch.sdpa for prefill when using a
868
# custom cross-attention mask. Because the mask is not a
@@ -1452,6 +1455,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1455
weight_loader(param, loaded_weight, shard_id)
1456
break
1457
else:
1458
+ orig_name = name
1459
+ name = maybe_remap_kv_scale_name(name, params_dict)
1460
+ if name is None:
1461
+ logger.debug("Missing name %s, orig name %s", name,
1462
+ orig_name)
1463
+ continue
1464
+
1465
param = params_dict.pop(name)
1466
weight_loader = getattr(param, "weight_loader",
1467
default_weight_loader)