@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a1fc18c030
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[ROCm][AMD][Model] llama 3.2 support upstreaming (#12421)

Signed-off-by: Aleksandr Malyshev <maleksan@amd.com> Co-authored-by: Aleksandr Malyshev <maleksan@amd.com>

Aleksandr Malyshev committed Jan 30, 2025 at 20:24 UTC a1fc18c030e4d0466f2b23cb7dd4d11ce4b85603
2 files changed +304 -88
vllm/attention/backends/rocm_flash_attn.py
+291 -85
@@ -90,6 +90,17 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
90 seq_lens: Optional[List[int]]
91 # seq_lens stored as a tensor.
92 seq_lens_tensor: Optional[torch.Tensor]
93 + # Maximum sequence length among prefill batch. 0 if there are decoding
94 + # requests only.
95 + max_prefill_seq_len: int
96 + # Maximum sequence length among decode batch. 0 if there are prefill
97 + # requests only.
98 + max_decode_seq_len: int
99 +
100 + # Whether or not if cuda graph is enabled.
101 + # Cuda-graph is currently enabled for decoding only.
102 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
103 + use_cuda_graph: bool
104
105 # NOTE(sang): Definition of context_len, query_len, and seq_len.
106 # |---------- N-1 iteration --------|
@@ -100,30 +111,18 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
111 # |-- query_len ---|
112
113 # Maximum query length in the batch. None for decoding.
103 - max_query_len: Optional[int]
104 - # Maximum sequence length among prefill batch. 0 if there are decoding
105 - # requests only.
106 - max_prefill_seq_len: int
107 - # Maximum sequence length among decode batch. 0 if there are prefill
108 - # requests only.
109 - max_decode_seq_len: int
114 + max_query_len: Optional[int] = None
115 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
116 # the batch, used to index into subquery. E.g., if the subquery length
117 # is [4, 6], it is [0, 4, 10].
113 - query_start_loc: Optional[torch.Tensor]
118 + query_start_loc: Optional[torch.Tensor] = None
119 # (batch_size + 1,). The cumulative sequence lengths of the sequences in
120 # the batch, used to index into sequence. E.g., if the sequence length is
121 # [4, 6], it is [0, 4, 10].
117 - seq_start_loc: Optional[torch.Tensor]
118 -
119 - # Whether or not if cuda graph is enabled.
120 - # Cuda-graph is currently enabled for decoding only.
121 - # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
122 - use_cuda_graph: bool
123 -
122 + seq_start_loc: Optional[torch.Tensor] = None
123 # (batch_size,) A tensor of context lengths (tokens that are computed
124 # so far).
126 - context_lens_tensor: Optional[torch.Tensor]
125 + context_lens_tensor: Optional[torch.Tensor] = None
126
127 # Max number of query tokens among request in the batch.
128 max_decode_query_len: Optional[int] = None
@@ -131,6 +130,23 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
130 _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
131 _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
132
133 + # Begin encoder attn & enc/dec cross-attn fields...
134 +
135 + # Encoder sequence lengths representation
136 + encoder_seq_lens: Optional[List[int]] = None
137 + encoder_seq_lens_tensor: Optional[torch.Tensor] = None
138 +
139 + # Maximum sequence length among encoder sequences
140 + max_encoder_seq_len: Optional[int] = None
141 +
142 + # Number of tokens input to encoder
143 + num_encoder_tokens: Optional[int] = None
144 +
145 + # Cross-attention memory-mapping data structures: slot mapping
146 + # and block tables
147 + cross_slot_mapping: Optional[torch.Tensor] = None
148 + cross_block_tables: Optional[torch.Tensor] = None
149 +
150 @property
151 def prefill_metadata(self) -> Optional["ROCmFlashAttentionMetadata"]:
152 if self.num_prefills == 0:
@@ -141,10 +157,7 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
157
158 assert self.seq_lens is not None
159 assert self.seq_lens_tensor is not None
144 - assert self.query_start_loc is not None
145 - assert self.context_lens_tensor is not None
160 assert self.block_tables is not None
147 - assert self.seq_start_loc is not None
161
162 self._cached_prefill_metadata = ROCmFlashAttentionMetadata(
163 num_prefills=self.num_prefills,
@@ -159,12 +172,20 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
172 max_query_len=self.max_query_len,
173 max_prefill_seq_len=self.max_prefill_seq_len,
174 max_decode_seq_len=0,
162 - query_start_loc=self.query_start_loc[:self.num_prefills + 1],
163 - seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
164 - context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
175 + query_start_loc=None if self.query_start_loc is None else
176 + self.query_start_loc[:self.num_prefills + 1],
177 + seq_start_loc=None if self.seq_start_loc is None else
178 + self.seq_start_loc[:self.num_prefills + 1],
179 + context_lens_tensor=None if self.context_lens_tensor is None else
180 + self.context_lens_tensor[:self.num_prefills],
181 block_tables=self.block_tables[:self.num_prefills],
182 use_cuda_graph=False,
167 - )
183 + # Begin encoder & cross attn fields below...
184 + encoder_seq_lens=self.encoder_seq_lens,
185 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
186 + max_encoder_seq_len=self.max_encoder_seq_len,
187 + cross_slot_mapping=self.cross_slot_mapping,
188 + cross_block_tables=self.cross_block_tables)
189 return self._cached_prefill_metadata
190
191 @property
@@ -194,7 +215,12 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
215 context_lens_tensor=None,
216 block_tables=self.block_tables[self.num_prefills:],
217 use_cuda_graph=self.use_cuda_graph,
197 - )
218 + # Begin encoder & cross attn fields below...
219 + encoder_seq_lens=self.encoder_seq_lens,
220 + encoder_seq_lens_tensor=self.encoder_seq_lens_tensor,
221 + max_encoder_seq_len=self.max_encoder_seq_len,
222 + cross_slot_mapping=self.cross_slot_mapping,
223 + cross_block_tables=self.cross_block_tables)
224 # Batch may be composed of prefill|decodes, adjust query start indices
225 # to refer to the start of decodes when the two are split apart.
226 # E.g. in tokens:[3 prefills|6 decodes], query_start_loc=[3,9] => [0,6].
@@ -304,6 +330,97 @@ def _make_alibi_bias(alibi_slopes: torch.Tensor,
330 return attn_biases
331
332
333 +def _get_seq_len_block_table_args(
334 + attn_metadata: ROCmFlashAttentionMetadata,
335 + attn_type: str,
336 +) -> tuple:
337 + '''
338 + The particular choice of sequence-length
339 + attributes which should be extracted from attn_metadata is dependent
340 + on the type of attention operation.
341 +
342 + Decoder attn -> select entirely decoder self-attention-related fields
343 + Encoder/decoder cross-attn -> select encoder sequence lengths
344 + Encoder attn -> select encoder sequence lengths fields
345 +
346 + Arguments:
347 +
348 + * attn_metadata: Attention metadata structure associated with attention op
349 + * attn_type: encoder attention, decoder self-attention,
350 + encoder/decoder cross-attention
351 +
352 + Returns:
353 +
354 + * Appropriate sequence-lengths tensors for query and key
355 + * Appropriate max sequence-length scalar
356 + '''
357 +
358 + partial_prefix_sum = 0
359 + if attn_type == AttentionType.ENCODER:
360 + assert attn_metadata.encoder_seq_lens is not None
361 + assert attn_metadata.encoder_seq_lens_tensor is not None
362 + query_seq_start_loc = torch.tensor(
363 + [0] + [
364 + partial_prefix_sum := partial_prefix_sum + i
365 + for i in attn_metadata.encoder_seq_lens
366 + ],
367 + device=attn_metadata.encoder_seq_lens_tensor.device,
368 + dtype=attn_metadata.encoder_seq_lens_tensor.dtype)
369 + causal_mask = False
370 +
371 + # No block tables associated with encoder attention
372 + return (query_seq_start_loc, attn_metadata.max_encoder_seq_len,
373 + query_seq_start_loc, attn_metadata.max_encoder_seq_len,
374 + attn_metadata.encoder_seq_lens, causal_mask)
375 + elif attn_type == AttentionType.DECODER:
376 + # Decoder self-attention
377 + # Choose max_seq_len based on whether we are in prompt_run
378 + assert attn_metadata.seq_lens is not None
379 + assert attn_metadata.seq_lens_tensor is not None
380 + query_seq_start_loc = torch.tensor(
381 + [0] + [
382 + partial_prefix_sum := partial_prefix_sum + i
383 + for i in attn_metadata.seq_lens
384 + ],
385 + device=attn_metadata.seq_lens_tensor.device,
386 + dtype=attn_metadata.seq_lens_tensor.dtype)
387 + max_seq_len = attn_metadata.max_prefill_seq_len
388 + causal_mask = True
389 +
390 + return (query_seq_start_loc, max_seq_len, query_seq_start_loc,
391 + max_seq_len, attn_metadata.seq_lens, causal_mask)
392 + elif attn_type == AttentionType.ENCODER_DECODER:
393 + assert attn_metadata.seq_lens is not None
394 + assert attn_metadata.encoder_seq_lens_tensor is not None
395 + query_start_loc = torch.tensor(
396 + [0] + [
397 + partial_prefix_sum := partial_prefix_sum + i
398 + for i in attn_metadata.seq_lens
399 + ],
400 + device=attn_metadata.encoder_seq_lens_tensor.device,
401 + dtype=attn_metadata.encoder_seq_lens_tensor.dtype)
402 +
403 + partial_prefix_sum = 0
404 + assert attn_metadata.encoder_seq_lens is not None
405 + assert attn_metadata.seq_lens_tensor is not None
406 + key_seq_start_loc = torch.tensor(
407 + [0] + [
408 + partial_prefix_sum := partial_prefix_sum + i
409 + for i in attn_metadata.encoder_seq_lens
410 + ],
411 + device=attn_metadata.seq_lens_tensor.device,
412 + dtype=attn_metadata.seq_lens_tensor.dtype)
413 + causal_mask = False
414 +
415 + # Enc/dec cross-attention KVs match encoder sequence length;
416 + # cross-attention utilizes special "cross" block tables
417 + return (query_start_loc, attn_metadata.max_prefill_seq_len,
418 + key_seq_start_loc, attn_metadata.max_encoder_seq_len,
419 + attn_metadata.seq_lens, causal_mask)
420 + else:
421 + raise AttributeError(f"Invalid attention type {str(attn_type)}")
422 +
423 +
424 class ROCmFlashAttentionImpl(AttentionImpl):
425 """
426 If the input tensors contain prompt tokens, the layout is as follows:
@@ -346,10 +463,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
463 if blocksparse_params is not None:
464 raise ValueError(
465 "ROCmFlashAttention does not support blocksparse attention.")
349 - if logits_soft_cap is not None:
350 - raise ValueError(
351 - "ROCmFlashAttention does not support attention logits soft "
352 - "capping.")
466 +
467 + if logits_soft_cap is None:
468 + # In flash-attn, setting logits_soft_cap as 0 means no soft cap.
469 + self.logits_soft_cap = 0.0
470 + else:
471 + self.logits_soft_cap = logits_soft_cap
472 + self.attn_type = attn_type
473 self.num_heads = num_heads
474 self.head_size = head_size
475 self.scale = float(scale)
@@ -374,6 +494,14 @@ class ROCmFlashAttentionImpl(AttentionImpl):
494 # NOTE: Allow for switching between Triton and CK. Defaulting to triton.
495 self.use_triton_flash_attn = envs.VLLM_USE_TRITON_FLASH_ATTN
496 if self.use_triton_flash_attn:
497 + if logits_soft_cap is not None:
498 + raise ValueError(
499 + "ROCm Triton FlashAttention does not support attention"
500 + "logits soft capping."
501 + " please try using the ROCm CK "
502 + "FA backend instead by setting the env var "
503 + "`VLLM_USE_TRITON_FLASH_ATTN=0`")
504 +
505 from vllm.attention.ops.triton_flash_attention import ( # noqa: F401
506 triton_attention)
507 self.attn_func = triton_attention
@@ -398,14 +526,13 @@ class ROCmFlashAttentionImpl(AttentionImpl):
526 self.use_naive_attn = True
527
528 if self.use_naive_attn:
401 - self.attn_func = _sdpa_attention
402 - logger.debug("Using naive attention in ROCmBackend")
529 + if logits_soft_cap is not None:
530 + raise ValueError(
531 + "ROCm Naive FlashAttention does not support"
532 + "attention logits soft capping.")
533
404 - if attn_type != AttentionType.DECODER:
405 - raise NotImplementedError("Encoder self-attention and "
406 - "encoder/decoder cross-attention "
407 - "are not implemented for "
408 - "ROCmFlashAttentionImpl")
534 + self.attn_func = _sdpa_attention
535 + logger.debug("Using naive (SDPA) attention in ROCmBackend")
536
537 def repeat_kv(self, x: torch.Tensor, n_rep: int) -> torch.Tensor:
538 """torch.repeat_interleave(x, dim=1, repeats=n_rep)"""
@@ -427,6 +554,37 @@ class ROCmFlashAttentionImpl(AttentionImpl):
554 ) -> torch.Tensor:
555 """Forward pass with FlashAttention and PagedAttention.
556
557 + For decoder-only models: query, key and value must be non-None.
558 +
559 + For encoder/decoder models:
560 + * ROCmFlashAttentionImpl.forward() may be invoked for both self- and
561 + cross-attention layers.
562 + * For self-attention: query, key and value must be non-None.
563 + * For cross-attention:
564 + * Query must be non-None
565 + * During prefill, key and value must be non-None; key and value
566 + get cached for use during decode.
567 + * During decode, key and value may be None, since:
568 + (1) key and value tensors were cached during prefill, and
569 + (2) cross-attention key and value tensors do not grow during
570 + decode
571 +
572 + A note on how the attn_type (attention type enum) argument impacts
573 + attention forward() behavior:
574 +
575 + * DECODER: normal decoder-only behavior;
576 + use decoder self-attention block table
577 + * ENCODER: no KV caching; pass encoder sequence
578 + attributes (encoder_seq_lens/encoder_seq_lens_tensor/
579 + max_encoder_seq_len) to kernel, in lieu of decoder
580 + sequence attributes (seq_lens/seq_lens_tensor/max_seq_len)
581 + * ENCODER_DECODER: cross-attention behavior;
582 + use cross-attention block table for caching KVs derived
583 + from encoder hidden states; since KV sequence lengths
584 + will match encoder sequence lengths, pass encoder sequence
585 + attributes to kernel (encoder_seq_lens/encoder_seq_lens_tensor/
586 + max_encoder_seq_len)
587 +
588 Args:
589 query: shape = [num_tokens, num_heads * head_size]
590 key: shape = [num_tokens, num_kv_heads * head_size]
@@ -435,54 +593,80 @@ class ROCmFlashAttentionImpl(AttentionImpl):
593 NOTE: kv_cache will be an empty tensor with shape [0]
594 for profiling run.
595 attn_metadata: Metadata for attention.
596 + attn_type: Select attention type, between encoder attention,
597 + decoder self-attention, or encoder/decoder cross-
598 + attention. Defaults to decoder self-attention,
599 + which is the vLLM default generally
600 Returns:
601 shape = [num_tokens, num_heads * head_size]
602 """
441 - # Reminder: Please update docs/source/features/compatibility_matrix.md
442 - # If the feature combo become valid
443 - num_tokens, hidden_size = query.shape
444 - # Reshape the query, key, and value tensors.
603 query = query.view(-1, self.num_heads, self.head_size)
446 - key = key.view(-1, self.num_kv_heads, self.head_size)
447 - value = value.view(-1, self.num_kv_heads, self.head_size)
604 + if key is not None:
605 + assert value is not None
606 + key = key.view(-1, self.num_kv_heads, self.head_size)
607 + value = value.view(-1, self.num_kv_heads, self.head_size)
608 + else:
609 + assert value is None
610
449 - if kv_cache.numel() > 0:
611 + if self.attn_type != AttentionType.ENCODER and kv_cache.numel() > 0:
612 key_cache, value_cache = PagedAttention.split_kv_cache(
613 kv_cache, self.num_kv_heads, self.head_size)
614
453 - # Reshape the input keys and values and store them in the cache.
454 - # If kv_cache is not provided, the new key and value tensors are
455 - # not cached. This happens during the initial memory profiling run.
456 - PagedAttention.write_to_paged_cache(
457 - key,
458 - value,
459 - key_cache,
460 - value_cache,
461 - attn_metadata.slot_mapping,
462 - self.kv_cache_dtype,
463 - layer._k_scale,
464 - layer._v_scale,
465 - )
466 -
467 - num_prefill_tokens = attn_metadata.num_prefill_tokens
468 - num_decode_tokens = attn_metadata.num_decode_tokens
469 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens
470 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens
615 + if key is not None and value is not None:
616 + # Reshape the input keys and values and store them in the
617 + # cache. If kv_cache is not provided, the new key and value
618 + # tensors are not cached. This happens during the initial
619 + # memory profiling run.
620 + PagedAttention.write_to_paged_cache(
621 + key,
622 + value,
623 + key_cache,
624 + value_cache,
625 + attn_metadata.slot_mapping
626 + if self.attn_type != AttentionType.ENCODER_DECODER else
627 + attn_metadata.cross_slot_mapping,
628 + self.kv_cache_dtype,
629 + layer._k_scale,
630 + layer._v_scale,
631 + )
632 +
633 + if self.attn_type != AttentionType.ENCODER:
634 + num_prefill_tokens = attn_metadata.num_prefill_tokens
635 + else:
636 + assert attn_metadata.num_encoder_tokens is not None
637 + num_prefill_tokens = attn_metadata.num_encoder_tokens
638
639 output = torch.empty_like(query)
640 # Query for decode. KV is not needed because it is already cached.
641 decode_query = query[num_prefill_tokens:]
642 # QKV for prefill.
643 query = query[:num_prefill_tokens]
477 - key = key[:num_prefill_tokens]
478 - value = value[:num_prefill_tokens]
644
480 - assert query.shape[0] == num_prefill_tokens
481 - assert decode_query.shape[0] == num_decode_tokens
645 + if key is not None and value is not None \
646 + and self.attn_type != AttentionType.ENCODER_DECODER:
647 + key = key[:num_prefill_tokens]
648 + value = value[:num_prefill_tokens]
649
650 if prefill_meta := attn_metadata.prefill_metadata:
651 # Prompt run.
485 - assert prefill_meta.seq_lens is not None
652 + # normal attention and DECODER
653 + if self.attn_type == AttentionType.DECODER and (
654 + kv_cache.numel() == 0 or prefill_meta.block_tables is None
655 + or prefill_meta.block_tables.numel() == 0):
656 + (query_seq_start_loc, query_max_seq_len, key_seq_start_loc,
657 + key_max_seq_len, seq_lens,
658 + causal_mask) = (prefill_meta.seq_start_loc,
659 + prefill_meta.max_prefill_seq_len,
660 + prefill_meta.seq_start_loc,
661 + prefill_meta.max_prefill_seq_len,
662 + attn_metadata.seq_lens, True)
663 + # prefix-enabled attention and ENCODER/ENCODER_DECODER
664 + else:
665 + (query_seq_start_loc, query_max_seq_len, key_seq_start_loc,
666 + key_max_seq_len, seq_lens,
667 + causal_mask) = _get_seq_len_block_table_args(
668 + prefill_meta, self.attn_type)
669 + # Prompt run.
670 if kv_cache.numel() == 0 or prefill_meta.block_tables.numel() == 0:
671 # triton attention
672 # When block_tables are not filled, it means q and k are the
@@ -493,18 +677,18 @@ class ROCmFlashAttentionImpl(AttentionImpl):
677 attn_masks = _make_alibi_bias(
678 self.alibi_slopes,
679 query.dtype,
496 - attn_metadata.seq_lens,
680 + seq_lens,
681 make_attn_mask=False) # type: ignore
682 out, _ = self.attn_func(
683 query,
684 key,
685 value,
686 None,
503 - prefill_meta.seq_start_loc,
504 - prefill_meta.seq_start_loc,
505 - prefill_meta.max_prefill_seq_len,
506 - prefill_meta.max_prefill_seq_len,
507 - True,
687 + query_seq_start_loc,
688 + key_seq_start_loc,
689 + query_max_seq_len,
690 + key_max_seq_len,
691 + causal_mask,
692 self.scale,
693 attn_masks[0][None]
694 if attn_masks is not None else None,
@@ -528,11 +712,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
712 query,
713 key,
714 value,
531 - prefill_meta.seq_lens,
532 - num_tokens,
715 + query_seq_start_loc,
716 + num_prefill_tokens,
717 self.num_heads,
718 self.head_size,
719 self.scale,
720 + causal_mask,
721 attn_masks,
722 )
723 else:
@@ -540,19 +725,23 @@ class ROCmFlashAttentionImpl(AttentionImpl):
725 q=query,
726 k=key,
727 v=value,
543 - cu_seqlens_q=prefill_meta.seq_start_loc,
544 - cu_seqlens_k=prefill_meta.seq_start_loc,
728 + cu_seqlens_q=query_seq_start_loc,
729 + cu_seqlens_k=key_seq_start_loc,
730 max_seqlen_q=prefill_meta.max_prefill_seq_len,
546 - max_seqlen_k=prefill_meta.max_prefill_seq_len,
731 + max_seqlen_k=key_max_seq_len,
732 softmax_scale=self.scale,
733 causal=True,
734 window_size=self.sliding_window,
735 alibi_slopes=self.alibi_slopes,
736 + softcap=self.logits_soft_cap,
737 )
738
739 # common code for prefill
740 assert output[:num_prefill_tokens].shape == out.shape
555 - output[:num_prefill_tokens] = out
741 + if output.shape[0] > num_prefill_tokens:
742 + output[:num_prefill_tokens] = out
743 + else:
744 + output = out
745 else:
746 # prefix-enabled attention
747 output[:num_prefill_tokens] = PagedAttention.forward_prefix(
@@ -583,7 +772,10 @@ class ROCmFlashAttentionImpl(AttentionImpl):
772 decode_query.dtype, head_size, block_size, gqa_ratio,
773 decode_meta.max_decode_seq_len)
774 if use_custom:
586 - max_seq_len = decode_meta.max_decode_seq_len
775 + max_seq_len = (decode_meta.max_decode_seq_len if self.attn_type
776 + != AttentionType.ENCODER_DECODER else
777 + decode_meta.max_encoder_seq_len)
778 + assert max_seq_len is not None
779 max_num_partitions = (
780 (max_seq_len + _PARTITION_SIZE_ROCM - 1) //
781 _PARTITION_SIZE_ROCM)
@@ -599,8 +791,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
791 device=output.device,
792 )
793 max_logits = torch.empty_like(exp_sums)
794 + if num_prefill_tokens > 0:
795 + out = output[num_prefill_tokens:]
796 + else:
797 + out = output
798 ops.paged_attention_rocm(
603 - output[num_prefill_tokens:],
799 + out,
800 exp_sums,
801 max_logits,
802 tmp_output,
@@ -609,8 +805,12 @@ class ROCmFlashAttentionImpl(AttentionImpl):
805 value_cache,
806 self.num_kv_heads,
807 self.scale,
612 - decode_meta.block_tables,
613 - decode_meta.seq_lens_tensor,
808 + decode_meta.block_tables
809 + if self.attn_type != AttentionType.ENCODER_DECODER else
810 + decode_meta.cross_block_tables,
811 + decode_meta.seq_lens_tensor
812 + if self.attn_type != AttentionType.ENCODER_DECODER else
813 + decode_meta.encoder_seq_lens_tensor,
814 block_size,
815 max_seq_len,
816 self.alibi_slopes,
@@ -623,9 +823,15 @@ class ROCmFlashAttentionImpl(AttentionImpl):
823 decode_query,
824 key_cache,
825 value_cache,
626 - decode_meta.block_tables,
627 - decode_meta.seq_lens_tensor,
628 - decode_meta.max_decode_seq_len,
826 + decode_meta.block_tables
827 + if self.attn_type != AttentionType.ENCODER_DECODER else
828 + decode_meta.cross_block_tables,
829 + decode_meta.seq_lens_tensor
830 + if self.attn_type != AttentionType.ENCODER_DECODER else
831 + decode_meta.encoder_seq_lens_tensor,
832 + decode_meta.max_decode_seq_len
833 + if self.attn_type != AttentionType.ENCODER_DECODER else
834 + decode_meta.max_encoder_seq_len,
835 self.kv_cache_dtype,
836 self.num_kv_heads,
837 self.scale,
@@ -635,7 +841,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
841 )
842
843 # Reshape the output tensor.
638 - return output.view(num_tokens, hidden_size)
844 + return output.view(-1, self.num_heads * self.head_size)
845
846
847 def _sdpa_attention(
vllm/model_executor/models/mllama.py
+13 -3
@@ -48,7 +48,8 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
48 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
49 from vllm.model_executor.layers.vocab_parallel_embedding import (
50 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
51 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
51 +from vllm.model_executor.model_loader.weight_utils import (
52 + default_weight_loader, maybe_remap_kv_scale_name)
53 from vllm.model_executor.sampling_metadata import SamplingMetadata
54 from vllm.multimodal import MULTIMODAL_REGISTRY
55 from vllm.sequence import SequenceData
@@ -847,7 +848,8 @@ class MllamaTextCrossAttention(nn.Module):
848 i,
849 i,
850 )
850 - elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
851 + elif self.attn.backend in (_Backend.XFORMERS, _Backend.ROCM_FLASH,
852 + _Backend.TORCH_SDPA):
853 key_cache, value_cache = PagedAttention.split_kv_cache(
854 kv_cache, self.num_local_key_value_heads, self.head_dim)
855 cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -859,7 +861,8 @@ class MllamaTextCrossAttention(nn.Module):
861 raise ValueError(
862 f"Unsupported Attention backend {self.attn.backend} "
863 "enum found. Expected the Attention backend to be "
862 - "FLASH_ATTN, FLASH_ATTN_VLLM_V1, XFORMERS or TORCH_SDPA.")
864 + "FLASH_ATTN, FLASH_ATTN_VLLM_V1, "
865 + "XFORMERS or TORCH_SDPA.")
866
867 # We have to call torch.sdpa for prefill when using a
868 # custom cross-attention mask. Because the mask is not a
@@ -1452,6 +1455,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1455 weight_loader(param, loaded_weight, shard_id)
1456 break
1457 else:
1458 + orig_name = name
1459 + name = maybe_remap_kv_scale_name(name, params_dict)
1460 + if name is None:
1461 + logger.debug("Missing name %s, orig name %s", name,
1462 + orig_name)
1463 + continue
1464 +
1465 param = params_dict.pop(name)
1466 weight_loader = getattr(param, "weight_loader",
1467 default_weight_loader)