@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eebad39f26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[torch.compile] support all attention backends (#10558)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Nov 22, 2024 at 14:04 UTC eebad39f265606cfe35af4d1e0bea678516648a3
77 files changed +879 -651
tests/kernels/test_encoder_decoder_attn.py
+26 -11
@@ -18,8 +18,10 @@ from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
18 from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19 from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20 global_force_attn_backend_context_manager)
21 +from vllm.config import VllmConfig
22 from vllm.forward_context import set_forward_context
23 from vllm.platforms import current_platform
24 +from vllm.plugins import set_current_vllm_config
25
26 # List of support backends for encoder/decoder models
27 LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
@@ -594,6 +596,7 @@ def _run_encoder_attention_test(
596 encoder_test_params: PhaseTestParameters,
597 attn_metadata: AttentionMetadata,
598 test_pt: TestPoint,
599 + vllm_config: VllmConfig,
600 ) -> torch.Tensor:
601 '''
602 Run encoder attention.
@@ -623,7 +626,7 @@ def _run_encoder_attention_test(
626 attn_type = AttentionType.ENCODER
627 packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
628 assert packed_qkv is not None
626 - with set_forward_context(attn_metadata):
629 + with set_forward_context(attn_metadata, vllm_config):
630 # In the test setup the shape of the query is
631 # [batch_size, seq_len, num_heads, head_size]. However
632 # the attention backend expect the shape to be
@@ -648,6 +651,7 @@ def _run_decoder_self_attention_test(
651 decoder_test_params: PhaseTestParameters,
652 attn_metadata: AttentionMetadata,
653 test_pt: TestPoint,
654 + vllm_config: VllmConfig,
655 ) -> torch.Tensor:
656 '''
657 Run decoder self-attention test.
@@ -677,7 +681,7 @@ def _run_decoder_self_attention_test(
681 kv_cache = test_rsrcs.kv_cache
682 packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
683 assert packed_qkv is not None
680 - with set_forward_context(attn_metadata):
684 + with set_forward_context(attn_metadata, vllm_config):
685 # In the test setup the shape of the query is
686 # [batch_size, seq_len, num_heads, head_size]. However
687 # the attention backend expect the shape to be
@@ -701,6 +705,7 @@ def _run_encoder_decoder_cross_attention_test(
705 cross_test_params: Optional[PhaseTestParameters],
706 attn_metadata: AttentionMetadata,
707 test_pt: TestPoint,
708 + vllm_config: VllmConfig,
709 ) -> torch.Tensor:
710 '''
711 Run encoder/decoder cross-attention test.
@@ -748,7 +753,7 @@ def _run_encoder_decoder_cross_attention_test(
753 cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
754 key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
755 value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
751 - with set_forward_context(attn_metadata):
756 + with set_forward_context(attn_metadata, vllm_config):
757 # In the test setup the shape of the query is
758 # [batch_size, seq_len, num_heads, head_size]. However
759 # the attention backend expect the shape to be
@@ -839,7 +844,9 @@ def test_encoder_only(
844
845 # Attention scale factor, attention backend instance, attention wrapper
846 # instance, KV cache init
842 - test_rsrcs = _make_test_resources(test_pt)
847 + vllm_config = VllmConfig()
848 + with set_current_vllm_config(vllm_config):
849 + test_rsrcs = _make_test_resources(test_pt)
850
851 # Construct encoder attention test params (only used
852 # during prefill)
@@ -863,7 +870,8 @@ def test_encoder_only(
870 test_rsrcs.attn,
871 enc_test_params,
872 prephase_attn_metadata,
866 - test_pt=test_pt))
873 + test_pt=test_pt,
874 + vllm_config=vllm_config))
875
876 # - Is encoder attention result correct?
877 assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
@@ -960,7 +968,9 @@ def test_e2e_enc_dec_attn(
968
969 # Attention scale factor, attention backend instance, attention wrapper
970 # instance, KV cache init
963 - test_rsrcs = _make_test_resources(test_pt)
971 + vllm_config = VllmConfig()
972 + with set_current_vllm_config(vllm_config):
973 + test_rsrcs = _make_test_resources(test_pt)
974
975 # Construct encoder attention test params (only used
976 # during prefill)
@@ -1011,7 +1021,8 @@ def test_e2e_enc_dec_attn(
1021 enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1022 enc_test_params,
1023 prephase_attn_metadata,
1014 - test_pt=test_pt)
1024 + test_pt=test_pt,
1025 + vllm_config=vllm_config)
1026
1027 # - Is encoder attention result correct?
1028 assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
@@ -1023,7 +1034,8 @@ def test_e2e_enc_dec_attn(
1034 test_rsrcs,
1035 prephase_dec_test_params,
1036 prephase_attn_metadata,
1026 - test_pt=test_pt)
1037 + test_pt=test_pt,
1038 + vllm_config=vllm_config)
1039
1040 # - Is prefill decoder self-attention correct?
1041 assert_actual_matches_ideal(prephase_dec_test_params,
@@ -1037,7 +1049,8 @@ def test_e2e_enc_dec_attn(
1049 prephase_dec_test_params,
1050 prephase_cross_test_params,
1051 prephase_attn_metadata,
1040 - test_pt=test_pt)
1052 + test_pt=test_pt,
1053 + vllm_config=vllm_config)
1054
1055 # - Is prefill encoder/decoder cross-attention correct?
1056 assert_actual_matches_ideal(prephase_cross_test_params,
@@ -1061,7 +1074,8 @@ def test_e2e_enc_dec_attn(
1074 test_rsrcs,
1075 decphase_dec_test_params,
1076 decphase_attn_metadata,
1064 - test_pt=test_pt)
1077 + test_pt=test_pt,
1078 + vllm_config=vllm_config)
1079
1080 # - Is decode-phase decoder self-attention correct?
1081 assert_actual_matches_ideal(decphase_dec_test_params,
@@ -1075,7 +1089,8 @@ def test_e2e_enc_dec_attn(
1089 decphase_dec_test_params,
1090 None,
1091 decphase_attn_metadata,
1078 - test_pt=test_pt)
1092 + test_pt=test_pt,
1093 + vllm_config=vllm_config)
1094
1095 # - Is decode-phase encoder/decoder cross-attention correct?
1096 assert_actual_matches_ideal(decphase_cross_test_params,
vllm/attention/backends/abstract.py
+14 -9
@@ -1,7 +1,6 @@
1 from abc import ABC, abstractmethod
2 from contextlib import contextmanager
3 from dataclasses import dataclass, fields
4 -from enum import Enum, auto
4 from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
5 Tuple, Type, TypeVar)
6
@@ -15,13 +14,19 @@ if TYPE_CHECKING:
14 ModelRunnerInputBuilderBase)
15
16
18 -class AttentionType(Enum):
19 - DECODER = auto() # Decoder attention between previous layer Q/K/V
20 - ENCODER = auto(
21 - ) # Encoder attention between previous layer Q/K/V for encoder-decoder
22 - ENCODER_ONLY = auto() # Encoder attention between previous layer Q/K/V
23 - ENCODER_DECODER = auto(
24 - ) # Attention between dec. Q and enc. K/V for encoder-decoder
17 +class AttentionType:
18 + """
19 + Attention type.
20 + Use string to be compatible with `torch.compile`.
21 + """
22 + # Decoder attention between previous layer Q/K/V
23 + DECODER = "decoder"
24 + # Encoder attention between previous layer Q/K/V for encoder-decoder
25 + ENCODER = "encoder"
26 + # Encoder attention between previous layer Q/K/V
27 + ENCODER_ONLY = "encoder_only"
28 + # Attention between dec. Q and enc. K/V for encoder-decoder
29 + ENCODER_DECODER = "encoder_decoder"
30
31
32 class AttentionBackend(ABC):
@@ -241,6 +246,6 @@ class AttentionImpl(ABC, Generic[T]):
246 attn_metadata: T,
247 k_scale: float = 1.0,
248 v_scale: float = 1.0,
244 - attn_type: AttentionType = AttentionType.DECODER,
249 + attn_type: str = AttentionType.DECODER,
250 ) -> torch.Tensor:
251 raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+1 -1
@@ -354,7 +354,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
354 attn_metadata: BlocksparseFlashAttentionMetadata,
355 k_scale: float = 1.0,
356 v_scale: float = 1.0,
357 - attn_type: AttentionType = AttentionType.DECODER,
357 + attn_type: str = AttentionType.DECODER,
358 ) -> torch.Tensor:
359 """Forward pass with FlashAttention and PagedAttention.
360
vllm/attention/backends/flash_attn.py
+172 -240
@@ -16,10 +16,8 @@ from vllm.attention.backends.utils import (
16 compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
17 get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
18 is_all_encoder_attn_metadata_set, is_block_tables_empty)
19 -from vllm.forward_context import get_forward_context
19 from vllm.multimodal import MultiModalPlaceholderMap
21 -from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
22 - make_tensor_with_pad)
20 +from vllm.utils import async_tensor_h2d, make_tensor_with_pad
21
22 if TYPE_CHECKING:
23 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -639,7 +637,7 @@ class FlashAttentionImpl(AttentionImpl):
637 attn_metadata: FlashAttentionMetadata,
638 k_scale: float = 1.0,
639 v_scale: float = 1.0,
642 - attn_type: AttentionType = AttentionType.DECODER,
640 + attn_type: str = AttentionType.DECODER,
641 ) -> torch.Tensor:
642 """Forward pass with FlashAttention.
643
@@ -668,23 +666,174 @@ class FlashAttentionImpl(AttentionImpl):
666 "requires setting cross-attention "
667 "metadata attributes.")
668
671 - output = torch.ops.vllm.unified_flash_attention(
672 - query,
673 - key,
674 - value,
675 - self.num_heads,
676 - self.head_size,
677 - self.num_kv_heads,
678 - kv_cache,
679 - self.kv_cache_dtype,
680 - k_scale,
681 - v_scale,
682 - self.scale,
683 - attn_type.value,
684 - self.sliding_window,
685 - self.alibi_slopes,
686 - self.logits_soft_cap,
687 - )
669 + num_heads: int = self.num_heads
670 + head_size: int = self.head_size
671 + num_kv_heads: int = self.num_kv_heads
672 + kv_cache_dtype: str = self.kv_cache_dtype
673 + softmax_scale: float = self.scale
674 + window_size = self.sliding_window
675 + alibi_slopes: Optional[torch.Tensor] = self.alibi_slopes
676 + logits_soft_cap: Optional[float] = self.logits_soft_cap
677 +
678 + num_tokens, hidden_size = query.shape
679 +
680 + # Reshape the query, key, and value tensors.
681 + query = query.view(-1, num_heads, head_size)
682 + if (key is not None) and (value is not None):
683 + key = key.view(-1, num_kv_heads, head_size)
684 + value = value.view(-1, num_kv_heads, head_size)
685 +
686 + if kv_cache.numel() > 0:
687 + key_cache = kv_cache[0]
688 + value_cache = kv_cache[1]
689 + # We skip updating the KV cache under two conditions:
690 + # a. When the Attention Type is ENCODER. In this phase, we compute
691 + # only the encoder attention without updating the cache.
692 + # b. When both Key and Value are None. This occurs during
693 + # cross-attention computation in the decoding phase, where the
694 + # KV cache is already populated with the cross-attention
695 + # tensor. Thus, we skip cache updates during this time.
696 + if (attn_type != AttentionType.ENCODER) and (key is not None) and (
697 + value is not None):
698 + if attn_type == AttentionType.ENCODER_DECODER:
699 + # Update cross-attention KV cache (prefill-only)
700 + updated_slot_mapping = attn_metadata.cross_slot_mapping
701 + else:
702 + # Update self-attention KV cache (prefill/decode)
703 + updated_slot_mapping = attn_metadata.slot_mapping
704 +
705 + # Reshape the input keys and values and store them in the cache.
706 + # If kv_cache is not provided, the new key and value tensors are
707 + # not cached. This happens during the initial memory
708 + # profiling run.
709 + torch.ops._C_cache_ops.reshape_and_cache_flash(
710 + key,
711 + value,
712 + kv_cache[0],
713 + kv_cache[1],
714 + updated_slot_mapping.flatten(), # type: ignore[union-attr]
715 + kv_cache_dtype,
716 + k_scale,
717 + v_scale,
718 + )
719 +
720 + (num_prefill_query_tokens, num_prefill_kv_tokens,
721 + num_decode_query_tokens) = \
722 + get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
723 + decode_query = query[num_prefill_query_tokens:]
724 + # QKV for prefill.
725 + query = query[:num_prefill_query_tokens]
726 + assert query.shape[0] == num_prefill_query_tokens
727 + assert decode_query.shape[0] == num_decode_query_tokens
728 +
729 + prefill_output: Optional[torch.Tensor] = None
730 + decode_output: Optional[torch.Tensor] = None
731 + if prefill_meta := attn_metadata.prefill_metadata:
732 + # Prompt run.
733 + if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
734 + or prefill_meta.block_tables.numel() == 0):
735 + # normal attention
736 + # When block_tables are not filled, it means q and k are the
737 + # prompt, and they have the same length.
738 + q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
739 + _get_query_key_seq_metadata(prefill_meta, True, attn_type)
740 +
741 + key = key[:num_prefill_kv_tokens]
742 + value = value[:num_prefill_kv_tokens]
743 +
744 + prefill_output = flash_attn_varlen_func(
745 + q=query,
746 + k=key,
747 + v=value,
748 + cu_seqlens_q=q_seq_start_loc,
749 + cu_seqlens_k=k_seq_start_loc,
750 + max_seqlen_q=q_seq_len,
751 + max_seqlen_k=k_seq_len,
752 + softmax_scale=softmax_scale,
753 + causal=_get_causal_option(attn_type),
754 + window_size=window_size,
755 + alibi_slopes=alibi_slopes,
756 + softcap=logits_soft_cap,
757 + )
758 + else:
759 + # prefix-enabled attention
760 + assert attn_type == AttentionType.DECODER, (
761 + "Only decoder-only models support prefix caching")
762 + assert prefill_meta.seq_lens is not None
763 + max_seq_len = max(prefill_meta.seq_lens)
764 + prefill_output = flash_attn_varlen_func( # noqa
765 + q=query,
766 + k=key_cache,
767 + v=value_cache,
768 + cu_seqlens_q=prefill_meta.query_start_loc,
769 + max_seqlen_q=prefill_meta.max_query_len,
770 + cu_seqlens_k=prefill_meta.seq_start_loc,
771 + max_seqlen_k=max_seq_len,
772 + softmax_scale=softmax_scale,
773 + causal=True,
774 + window_size=window_size,
775 + alibi_slopes=alibi_slopes,
776 + block_table=prefill_meta.block_tables,
777 + softcap=logits_soft_cap,
778 + )
779 +
780 + if decode_meta := attn_metadata.decode_metadata:
781 + # Decoding run.
782 + # Use flash_attn_varlen_func kernel for speculative decoding
783 + # because different queries might have different lengths.
784 +
785 + assert decode_meta.max_decode_query_len is not None
786 + # use only for actual varlen decoding
787 + if decode_meta.max_decode_query_len > 1:
788 + assert attn_type == AttentionType.DECODER, (
789 + "Only decoder-only models support max_decode_query_len > 1"
790 + )
791 + decode_output = flash_attn_varlen_func(
792 + q=decode_query,
793 + k=key_cache,
794 + v=value_cache,
795 + cu_seqlens_q=decode_meta.query_start_loc,
796 + max_seqlen_q=decode_meta.max_decode_query_len,
797 + cu_seqlens_k=decode_meta.seq_start_loc,
798 + max_seqlen_k=decode_meta.max_decode_seq_len,
799 + softmax_scale=softmax_scale,
800 + causal=True,
801 + window_size=window_size,
802 + alibi_slopes=alibi_slopes,
803 + softcap=logits_soft_cap,
804 + block_table=decode_meta.block_tables,
805 + )
806 + else:
807 + # Use flash_attn_with_kvcache for normal decoding.
808 + (
809 + seq_lens_arg,
810 + _,
811 + block_tables_arg,
812 + ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
813 + decode_output = flash_attn_with_kvcache(
814 + q=decode_query.unsqueeze(1),
815 + k_cache=key_cache,
816 + v_cache=value_cache,
817 + block_table=block_tables_arg,
818 + cache_seqlens=seq_lens_arg,
819 + softmax_scale=softmax_scale,
820 + causal=True,
821 + window_size=window_size,
822 + alibi_slopes=alibi_slopes,
823 + softcap=logits_soft_cap,
824 + ).squeeze(1)
825 +
826 + if prefill_output is None:
827 + assert decode_output is not None
828 + return decode_output.view(num_decode_query_tokens, hidden_size)
829 + if decode_output is None:
830 + assert prefill_output is not None
831 + return prefill_output.view(num_prefill_query_tokens, hidden_size)
832 +
833 + assert decode_meta is not None
834 + decode_output = decode_output.squeeze(1)
835 + output = torch.cat([prefill_output, decode_output], dim=0)
836 + return output.view(num_tokens, hidden_size)
837
838 return output
839
@@ -692,7 +841,7 @@ class FlashAttentionImpl(AttentionImpl):
841 def _get_query_key_seq_metadata(
842 attn_metadata,
843 is_prompt: bool,
695 - attn_type: AttentionType,
844 + attn_type: str,
845 ) -> tuple:
846 """
847 Returns sequence metadata for key and query based on the specified
@@ -754,7 +903,7 @@ def _get_query_key_seq_metadata(
903 raise AttributeError(f"Invalid attention type {str(attn_type)}")
904
905
757 -def _get_causal_option(attn_type: AttentionType) -> bool:
906 +def _get_causal_option(attn_type: str) -> bool:
907 """
908 Determine whether the given attention type is suitable for causal
909 attention mechanisms.
@@ -770,220 +919,3 @@ def _get_causal_option(attn_type: AttentionType) -> bool:
919 return not (attn_type == AttentionType.ENCODER
920 or attn_type == AttentionType.ENCODER_ONLY
921 or attn_type == AttentionType.ENCODER_DECODER)
773 -
774 -
775 -def unified_flash_attention(
776 - query: torch.Tensor,
777 - key: torch.Tensor,
778 - value: torch.Tensor,
779 - num_heads: int,
780 - head_size: int,
781 - num_kv_heads: int,
782 - kv_cache: torch.Tensor,
783 - kv_cache_dtype: str,
784 - k_scale: float,
785 - v_scale: float,
786 - softmax_scale: float,
787 - attn_type_int_val: int,
788 - window_size: Optional[List[int]] = None,
789 - alibi_slopes: Optional[torch.Tensor] = None,
790 - logits_soft_cap: Optional[float] = None,
791 -) -> torch.Tensor:
792 -
793 - # Convert integer attn_type to enum
794 - try:
795 - attn_type = AttentionType(attn_type_int_val)
796 - except ValueError as err:
797 - raise AttributeError(
798 - f"Invalid attention type {str(attn_type_int_val)}") from err
799 -
800 - current_metadata = get_forward_context()
801 - assert current_metadata is not None
802 - assert isinstance(current_metadata, FlashAttentionMetadata)
803 - attn_metadata: FlashAttentionMetadata = current_metadata
804 -
805 - num_tokens, hidden_size = query.shape
806 -
807 - # Reshape the query, key, and value tensors.
808 - query = query.view(-1, num_heads, head_size)
809 - if (key is not None) and (value is not None):
810 - key = key.view(-1, num_kv_heads, head_size)
811 - value = value.view(-1, num_kv_heads, head_size)
812 -
813 - if kv_cache.numel() > 0:
814 - key_cache = kv_cache[0]
815 - value_cache = kv_cache[1]
816 - # We skip updating the KV cache under two conditions:
817 - # a. When the Attention Type is ENCODER. In this phase, we compute
818 - # only the encoder attention without updating the cache.
819 - # b. When both Key and Value are None. This occurs during
820 - # cross-attention computation in the decoding phase, where the KV
821 - # cache is already populated with the cross-attention tensor.
822 - # Thus, we skip cache updates during this time.
823 - if (attn_type != AttentionType.ENCODER) and (key is not None) and (
824 - value is not None):
825 - if attn_type == AttentionType.ENCODER_DECODER:
826 - # Update cross-attention KV cache (prefill-only)
827 - updated_slot_mapping = attn_metadata.cross_slot_mapping
828 - else:
829 - # Update self-attention KV cache (prefill/decode)
830 - updated_slot_mapping = attn_metadata.slot_mapping
831 -
832 - # Reshape the input keys and values and store them in the cache.
833 - # If kv_cache is not provided, the new key and value tensors are
834 - # not cached. This happens during the initial memory profiling run.
835 - torch.ops._C_cache_ops.reshape_and_cache_flash(
836 - key,
837 - value,
838 - kv_cache[0],
839 - kv_cache[1],
840 - updated_slot_mapping.flatten(), # type: ignore[union-attr]
841 - kv_cache_dtype,
842 - k_scale,
843 - v_scale,
844 - )
845 -
846 - (num_prefill_query_tokens, num_prefill_kv_tokens,
847 - num_decode_query_tokens) = \
848 - get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
849 - decode_query = query[num_prefill_query_tokens:]
850 - # QKV for prefill.
851 - query = query[:num_prefill_query_tokens]
852 - assert query.shape[0] == num_prefill_query_tokens
853 - assert decode_query.shape[0] == num_decode_query_tokens
854 -
855 - prefill_output: Optional[torch.Tensor] = None
856 - decode_output: Optional[torch.Tensor] = None
857 - if prefill_meta := attn_metadata.prefill_metadata:
858 - # Prompt run.
859 - if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
860 - or prefill_meta.block_tables.numel() == 0):
861 - # normal attention
862 - # When block_tables are not filled, it means q and k are the
863 - # prompt, and they have the same length.
864 - q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
865 - _get_query_key_seq_metadata(prefill_meta, True, attn_type)
866 -
867 - key = key[:num_prefill_kv_tokens]
868 - value = value[:num_prefill_kv_tokens]
869 -
870 - prefill_output = flash_attn_varlen_func(
871 - q=query,
872 - k=key,
873 - v=value,
874 - cu_seqlens_q=q_seq_start_loc,
875 - cu_seqlens_k=k_seq_start_loc,
876 - max_seqlen_q=q_seq_len,
877 - max_seqlen_k=k_seq_len,
878 - softmax_scale=softmax_scale,
879 - causal=_get_causal_option(attn_type),
880 - window_size=window_size,
881 - alibi_slopes=alibi_slopes,
882 - softcap=logits_soft_cap,
883 - )
884 - else:
885 - # prefix-enabled attention
886 - assert attn_type == AttentionType.DECODER, (
887 - "Only decoder-only models support prefix caching")
888 - assert prefill_meta.seq_lens is not None
889 - max_seq_len = max(prefill_meta.seq_lens)
890 - prefill_output = flash_attn_varlen_func( # noqa
891 - q=query,
892 - k=key_cache,
893 - v=value_cache,
894 - cu_seqlens_q=prefill_meta.query_start_loc,
895 - max_seqlen_q=prefill_meta.max_query_len,
896 - cu_seqlens_k=prefill_meta.seq_start_loc,
897 - max_seqlen_k=max_seq_len,
898 - softmax_scale=softmax_scale,
899 - causal=True,
900 - window_size=window_size,
901 - alibi_slopes=alibi_slopes,
902 - block_table=prefill_meta.block_tables,
903 - softcap=logits_soft_cap,
904 - )
905 -
906 - if decode_meta := attn_metadata.decode_metadata:
907 - # Decoding run.
908 - # Use flash_attn_varlen_func kernel for speculative decoding
909 - # because different queries might have different lengths.
910 -
911 - assert decode_meta.max_decode_query_len is not None
912 - # use only for actual varlen decoding
913 - if decode_meta.max_decode_query_len > 1:
914 - assert attn_type == AttentionType.DECODER, (
915 - "Only decoder-only models support max_decode_query_len > 1")
916 - decode_output = flash_attn_varlen_func(
917 - q=decode_query,
918 - k=key_cache,
919 - v=value_cache,
920 - cu_seqlens_q=decode_meta.query_start_loc,
921 - max_seqlen_q=decode_meta.max_decode_query_len,
922 - cu_seqlens_k=decode_meta.seq_start_loc,
923 - max_seqlen_k=decode_meta.max_decode_seq_len,
924 - softmax_scale=softmax_scale,
925 - causal=True,
926 - window_size=window_size,
927 - alibi_slopes=alibi_slopes,
928 - softcap=logits_soft_cap,
929 - block_table=decode_meta.block_tables,
930 - )
931 - else:
932 - # Use flash_attn_with_kvcache for normal decoding.
933 - (
934 - seq_lens_arg,
935 - _,
936 - block_tables_arg,
937 - ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
938 - decode_output = flash_attn_with_kvcache(
939 - q=decode_query.unsqueeze(1),
940 - k_cache=key_cache,
941 - v_cache=value_cache,
942 - block_table=block_tables_arg,
943 - cache_seqlens=seq_lens_arg,
944 - softmax_scale=softmax_scale,
945 - causal=True,
946 - window_size=window_size,
947 - alibi_slopes=alibi_slopes,
948 - softcap=logits_soft_cap,
949 - ).squeeze(1)
950 -
951 - if prefill_output is None:
952 - assert decode_output is not None
953 - return decode_output.view(num_decode_query_tokens, hidden_size)
954 - if decode_output is None:
955 - assert prefill_output is not None
956 - return prefill_output.view(num_prefill_query_tokens, hidden_size)
957 -
958 - assert decode_meta is not None
959 - decode_output = decode_output.squeeze(1)
960 - output = torch.cat([prefill_output, decode_output], dim=0)
961 - return output.view(num_tokens, hidden_size)
962 -
963 -
964 -def unified_flash_attention_fake(
965 - query: torch.Tensor,
966 - key: torch.Tensor,
967 - value: torch.Tensor,
968 - num_heads: int,
969 - head_size: int,
970 - num_kv_heads: int,
971 - kv_cache: torch.Tensor,
972 - kv_cache_dtype: str,
973 - k_scale: float,
974 - v_scale: float,
975 - softmax_scale: float,
976 - attn_type_int_val: int,
977 - window_size: Optional[List[int]] = None,
978 - alibi_slopes: Optional[torch.Tensor] = None,
979 - logits_soft_cap: Optional[float] = None,
980 -) -> torch.Tensor:
981 - return torch.empty_like(query)
982 -
983 -
984 -direct_register_custom_op(
985 - op_name="unified_flash_attention",
986 - op_func=unified_flash_attention,
987 - mutates_args=["kv_cache"],
988 - fake_impl=unified_flash_attention_fake,
989 -)
vllm/attention/backends/flashinfer.py
+111 -169
@@ -30,9 +30,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
30 compute_slot_mapping_start_idx,
31 is_block_tables_empty)
32 from vllm.attention.ops.paged_attn import PagedAttention
33 -from vllm.forward_context import get_forward_context
34 -from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
35 - get_kv_cache_torch_dtype, make_tensor_with_pad)
33 +from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
34 + make_tensor_with_pad)
35
36 if TYPE_CHECKING:
37 from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -774,7 +773,7 @@ class FlashInferImpl(AttentionImpl):
773 attn_metadata: FlashInferMetadata,
774 k_scale: float = 1.0,
775 v_scale: float = 1.0,
777 - attn_type: AttentionType = AttentionType.DECODER,
776 + attn_type: str = AttentionType.DECODER,
777 ) -> torch.Tensor:
778 if attn_type != AttentionType.DECODER:
779 raise NotImplementedError("Encoder self-attention and "
@@ -782,174 +781,117 @@ class FlashInferImpl(AttentionImpl):
781 "are not implemented for "
782 "FlashInferImpl")
783
785 - return torch.ops.vllm.unified_flash_infer(
786 - query,
787 - key,
788 - value,
789 - self.num_heads,
790 - self.head_size,
791 - self.num_kv_heads,
792 - kv_cache,
793 - self.kv_cache_dtype,
794 - k_scale,
795 - v_scale,
796 - self.scale,
797 - self.sliding_window,
798 - self.alibi_slopes,
799 - self.logits_soft_cap,
800 - )
801 -
802 -
803 -def unified_flash_infer(
804 - query: torch.Tensor,
805 - key: torch.Tensor,
806 - value: torch.Tensor,
807 - num_heads: int,
808 - head_size: int,
809 - num_kv_heads: int,
810 - kv_cache: torch.Tensor,
811 - kv_cache_dtype: str,
812 - k_scale: float,
813 - v_scale: float,
814 - softmax_scale: float,
815 - window_size: Optional[List[int]] = None,
816 - alibi_slopes: Optional[torch.Tensor] = None,
817 - logits_soft_cap: Optional[float] = None,
818 -) -> torch.Tensor:
819 -
820 - current_metadata = get_forward_context()
821 - assert current_metadata is not None
822 - assert isinstance(current_metadata, FlashInferMetadata)
823 - attn_metadata: FlashInferMetadata = current_metadata
824 -
825 - num_tokens, hidden_size = query.shape
826 - query = query.view(-1, num_heads, head_size)
827 - key = key.view(-1, num_kv_heads, head_size)
828 - value = value.view(-1, num_kv_heads, head_size)
829 -
830 - if kv_cache.numel() > 0:
831 - # Use the same reshape and cache kernel as flash attention.
832 - ops.reshape_and_cache_flash(
833 - key,
834 - value,
835 - kv_cache[:, 0],
836 - kv_cache[:, 1],
837 - attn_metadata.slot_mapping.flatten(),
838 - kv_cache_dtype,
839 - k_scale,
840 - v_scale,
841 - )
842 - # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
843 - # to process the cache when the kv_cache_dtype is fp8
844 - if kv_cache_dtype.startswith("fp8"):
845 - torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
846 - kv_cache_dtype)
847 - kv_cache = kv_cache.view(torch_dtype)
848 -
849 - num_prefill_tokens = attn_metadata.num_prefill_tokens
850 - num_decode_tokens = attn_metadata.num_decode_tokens
851 - assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
852 - f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
853 - assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
854 - f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
855 - query = query.contiguous() # Flashinfer requires query to be contiguous
856 - # Query for decode. KV is not needed because it is already cached.
857 - # QKV for prefill.
858 - decode_query = query[num_prefill_tokens:]
859 - query = query[:num_prefill_tokens]
860 -
861 - key = key[:num_prefill_tokens]
862 - value = value[:num_prefill_tokens]
863 -
864 - assert query.shape[0] == num_prefill_tokens
865 - assert decode_query.shape[0] == num_decode_tokens
866 -
867 - window_left = window_size[0] if window_size is not None else -1
868 -
869 - prefill_output: Optional[torch.Tensor] = None
870 - decode_output: Optional[torch.Tensor] = None
871 - if prefill_meta := attn_metadata.prefill_metadata:
872 - # We will use flash attention for prefill
873 - # when kv_cache is not provided.
874 - # This happens when vllm runs the profiling to
875 - # determine the number of blocks.
876 - if kv_cache.numel() == 0:
877 - prefill_output = flash_attn_varlen_func(
878 - q=query,
879 - k=key,
880 - v=value,
881 - cu_seqlens_q=prefill_meta.seq_start_loc,
882 - cu_seqlens_k=prefill_meta.seq_start_loc,
883 - max_seqlen_q=prefill_meta.max_prefill_seq_len,
884 - max_seqlen_k=prefill_meta.max_prefill_seq_len,
885 - softmax_scale=softmax_scale,
886 - causal=True,
887 - window_size=window_size,
888 - alibi_slopes=alibi_slopes,
784 + num_heads: int = self.num_heads
785 + head_size: int = self.head_size
786 + num_kv_heads: int = self.num_kv_heads
787 + kv_cache_dtype: str = self.kv_cache_dtype
788 + softmax_scale: float = self.scale
789 + window_size = self.sliding_window
790 + alibi_slopes = self.alibi_slopes
791 + logits_soft_cap = self.logits_soft_cap
792 +
793 + num_tokens, hidden_size = query.shape
794 + query = query.view(-1, num_heads, head_size)
795 + key = key.view(-1, num_kv_heads, head_size)
796 + value = value.view(-1, num_kv_heads, head_size)
797 +
798 + if kv_cache.numel() > 0:
799 + # Use the same reshape and cache kernel as flash attention.
800 + ops.reshape_and_cache_flash(
801 + key,
802 + value,
803 + kv_cache[:, 0],
804 + kv_cache[:, 1],
805 + attn_metadata.slot_mapping.flatten(),
806 + kv_cache_dtype,
807 + k_scale,
808 + v_scale,
809 )
890 - else:
891 - assert prefill_meta is not None
892 - assert prefill_meta.prefill_wrapper is not None
893 - prefill_output = prefill_meta.prefill_wrapper.forward(
894 - query,
810 + # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
811 + # to process the cache when the kv_cache_dtype is fp8
812 + if kv_cache_dtype.startswith("fp8"):
813 + torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
814 + kv_cache_dtype)
815 + kv_cache = kv_cache.view(torch_dtype)
816 +
817 + num_prefill_tokens = attn_metadata.num_prefill_tokens
818 + num_decode_tokens = attn_metadata.num_decode_tokens
819 + assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
820 + f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
821 + assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
822 + f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
823 + query = query.contiguous(
824 + ) # Flashinfer requires query to be contiguous
825 + # Query for decode. KV is not needed because it is already cached.
826 + # QKV for prefill.
827 + decode_query = query[num_prefill_tokens:]
828 + query = query[:num_prefill_tokens]
829 +
830 + key = key[:num_prefill_tokens]
831 + value = value[:num_prefill_tokens]
832 +
833 + assert query.shape[0] == num_prefill_tokens
834 + assert decode_query.shape[0] == num_decode_tokens
835 +
836 + window_left = window_size[0] if window_size is not None else -1
837 +
838 + prefill_output: Optional[torch.Tensor] = None
839 + decode_output: Optional[torch.Tensor] = None
840 + if prefill_meta := attn_metadata.prefill_metadata:
841 + # We will use flash attention for prefill
842 + # when kv_cache is not provided.
843 + # This happens when vllm runs the profiling to
844 + # determine the number of blocks.
845 + if kv_cache.numel() == 0:
846 + prefill_output = flash_attn_varlen_func(
847 + q=query,
848 + k=key,
849 + v=value,
850 + cu_seqlens_q=prefill_meta.seq_start_loc,
851 + cu_seqlens_k=prefill_meta.seq_start_loc,
852 + max_seqlen_q=prefill_meta.max_prefill_seq_len,
853 + max_seqlen_k=prefill_meta.max_prefill_seq_len,
854 + softmax_scale=softmax_scale,
855 + causal=True,
856 + window_size=window_size,
857 + alibi_slopes=alibi_slopes,
858 + )
859 + else:
860 + assert prefill_meta is not None
861 + assert prefill_meta.prefill_wrapper is not None
862 + prefill_output = prefill_meta.prefill_wrapper.forward(
863 + query,
864 + kv_cache,
865 + logits_soft_cap=logits_soft_cap,
866 + causal=True,
867 + k_scale=k_scale,
868 + v_scale=v_scale,
869 + window_left=window_left)
870 + if decode_meta := attn_metadata.decode_metadata:
871 + assert decode_meta is not None
872 + assert decode_meta.decode_wrapper is not None
873 + decode_output = decode_meta.decode_wrapper.forward(
874 + decode_query,
875 kv_cache,
876 + sm_scale=softmax_scale,
877 logits_soft_cap=logits_soft_cap,
897 - causal=True,
878 k_scale=k_scale,
879 v_scale=v_scale,
880 window_left=window_left)
901 - if decode_meta := attn_metadata.decode_metadata:
902 - assert attn_metadata.decode_metadata is not None
903 - assert attn_metadata.decode_metadata.decode_wrapper is not None
904 - decode_output = attn_metadata.decode_metadata.decode_wrapper.forward(
905 - decode_query,
906 - kv_cache,
907 - sm_scale=softmax_scale,
908 - logits_soft_cap=logits_soft_cap,
909 - k_scale=k_scale,
910 - v_scale=v_scale,
911 - window_left=window_left)
912 -
913 - if prefill_output is None and decode_output is not None:
914 - # Decode only batch.
915 - output, num_tokens = decode_output, num_decode_tokens
916 - elif decode_output is None and prefill_output is not None:
917 - # Prefill only batch.
918 - output, num_tokens = prefill_output, num_prefill_tokens
919 - else:
920 - # Chunked prefill batch does not work with speculative decoding in
921 - # FlashInfer backend, so the query length for decode should be 1.
922 - assert prefill_output is not None
923 - assert decode_output is not None
924 - assert decode_meta is not None
925 - assert decode_meta.decode_query_len == 1
926 - decode_output = decode_output.squeeze(1)
927 - output = torch.cat([prefill_output, decode_output], dim=0)
928 - return output.view(num_tokens, hidden_size)
929 -
930 -
931 -def unified_flash_infer_fake(
932 - query: torch.Tensor,
933 - key: torch.Tensor,
934 - value: torch.Tensor,
935 - num_heads: int,
936 - head_size: int,
937 - num_kv_heads: int,
938 - kv_cache: torch.Tensor,
939 - kv_cache_dtype: str,
940 - k_scale: float,
941 - v_scale: float,
942 - softmax_scale: float,
943 - window_size: Optional[List[int]] = None,
944 - alibi_slopes: Optional[torch.Tensor] = None,
945 - logits_soft_cap: Optional[float] = None,
946 -) -> torch.Tensor:
947 - return torch.empty_like(query).contiguous()
948 -
949 -
950 -direct_register_custom_op(
951 - op_name="unified_flash_infer",
952 - op_func=unified_flash_infer,
953 - mutates_args=["kv_cache"],
954 - fake_impl=unified_flash_infer_fake,
955 -)
881 +
882 + if prefill_output is None and decode_output is not None:
883 + # Decode only batch.
884 + output, num_tokens = decode_output, num_decode_tokens
885 + elif decode_output is None and prefill_output is not None:
886 + # Prefill only batch.
887 + output, num_tokens = prefill_output, num_prefill_tokens
888 + else:
889 + # Chunked prefill batch does not work with speculative decoding in
890 + # FlashInfer backend, so the query length for decode should be 1.
891 + assert prefill_output is not None
892 + assert decode_output is not None
893 + assert decode_meta is not None
894 + assert decode_meta.decode_query_len == 1
895 + decode_output = decode_output.squeeze(1)
896 + output = torch.cat([prefill_output, decode_output], dim=0)
897 + return output.view(num_tokens, hidden_size)
vllm/attention/backends/hpu_attn.py
+1 -1
@@ -140,7 +140,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
140 attn_metadata: HPUAttentionMetadata,
141 k_scale: float = 1.0,
142 v_scale: float = 1.0,
143 - attn_type: AttentionType = AttentionType.DECODER,
143 + attn_type: str = AttentionType.DECODER,
144 ) -> torch.Tensor:
145 """Forward pass with xFormers and PagedAttention.
146
vllm/attention/backends/ipex_attn.py
+1 -1
@@ -172,7 +172,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
172 attn_metadata: IpexAttnMetadata, # type: ignore
173 k_scale: float = 1.0,
174 v_scale: float = 1.0,
175 - attn_type: AttentionType = AttentionType.DECODER,
175 + attn_type: str = AttentionType.DECODER,
176 ) -> torch.Tensor:
177 """Forward pass with IPEX varlen_attention and PagedAttention.
178
vllm/attention/backends/pallas.py
+1 -1
@@ -150,7 +150,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
150 attn_metadata: PallasMetadata,
151 k_scale: float = 1.0,
152 v_scale: float = 1.0,
153 - attn_type: AttentionType = AttentionType.DECODER,
153 + attn_type: str = AttentionType.DECODER,
154 ) -> torch.Tensor:
155 """Forward pass with Pallas attention.
156
vllm/attention/backends/rocm_flash_attn.py
+1 -1
@@ -414,7 +414,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
414 attn_metadata: ROCmFlashAttentionMetadata,
415 k_scale: float = 1.0,
416 v_scale: float = 1.0,
417 - attn_type: AttentionType = AttentionType.DECODER,
417 + attn_type: str = AttentionType.DECODER,
418 ) -> torch.Tensor:
419 """Forward pass with FlashAttention and PagedAttention.
420
vllm/attention/backends/torch_sdpa.py
+6 -6
@@ -141,7 +141,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
141
142 def get_seq_lens(
143 self,
144 - attn_type: AttentionType,
144 + attn_type: str,
145 ):
146 '''
147 Extract appropriate sequence lengths from attention metadata
@@ -174,7 +174,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
174
175 def get_attn_bias(
176 self,
177 - attn_type: AttentionType,
177 + attn_type: str,
178 ) -> Optional[List[torch.Tensor]]:
179 '''
180 Extract appropriate attention bias from attention metadata
@@ -203,7 +203,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
203 def set_attn_bias(
204 self,
205 attn_bias: List[torch.Tensor],
206 - attn_type: AttentionType,
206 + attn_type: str,
207 ) -> None:
208 '''
209 Update appropriate attention bias field of attention metadata,
@@ -229,7 +229,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
229
230 def get_seq_len_block_table_args(
231 self,
232 - attn_type: AttentionType,
232 + attn_type: str,
233 ) -> tuple:
234 '''
235 The particular choice of sequence-length- and block-table-related
@@ -426,7 +426,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
426 attn_metadata: TorchSDPAMetadata, # type: ignore
427 k_scale: float = 1.0,
428 v_scale: float = 1.0,
429 - attn_type: AttentionType = AttentionType.DECODER,
429 + attn_type: str = AttentionType.DECODER,
430 ) -> torch.Tensor:
431 """Forward pass with torch SDPA and PagedAttention.
432
@@ -574,7 +574,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
574 key: torch.Tensor,
575 value: torch.Tensor,
576 attn_metadata: TorchSDPAMetadata,
577 - attn_type: AttentionType = AttentionType.DECODER,
577 + attn_type: str = AttentionType.DECODER,
578 ) -> None:
579 if self.num_kv_heads != self.num_heads:
580 key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
vllm/attention/backends/utils.py
+2 -2
@@ -478,7 +478,7 @@ def is_all_cross_attn_metadata_set(attn_metadata):
478 def get_seq_len_block_table_args(
479 attn_metadata,
480 is_prompt: bool,
481 - attn_type: AttentionType,
481 + attn_type: str,
482 ) -> tuple:
483 '''
484 The particular choice of sequence-length- and block-table-related
@@ -529,7 +529,7 @@ def get_seq_len_block_table_args(
529
530 def get_num_prefill_decode_query_kv_tokens(
531 attn_metadata,
532 - attn_type: AttentionType,
532 + attn_type: str,
533 ) -> Tuple[int, int, int]:
534 """
535 Calculate the number of prefill and decode tokens for query, key/value
vllm/attention/backends/xformers.py
+4 -4
@@ -284,7 +284,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
284
285 def _get_attn_bias(
286 attn_metadata: XFormersMetadata,
287 - attn_type: AttentionType,
287 + attn_type: str,
288 ) -> Optional[AttentionBias]:
289 '''
290 Extract appropriate attention bias from attention metadata
@@ -314,7 +314,7 @@ def _get_attn_bias(
314 def _set_attn_bias(
315 attn_metadata: XFormersMetadata,
316 attn_bias: List[Optional[AttentionBias]],
317 - attn_type: AttentionType,
317 + attn_type: str,
318 ) -> None:
319 '''
320 Update appropriate attention bias field of attention metadata,
@@ -416,7 +416,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
416 attn_metadata: "XFormersMetadata",
417 k_scale: float = 1.0,
418 v_scale: float = 1.0,
419 - attn_type: AttentionType = AttentionType.DECODER,
419 + attn_type: str = AttentionType.DECODER,
420 ) -> torch.Tensor:
421 """Forward pass with xFormers and PagedAttention.
422
@@ -617,7 +617,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
617 key: torch.Tensor,
618 value: torch.Tensor,
619 attn_metadata: XFormersMetadata,
620 - attn_type: AttentionType = AttentionType.DECODER,
620 + attn_type: str = AttentionType.DECODER,
621 ) -> torch.Tensor:
622 """Attention for 1D query of multiple prompts. Multiple prompt
623 tokens are flattened in to `query` input.
vllm/attention/layer.py
+72 -9
@@ -4,12 +4,17 @@ from typing import Any, Dict, List, Optional
4 import torch
5 import torch.nn as nn
6
7 +import vllm.envs as envs
8 from vllm.attention import AttentionMetadata, AttentionType
9 from vllm.attention.selector import get_attn_backend
10 from vllm.config import CacheConfig
11 +from vllm.forward_context import ForwardContext, get_forward_context
12 from vllm.model_executor.layers.quantization.base_config import (
13 QuantizationConfig)
14 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15 +from vllm.platforms import current_platform
16 +from vllm.plugins import get_current_vllm_config
17 +from vllm.utils import direct_register_custom_op
18
19
20 class Attention(nn.Module):
@@ -86,6 +91,18 @@ class Attention(nn.Module):
91 alibi_slopes, sliding_window, kv_cache_dtype,
92 blocksparse_params, logits_soft_cap)
93
94 + # For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
95 + # torch.compile works by registering the attention as one giant
96 + # opaque custom op. For other platforms, we directly call them
97 + # and let torch.compile handle them.
98 + self.use_direct_call = envs.VLLM_USE_V1 or not (
99 + current_platform.is_cuda_alike() or current_platform.is_cpu())
100 + compilation_config = get_current_vllm_config().compilation_config
101 + if prefix in compilation_config.static_forward_context:
102 + raise ValueError(f"Duplicate layer name: {prefix}")
103 + compilation_config.static_forward_context[prefix] = self
104 + self.layer_name = prefix
105 +
106 def forward(
107 self,
108 query: torch.Tensor,
@@ -93,17 +110,22 @@ class Attention(nn.Module):
110 value: torch.Tensor,
111 kv_cache: torch.Tensor,
112 attn_metadata: AttentionMetadata,
96 - attn_type: AttentionType = AttentionType.DECODER,
113 + attn_type: str = AttentionType.DECODER,
114 ) -> torch.Tensor:
115
99 - return self.impl.forward(query,
100 - key,
101 - value,
102 - kv_cache,
103 - attn_metadata,
104 - self._k_scale,
105 - self._v_scale,
106 - attn_type=attn_type)
116 + if self.use_direct_call:
117 + return self.impl.forward(query,
118 + key,
119 + value,
120 + kv_cache,
121 + attn_metadata,
122 + self._k_scale,
123 + self._v_scale,
124 + attn_type=attn_type)
125 + else:
126 + return torch.ops.vllm.unified_attention(query, key, value,
127 + kv_cache, attn_type,
128 + self.layer_name)
129
130 def extra_repr(self) -> str:
131 s = f"head_size={self.impl.head_size}" # type: ignore
@@ -112,3 +134,44 @@ class Attention(nn.Module):
134 s += f", scale={self.impl.scale}" # type: ignore
135 s += f", backend={self.impl.__class__.__name__}"
136 return s
137 +
138 +
139 +def unified_attention(
140 + query: torch.Tensor,
141 + key: torch.Tensor,
142 + value: torch.Tensor,
143 + kv_cache: torch.Tensor,
144 + attn_type: str,
145 + layer_name: str,
146 +) -> torch.Tensor:
147 + forward_context: ForwardContext = get_forward_context()
148 + attn_metadata = forward_context.dynamic_forward_context
149 + self = forward_context.static_forward_context[layer_name]
150 + return self.impl.forward(query,
151 + key,
152 + value,
153 + kv_cache,
154 + attn_metadata,
155 + self._k_scale,
156 + self._v_scale,
157 + attn_type=attn_type)
158 +
159 +
160 +def unified_attention_fake(
161 + query: torch.Tensor,
162 + key: torch.Tensor,
163 + value: torch.Tensor,
164 + kv_cache: torch.Tensor,
165 + attn_type: str,
166 + layer_name: str,
167 +) -> torch.Tensor:
168 + return torch.empty_like(query).contiguous()
169 +
170 +
171 +direct_register_custom_op(
172 + op_name="unified_attention",
173 + op_func=unified_attention,
174 + mutates_args=["kv_cache"],
175 + fake_impl=unified_attention_fake,
176 + dispatch_key=current_platform.dispatch_key,
177 +)
vllm/config.py
+7 -2
@@ -2135,8 +2135,7 @@ class CompilationConfig(BaseModel):
2135 backend: str = ""
2136 custom_ops: List[str] = Field(default_factory=list)
2137 splitting_ops: List[str] = Field(default_factory=lambda: [
2138 - "vllm.unified_flash_attention",
2139 - "vllm.unified_flash_infer",
2138 + "vllm.unified_attention",
2139 "vllm.unified_v1_flash_attention",
2140 ])
2141
@@ -2197,6 +2196,11 @@ class CompilationConfig(BaseModel):
2196 enabled_custom_ops: Counter[str] = PrivateAttr
2197 disabled_custom_ops: Counter[str] = PrivateAttr
2198
2199 + # Per-model forward context
2200 + # Mainly used to store attention cls
2201 + # Map from layer name to the attention cls
2202 + static_forward_context: Dict[str, Any] = PrivateAttr
2203 +
2204 @classmethod
2205 def from_cli(cls, cli_value: str) -> "CompilationConfig":
2206 """Parse the CLI value for the compilation config."""
@@ -2228,6 +2232,7 @@ class CompilationConfig(BaseModel):
2232
2233 self.enabled_custom_ops = Counter()
2234 self.disabled_custom_ops = Counter()
2235 + self.static_forward_context = {}
2236
2237 def init_backend(self) -> Union[str, Callable]:
2238 if self.level == CompilationLevel.NO_COMPILATION:
vllm/forward_context.py
+22 -5
@@ -1,21 +1,38 @@
1 from contextlib import contextmanager
2 -from typing import Any
2 +from dataclasses import dataclass
3 +from typing import Any, Dict, Optional
4
4 -_forward_context: Any = None
5 +from vllm.config import VllmConfig
6
7
7 -def get_forward_context() -> Any:
8 +@dataclass
9 +class ForwardContext:
10 + static_forward_context: Dict[str, Any]
11 + # TODO: extend to support per-layer dynamic forward context
12 + dynamic_forward_context: Any
13 +
14 +
15 +_forward_context: Optional[ForwardContext] = None
16 +
17 +
18 +def get_forward_context() -> ForwardContext:
19 """Get the current forward context."""
20 + assert _forward_context is not None, (
21 + "Forward context is not set. "
22 + "Please use `set_forward_context` to set the forward context.")
23 return _forward_context
24
25
26 @contextmanager
13 -def set_forward_context(context: Any):
27 +def set_forward_context(context: Any, vllm_config: VllmConfig):
28 """A context manager that stores the current forward context,
29 can be attention metadata, etc."""
30 global _forward_context
31 prev_context = _forward_context
18 - _forward_context = context
32 + _forward_context = ForwardContext(
33 + static_forward_context=vllm_config.compilation_config.
34 + static_forward_context,
35 + dynamic_forward_context=context)
36 try:
37 yield
38 finally:
vllm/model_executor/models/arctic.py
+11 -4
@@ -223,6 +223,7 @@ class ArcticAttention(nn.Module):
223 layer_idx: Optional[int] = None,
224 cache_config: Optional[CacheConfig] = None,
225 quant_config: Optional[QuantizationConfig] = None,
226 + prefix: str = "",
227 ):
228 super().__init__()
229 self.config = config
@@ -274,7 +275,8 @@ class ArcticAttention(nn.Module):
275 self.scaling,
276 num_kv_heads=self.num_kv_heads,
277 cache_config=cache_config,
277 - quant_config=quant_config)
278 + quant_config=quant_config,
279 + prefix=f"{prefix}.attn")
280
281 def forward(
282 self,
@@ -299,6 +301,7 @@ class ArcticDecoderLayer(nn.Module):
301 layer_idx: int,
302 cache_config: Optional[CacheConfig] = None,
303 quant_config: Optional[QuantizationConfig] = None,
304 + prefix: str = "",
305 ) -> None:
306 super().__init__()
307 self.layer_idx = layer_idx
@@ -308,7 +311,8 @@ class ArcticDecoderLayer(nn.Module):
311 self.self_attn = ArcticAttention(config,
312 layer_idx,
313 cache_config,
311 - quant_config=quant_config)
314 + quant_config=quant_config,
315 + prefix=f"{prefix}.self_attn")
316 self.block_sparse_moe = ArcticMoE(
317 config,
318 layer_id=layer_idx,
@@ -380,8 +384,11 @@ class ArcticModel(nn.Module):
384 org_num_embeddings=self.vocab_size)
385 self.start_layer, self.end_layer, self.layers = make_layers(
386 config.num_hidden_layers,
383 - lambda prefix: ArcticDecoderLayer(config, int(
384 - prefix.split(".")[-1]), cache_config, quant_config),
387 + lambda prefix: ArcticDecoderLayer(config,
388 + int(prefix.split(".")[-1]),
389 + cache_config,
390 + quant_config,
391 + prefix=prefix),
392 prefix=f"{prefix}.layers")
393 self._attn_implementation = config._attn_implementation
394 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/baichuan.py
+13 -5
@@ -116,6 +116,7 @@ class BaiChuanAttention(nn.Module):
116 max_position_embeddings: int = 8192,
117 cache_config: Optional[CacheConfig] = None,
118 quant_config: Optional[QuantizationConfig] = None,
119 + prefix: str = "",
120 ):
121 super().__init__()
122 self.hidden_size = hidden_size
@@ -158,7 +159,8 @@ class BaiChuanAttention(nn.Module):
159 self.head_dim,
160 scaling,
161 alibi_slopes=alibi_slopes,
161 - quant_config=quant_config)
162 + quant_config=quant_config,
163 + prefix=f"{prefix}.attn")
164 else:
165 self.rotary_emb = get_rope(
166 self.head_dim,
@@ -171,7 +173,8 @@ class BaiChuanAttention(nn.Module):
173 self.head_dim,
174 self.scaling,
175 cache_config=cache_config,
174 - quant_config=quant_config)
176 + quant_config=quant_config,
177 + prefix=f"{prefix}.attn")
178
179 def forward(
180 self,
@@ -195,7 +198,8 @@ class BaiChuanDecoderLayer(nn.Module):
198 config: PretrainedConfig,
199 position_embedding: str,
200 cache_config: Optional[CacheConfig] = None,
198 - quant_config: Optional[QuantizationConfig] = None):
201 + quant_config: Optional[QuantizationConfig] = None,
202 + prefix: str = ""):
203 super().__init__()
204 self.hidden_size = config.hidden_size
205 rope_theta = getattr(config, "rope_theta", 10000)
@@ -209,6 +213,7 @@ class BaiChuanDecoderLayer(nn.Module):
213 max_position_embeddings=max_position_embeddings,
214 cache_config=cache_config,
215 quant_config=quant_config,
216 + prefix=f"{prefix}.self_attn",
217 )
218 self.mlp = BaiChuanMLP(
219 hidden_size=self.hidden_size,
@@ -275,8 +280,11 @@ class BaiChuanModel(nn.Module):
280 )
281 self.start_layer, self.end_layer, self.layers = make_layers(
282 config.num_hidden_layers,
278 - lambda prefix: BaiChuanDecoderLayer(config, position_embedding,
279 - cache_config, quant_config),
283 + lambda prefix: BaiChuanDecoderLayer(config,
284 + position_embedding,
285 + cache_config,
286 + quant_config,
287 + prefix=prefix),
288 prefix=f"{prefix}.layers",
289 )
290 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/bart.py
+35 -13
@@ -126,6 +126,7 @@ class BartEncoderAttention(nn.Module):
126 config: Optional[BartConfig] = None,
127 cache_config: Optional[CacheConfig] = None,
128 quant_config: Optional[QuantizationConfig] = None,
129 + prefix: str = "",
130 ):
131 super().__init__()
132 self.d_model = config.d_model
@@ -178,7 +179,8 @@ class BartEncoderAttention(nn.Module):
179 self.scaling,
180 num_kv_heads=self.num_kv_heads,
181 cache_config=cache_config,
181 - quant_config=quant_config)
182 + quant_config=quant_config,
183 + prefix=f"{prefix}.attn")
184
185 def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
186 attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -208,6 +210,7 @@ class BartDecoderSelfAttention(nn.Module):
210 config: Optional[BartConfig] = None,
211 cache_config: Optional[CacheConfig] = None,
212 quant_config: Optional[QuantizationConfig] = None,
213 + prefix: str = "",
214 ):
215 super().__init__()
216 self.d_model = config.d_model
@@ -260,7 +263,8 @@ class BartDecoderSelfAttention(nn.Module):
263 self.scaling,
264 num_kv_heads=self.num_kv_heads,
265 cache_config=cache_config,
263 - quant_config=quant_config)
266 + quant_config=quant_config,
267 + prefix=f"{prefix}.attn")
268
269 def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
270 attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -290,6 +294,7 @@ class BartCrossAttention(nn.Module):
294 config: Optional[BartConfig] = None,
295 cache_config: Optional[CacheConfig] = None,
296 quant_config: Optional[QuantizationConfig] = None,
297 + prefix: str = "",
298 ):
299 super().__init__()
300 self.d_model = config.d_model
@@ -342,7 +347,8 @@ class BartCrossAttention(nn.Module):
347 self.scaling,
348 num_kv_heads=self.num_kv_heads,
349 cache_config=cache_config,
345 - quant_config=quant_config)
350 + quant_config=quant_config,
351 + prefix=f"{prefix}.attn")
352
353 def forward(
354 self,
@@ -384,6 +390,7 @@ class BartEncoderLayer(nn.Module):
390 config: BartConfig,
391 cache_config: Optional[CacheConfig] = None,
392 quant_config: Optional[QuantizationConfig] = None,
393 + prefix: str = "",
394 ):
395 super().__init__()
396 self.embed_dim = config.d_model
@@ -393,7 +400,9 @@ class BartEncoderLayer(nn.Module):
400 num_heads=config.encoder_attention_heads,
401 config=config,
402 cache_config=cache_config,
396 - quant_config=quant_config)
403 + quant_config=quant_config,
404 + prefix=f"{prefix}.self_attn",
405 + )
406 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
407 self.activation_fn = get_act_fn(config.activation_function)
408
@@ -464,6 +473,7 @@ class BartDecoderLayer(nn.Module):
473 config: BartConfig,
474 cache_config: Optional[CacheConfig] = None,
475 quant_config: Optional[QuantizationConfig] = None,
476 + prefix: str = "",
477 ):
478 super().__init__()
479 self.embed_dim = config.d_model
@@ -473,7 +483,9 @@ class BartDecoderLayer(nn.Module):
483 num_heads=config.decoder_attention_heads,
484 config=config,
485 cache_config=cache_config,
476 - quant_config=quant_config)
486 + quant_config=quant_config,
487 + prefix=f"{prefix}.self_attn",
488 + )
489 self.activation_fn = get_act_fn(config.activation_function)
490
491 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
@@ -486,6 +498,7 @@ class BartDecoderLayer(nn.Module):
498 self.embed_dim,
499 config.decoder_attention_heads,
500 config=config,
501 + prefix=f"{prefix}.encoder_attn",
502 )
503 self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
504
@@ -578,7 +591,8 @@ class BartEncoder(nn.Module):
591 cache_config: Optional[CacheConfig] = None,
592 quant_config: Optional[QuantizationConfig] = None,
593 lora_config: Optional[LoRAConfig] = None,
581 - embed_tokens: Optional[nn.Embedding] = None):
594 + embed_tokens: Optional[nn.Embedding] = None,
595 + prefix: str = ""):
596 super().__init__()
597
598 self.cache_config = cache_config
@@ -599,9 +613,13 @@ class BartEncoder(nn.Module):
613 config.max_position_embeddings,
614 embed_dim,
615 )
602 - self.layers = nn.ModuleList(
603 - [BartEncoderLayer(config,cache_config,quant_config) \
604 - for _ in range(config.encoder_layers)])
616 + self.layers = nn.ModuleList([
617 + BartEncoderLayer(config,
618 + cache_config,
619 + quant_config,
620 + prefix=f"{prefix}.layers.{layer_idx}")
621 + for layer_idx in range(config.encoder_layers)
622 + ])
623
624 self.layernorm_embedding = nn.LayerNorm(embed_dim)
625
@@ -661,6 +679,7 @@ class BartDecoder(nn.Module):
679 quant_config: Optional[QuantizationConfig] = None,
680 lora_config: Optional[LoRAConfig] = None,
681 embed_tokens: Optional[nn.Embedding] = None,
682 + prefix: str = "",
683 ):
684 super().__init__()
685 self.cache_config = cache_config
@@ -683,8 +702,9 @@ class BartDecoder(nn.Module):
702 )
703
704 self.layers = nn.ModuleList(
686 - [BartDecoderLayer(config,cache_config,quant_config) \
687 - for _ in range(config.decoder_layers)])
705 + [BartDecoderLayer(config,cache_config,quant_config,
706 + prefix=f"{prefix}.layers.{layer_idx}") \
707 + for layer_idx in range(config.decoder_layers)])
708
709 self.layernorm_embedding = nn.LayerNorm(config.d_model)
710
@@ -759,10 +779,12 @@ class BartModel(nn.Module):
779
780 self.encoder = BartEncoder(config,
781 cache_config,
762 - quant_config=quant_config)
782 + quant_config=quant_config,
783 + prefix=f"{prefix}.encoder")
784 self.decoder = BartDecoder(config,
785 cache_config,
765 - quant_config=quant_config)
786 + quant_config=quant_config,
787 + prefix=f"{prefix}.decoder")
788
789 def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
790 encoder_input_ids: torch.Tensor,
vllm/model_executor/models/bloom.py
+10 -4
@@ -78,6 +78,7 @@ class BloomAttention(nn.Module):
78 config: BloomConfig,
79 cache_config: Optional[CacheConfig] = None,
80 quant_config: Optional[QuantizationConfig] = None,
81 + prefix: str = "",
82 ):
83 super().__init__()
84 self.hidden_size = config.hidden_size
@@ -116,7 +117,8 @@ class BloomAttention(nn.Module):
117 scaling,
118 alibi_slopes=alibi_slopes,
119 cache_config=cache_config,
119 - quant_config=quant_config)
120 + quant_config=quant_config,
121 + prefix=f"{prefix}.attn")
122
123 def forward(
124 self,
@@ -168,14 +170,17 @@ class BloomBlock(nn.Module):
170 config: BloomConfig,
171 cache_config: Optional[CacheConfig] = None,
172 quant_config: Optional[QuantizationConfig] = None,
173 + prefix: str = "",
174 ):
175 super().__init__()
176 hidden_size = config.hidden_size
177
178 self.input_layernorm = nn.LayerNorm(hidden_size,
179 eps=config.layer_norm_epsilon)
177 - self.self_attention = BloomAttention(config, cache_config,
178 - quant_config)
180 + self.self_attention = BloomAttention(config,
181 + cache_config,
182 + quant_config,
183 + prefix=f"{prefix}.self_attention")
184 self.post_attention_layernorm = nn.LayerNorm(
185 hidden_size, eps=config.layer_norm_epsilon)
186 self.mlp = BloomMLP(config, quant_config)
@@ -242,7 +247,8 @@ class BloomModel(nn.Module):
247 # Transformer blocks
248 self.start_layer, self.end_layer, self.h = make_layers(
249 config.num_hidden_layers,
245 - lambda prefix: BloomBlock(config, cache_config, quant_config),
250 + lambda prefix: BloomBlock(
251 + config, cache_config, quant_config, prefix=prefix),
252 prefix=f"{prefix}.h")
253
254 # Final Layer Norm
vllm/model_executor/models/chameleon.py
+9 -2
@@ -223,6 +223,7 @@ class ChameleonAttention(nn.Module):
223 quant_config: Optional[QuantizationConfig] = None,
224 bias: bool = False,
225 cache_config: Optional[CacheConfig] = None,
226 + prefix: str = "",
227 ) -> None:
228 super().__init__()
229 self.hidden_size = hidden_size
@@ -276,7 +277,8 @@ class ChameleonAttention(nn.Module):
277 self.scaling,
278 num_kv_heads=self.num_kv_heads,
279 cache_config=cache_config,
279 - quant_config=quant_config)
280 + quant_config=quant_config,
281 + prefix=f"{prefix}.attn")
282
283 def _apply_qk_norm(self, q: torch.Tensor,
284 k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -313,6 +315,7 @@ class ChameleonDecoderLayer(nn.Module):
315 config: ChameleonConfig,
316 cache_config: Optional[CacheConfig] = None,
317 quant_config: Optional[QuantizationConfig] = None,
318 + prefix: str = "",
319 ) -> None:
320 super().__init__()
321 self.hidden_size = config.hidden_size
@@ -336,6 +339,7 @@ class ChameleonDecoderLayer(nn.Module):
339 quant_config=quant_config,
340 bias=False,
341 cache_config=cache_config,
342 + prefix=f"{prefix}.self_attn",
343 )
344 self.mlp = ChameleonMLP(
345 hidden_size=self.hidden_size,
@@ -386,6 +390,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
390 config: ChameleonConfig,
391 cache_config: Optional[CacheConfig] = None,
392 quant_config: Optional[QuantizationConfig] = None,
393 + prefix: str = "",
394 ) -> None:
395 super().__init__()
396 self.hidden_size = config.hidden_size
@@ -409,6 +414,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
414 quant_config=quant_config,
415 bias=False,
416 cache_config=cache_config,
417 + prefix=f"{prefix}.self_attn",
418 )
419 self.mlp = ChameleonMLP(
420 hidden_size=self.hidden_size,
@@ -855,7 +861,8 @@ class ChameleonModel(nn.Module):
861 config.num_hidden_layers,
862 lambda prefix: decoder_layer(config=config,
863 cache_config=cache_config,
858 - quant_config=quant_config),
864 + quant_config=quant_config,
865 + prefix=prefix),
866 prefix=f"{prefix}.layers",
867 )
868
vllm/model_executor/models/chatglm.py
+19 -6
@@ -230,6 +230,7 @@ class GLMAttention(nn.Module):
230 config: ChatGLMConfig,
231 cache_config: Optional[CacheConfig] = None,
232 quant_config: Optional[QuantizationConfig] = None,
233 + prefix: str = "",
234 ):
235 super().__init__()
236 self.hidden_size = config.hidden_size
@@ -285,7 +286,8 @@ class GLMAttention(nn.Module):
286 self.scaling,
287 num_kv_heads=self.num_kv_heads,
288 cache_config=cache_config,
288 - quant_config=quant_config)
289 + quant_config=quant_config,
290 + prefix=f"{prefix}.attn")
291
292 def forward(
293 self,
@@ -364,6 +366,7 @@ class GLMBlock(nn.Module):
366 config: ChatGLMConfig,
367 cache_config: Optional[CacheConfig] = None,
368 quant_config: Optional[QuantizationConfig] = None,
369 + prefix: str = "",
370 ):
371 super().__init__()
372 self.apply_residual_connection_post_layernorm = (
@@ -377,7 +380,10 @@ class GLMBlock(nn.Module):
380 eps=config.layernorm_epsilon)
381
382 # Self attention.
380 - self.self_attention = GLMAttention(config, cache_config, quant_config)
383 + self.self_attention = GLMAttention(config,
384 + cache_config,
385 + quant_config,
386 + prefix=f"{prefix}.self_attention")
387 self.hidden_dropout = config.hidden_dropout
388
389 # Layernorm on the attention output
@@ -446,7 +452,8 @@ class GLMTransformer(nn.Module):
452 # Transformer layers.
453 self.start_layer, self.end_layer, self.layers = make_layers(
454 self.num_layers,
449 - lambda prefix: GLMBlock(config, cache_config, quant_config),
455 + lambda prefix: GLMBlock(
456 + config, cache_config, quant_config, prefix=prefix),
457 prefix=f"{prefix}.layers",
458 )
459
@@ -500,16 +507,22 @@ class ChatGLMModel(nn.Module):
507 self.num_layers = config.num_layers
508 self.multi_query_group_num = config.multi_query_group_num
509 self.kv_channels = config.kv_channels
503 - self.encoder = GLMTransformer(config, cache_config, quant_config)
510 + self.encoder = GLMTransformer(config,
511 + cache_config,
512 + quant_config,
513 + prefix=f"{prefix}.encoder")
514
515 self.output_layer = ParallelLMHead(config.padded_vocab_size,
516 config.hidden_size,
507 - quant_config=quant_config)
517 + quant_config=quant_config,
518 + prefix=f"{prefix}.output_layer")
519
520 vision_config_flag = getattr(config, 'vision_config', None)
521 if vision_config_flag is not None:
522 self.vision_config = Namespace(**config.vision_config)
512 - self.vision = EVA2CLIPModel(self.config, quant_config)
523 + self.vision = EVA2CLIPModel(self.config,
524 + quant_config,
525 + prefix=f"{prefix}.vision")
526 else:
527 self.vision = None
528
vllm/model_executor/models/commandr.py
+9 -5
@@ -120,6 +120,7 @@ class CohereAttention(nn.Module):
120 config: CohereConfig,
121 cache_config: Optional[CacheConfig] = None,
122 quant_config: Optional[QuantizationConfig] = None,
123 + prefix: str = "",
124 ):
125 super().__init__()
126 tp_size = get_tensor_model_parallel_world_size()
@@ -175,7 +176,8 @@ class CohereAttention(nn.Module):
176 self.scaling,
177 num_kv_heads=self.num_kv_heads,
178 cache_config=cache_config,
178 - quant_config=quant_config)
179 + quant_config=quant_config,
180 + prefix=f"{prefix}.attn")
181 if self.use_qk_norm:
182 self.q_norm = LayerNorm(param_shape=(self.num_heads,
183 self.head_dim),
@@ -215,13 +217,15 @@ class CohereDecoderLayer(nn.Module):
217 def __init__(self,
218 config: CohereConfig,
219 cache_config: Optional[CacheConfig] = None,
218 - quant_config: Optional[QuantizationConfig] = None):
220 + quant_config: Optional[QuantizationConfig] = None,
221 + prefix: str = ""):
222 super().__init__()
223 self.hidden_size = config.hidden_size
224
225 self.self_attn = CohereAttention(config,
226 cache_config,
224 - quant_config=quant_config)
227 + quant_config=quant_config,
228 + prefix=f"{prefix}.self_attn")
229
230 self.mlp = CohereMLP(config, quant_config=quant_config)
231 self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
@@ -271,8 +275,8 @@ class CohereModel(nn.Module):
275 config.hidden_size)
276 self.start_layer, self.end_layer, self.layers = make_layers(
277 config.num_hidden_layers,
274 - lambda prefix: CohereDecoderLayer(config, cache_config,
275 - quant_config),
278 + lambda prefix: CohereDecoderLayer(
279 + config, cache_config, quant_config, prefix=prefix),
280 prefix=f"{prefix}.layers")
281 self.norm = LayerNorm(param_shape=(config.hidden_size),
282 eps=config.layer_norm_eps)
vllm/model_executor/models/dbrx.py
+16 -5
@@ -154,6 +154,7 @@ class DbrxAttention(nn.Module):
154 config: DbrxConfig,
155 cache_config: Optional[CacheConfig] = None,
156 quant_config: Optional[QuantizationConfig] = None,
157 + prefix: str = "",
158 ):
159 super().__init__()
160 self.d_model = config.d_model
@@ -208,7 +209,8 @@ class DbrxAttention(nn.Module):
209 self.scaling,
210 num_kv_heads=self.num_kv_heads,
211 cache_config=cache_config,
211 - quant_config=quant_config)
212 + quant_config=quant_config,
213 + prefix=f"{prefix}.attn")
214
215 def forward(
216 self,
@@ -234,10 +236,14 @@ class DbrxFusedNormAttention(nn.Module):
236 config: DbrxConfig,
237 cache_config: Optional[CacheConfig] = None,
238 quant_config: Optional[QuantizationConfig] = None,
239 + prefix: str = "",
240 ):
241 super().__init__()
242 self.d_model = config.d_model
240 - self.attn = DbrxAttention(config, cache_config, quant_config)
243 + self.attn = DbrxAttention(config,
244 + cache_config,
245 + quant_config,
246 + prefix=f"{prefix}.attn")
247 self.norm_1 = nn.LayerNorm(self.d_model)
248 self.norm_2 = nn.LayerNorm(self.d_model)
249
@@ -269,10 +275,14 @@ class DbrxBlock(nn.Module):
275 config: DbrxConfig,
276 cache_config: Optional[CacheConfig] = None,
277 quant_config: Optional[QuantizationConfig] = None,
278 + prefix: str = "",
279 ):
280 super().__init__()
274 - self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
275 - quant_config)
281 + self.norm_attn_norm = DbrxFusedNormAttention(
282 + config,
283 + cache_config,
284 + quant_config,
285 + prefix=f"{prefix}.norm_attn_norm")
286 self.ffn = DbrxMoE(config, quant_config)
287
288 def forward(
@@ -308,7 +318,8 @@ class DbrxModel(nn.Module):
318 )
319 self.start_layer, self.end_layer, self.blocks = make_layers(
320 config.n_layers,
311 - lambda prefix: DbrxBlock(config, cache_config, quant_config),
321 + lambda prefix: DbrxBlock(
322 + config, cache_config, quant_config, prefix=prefix),
323 prefix=f"{prefix}.blocks",
324 )
325 self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
vllm/model_executor/models/deepseek.py
+7 -2
@@ -184,6 +184,7 @@ class DeepseekAttention(nn.Module):
184 max_position_embeddings: int = 8192,
185 cache_config: Optional[CacheConfig] = None,
186 quant_config: Optional[QuantizationConfig] = None,
187 + prefix: str = "",
188 ) -> None:
189 super().__init__()
190 self.hidden_size = hidden_size
@@ -236,7 +237,8 @@ class DeepseekAttention(nn.Module):
237 self.scaling,
238 num_kv_heads=self.num_kv_heads,
239 cache_config=cache_config,
239 - quant_config=quant_config)
240 + quant_config=quant_config,
241 + prefix=f"{prefix}.attn")
242
243 def forward(
244 self,
@@ -261,6 +263,7 @@ class DeepseekDecoderLayer(nn.Module):
263 layer_idx: int,
264 cache_config: Optional[CacheConfig] = None,
265 quant_config: Optional[QuantizationConfig] = None,
266 + prefix: str = "",
267 ) -> None:
268 super().__init__()
269 self.hidden_size = config.hidden_size
@@ -277,6 +280,7 @@ class DeepseekDecoderLayer(nn.Module):
280 max_position_embeddings=max_position_embeddings,
281 cache_config=cache_config,
282 quant_config=quant_config,
283 + prefix=f"{prefix}.self_attn",
284 )
285 if (config.n_routed_experts is not None
286 and layer_idx >= config.first_k_dense_replace
@@ -346,7 +350,8 @@ class DeepseekModel(nn.Module):
350 lambda prefix: DeepseekDecoderLayer(config,
351 int(prefix.split(".")[-1]),
352 cache_config,
349 - quant_config=quant_config),
353 + quant_config=quant_config,
354 + prefix=prefix),
355 prefix=f"{prefix}.layers")
356 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
357 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/deepseek_v2.py
+2 -1
@@ -268,7 +268,8 @@ class DeepseekV2Attention(nn.Module):
268 self.scaling,
269 num_kv_heads=self.num_local_heads,
270 cache_config=cache_config,
271 - quant_config=quant_config)
271 + quant_config=quant_config,
272 + prefix=f"{prefix}.attn")
273
274 def forward(
275 self,
vllm/model_executor/models/exaone.py
+2 -1
@@ -174,6 +174,7 @@ class ExaoneAttention(nn.Module):
174 num_kv_heads=self.num_kv_heads,
175 cache_config=cache_config,
176 quant_config=quant_config,
177 + prefix=f"{prefix}.attn",
178 )
179
180 def forward(
@@ -219,7 +220,7 @@ class ExaoneBlockAttention(nn.Module):
220 quant_config=quant_config,
221 bias=bias,
222 cache_config=cache_config,
222 - prefix=prefix,
223 + prefix=f"{prefix}.attention",
224 )
225
226 def forward(
vllm/model_executor/models/falcon.py
+15 -7
@@ -84,6 +84,7 @@ class FalconAttention(nn.Module):
84 config: FalconConfig,
85 cache_config: Optional[CacheConfig] = None,
86 quant_config: Optional[QuantizationConfig] = None,
87 + prefix: str = "",
88 ):
89 super().__init__()
90
@@ -158,7 +159,8 @@ class FalconAttention(nn.Module):
159 self.head_dim,
160 self.inv_norm_factor,
161 num_kv_heads=self.num_kv_heads,
161 - quant_config=quant_config)
162 + quant_config=quant_config,
163 + prefix=f"{prefix}.attn")
164 elif self.use_alibi:
165 tp_rank = get_tensor_model_parallel_rank()
166 head_start = tp_rank * self.num_heads
@@ -171,14 +173,16 @@ class FalconAttention(nn.Module):
173 self.inv_norm_factor,
174 num_kv_heads=self.num_kv_heads,
175 alibi_slopes=alibi_slopes,
174 - quant_config=quant_config)
176 + quant_config=quant_config,
177 + prefix=f"{prefix}.attn")
178 else:
179 self.attn = Attention(self.num_heads,
180 self.head_dim,
181 scale=self.inv_norm_factor,
182 num_kv_heads=self.num_kv_heads,
183 cache_config=cache_config,
181 - quant_config=quant_config)
184 + quant_config=quant_config,
185 + prefix=f"{prefix}.attn")
186
187 def forward(
188 self,
@@ -241,12 +245,16 @@ class FalconDecoderLayer(nn.Module):
245 config: FalconConfig,
246 cache_config: Optional[CacheConfig] = None,
247 quant_config: Optional[QuantizationConfig] = None,
248 + prefix: str = "",
249 ):
250 super().__init__()
251 hidden_size = config.hidden_size
252 self.num_heads = config.num_attention_heads
248 - self.self_attention = FalconAttention(config, cache_config,
249 - quant_config)
253 + self.self_attention = FalconAttention(
254 + config,
255 + cache_config,
256 + quant_config,
257 + prefix=f"{prefix}.self_attention")
258 self.mlp = FalconMLP(config, quant_config)
259 self.config = config
260
@@ -357,8 +365,8 @@ class FalconModel(nn.Module):
365 # Transformer blocks
366 self.start_layer, self.end_layer, self.h = make_layers(
367 config.num_hidden_layers,
360 - lambda prefix: FalconDecoderLayer(config, cache_config,
361 - quant_config),
368 + lambda prefix: FalconDecoderLayer(
369 + config, cache_config, quant_config, prefix=prefix),
370 prefix=f"{prefix}.h")
371
372 # Final Layer Norm
vllm/model_executor/models/florence2.py
+6 -4
@@ -35,10 +35,12 @@ class Florence2LanguageModel(nn.Module):
35 self.shared = BartScaledWordEmbedding(self.vocab_size, config.d_model)
36 self.encoder = BartEncoder(config,
37 cache_config=cache_config,
38 - quant_config=quant_config)
38 + quant_config=quant_config,
39 + prefix=f"{prefix}.encoder")
40 self.decoder = BartDecoder(config,
41 cache_config=cache_config,
41 - quant_config=quant_config)
42 + quant_config=quant_config,
43 + prefix=f"{prefix}.decoder")
44
45 if self.config.tie_word_embeddings:
46 self.encoder.embed_tokens.weight = self.shared.weight
@@ -99,7 +101,7 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
101
102 self.config = config
103 self.model = Florence2LanguageModel(vllm_config=vllm_config,
102 - prefix=prefix)
104 + prefix=f"{prefix}.model")
105 embed_scale = math.sqrt(
106 config.d_model) if config.scale_embedding else 1.0
107
@@ -198,7 +200,7 @@ class Florence2ForConditionalGeneration(nn.Module):
200 # TODO(Isotr0py): Add vision backbone
201 self.language_model = Florence2LanguageForConditionalGeneration(
202 vllm_config=vllm_config.with_hf_config(config.text_config),
201 - prefix=prefix,
203 + prefix=f"{prefix}.language_model",
204 )
205
206 @property
vllm/model_executor/models/gemma.py
+2 -1
@@ -174,7 +174,8 @@ class GemmaAttention(nn.Module):
174 self.scaling,
175 num_kv_heads=self.num_kv_heads,
176 cache_config=cache_config,
177 - quant_config=quant_config)
177 + quant_config=quant_config,
178 + prefix=f"{prefix}.attn")
179
180 def forward(
181 self,
vllm/model_executor/models/gemma2.py
+11 -4
@@ -95,7 +95,8 @@ class Gemma2Attention(nn.Module):
95 rope_theta: float,
96 cache_config: Optional[CacheConfig] = None,
97 quant_config: Optional[QuantizationConfig] = None,
98 - attn_logits_soft_cap: Optional[float] = None) -> None:
98 + attn_logits_soft_cap: Optional[float] = None,
99 + prefix: str = "") -> None:
100 super().__init__()
101 self.layer_idx = layer_idx
102 self.config = config
@@ -154,7 +155,8 @@ class Gemma2Attention(nn.Module):
155 num_kv_heads=self.num_kv_heads,
156 cache_config=cache_config,
157 quant_config=quant_config,
157 - logits_soft_cap=attn_logits_soft_cap)
158 + logits_soft_cap=attn_logits_soft_cap,
159 + prefix=f"{prefix}.attn")
160
161 def forward(
162 self,
@@ -179,6 +181,7 @@ class Gemma2DecoderLayer(nn.Module):
181 config: Gemma2Config,
182 cache_config: Optional[CacheConfig] = None,
183 quant_config: Optional[QuantizationConfig] = None,
184 + prefix: str = "",
185 ) -> None:
186 super().__init__()
187 self.hidden_size = config.hidden_size
@@ -194,6 +197,7 @@ class Gemma2DecoderLayer(nn.Module):
197 cache_config=cache_config,
198 quant_config=quant_config,
199 attn_logits_soft_cap=config.attn_logit_softcapping,
200 + prefix=f"{prefix}.self_attn",
201 )
202 self.hidden_size = config.hidden_size
203 self.mlp = Gemma2MLP(
@@ -257,8 +261,11 @@ class Gemma2Model(nn.Module):
261 )
262 self.start_layer, self.end_layer, self.layers = make_layers(
263 config.num_hidden_layers,
260 - lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[
261 - -1]), config, cache_config, quant_config),
264 + lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[-1]),
265 + config,
266 + cache_config,
267 + quant_config,
268 + prefix=prefix),
269 prefix=f"{prefix}.layers")
270 self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
271
vllm/model_executor/models/glm4_vision_encoder.py
+13 -4
@@ -56,6 +56,7 @@ class Attention(nn.Module):
56 self,
57 config,
58 quant_config: Optional[QuantizationConfig] = None,
59 + prefix: str = '',
60 ):
61 super().__init__()
62 self.hidden_size = config.hidden_size
@@ -135,11 +136,14 @@ class TransformerLayer(nn.Module):
136 self,
137 config,
138 quant_config: Optional[QuantizationConfig] = None,
139 + prefix: str = '',
140 ):
141 super().__init__()
142 self.input_layernorm = LayerNorm(config.hidden_size,
143 eps=config.layer_norm_eps)
142 - self.attention = Attention(config, quant_config=quant_config)
144 + self.attention = Attention(config,
145 + quant_config=quant_config,
146 + prefix=f"{prefix}.attention")
147 self.mlp = MLP(config, quant_config=quant_config)
148 self.post_attention_layernorm = LayerNorm(config.hidden_size,
149 eps=config.layer_norm_eps)
@@ -161,11 +165,14 @@ class Transformer(nn.Module):
165 self,
166 config,
167 quant_config: Optional[QuantizationConfig] = None,
168 + prefix: str = '',
169 ):
170 super().__init__()
171 self.layers = nn.ModuleList([
167 - TransformerLayer(config, quant_config=quant_config)
168 - for _ in range(config.num_hidden_layers)
172 + TransformerLayer(config,
173 + quant_config=quant_config,
174 + prefix=f"{prefix}.layer.{layer_idx}")
175 + for layer_idx in range(config.num_hidden_layers)
176 ])
177
178 def forward(self, hidden_states):
@@ -252,12 +259,14 @@ class EVA2CLIPModel(nn.Module):
259 self,
260 config,
261 quant_config: Optional[QuantizationConfig] = None,
262 + prefix: str = '',
263 ):
264 super().__init__()
265 vision_config = Namespace(**config.vision_config)
266 self.patch_embedding = PatchEmbedding(vision_config)
267 self.transformer = Transformer(vision_config,
260 - quant_config=quant_config)
268 + quant_config=quant_config,
269 + prefix=f"{prefix}.transformer")
270 self.linear_proj = GLU(config,
271 in_features=config.hidden_size,
272 quant_config=quant_config)
vllm/model_executor/models/gpt2.py
+2 -1
@@ -84,7 +84,8 @@ class GPT2Attention(nn.Module):
84 self.head_dim,
85 scale=self.scale,
86 cache_config=cache_config,
87 - quant_config=quant_config)
87 + quant_config=quant_config,
88 + prefix=f"{prefix}.attn")
89
90 def forward(
91 self,
vllm/model_executor/models/gpt_bigcode.py
+10 -3
@@ -52,6 +52,7 @@ class GPTBigCodeAttention(nn.Module):
52 config: GPTBigCodeConfig,
53 cache_config: Optional[CacheConfig] = None,
54 quant_config: Optional[QuantizationConfig] = None,
55 + prefix: str = "",
56 ):
57 super().__init__()
58 self.hidden_size = config.hidden_size
@@ -92,7 +93,8 @@ class GPTBigCodeAttention(nn.Module):
93 scale=self.scale,
94 num_kv_heads=self.num_kv_heads,
95 cache_config=cache_config,
95 - quant_config=quant_config)
96 + quant_config=quant_config,
97 + prefix=f"{prefix}.attn")
98
99 def forward(
100 self,
@@ -151,6 +153,7 @@ class GPTBigCodeBlock(nn.Module):
153 config: GPTBigCodeConfig,
154 cache_config: Optional[CacheConfig] = None,
155 quant_config: Optional[QuantizationConfig] = None,
156 + prefix: str = "",
157 ):
158 super().__init__()
159 hidden_size = config.hidden_size
@@ -158,7 +161,10 @@ class GPTBigCodeBlock(nn.Module):
161 hidden_size)
162
163 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
161 - self.attn = GPTBigCodeAttention(config, cache_config, quant_config)
164 + self.attn = GPTBigCodeAttention(config,
165 + cache_config,
166 + quant_config,
167 + prefix=f"{prefix}.attn")
168 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
169 self.mlp = GPTBigMLP(inner_dim, config, quant_config)
170
@@ -210,7 +216,8 @@ class GPTBigCodeModel(nn.Module):
216 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
217 self.start_layer, self.end_layer, self.h = make_layers(
218 config.num_hidden_layers,
213 - lambda prefix: GPTBigCodeBlock(config, cache_config, quant_config),
219 + lambda prefix: GPTBigCodeBlock(
220 + config, cache_config, quant_config, prefix=prefix),
221 prefix=f"{prefix}.h",
222 )
223 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
vllm/model_executor/models/gpt_j.py
+10 -3
@@ -53,6 +53,7 @@ class GPTJAttention(nn.Module):
53 config: GPTJConfig,
54 cache_config: Optional[CacheConfig] = None,
55 quant_config: Optional[QuantizationConfig] = None,
56 + prefix: str = "",
57 ):
58 super().__init__()
59 self.total_num_heads = config.num_attention_heads
@@ -94,7 +95,8 @@ class GPTJAttention(nn.Module):
95 self.head_size,
96 scaling,
97 cache_config=cache_config,
97 - quant_config=quant_config)
98 + quant_config=quant_config,
99 + prefix=f"{prefix}.attn")
100
101 def forward(
102 self,
@@ -147,12 +149,16 @@ class GPTJBlock(nn.Module):
149 config: GPTJConfig,
150 cache_config: Optional[CacheConfig] = None,
151 quant_config: Optional[QuantizationConfig] = None,
152 + prefix: str = "",
153 ):
154 super().__init__()
155 inner_dim = (4 * config.n_embd
156 if config.n_inner is None else config.n_inner)
157 self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
155 - self.attn = GPTJAttention(config, cache_config, quant_config)
158 + self.attn = GPTJAttention(config,
159 + cache_config,
160 + quant_config,
161 + prefix=f"{prefix}.attn")
162 self.mlp = GPTJMLP(inner_dim, config, quant_config)
163
164 def forward(
@@ -193,7 +199,8 @@ class GPTJModel(nn.Module):
199 )
200 self.start_layer, self.end_layer, self.h = make_layers(
201 config.n_layer,
196 - lambda prefix: GPTJBlock(config, cache_config, quant_config),
202 + lambda prefix: GPTJBlock(
203 + config, cache_config, quant_config, prefix=prefix),
204 prefix=f"{prefix}.h",
205 )
206 self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
vllm/model_executor/models/gpt_neox.py
+10 -3
@@ -52,6 +52,7 @@ class GPTNeoXAttention(nn.Module):
52 config: GPTNeoXConfig,
53 cache_config: Optional[CacheConfig] = None,
54 quant_config: Optional[QuantizationConfig] = None,
55 + prefix: str = "",
56 ):
57 super().__init__()
58 self.total_num_heads = config.num_attention_heads
@@ -94,7 +95,8 @@ class GPTNeoXAttention(nn.Module):
95 self.head_size,
96 scaling,
97 cache_config=cache_config,
97 - quant_config=quant_config)
98 + quant_config=quant_config,
99 + prefix=f"{prefix}.attn")
100
101 def forward(
102 self,
@@ -145,6 +147,7 @@ class GPTNeoXLayer(nn.Module):
147 config: GPTNeoXConfig,
148 cache_config: Optional[CacheConfig] = None,
149 quant_config: Optional[QuantizationConfig] = None,
150 + prefix: str = "",
151 ):
152 super().__init__()
153 self.use_parallel_residual = config.use_parallel_residual
@@ -152,7 +155,10 @@ class GPTNeoXLayer(nn.Module):
155 eps=config.layer_norm_eps)
156 self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
157 eps=config.layer_norm_eps)
155 - self.attention = GPTNeoXAttention(config, cache_config, quant_config)
158 + self.attention = GPTNeoXAttention(config,
159 + cache_config,
160 + quant_config,
161 + prefix=f"{prefix}.attention")
162 self.mlp = GPTNeoXMLP(config, quant_config)
163
164 def forward(
@@ -205,7 +211,8 @@ class GPTNeoXModel(nn.Module):
211 )
212 self.start_layer, self.end_layer, self.layers = make_layers(
213 config.num_hidden_layers,
208 - lambda prefix: GPTNeoXLayer(config, cache_config, quant_config),
214 + lambda prefix: GPTNeoXLayer(
215 + config, cache_config, quant_config, prefix=prefix),
216 prefix=f"{prefix}.layers",
217 )
218 self.final_layer_norm = nn.LayerNorm(config.hidden_size,
vllm/model_executor/models/granite.py
+2 -1
@@ -161,7 +161,8 @@ class GraniteAttention(nn.Module):
161 self.scaling,
162 num_kv_heads=self.num_kv_heads,
163 cache_config=cache_config,
164 - quant_config=quant_config)
164 + quant_config=quant_config,
165 + prefix=f"{prefix}.attn")
166
167 def forward(
168 self,
vllm/model_executor/models/granitemoe.py
+2 -1
@@ -164,7 +164,8 @@ class GraniteMoeAttention(nn.Module):
164 self.scaling,
165 num_kv_heads=self.num_kv_heads,
166 cache_config=cache_config,
167 - quant_config=quant_config)
167 + quant_config=quant_config,
168 + prefix=f"{prefix}.attn")
169
170 def forward(
171 self,
vllm/model_executor/models/internlm2.py
+15 -6
@@ -1,5 +1,5 @@
1 from functools import partial
2 -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
2 +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
3
4 import torch
5 from torch import nn
@@ -250,7 +250,12 @@ class InternLMDecoderLayer(nn.Module):
250 @support_torch_compile
251 class InternLM2Model(nn.Module):
252
253 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
253 + def __init__(
254 + self,
255 + *,
256 + vllm_config: VllmConfig,
257 + prefix: str = "",
258 + layer_type: Type[InternLMDecoderLayer] = InternLMDecoderLayer):
259 super().__init__()
260
261 config = vllm_config.model_config.hf_config
@@ -266,7 +271,7 @@ class InternLM2Model(nn.Module):
271 )
272 self.start_layer, self.end_layer, self.layers = make_layers(
273 config.num_hidden_layers,
269 - lambda prefix: InternLMDecoderLayer(
274 + lambda prefix: layer_type(
275 config, cache_config, quant_config, prefix=prefix),
276 prefix=f"{prefix}.layers")
277 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -316,14 +321,18 @@ class InternLM2Model(nn.Module):
321
322 class InternLM2ForCausalLM(nn.Module, SupportsPP):
323
319 - def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
324 + def __init__(self,
325 + *,
326 + vllm_config: VllmConfig,
327 + prefix: str = "",
328 + model_type: Type[InternLM2Model] = InternLM2Model):
329 super().__init__()
330 config = vllm_config.model_config.hf_config
331 quant_config = vllm_config.quant_config
332 self.config = config
333 self.quant_config = quant_config
325 - self.model = InternLM2Model(vllm_config=vllm_config,
326 - prefix=maybe_prefix(prefix, "model"))
334 + self.model = model_type(vllm_config=vllm_config,
335 + prefix=maybe_prefix(prefix, "model"))
336 self.output = ParallelLMHead(config.vocab_size,
337 config.hidden_size,
338 quant_config=quant_config,
vllm/model_executor/models/internlm2_ve.py
+6 -17
@@ -14,8 +14,6 @@ from vllm.model_executor.models.internlm2 import (InternLM2Attention,
14 InternLM2MLP, InternLM2Model)
15 from vllm.sequence import IntermediateTensors
16
17 -from .utils import make_layers, maybe_prefix
18 -
17
18 class InternLM2VEDecoderLayer(nn.Module):
19
@@ -105,17 +103,9 @@ class InternLM2VEDecoderLayer(nn.Module):
103 class InternLM2VEModel(InternLM2Model):
104
105 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
108 - super().__init__(vllm_config=vllm_config, prefix=prefix)
109 -
110 - config = vllm_config.model_config.hf_config
111 - cache_config = vllm_config.cache_config
112 - quant_config = vllm_config.quant_config
113 -
114 - self.start_layer, self.end_layer, self.layers = make_layers(
115 - config.num_hidden_layers,
116 - lambda prefix: InternLM2VEDecoderLayer(
117 - config, cache_config, quant_config, prefix=prefix),
118 - prefix=f"{prefix}.layers")
106 + super().__init__(vllm_config=vllm_config,
107 + prefix=prefix,
108 + layer_type=InternLM2VEDecoderLayer)
109
110 def forward(
111 self,
@@ -159,7 +149,6 @@ class InternLM2VEModel(InternLM2Model):
149 class InternLM2VEForCausalLM(InternLM2ForCausalLM):
150
151 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
162 - super().__init__(vllm_config=vllm_config, prefix=prefix)
163 -
164 - self.model = InternLM2VEModel(vllm_config=vllm_config,
165 - prefix=maybe_prefix(prefix, "model"))
152 + super().__init__(vllm_config=vllm_config,
153 + prefix=prefix,
154 + model_type=InternLM2VEModel)
vllm/model_executor/models/jais.py
+10 -3
@@ -76,6 +76,7 @@ class JAISAttention(nn.Module):
76 config: JAISConfig,
77 cache_config: Optional[CacheConfig] = None,
78 quant_config: Optional[QuantizationConfig] = None,
79 + prefix: str = "",
80 ):
81 super().__init__()
82 self.hidden_size = config.hidden_size
@@ -114,7 +115,8 @@ class JAISAttention(nn.Module):
115 scale=self.scale,
116 alibi_slopes=alibi_slopes,
117 cache_config=cache_config,
117 - quant_config=quant_config)
118 + quant_config=quant_config,
119 + prefix=f"{prefix}.attn")
120
121 def forward(
122 self,
@@ -178,6 +180,7 @@ class JAISBlock(nn.Module):
180 config: JAISConfig,
181 cache_config: Optional[CacheConfig] = None,
182 quant_config: Optional[QuantizationConfig] = None,
183 + prefix: str = "",
184 ):
185 super().__init__()
186 hidden_size = config.hidden_size
@@ -185,7 +188,10 @@ class JAISBlock(nn.Module):
188 hidden_size)
189
190 self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
188 - self.attn = JAISAttention(config, cache_config, quant_config)
191 + self.attn = JAISAttention(config,
192 + cache_config,
193 + quant_config,
194 + prefix=f"{prefix}.attn")
195 self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
196 self.mlp = JAISMLP(inner_dim, config, quant_config)
197
@@ -241,7 +247,8 @@ class JAISModel(nn.Module):
247 config.num_hidden_layers,
248 lambda prefix: JAISBlock(config=config,
249 cache_config=cache_config,
244 - quant_config=quant_config),
250 + quant_config=quant_config,
251 + prefix=prefix),
252 prefix=f"{prefix}.h",
253 )
254
vllm/model_executor/models/jamba.py
+6 -2
@@ -102,7 +102,8 @@ class JambaMambaDecoderLayer(nn.Module):
102 config: JambaConfig,
103 layer_idx: int,
104 cache_config: Optional[CacheConfig] = None,
105 - quant_config: Optional[QuantizationConfig] = None) -> None:
105 + quant_config: Optional[QuantizationConfig] = None,
106 + prefix: str = "") -> None:
107 super().__init__()
108 self.config = config
109 self.mamba = MambaMixer(hidden_size= config.hidden_size,
@@ -157,6 +158,7 @@ class JambaAttentionDecoderLayer(nn.Module):
158 layer_idx: int,
159 cache_config: Optional[CacheConfig] = None,
160 quant_config: Optional[QuantizationConfig] = None,
161 + prefix: str = "",
162 ) -> None:
163 super().__init__()
164 self.hidden_size = config.hidden_size
@@ -198,6 +200,7 @@ class JambaAttentionDecoderLayer(nn.Module):
200 self.scaling,
201 num_kv_heads=self.num_kv_heads,
202 cache_config=cache_config,
203 + prefix=f"{prefix}.attn",
204 )
205
206 num_experts = config.layers_num_experts[layer_idx]
@@ -287,7 +290,8 @@ class JambaModel(nn.Module):
290 layer_class(config,
291 layer_idx=i,
292 cache_config=cache_config,
290 - quant_config=quant_config))
293 + quant_config=quant_config,
294 + prefix=f"{prefix}.layers.{i}"))
295 self.layers = nn.ModuleList(decoder_layers)
296 self.final_layernorm = RMSNorm(config.hidden_size,
297 eps=config.rms_norm_eps)
vllm/model_executor/models/llama.py
+1
@@ -174,6 +174,7 @@ class LlamaAttention(nn.Module):
174 num_kv_heads=self.num_kv_heads,
175 cache_config=cache_config,
176 quant_config=quant_config,
177 + prefix=f"{prefix}.attn",
178 )
179
180 def forward(
vllm/model_executor/models/minicpm.py
+8 -3
@@ -192,6 +192,7 @@ class MiniCPMAttention(nn.Module):
192 max_position_embeddings: int = 8192,
193 cache_config: Optional[CacheConfig] = None,
194 quant_config: Optional[QuantizationConfig] = None,
195 + prefix: str = "",
196 ) -> None:
197 super().__init__()
198 self.hidden_size = hidden_size
@@ -246,7 +247,8 @@ class MiniCPMAttention(nn.Module):
247 self.scaling,
248 num_kv_heads=self.num_kv_heads,
249 cache_config=cache_config,
249 - quant_config=quant_config)
250 + quant_config=quant_config,
251 + prefix=f"{prefix}.attn")
252
253 def forward(
254 self,
@@ -273,6 +275,7 @@ class MiniCPMDecoderLayer(nn.Module):
275 config: PretrainedConfig,
276 cache_config: Optional[CacheConfig] = None,
277 quant_config: Optional[QuantizationConfig] = None,
278 + prefix: str = "",
279 ) -> None:
280 super().__init__()
281 self.config = config
@@ -283,6 +286,7 @@ class MiniCPMDecoderLayer(nn.Module):
286 self.rope_scaling = getattr(config, "rope_scaling", None)
287 self.max_position_embeddings = getattr(config,
288 "max_position_embeddings", 8192)
289 + self.prefix = prefix
290 self._init_attn_block()
291 self._init_ffn_block()
292
@@ -298,6 +302,7 @@ class MiniCPMDecoderLayer(nn.Module):
302 max_position_embeddings=self.max_position_embeddings,
303 cache_config=self.cache_config,
304 quant_config=self.quant_config,
305 + prefix=f"{self.prefix}.self_attn",
306 )
307
308 def _init_ffn_block(self):
@@ -388,8 +393,8 @@ class MiniCPMModel(nn.Module):
393 ):
394 self.start_layer, self.end_layer, self.layers = make_layers(
395 config.num_hidden_layers,
391 - lambda prefix: MiniCPMDecoderLayer(config, cache_config,
392 - quant_config),
396 + lambda prefix: MiniCPMDecoderLayer(
397 + config, cache_config, quant_config, prefix=prefix),
398 prefix=f"{prefix}.layers")
399
400 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/minicpm3.py
+6 -3
@@ -60,6 +60,7 @@ class MiniCPM3Attention(nn.Module):
60 max_position_embeddings: int = 8192,
61 cache_config: Optional[CacheConfig] = None,
62 quant_config: Optional[QuantizationConfig] = None,
63 + prefix: str = "",
64 ) -> None:
65 super().__init__()
66 self.hidden_size = hidden_size
@@ -119,7 +120,8 @@ class MiniCPM3Attention(nn.Module):
120 self.scaling,
121 num_kv_heads=self.num_local_heads,
122 cache_config=cache_config,
122 - quant_config=quant_config)
123 + quant_config=quant_config,
124 + prefix=f"{prefix}.attn")
125
126 def forward(
127 self,
@@ -195,6 +197,7 @@ class MiniCPM3DecoderLayer(MiniCPMDecoderLayer):
197 max_position_embeddings=self.max_position_embeddings,
198 cache_config=self.cache_config,
199 quant_config=self.quant_config,
200 + prefix=f"{self.prefix}.self_attn",
201 )
202
203
@@ -209,8 +212,8 @@ class MiniCPM3Model(MiniCPMModel):
212 ):
213 self.start_layer, self.end_layer, self.layers = make_layers(
214 config.num_hidden_layers,
212 - lambda prefix: MiniCPM3DecoderLayer(config, cache_config,
213 - quant_config),
215 + lambda prefix: MiniCPM3DecoderLayer(
216 + config, cache_config, quant_config, prefix=prefix),
217 prefix=f"{prefix}.layers")
218
219
vllm/model_executor/models/mixtral.py
+2 -1
@@ -166,7 +166,8 @@ class MixtralAttention(nn.Module):
166 self.scaling,
167 num_kv_heads=self.num_kv_heads,
168 cache_config=cache_config,
169 - quant_config=quant_config)
169 + quant_config=quant_config,
170 + prefix=f"{prefix}.attn")
171
172 def forward(
173 self,
vllm/model_executor/models/mixtral_quant.py
+9 -3
@@ -170,6 +170,7 @@ class MixtralAttention(nn.Module):
170 rope_theta: float = 10000,
171 quant_config: Optional[QuantizationConfig] = None,
172 cache_config: Optional[CacheConfig] = None,
173 + prefix: str = "",
174 ) -> None:
175 super().__init__()
176 self.hidden_size = hidden_size
@@ -219,7 +220,8 @@ class MixtralAttention(nn.Module):
220 self.scaling,
221 num_kv_heads=self.num_kv_heads,
222 cache_config=cache_config,
222 - quant_config=quant_config)
223 + quant_config=quant_config,
224 + prefix=f"{prefix}.attn")
225
226 def forward(
227 self,
@@ -243,6 +245,7 @@ class MixtralDecoderLayer(nn.Module):
245 config: MixtralConfig,
246 cache_config: Optional[CacheConfig] = None,
247 quant_config: Optional[QuantizationConfig] = None,
248 + prefix: str = "",
249 ) -> None:
250 super().__init__()
251 self.hidden_size = config.hidden_size
@@ -255,7 +258,9 @@ class MixtralDecoderLayer(nn.Module):
258 num_kv_heads=config.num_key_value_heads,
259 rope_theta=rope_theta,
260 cache_config=cache_config,
258 - quant_config=quant_config)
261 + quant_config=quant_config,
262 + prefix=f"{prefix}.self_attn",
263 + )
264 self.block_sparse_moe = MixtralMoE(config=config,
265 quant_config=quant_config)
266 self.input_layernorm = RMSNorm(config.hidden_size,
@@ -311,7 +316,8 @@ class MixtralModel(nn.Module):
316 self.start_layer, self.end_layer, self.layers = make_layers(
317 config.num_hidden_layers,
318 lambda prefix: MixtralDecoderLayer(
314 - config, cache_config, quant_config=quant_config),
319 + config, cache_config, quant_config=quant_config, prefix=prefix
320 + ),
321 prefix=f"{prefix}.layers")
322 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
323 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/molmo.py
+10 -3
@@ -370,6 +370,7 @@ class MolmoAttention(nn.Module):
370 config: PretrainedConfig,
371 cache_config: Optional[CacheConfig] = None,
372 quant_config: Optional[QuantizationConfig] = None,
373 + prefix: str = "",
374 ) -> None:
375 super().__init__()
376 self.hidden_size = config.hidden_size
@@ -427,7 +428,8 @@ class MolmoAttention(nn.Module):
428 self.scaling,
429 num_kv_heads=self.num_kv_heads,
430 cache_config=cache_config,
430 - quant_config=quant_config)
431 + quant_config=quant_config,
432 + prefix=f"{prefix}.attn")
433
434 # Attention output projection.
435 self.o_proj = RowParallelLinear(
@@ -517,10 +519,14 @@ class MolmoDecoderLayer(nn.Module):
519 config: PretrainedConfig,
520 cache_config: Optional[CacheConfig] = None,
521 quant_config: Optional[QuantizationConfig] = None,
522 + prefix: str = "",
523 ) -> None:
524 super().__init__()
525 # Attention block.
523 - self.self_attn = MolmoAttention(config, cache_config, quant_config)
526 + self.self_attn = MolmoAttention(config,
527 + cache_config,
528 + quant_config,
529 + prefix=f"{prefix}.self_attn")
530
531 # MLP block.
532 self.mlp = MolmoMLP(config, quant_config=quant_config)
@@ -738,7 +744,8 @@ class MolmoModel(nn.Module):
744 else MolmoDecoderLayer
745 self.start_layer, self.end_layer, self.layers = make_layers(
746 config.num_hidden_layers,
741 - lambda prefix: decoder_layer(config, cache_config, quant_config),
747 + lambda prefix: decoder_layer(
748 + config, cache_config, quant_config, prefix=prefix),
749 prefix=f"{prefix}.layers",
750 )
751
vllm/model_executor/models/mpt.py
+10 -3
@@ -50,6 +50,7 @@ class MPTAttention(nn.Module):
50 config: MPTConfig,
51 cache_config: Optional[CacheConfig] = None,
52 quant_config: Optional[QuantizationConfig] = None,
53 + prefix: str = "",
54 ):
55 super().__init__()
56 self.d_model = config.d_model
@@ -115,7 +116,8 @@ class MPTAttention(nn.Module):
116 alibi_slopes=alibi_slopes,
117 num_kv_heads=self.num_kv_heads,
118 cache_config=cache_config,
118 - quant_config=quant_config)
119 + quant_config=quant_config,
120 + prefix=f"{prefix}.attn")
121
122 def forward(
123 self,
@@ -176,11 +178,15 @@ class MPTBlock(nn.Module):
178 config: MPTConfig,
179 cache_config: Optional[CacheConfig] = None,
180 quant_config: Optional[QuantizationConfig] = None,
181 + prefix: str = "",
182 ):
183 super().__init__()
184 hidden_size = config.d_model
185 self.norm_1 = nn.LayerNorm(hidden_size)
183 - self.attn = MPTAttention(config, cache_config, quant_config)
186 + self.attn = MPTAttention(config,
187 + cache_config,
188 + quant_config,
189 + prefix=f"{prefix}.attn")
190 self.norm_2 = nn.LayerNorm(hidden_size)
191 self.ffn = MPTMLP(config, quant_config)
192
@@ -224,7 +230,8 @@ class MPTModel(nn.Module):
230 )
231 self.start_layer, self.end_layer, self.blocks = make_layers(
232 config.n_layers,
227 - lambda prefix: MPTBlock(config, cache_config, quant_config),
233 + lambda prefix: MPTBlock(
234 + config, cache_config, quant_config, prefix=prefix),
235 prefix=f"{prefix}.blocks")
236 self.norm_f = nn.LayerNorm(config.d_model)
237 if config.no_bias:
vllm/model_executor/models/nemotron.py
+2 -1
@@ -195,7 +195,8 @@ class NemotronAttention(nn.Module):
195 self.scaling,
196 num_kv_heads=self.num_kv_heads,
197 cache_config=cache_config,
198 - quant_config=quant_config)
198 + quant_config=quant_config,
199 + prefix=f"{prefix}.attn")
200
201 def forward(
202 self,
vllm/model_executor/models/olmo.py
+11 -5
@@ -62,6 +62,7 @@ class OlmoAttention(nn.Module):
62 config: OlmoConfig,
63 cache_config: Optional[CacheConfig] = None,
64 quant_config: Optional[QuantizationConfig] = None,
65 + prefix: str = "",
66 ):
67 super().__init__()
68 self.config = config
@@ -101,7 +102,8 @@ class OlmoAttention(nn.Module):
102 self.head_dim,
103 scale=self.scaling,
104 cache_config=cache_config,
104 - quant_config=quant_config)
105 + quant_config=quant_config,
106 + prefix=f"{prefix}.attn")
107
108 # Attention output projection.
109 self.o_proj = RowParallelLinear(
@@ -184,10 +186,14 @@ class OlmoDecoderLayer(nn.Module):
186 def __init__(self,
187 config: OlmoConfig,
188 cache_config: Optional[CacheConfig] = None,
187 - quant_config: Optional[QuantizationConfig] = None):
189 + quant_config: Optional[QuantizationConfig] = None,
190 + prefix: str = ""):
191 super().__init__()
192 # Attention block.
190 - self.self_attn = OlmoAttention(config, cache_config, quant_config)
193 + self.self_attn = OlmoAttention(config,
194 + cache_config,
195 + quant_config,
196 + prefix=f"{prefix}.self_attn")
197
198 # MLP block.
199 self.mlp = OlmoMLP(config, quant_config)
@@ -238,8 +244,8 @@ class OlmoModel(nn.Module):
244 config.hidden_size)
245 self.start_layer, self.end_layer, self.layers = make_layers(
246 config.num_hidden_layers,
241 - lambda prefix: OlmoDecoderLayer(config, cache_config, quant_config
242 - ),
247 + lambda prefix: OlmoDecoderLayer(
248 + config, cache_config, quant_config, prefix=prefix),
249 prefix=f"{prefix}.layers")
250 self.norm = nn.LayerNorm(config.hidden_size,
251 elementwise_affine=False,
vllm/model_executor/models/olmoe.py
+10 -3
@@ -102,6 +102,7 @@ class OlmoeAttention(nn.Module):
102 max_position_embeddings: int = 4096,
103 cache_config: Optional[CacheConfig] = None,
104 quant_config: Optional[QuantizationConfig] = None,
105 + prefix: str = "",
106 ) -> None:
107 super().__init__()
108 self.hidden_size = hidden_size
@@ -156,7 +157,8 @@ class OlmoeAttention(nn.Module):
157 self.scaling,
158 num_kv_heads=self.num_kv_heads,
159 cache_config=cache_config,
159 - quant_config=quant_config)
160 + quant_config=quant_config,
161 + prefix=f"{prefix}.attn")
162
163 def forward(
164 self,
@@ -182,6 +184,7 @@ class OlmoeDecoderLayer(nn.Module):
184 layer_idx: int,
185 cache_config: Optional[CacheConfig] = None,
186 quant_config: Optional[QuantizationConfig] = None,
187 + prefix: str = "",
188 ) -> None:
189 super().__init__()
190 self.hidden_size = config.hidden_size
@@ -199,6 +202,7 @@ class OlmoeDecoderLayer(nn.Module):
202 max_position_embeddings=max_position_embeddings,
203 cache_config=cache_config,
204 quant_config=quant_config,
205 + prefix=f"{prefix}.self_attn",
206 )
207
208 self.mlp = OlmoeMoE(
@@ -260,8 +264,11 @@ class OlmoeModel(nn.Module):
264 )
265 self.start_layer, self.end_layer, self.layers = make_layers(
266 config.num_hidden_layers,
263 - lambda prefix: OlmoeDecoderLayer(config, int(
264 - prefix.split(".")[-1]), cache_config, quant_config),
267 + lambda prefix: OlmoeDecoderLayer(config,
268 + int(prefix.split(".")[-1]),
269 + cache_config,
270 + quant_config,
271 + prefix=prefix),
272 prefix=f"{prefix}.layers")
273 self.norm = RMSNorm(config.hidden_size, eps=1e-5)
274
vllm/model_executor/models/orion.py
+6 -5
@@ -75,6 +75,7 @@ class OrionAttention(nn.Module):
75 max_position_embeddings: int = 8192,
76 cache_config: Optional[CacheConfig] = None,
77 quant_config: Optional[QuantizationConfig] = None,
78 + prefix: str = "",
79 ) -> None:
80 super().__init__()
81 self.hidden_size = hidden_size
@@ -126,7 +127,8 @@ class OrionAttention(nn.Module):
127 self.scaling,
128 num_kv_heads=self.num_kv_heads,
129 cache_config=cache_config,
129 - quant_config=quant_config)
130 + quant_config=quant_config,
131 + prefix=f"{prefix}.attn")
132
133 def forward(
134 self,
@@ -150,6 +152,7 @@ class OrionDecoderLayer(nn.Module):
152 config: PretrainedConfig,
153 cache_config: Optional[CacheConfig] = None,
154 quant_config: Optional[QuantizationConfig] = None,
155 + prefix: str = "",
156 ) -> None:
157 super().__init__()
158 self.hidden_size = config.hidden_size
@@ -166,6 +169,7 @@ class OrionDecoderLayer(nn.Module):
169 max_position_embeddings=max_position_embeddings,
170 cache_config=cache_config,
171 quant_config=quant_config,
172 + prefix=f"{prefix}.self_attn",
173 )
174 self.mlp = OrionMLP(
175 hidden_size=self.hidden_size,
@@ -226,10 +230,7 @@ class OrionModel(nn.Module):
230 self.start_layer, self.end_layer, self.layers = make_layers(
231 config.num_hidden_layers,
232 lambda prefix: OrionDecoderLayer(
229 - config,
230 - cache_config,
231 - quant_config,
232 - ),
233 + config, cache_config, quant_config, prefix=prefix),
234 prefix=f"{prefix}.layers")
235 self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
236 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/persimmon.py
+10 -6
@@ -75,7 +75,8 @@ class PersimmonAttention(nn.Module):
75 def __init__(self,
76 config: PersimmonConfig,
77 cache_config: Optional[CacheConfig] = None,
78 - quant_config: Optional[QuantizationConfig] = None):
78 + quant_config: Optional[QuantizationConfig] = None,
79 + prefix: str = ""):
80 super().__init__()
81 self.config = config
82 tensor_parallel_world_size = get_tensor_model_parallel_world_size()
@@ -122,7 +123,8 @@ class PersimmonAttention(nn.Module):
123 self.head_dim,
124 scale=self.scaling,
125 cache_config=cache_config,
125 - quant_config=quant_config)
126 + quant_config=quant_config,
127 + prefix=f"{prefix}.attn")
128
129 def _split_heads(self, x: torch.Tensor) -> torch.Tensor:
130 # [seq_length, hidden_size] -> [seq_length, num_heads, head_dim]
@@ -167,12 +169,14 @@ class PersimmonDecoderLayer(nn.Module):
169 def __init__(self,
170 config: PersimmonConfig,
171 cache_config: Optional[CacheConfig] = None,
170 - quant_config: Optional[QuantizationConfig] = None):
172 + quant_config: Optional[QuantizationConfig] = None,
173 + prefix: str = ""):
174 super().__init__()
175 self.hidden_size = config.hidden_size
176 self.self_attn = PersimmonAttention(config=config,
177 cache_config=cache_config,
175 - quant_config=quant_config)
178 + quant_config=quant_config,
179 + prefix=f"{prefix}.self_attn")
180 self.mlp = PersimmonMLP(config, quant_config=quant_config)
181 self.input_layernorm = nn.LayerNorm(config.hidden_size,
182 eps=config.layer_norm_eps)
@@ -226,8 +230,8 @@ class PersimmonModel(nn.Module):
230 config.hidden_size)
231 self.start_layer, self.end_layer, self.layers = make_layers(
232 config.num_hidden_layers,
229 - lambda prefix: PersimmonDecoderLayer(config, cache_config,
230 - quant_config),
233 + lambda prefix: PersimmonDecoderLayer(
234 + config, cache_config, quant_config, prefix=prefix),
235 prefix=f"{prefix}.layers")
236 self.final_layernorm = nn.LayerNorm(config.hidden_size,
237 eps=config.layer_norm_eps)
vllm/model_executor/models/phi.py
+12 -5
@@ -69,7 +69,8 @@ class PhiAttention(nn.Module):
69 def __init__(self,
70 config: PhiConfig,
71 cache_config: Optional[CacheConfig] = None,
72 - quant_config: Optional[QuantizationConfig] = None):
72 + quant_config: Optional[QuantizationConfig] = None,
73 + prefix: str = ""):
74 super().__init__()
75 self.total_num_heads = config.num_attention_heads
76 self.hidden_size = config.hidden_size
@@ -116,7 +117,8 @@ class PhiAttention(nn.Module):
117 self.head_size,
118 scaling,
119 cache_config=cache_config,
119 - quant_config=quant_config)
120 + quant_config=quant_config,
121 + prefix=f"{prefix}.attn")
122
123 def forward(
124 self,
@@ -167,11 +169,15 @@ class PhiLayer(nn.Module):
169 def __init__(self,
170 config: PhiConfig,
171 cache_config: Optional[CacheConfig] = None,
170 - quant_config: Optional[QuantizationConfig] = None):
172 + quant_config: Optional[QuantizationConfig] = None,
173 + prefix: str = ""):
174 super().__init__()
175 self.input_layernorm = nn.LayerNorm(config.hidden_size,
176 eps=config.layer_norm_eps)
174 - self.self_attn = PhiAttention(config, cache_config, quant_config)
177 + self.self_attn = PhiAttention(config,
178 + cache_config,
179 + quant_config,
180 + prefix=f"{prefix}.self_attn")
181 self.mlp = PhiMLP(config, quant_config)
182
183 def forward(
@@ -210,7 +216,8 @@ class PhiModel(nn.Module):
216 config.hidden_size)
217 self.start_layer, self.end_layer, self.layers = make_layers(
218 config.num_hidden_layers,
213 - lambda prefix: PhiLayer(config, cache_config, quant_config),
219 + lambda prefix: PhiLayer(
220 + config, cache_config, quant_config, prefix=prefix),
221 prefix=f"{prefix}.layers")
222 self.final_layernorm = nn.LayerNorm(config.hidden_size,
223 eps=config.layer_norm_eps)
vllm/model_executor/models/phi3_small.py
+15 -11
@@ -117,6 +117,7 @@ class Phi3SmallSelfAttention(nn.Module):
117 layer_idx: int,
118 cache_config: Optional[CacheConfig] = None,
119 quant_config: Optional[QuantizationConfig] = None,
120 + prefix: str = "",
121 ) -> None:
122 super().__init__()
123 self.layer_idx = layer_idx
@@ -214,15 +215,14 @@ class Phi3SmallSelfAttention(nn.Module):
215 "homo_head": self.homo_heads
216 }
217
217 - self.attn = Attention(
218 - self.num_heads_per_partition,
219 - self.head_dim,
220 - self.scale,
221 - num_kv_heads=self.num_kv_heads_per_partion,
222 - cache_config=cache_config,
223 - quant_config=quant_config,
224 - blocksparse_params=bs_params,
225 - )
218 + self.attn = Attention(self.num_heads_per_partition,
219 + self.head_dim,
220 + self.scale,
221 + num_kv_heads=self.num_kv_heads_per_partion,
222 + cache_config=cache_config,
223 + quant_config=quant_config,
224 + blocksparse_params=bs_params,
225 + prefix=f"{prefix}.attn")
226
227 def forward(
228 self,
@@ -259,13 +259,15 @@ class Phi3SmallDecoderLayer(nn.Module):
259 layer_idx: int,
260 cache_config: Optional[CacheConfig] = None,
261 quant_config: Optional[QuantizationConfig] = None,
262 + prefix: str = "",
263 ):
264 super().__init__()
265 self.hidden_size = config.hidden_size
266 self.self_attn = Phi3SmallSelfAttention(config,
267 layer_idx,
268 cache_config=cache_config,
268 - quant_config=quant_config)
269 + quant_config=quant_config,
270 + prefix=f"{prefix}.self_attn")
271 self.mlp = Phi3SmallMLP(config, quant_config)
272
273 self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -315,7 +317,9 @@ class Phi3SmallModel(nn.Module):
317 config.num_hidden_layers,
318 lambda prefix: Phi3SmallDecoderLayer(config,
319 int(prefix.split('.')[-1]),
318 - cache_config, quant_config),
320 + cache_config,
321 + quant_config,
322 + prefix=prefix),
323 prefix=f"{prefix}.layers")
324
325 self.final_layernorm = nn.LayerNorm(config.hidden_size,
vllm/model_executor/models/phimoe.py
+6 -2
@@ -294,6 +294,7 @@ class PhiMoEAttention(nn.Module):
294 cache_config: Optional[CacheConfig] = None,
295 quant_config: Optional[QuantizationConfig] = None,
296 rope_scaling: Optional[dict] = None,
297 + prefix: str = "",
298 ) -> None:
299 super().__init__()
300 self.hidden_size = hidden_size
@@ -347,6 +348,7 @@ class PhiMoEAttention(nn.Module):
348 num_kv_heads=self.num_kv_heads,
349 cache_config=cache_config,
350 quant_config=quant_config,
351 + prefix=f"{prefix}.attn",
352 )
353
354 def forward(
@@ -371,6 +373,7 @@ class PhiMoEDecoderLayer(nn.Module):
373 config: PhiMoEConfig,
374 cache_config: Optional[CacheConfig] = None,
375 quant_config: Optional[QuantizationConfig] = None,
376 + prefix: str = "",
377 ) -> None:
378 super().__init__()
379 self.hidden_size = config.hidden_size
@@ -385,6 +388,7 @@ class PhiMoEDecoderLayer(nn.Module):
388 cache_config=cache_config,
389 quant_config=quant_config,
390 rope_scaling=config.rope_scaling,
391 + prefix=f"{prefix}.self_attn",
392 )
393 self.block_sparse_moe = PhiMoE(
394 num_experts=config.num_local_experts,
@@ -454,8 +458,8 @@ class PhiMoEModel(nn.Module):
458 )
459 self.start_layer, self.end_layer, self.layers = make_layers(
460 config.num_hidden_layers,
457 - lambda prefix: PhiMoEDecoderLayer(config, cache_config,
458 - quant_config),
461 + lambda prefix: PhiMoEDecoderLayer(
462 + config, cache_config, quant_config, prefix=prefix),
463 prefix=f"{prefix}.layers")
464 self.norm = nn.LayerNorm(config.hidden_size,
465 eps=config.rms_norm_eps,
vllm/model_executor/models/qwen.py
+8 -3
@@ -442,6 +442,7 @@ class QWenAttention(nn.Module):
442 rope_scaling: Optional[Dict[str, Any]] = None,
443 cache_config: Optional[CacheConfig] = None,
444 quant_config: Optional[QuantizationConfig] = None,
445 + prefix: str = "",
446 ):
447 super().__init__()
448 self.hidden_size = hidden_size
@@ -478,7 +479,8 @@ class QWenAttention(nn.Module):
479 self.head_dim,
480 self.scaling,
481 cache_config=cache_config,
481 - quant_config=quant_config)
482 + quant_config=quant_config,
483 + prefix=f"{prefix}.attn")
484
485 def forward(
486 self,
@@ -502,6 +504,7 @@ class QWenBlock(nn.Module):
504 config: PretrainedConfig,
505 cache_config: Optional[CacheConfig] = None,
506 quant_config: Optional[QuantizationConfig] = None,
507 + prefix: str = "",
508 ):
509 super().__init__()
510 self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -514,7 +517,8 @@ class QWenBlock(nn.Module):
517 rope_theta=rope_theta,
518 rope_scaling=rope_scaling,
519 cache_config=cache_config,
517 - quant_config=quant_config)
520 + quant_config=quant_config,
521 + prefix=f"{prefix}.attn")
522
523 self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
524
@@ -568,7 +572,8 @@ class QWenModel(nn.Module):
572 )
573 self.start_layer, self.end_layer, self.h = make_layers(
574 config.num_hidden_layers,
571 - lambda prefix: QWenBlock(config, cache_config, quant_config),
575 + lambda prefix: QWenBlock(
576 + config, cache_config, quant_config, prefix=prefix),
577 prefix=f"{prefix}.h")
578 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
579 self.make_empty_intermediate_tensors = (
vllm/model_executor/models/qwen2_moe.py
+7 -2
@@ -168,6 +168,7 @@ class Qwen2MoeAttention(nn.Module):
168 max_position_embeddings: int = 8192,
169 cache_config: Optional[CacheConfig] = None,
170 quant_config: Optional[QuantizationConfig] = None,
171 + prefix: str = "",
172 ) -> None:
173 super().__init__()
174 self.hidden_size = hidden_size
@@ -220,7 +221,8 @@ class Qwen2MoeAttention(nn.Module):
221 self.scaling,
222 num_kv_heads=self.num_kv_heads,
223 cache_config=cache_config,
223 - quant_config=quant_config)
224 + quant_config=quant_config,
225 + prefix=f"{prefix}.attn")
226
227 def forward(
228 self,
@@ -245,6 +247,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
247 layer_idx: int,
248 cache_config: Optional[CacheConfig] = None,
249 quant_config: Optional[QuantizationConfig] = None,
250 + prefix: str = "",
251 ) -> None:
252 super().__init__()
253 self.hidden_size = config.hidden_size
@@ -261,6 +264,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
264 max_position_embeddings=max_position_embeddings,
265 cache_config=cache_config,
266 quant_config=quant_config,
267 + prefix=f"{prefix}.self_attn",
268 )
269
270 # Note: Qwen/Qwen2-57B-A14B-Instruct does not have
@@ -336,7 +340,8 @@ class Qwen2MoeModel(nn.Module):
340 layer_idx=int(
341 prefix.split(".")[-1]),
342 cache_config=cache_config,
339 - quant_config=quant_config),
343 + quant_config=quant_config,
344 + prefix=prefix),
345 prefix=f"{prefix}.layers",
346 )
347 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/solar.py
+1
@@ -167,6 +167,7 @@ class SolarAttention(nn.Module):
167 num_kv_heads=self.num_kv_heads,
168 cache_config=cache_config,
169 quant_config=quant_config,
170 + prefix=f"{prefix}.attn",
171 )
172
173 def forward(
vllm/model_executor/models/stablelm.py
+11 -5
@@ -77,7 +77,8 @@ class StablelmAttention(nn.Module):
77 def __init__(self,
78 config: PretrainedConfig,
79 cache_config: Optional[CacheConfig] = None,
80 - quant_config: Optional[QuantizationConfig] = None) -> None:
80 + quant_config: Optional[QuantizationConfig] = None,
81 + prefix: str = "") -> None:
82 super().__init__()
83 self.config = config
84 self.hidden_size = config.hidden_size
@@ -131,7 +132,8 @@ class StablelmAttention(nn.Module):
132 self.scaling,
133 num_kv_heads=self.num_key_value_heads,
134 cache_config=cache_config,
134 - quant_config=quant_config)
135 + quant_config=quant_config,
136 + prefix=f"{prefix}.attn")
137
138 def forward(
139 self,
@@ -155,9 +157,13 @@ class StablelmDecoderLayer(nn.Module):
157 config: PretrainedConfig,
158 cache_config: Optional[CacheConfig] = None,
159 quant_config: Optional[QuantizationConfig] = None,
160 + prefix: str = "",
161 ) -> None:
162 super().__init__()
160 - self.self_attn = StablelmAttention(config, cache_config, quant_config)
163 + self.self_attn = StablelmAttention(config,
164 + cache_config,
165 + quant_config,
166 + prefix=f"{prefix}.self_attn")
167 self.mlp = StablelmMLP(config, quant_config)
168 norm_eps = getattr(config, "norm_eps",
169 getattr(config, "layer_norm_eps", 1e-05))
@@ -207,8 +213,8 @@ class StableLMEpochModel(nn.Module):
213 )
214 self.start_layer, self.end_layer, self.layers = make_layers(
215 config.num_hidden_layers,
210 - lambda prefix: StablelmDecoderLayer(config, cache_config,
211 - quant_config),
216 + lambda prefix: StablelmDecoderLayer(
217 + config, cache_config, quant_config, prefix=prefix),
218 prefix=f"{prefix}.layers",
219 )
220 norm_eps = getattr(config, "norm_eps",
vllm/model_executor/models/starcoder2.py
+10 -5
@@ -52,7 +52,8 @@ class Starcoder2Attention(nn.Module):
52 def __init__(self,
53 config: Starcoder2Config,
54 cache_config: Optional[CacheConfig] = None,
55 - quant_config: Optional[QuantizationConfig] = None):
55 + quant_config: Optional[QuantizationConfig] = None,
56 + prefix: str = ""):
57 super().__init__()
58 self.config = config
59
@@ -105,7 +106,8 @@ class Starcoder2Attention(nn.Module):
106 self.scaling,
107 num_kv_heads=self.num_kv_heads,
108 cache_config=cache_config,
108 - quant_config=quant_config)
109 + quant_config=quant_config,
110 + prefix=f"{prefix}.attn")
111
112 def forward(
113 self,
@@ -154,12 +156,14 @@ class Starcoder2DecoderLayer(nn.Module):
156 def __init__(self,
157 config: Starcoder2Config,
158 cache_config: Optional[CacheConfig] = None,
157 - quant_config: Optional[QuantizationConfig] = None):
159 + quant_config: Optional[QuantizationConfig] = None,
160 + prefix: str = ""):
161 super().__init__()
162 self.hidden_size = config.hidden_size
163 self.self_attn = Starcoder2Attention(config,
164 cache_config,
162 - quant_config=quant_config)
165 + quant_config=quant_config,
166 + prefix=f"{prefix}.self_attn")
167 self.mlp = Starcoder2MLP(config, quant_config=quant_config)
168 self.input_layernorm = nn.LayerNorm(config.hidden_size,
169 eps=config.norm_epsilon)
@@ -213,7 +217,8 @@ class Starcoder2Model(nn.Module):
217 self.start_layer, self.end_layer, self.layers = make_layers(
218 config.num_hidden_layers,
219 lambda prefix: Starcoder2DecoderLayer(
216 - config, cache_config, quant_config=quant_config),
220 + config, cache_config, quant_config=quant_config, prefix=prefix
221 + ),
222 prefix=f"{prefix}.layers",
223 )
224 self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
vllm/model_executor/models/xverse.py
+7 -3
@@ -93,6 +93,7 @@ class XverseAttention(nn.Module):
93 quant_config: Optional[QuantizationConfig] = None,
94 bias: bool = False,
95 cache_config: Optional[CacheConfig] = None,
96 + prefix: str = "",
97 ) -> None:
98 super().__init__()
99 self.hidden_size = hidden_size
@@ -138,7 +139,8 @@ class XverseAttention(nn.Module):
139 self.scaling,
140 num_kv_heads=self.num_kv_heads,
141 cache_config=cache_config,
141 - quant_config=quant_config)
142 + quant_config=quant_config,
143 + prefix=f"{prefix}.attn")
144
145 def forward(
146 self,
@@ -162,6 +164,7 @@ class XverseDecoderLayer(nn.Module):
164 config: PretrainedConfig,
165 cache_config: Optional[CacheConfig] = None,
166 quant_config: Optional[QuantizationConfig] = None,
167 + prefix: str = "",
168 ) -> None:
169 super().__init__()
170 self.hidden_size = config.hidden_size
@@ -180,6 +183,7 @@ class XverseDecoderLayer(nn.Module):
183 quant_config=quant_config,
184 bias=getattr(config, "bias", False),
185 cache_config=cache_config,
186 + prefix=f"{prefix}.self_attn",
187 )
188 self.mlp = XverseMLP(
189 hidden_size=self.hidden_size,
@@ -243,8 +247,8 @@ class XverseModel(nn.Module):
247 )
248 self.start_layer, self.end_layer, self.layers = make_layers(
249 config.num_hidden_layers,
246 - lambda prefix: XverseDecoderLayer(config, cache_config,
247 - quant_config),
250 + lambda prefix: XverseDecoderLayer(
251 + config, cache_config, quant_config, prefix=prefix),
252 prefix=f"{prefix}.layers",
253 )
254 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/platforms/cpu.py
+1
@@ -20,6 +20,7 @@ logger = init_logger(__name__)
20 class CpuPlatform(Platform):
21 _enum = PlatformEnum.CPU
22 device_type: str = "cpu"
23 + dispatch_key: str = "CPU"
24
25 @classmethod
26 def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/cuda.py
+1
@@ -121,6 +121,7 @@ def device_id_to_physical_device_id(device_id: int) -> int:
121 class CudaPlatform(Platform):
122 _enum = PlatformEnum.CUDA
123 device_type: str = "cuda"
124 + dispatch_key: str = "CUDA"
125
126 @classmethod
127 def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/hpu.py
+1
@@ -13,6 +13,7 @@ else:
13 class HpuPlatform(Platform):
14 _enum = PlatformEnum.HPU
15 device_type: str = "hpu"
16 + dispatch_key: str = "HPU"
17
18 @classmethod
19 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/interface.py
+4
@@ -57,6 +57,10 @@ class DeviceCapability(NamedTuple):
57 class Platform:
58 _enum: PlatformEnum
59 device_type: str
60 + # available dispatch keys:
61 + # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
62 + # use "CPU" as a fallback for platforms not registered in PyTorch
63 + dispatch_key: str = "CPU"
64
65 def is_cuda(self) -> bool:
66 return self._enum == PlatformEnum.CUDA
vllm/platforms/openvino.py
+1
@@ -18,6 +18,7 @@ logger = init_logger(__name__)
18 class OpenVinoPlatform(Platform):
19 _enum = PlatformEnum.OPENVINO
20 device_type: str = "openvino"
21 + dispatch_key: str = "CPU"
22
23 @classmethod
24 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/rocm.py
+1
@@ -36,6 +36,7 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
36 class RocmPlatform(Platform):
37 _enum = PlatformEnum.ROCM
38 device_type: str = "cuda"
39 + dispatch_key: str = "CUDA"
40
41 @classmethod
42 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/tpu.py
+1
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
17 class TpuPlatform(Platform):
18 _enum = PlatformEnum.TPU
19 device_type: str = "tpu"
20 + dispatch_key: str = "XLA"
21
22 @classmethod
23 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
17 class XPUPlatform(Platform):
18 _enum = PlatformEnum.XPU
19 device_type: str = "xpu"
20 + dispatch_key: str = "XPU"
21
22 @classmethod
23 def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/spec_decode/draft_model_runner.py
+2 -1
@@ -273,7 +273,8 @@ class TP1DraftModelRunner(ModelRunner):
273 if previous_hidden_states is not None else {}
274
275 # Run model
276 - with set_forward_context(model_input.attn_metadata):
276 + with set_forward_context(model_input.attn_metadata,
277 + self.vllm_config):
278 hidden_states = model_executable(
279 input_ids=model_input.input_tokens,
280 positions=model_input.input_positions,
vllm/utils.py
+2 -1
@@ -1573,6 +1573,7 @@ def direct_register_custom_op(
1573 mutates_args: List[str],
1574 fake_impl: Optional[Callable] = None,
1575 target_lib: Optional[Library] = None,
1576 + dispatch_key: str = "CUDA",
1577 ):
1578 """
1579 `torch.library.custom_op` can have significant overhead because it
@@ -1601,7 +1602,7 @@ def direct_register_custom_op(
1602 schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
1603 my_lib = target_lib or vllm_lib
1604 my_lib.define(op_name + schema_str)
1604 - my_lib.impl(op_name, op_func, "CUDA")
1605 + my_lib.impl(op_name, op_func, dispatch_key=dispatch_key)
1606 if fake_impl is not None:
1607 my_lib._register_fake(op_name, fake_impl)
1608
vllm/v1/attention/backends/flash_attn.py
+2 -1
@@ -173,7 +173,8 @@ def unified_v1_flash_attention(
173 alibi_slopes: Optional[torch.Tensor] = None,
174 logits_soft_cap: Optional[float] = None,
175 ) -> None:
176 - current_metadata = get_forward_context()
176 + context = get_forward_context()
177 + current_metadata = context.dynamic_forward_context
178 if current_metadata is None:
179 # Profiling run.
180 return
vllm/v1/worker/gpu_model_runner.py
+2 -2
@@ -447,7 +447,7 @@ class GPUModelRunner:
447
448 # Run the decoder.
449 # Use persistent buffers for CUDA graphs.
450 - with set_forward_context(attn_metadata):
450 + with set_forward_context(attn_metadata, self.vllm_config):
451 hidden_states = self.model(
452 input_ids=None,
453 positions=self.positions[:num_input_tokens],
@@ -523,7 +523,7 @@ class GPUModelRunner:
523 num_tokens: int,
524 kv_caches: List[torch.Tensor],
525 ) -> torch.Tensor:
526 - with set_forward_context(None):
526 + with set_forward_context(None, self.vllm_config):
527 hidden_states = model(
528 input_ids=None,
529 positions=self.positions[:num_tokens],
vllm/worker/embedding_model_runner.py
+1 -1
@@ -97,7 +97,7 @@ class EmbeddingModelRunner(
97 model_forward_end = torch.cuda.Event(enable_timing=True)
98 model_forward_start.record()
99
100 - with set_forward_context(model_input.attn_metadata):
100 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
101 hidden_or_intermediate_states = model_executable(
102 input_ids=model_input.input_tokens,
103 positions=model_input.input_positions,
vllm/worker/enc_dec_model_runner.py
+1 -1
@@ -176,7 +176,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
176 } if self.has_inner_state else {}
177
178 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
179 - with set_forward_context(model_input.attn_metadata):
179 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
180 hidden_or_intermediate_states = model_executable(
181 input_ids=model_input.input_tokens,
182 positions=model_input.input_positions,
vllm/worker/model_runner.py
+2 -2
@@ -1503,7 +1503,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1503 self._update_inputs_to_capture_for_enc_dec_model(
1504 capture_inputs)
1505
1506 - with set_forward_context(attn_metadata):
1506 + with set_forward_context(attn_metadata, self.vllm_config):
1507 graph_runner.capture(**capture_inputs)
1508 self.graph_memory_pool = graph_runner.graph.pool()
1509 self.graph_runners[virtual_engine][batch_size] = (
@@ -1649,7 +1649,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1649 model_forward_end = torch.cuda.Event(enable_timing=True)
1650 model_forward_start.record()
1651
1652 - with set_forward_context(model_input.attn_metadata):
1652 + with set_forward_context(model_input.attn_metadata, self.vllm_config):
1653 hidden_or_intermediate_states = model_executable(
1654 input_ids=model_input.input_tokens,
1655 positions=model_input.input_positions,