A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[torch.compile] support all attention backends (#10558)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Nov 22, 2024 at 14:04 UTC
eebad39f265606cfe35af4d1e0bea678516648a3
77 files changed
+879
-651
tests/kernels/test_encoder_decoder_attn.py
+26
-11
@@ -18,8 +18,10 @@ from vllm.attention import (Attention, AttentionBackend, AttentionMetadata,
18
from vllm.attention.backends.utils import STR_NOT_IMPL_ENC_DEC_ROCM_HIP
19
from vllm.attention.selector import (_Backend, _cached_get_attn_backend,
20
global_force_attn_backend_context_manager)
21
+from vllm.config import VllmConfig
22
from vllm.forward_context import set_forward_context
23
from vllm.platforms import current_platform
24
+from vllm.plugins import set_current_vllm_config
25
26
# List of support backends for encoder/decoder models
27
LIST_ENC_DEC_SUPPORTED_BACKENDS = [_Backend.XFORMERS, _Backend.FLASH_ATTN]
@@ -594,6 +596,7 @@ def _run_encoder_attention_test(
596
encoder_test_params: PhaseTestParameters,
597
attn_metadata: AttentionMetadata,
598
test_pt: TestPoint,
599
+ vllm_config: VllmConfig,
600
) -> torch.Tensor:
601
'''
602
Run encoder attention.
@@ -623,7 +626,7 @@ def _run_encoder_attention_test(
626
attn_type = AttentionType.ENCODER
627
packed_qkv = encoder_test_params.packed_qkvo.packed_qkv
628
assert packed_qkv is not None
626
- with set_forward_context(attn_metadata):
629
+ with set_forward_context(attn_metadata, vllm_config):
630
# In the test setup the shape of the query is
631
# [batch_size, seq_len, num_heads, head_size]. However
632
# the attention backend expect the shape to be
@@ -648,6 +651,7 @@ def _run_decoder_self_attention_test(
651
decoder_test_params: PhaseTestParameters,
652
attn_metadata: AttentionMetadata,
653
test_pt: TestPoint,
654
+ vllm_config: VllmConfig,
655
) -> torch.Tensor:
656
'''
657
Run decoder self-attention test.
@@ -677,7 +681,7 @@ def _run_decoder_self_attention_test(
681
kv_cache = test_rsrcs.kv_cache
682
packed_qkv = decoder_test_params.packed_qkvo.packed_qkv
683
assert packed_qkv is not None
680
- with set_forward_context(attn_metadata):
684
+ with set_forward_context(attn_metadata, vllm_config):
685
# In the test setup the shape of the query is
686
# [batch_size, seq_len, num_heads, head_size]. However
687
# the attention backend expect the shape to be
@@ -701,6 +705,7 @@ def _run_encoder_decoder_cross_attention_test(
705
cross_test_params: Optional[PhaseTestParameters],
706
attn_metadata: AttentionMetadata,
707
test_pt: TestPoint,
708
+ vllm_config: VllmConfig,
709
) -> torch.Tensor:
710
'''
711
Run encoder/decoder cross-attention test.
@@ -748,7 +753,7 @@ def _run_encoder_decoder_cross_attention_test(
753
cross_pckd_qkv = cross_test_params.packed_qkvo.packed_qkv
754
key = (None if cross_pckd_qkv is None else cross_pckd_qkv.key)
755
value = (None if cross_pckd_qkv is None else cross_pckd_qkv.value)
751
- with set_forward_context(attn_metadata):
756
+ with set_forward_context(attn_metadata, vllm_config):
757
# In the test setup the shape of the query is
758
# [batch_size, seq_len, num_heads, head_size]. However
759
# the attention backend expect the shape to be
@@ -839,7 +844,9 @@ def test_encoder_only(
844
845
# Attention scale factor, attention backend instance, attention wrapper
846
# instance, KV cache init
842
- test_rsrcs = _make_test_resources(test_pt)
847
+ vllm_config = VllmConfig()
848
+ with set_current_vllm_config(vllm_config):
849
+ test_rsrcs = _make_test_resources(test_pt)
850
851
# Construct encoder attention test params (only used
852
# during prefill)
@@ -863,7 +870,8 @@ def test_encoder_only(
870
test_rsrcs.attn,
871
enc_test_params,
872
prephase_attn_metadata,
866
- test_pt=test_pt))
873
+ test_pt=test_pt,
874
+ vllm_config=vllm_config))
875
876
# - Is encoder attention result correct?
877
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
@@ -960,7 +968,9 @@ def test_e2e_enc_dec_attn(
968
969
# Attention scale factor, attention backend instance, attention wrapper
970
# instance, KV cache init
963
- test_rsrcs = _make_test_resources(test_pt)
971
+ vllm_config = VllmConfig()
972
+ with set_current_vllm_config(vllm_config):
973
+ test_rsrcs = _make_test_resources(test_pt)
974
975
# Construct encoder attention test params (only used
976
# during prefill)
@@ -1011,7 +1021,8 @@ def test_e2e_enc_dec_attn(
1021
enc_pckd_act_out = _run_encoder_attention_test(test_rsrcs.attn,
1022
enc_test_params,
1023
prephase_attn_metadata,
1014
- test_pt=test_pt)
1024
+ test_pt=test_pt,
1025
+ vllm_config=vllm_config)
1026
1027
# - Is encoder attention result correct?
1028
assert_actual_matches_ideal(enc_test_params, enc_pckd_act_out,
@@ -1023,7 +1034,8 @@ def test_e2e_enc_dec_attn(
1034
test_rsrcs,
1035
prephase_dec_test_params,
1036
prephase_attn_metadata,
1026
- test_pt=test_pt)
1037
+ test_pt=test_pt,
1038
+ vllm_config=vllm_config)
1039
1040
# - Is prefill decoder self-attention correct?
1041
assert_actual_matches_ideal(prephase_dec_test_params,
@@ -1037,7 +1049,8 @@ def test_e2e_enc_dec_attn(
1049
prephase_dec_test_params,
1050
prephase_cross_test_params,
1051
prephase_attn_metadata,
1040
- test_pt=test_pt)
1052
+ test_pt=test_pt,
1053
+ vllm_config=vllm_config)
1054
1055
# - Is prefill encoder/decoder cross-attention correct?
1056
assert_actual_matches_ideal(prephase_cross_test_params,
@@ -1061,7 +1074,8 @@ def test_e2e_enc_dec_attn(
1074
test_rsrcs,
1075
decphase_dec_test_params,
1076
decphase_attn_metadata,
1064
- test_pt=test_pt)
1077
+ test_pt=test_pt,
1078
+ vllm_config=vllm_config)
1079
1080
# - Is decode-phase decoder self-attention correct?
1081
assert_actual_matches_ideal(decphase_dec_test_params,
@@ -1075,7 +1089,8 @@ def test_e2e_enc_dec_attn(
1089
decphase_dec_test_params,
1090
None,
1091
decphase_attn_metadata,
1078
- test_pt=test_pt)
1092
+ test_pt=test_pt,
1093
+ vllm_config=vllm_config)
1094
1095
# - Is decode-phase encoder/decoder cross-attention correct?
1096
assert_actual_matches_ideal(decphase_cross_test_params,
vllm/attention/backends/abstract.py
+14
-9
@@ -1,7 +1,6 @@
1
from abc import ABC, abstractmethod
2
from contextlib import contextmanager
3
from dataclasses import dataclass, fields
4
-from enum import Enum, auto
4
from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
5
Tuple, Type, TypeVar)
6
@@ -15,13 +14,19 @@ if TYPE_CHECKING:
14
ModelRunnerInputBuilderBase)
15
16
18
-class AttentionType(Enum):
19
- DECODER = auto() # Decoder attention between previous layer Q/K/V
20
- ENCODER = auto(
21
- ) # Encoder attention between previous layer Q/K/V for encoder-decoder
22
- ENCODER_ONLY = auto() # Encoder attention between previous layer Q/K/V
23
- ENCODER_DECODER = auto(
24
- ) # Attention between dec. Q and enc. K/V for encoder-decoder
17
+class AttentionType:
18
+ """
19
+ Attention type.
20
+ Use string to be compatible with `torch.compile`.
21
+ """
22
+ # Decoder attention between previous layer Q/K/V
23
+ DECODER = "decoder"
24
+ # Encoder attention between previous layer Q/K/V for encoder-decoder
25
+ ENCODER = "encoder"
26
+ # Encoder attention between previous layer Q/K/V
27
+ ENCODER_ONLY = "encoder_only"
28
+ # Attention between dec. Q and enc. K/V for encoder-decoder
29
+ ENCODER_DECODER = "encoder_decoder"
30
31
32
class AttentionBackend(ABC):
@@ -241,6 +246,6 @@ class AttentionImpl(ABC, Generic[T]):
246
attn_metadata: T,
247
k_scale: float = 1.0,
248
v_scale: float = 1.0,
244
- attn_type: AttentionType = AttentionType.DECODER,
249
+ attn_type: str = AttentionType.DECODER,
250
) -> torch.Tensor:
251
raise NotImplementedError
vllm/attention/backends/blocksparse_attn.py
+1
-1
@@ -354,7 +354,7 @@ class BlocksparseFlashAttentionImpl(AttentionImpl):
354
attn_metadata: BlocksparseFlashAttentionMetadata,
355
k_scale: float = 1.0,
356
v_scale: float = 1.0,
357
- attn_type: AttentionType = AttentionType.DECODER,
357
+ attn_type: str = AttentionType.DECODER,
358
) -> torch.Tensor:
359
"""Forward pass with FlashAttention and PagedAttention.
360
vllm/attention/backends/flash_attn.py
+172
-240
@@ -16,10 +16,8 @@ from vllm.attention.backends.utils import (
16
compute_slot_mapping_start_idx, get_num_prefill_decode_query_kv_tokens,
17
get_seq_len_block_table_args, is_all_cross_attn_metadata_set,
18
is_all_encoder_attn_metadata_set, is_block_tables_empty)
19
-from vllm.forward_context import get_forward_context
19
from vllm.multimodal import MultiModalPlaceholderMap
21
-from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
22
- make_tensor_with_pad)
20
+from vllm.utils import async_tensor_h2d, make_tensor_with_pad
21
22
if TYPE_CHECKING:
23
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -639,7 +637,7 @@ class FlashAttentionImpl(AttentionImpl):
637
attn_metadata: FlashAttentionMetadata,
638
k_scale: float = 1.0,
639
v_scale: float = 1.0,
642
- attn_type: AttentionType = AttentionType.DECODER,
640
+ attn_type: str = AttentionType.DECODER,
641
) -> torch.Tensor:
642
"""Forward pass with FlashAttention.
643
@@ -668,23 +666,174 @@ class FlashAttentionImpl(AttentionImpl):
666
"requires setting cross-attention "
667
"metadata attributes.")
668
671
- output = torch.ops.vllm.unified_flash_attention(
672
- query,
673
- key,
674
- value,
675
- self.num_heads,
676
- self.head_size,
677
- self.num_kv_heads,
678
- kv_cache,
679
- self.kv_cache_dtype,
680
- k_scale,
681
- v_scale,
682
- self.scale,
683
- attn_type.value,
684
- self.sliding_window,
685
- self.alibi_slopes,
686
- self.logits_soft_cap,
687
- )
669
+ num_heads: int = self.num_heads
670
+ head_size: int = self.head_size
671
+ num_kv_heads: int = self.num_kv_heads
672
+ kv_cache_dtype: str = self.kv_cache_dtype
673
+ softmax_scale: float = self.scale
674
+ window_size = self.sliding_window
675
+ alibi_slopes: Optional[torch.Tensor] = self.alibi_slopes
676
+ logits_soft_cap: Optional[float] = self.logits_soft_cap
677
+
678
+ num_tokens, hidden_size = query.shape
679
+
680
+ # Reshape the query, key, and value tensors.
681
+ query = query.view(-1, num_heads, head_size)
682
+ if (key is not None) and (value is not None):
683
+ key = key.view(-1, num_kv_heads, head_size)
684
+ value = value.view(-1, num_kv_heads, head_size)
685
+
686
+ if kv_cache.numel() > 0:
687
+ key_cache = kv_cache[0]
688
+ value_cache = kv_cache[1]
689
+ # We skip updating the KV cache under two conditions:
690
+ # a. When the Attention Type is ENCODER. In this phase, we compute
691
+ # only the encoder attention without updating the cache.
692
+ # b. When both Key and Value are None. This occurs during
693
+ # cross-attention computation in the decoding phase, where the
694
+ # KV cache is already populated with the cross-attention
695
+ # tensor. Thus, we skip cache updates during this time.
696
+ if (attn_type != AttentionType.ENCODER) and (key is not None) and (
697
+ value is not None):
698
+ if attn_type == AttentionType.ENCODER_DECODER:
699
+ # Update cross-attention KV cache (prefill-only)
700
+ updated_slot_mapping = attn_metadata.cross_slot_mapping
701
+ else:
702
+ # Update self-attention KV cache (prefill/decode)
703
+ updated_slot_mapping = attn_metadata.slot_mapping
704
+
705
+ # Reshape the input keys and values and store them in the cache.
706
+ # If kv_cache is not provided, the new key and value tensors are
707
+ # not cached. This happens during the initial memory
708
+ # profiling run.
709
+ torch.ops._C_cache_ops.reshape_and_cache_flash(
710
+ key,
711
+ value,
712
+ kv_cache[0],
713
+ kv_cache[1],
714
+ updated_slot_mapping.flatten(), # type: ignore[union-attr]
715
+ kv_cache_dtype,
716
+ k_scale,
717
+ v_scale,
718
+ )
719
+
720
+ (num_prefill_query_tokens, num_prefill_kv_tokens,
721
+ num_decode_query_tokens) = \
722
+ get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
723
+ decode_query = query[num_prefill_query_tokens:]
724
+ # QKV for prefill.
725
+ query = query[:num_prefill_query_tokens]
726
+ assert query.shape[0] == num_prefill_query_tokens
727
+ assert decode_query.shape[0] == num_decode_query_tokens
728
+
729
+ prefill_output: Optional[torch.Tensor] = None
730
+ decode_output: Optional[torch.Tensor] = None
731
+ if prefill_meta := attn_metadata.prefill_metadata:
732
+ # Prompt run.
733
+ if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
734
+ or prefill_meta.block_tables.numel() == 0):
735
+ # normal attention
736
+ # When block_tables are not filled, it means q and k are the
737
+ # prompt, and they have the same length.
738
+ q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
739
+ _get_query_key_seq_metadata(prefill_meta, True, attn_type)
740
+
741
+ key = key[:num_prefill_kv_tokens]
742
+ value = value[:num_prefill_kv_tokens]
743
+
744
+ prefill_output = flash_attn_varlen_func(
745
+ q=query,
746
+ k=key,
747
+ v=value,
748
+ cu_seqlens_q=q_seq_start_loc,
749
+ cu_seqlens_k=k_seq_start_loc,
750
+ max_seqlen_q=q_seq_len,
751
+ max_seqlen_k=k_seq_len,
752
+ softmax_scale=softmax_scale,
753
+ causal=_get_causal_option(attn_type),
754
+ window_size=window_size,
755
+ alibi_slopes=alibi_slopes,
756
+ softcap=logits_soft_cap,
757
+ )
758
+ else:
759
+ # prefix-enabled attention
760
+ assert attn_type == AttentionType.DECODER, (
761
+ "Only decoder-only models support prefix caching")
762
+ assert prefill_meta.seq_lens is not None
763
+ max_seq_len = max(prefill_meta.seq_lens)
764
+ prefill_output = flash_attn_varlen_func( # noqa
765
+ q=query,
766
+ k=key_cache,
767
+ v=value_cache,
768
+ cu_seqlens_q=prefill_meta.query_start_loc,
769
+ max_seqlen_q=prefill_meta.max_query_len,
770
+ cu_seqlens_k=prefill_meta.seq_start_loc,
771
+ max_seqlen_k=max_seq_len,
772
+ softmax_scale=softmax_scale,
773
+ causal=True,
774
+ window_size=window_size,
775
+ alibi_slopes=alibi_slopes,
776
+ block_table=prefill_meta.block_tables,
777
+ softcap=logits_soft_cap,
778
+ )
779
+
780
+ if decode_meta := attn_metadata.decode_metadata:
781
+ # Decoding run.
782
+ # Use flash_attn_varlen_func kernel for speculative decoding
783
+ # because different queries might have different lengths.
784
+
785
+ assert decode_meta.max_decode_query_len is not None
786
+ # use only for actual varlen decoding
787
+ if decode_meta.max_decode_query_len > 1:
788
+ assert attn_type == AttentionType.DECODER, (
789
+ "Only decoder-only models support max_decode_query_len > 1"
790
+ )
791
+ decode_output = flash_attn_varlen_func(
792
+ q=decode_query,
793
+ k=key_cache,
794
+ v=value_cache,
795
+ cu_seqlens_q=decode_meta.query_start_loc,
796
+ max_seqlen_q=decode_meta.max_decode_query_len,
797
+ cu_seqlens_k=decode_meta.seq_start_loc,
798
+ max_seqlen_k=decode_meta.max_decode_seq_len,
799
+ softmax_scale=softmax_scale,
800
+ causal=True,
801
+ window_size=window_size,
802
+ alibi_slopes=alibi_slopes,
803
+ softcap=logits_soft_cap,
804
+ block_table=decode_meta.block_tables,
805
+ )
806
+ else:
807
+ # Use flash_attn_with_kvcache for normal decoding.
808
+ (
809
+ seq_lens_arg,
810
+ _,
811
+ block_tables_arg,
812
+ ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
813
+ decode_output = flash_attn_with_kvcache(
814
+ q=decode_query.unsqueeze(1),
815
+ k_cache=key_cache,
816
+ v_cache=value_cache,
817
+ block_table=block_tables_arg,
818
+ cache_seqlens=seq_lens_arg,
819
+ softmax_scale=softmax_scale,
820
+ causal=True,
821
+ window_size=window_size,
822
+ alibi_slopes=alibi_slopes,
823
+ softcap=logits_soft_cap,
824
+ ).squeeze(1)
825
+
826
+ if prefill_output is None:
827
+ assert decode_output is not None
828
+ return decode_output.view(num_decode_query_tokens, hidden_size)
829
+ if decode_output is None:
830
+ assert prefill_output is not None
831
+ return prefill_output.view(num_prefill_query_tokens, hidden_size)
832
+
833
+ assert decode_meta is not None
834
+ decode_output = decode_output.squeeze(1)
835
+ output = torch.cat([prefill_output, decode_output], dim=0)
836
+ return output.view(num_tokens, hidden_size)
837
838
return output
839
@@ -692,7 +841,7 @@ class FlashAttentionImpl(AttentionImpl):
841
def _get_query_key_seq_metadata(
842
attn_metadata,
843
is_prompt: bool,
695
- attn_type: AttentionType,
844
+ attn_type: str,
845
) -> tuple:
846
"""
847
Returns sequence metadata for key and query based on the specified
@@ -754,7 +903,7 @@ def _get_query_key_seq_metadata(
903
raise AttributeError(f"Invalid attention type {str(attn_type)}")
904
905
757
-def _get_causal_option(attn_type: AttentionType) -> bool:
906
+def _get_causal_option(attn_type: str) -> bool:
907
"""
908
Determine whether the given attention type is suitable for causal
909
attention mechanisms.
@@ -770,220 +919,3 @@ def _get_causal_option(attn_type: AttentionType) -> bool:
919
return not (attn_type == AttentionType.ENCODER
920
or attn_type == AttentionType.ENCODER_ONLY
921
or attn_type == AttentionType.ENCODER_DECODER)
773
-
774
-
775
-def unified_flash_attention(
776
- query: torch.Tensor,
777
- key: torch.Tensor,
778
- value: torch.Tensor,
779
- num_heads: int,
780
- head_size: int,
781
- num_kv_heads: int,
782
- kv_cache: torch.Tensor,
783
- kv_cache_dtype: str,
784
- k_scale: float,
785
- v_scale: float,
786
- softmax_scale: float,
787
- attn_type_int_val: int,
788
- window_size: Optional[List[int]] = None,
789
- alibi_slopes: Optional[torch.Tensor] = None,
790
- logits_soft_cap: Optional[float] = None,
791
-) -> torch.Tensor:
792
-
793
- # Convert integer attn_type to enum
794
- try:
795
- attn_type = AttentionType(attn_type_int_val)
796
- except ValueError as err:
797
- raise AttributeError(
798
- f"Invalid attention type {str(attn_type_int_val)}") from err
799
-
800
- current_metadata = get_forward_context()
801
- assert current_metadata is not None
802
- assert isinstance(current_metadata, FlashAttentionMetadata)
803
- attn_metadata: FlashAttentionMetadata = current_metadata
804
-
805
- num_tokens, hidden_size = query.shape
806
-
807
- # Reshape the query, key, and value tensors.
808
- query = query.view(-1, num_heads, head_size)
809
- if (key is not None) and (value is not None):
810
- key = key.view(-1, num_kv_heads, head_size)
811
- value = value.view(-1, num_kv_heads, head_size)
812
-
813
- if kv_cache.numel() > 0:
814
- key_cache = kv_cache[0]
815
- value_cache = kv_cache[1]
816
- # We skip updating the KV cache under two conditions:
817
- # a. When the Attention Type is ENCODER. In this phase, we compute
818
- # only the encoder attention without updating the cache.
819
- # b. When both Key and Value are None. This occurs during
820
- # cross-attention computation in the decoding phase, where the KV
821
- # cache is already populated with the cross-attention tensor.
822
- # Thus, we skip cache updates during this time.
823
- if (attn_type != AttentionType.ENCODER) and (key is not None) and (
824
- value is not None):
825
- if attn_type == AttentionType.ENCODER_DECODER:
826
- # Update cross-attention KV cache (prefill-only)
827
- updated_slot_mapping = attn_metadata.cross_slot_mapping
828
- else:
829
- # Update self-attention KV cache (prefill/decode)
830
- updated_slot_mapping = attn_metadata.slot_mapping
831
-
832
- # Reshape the input keys and values and store them in the cache.
833
- # If kv_cache is not provided, the new key and value tensors are
834
- # not cached. This happens during the initial memory profiling run.
835
- torch.ops._C_cache_ops.reshape_and_cache_flash(
836
- key,
837
- value,
838
- kv_cache[0],
839
- kv_cache[1],
840
- updated_slot_mapping.flatten(), # type: ignore[union-attr]
841
- kv_cache_dtype,
842
- k_scale,
843
- v_scale,
844
- )
845
-
846
- (num_prefill_query_tokens, num_prefill_kv_tokens,
847
- num_decode_query_tokens) = \
848
- get_num_prefill_decode_query_kv_tokens(attn_metadata, attn_type)
849
- decode_query = query[num_prefill_query_tokens:]
850
- # QKV for prefill.
851
- query = query[:num_prefill_query_tokens]
852
- assert query.shape[0] == num_prefill_query_tokens
853
- assert decode_query.shape[0] == num_decode_query_tokens
854
-
855
- prefill_output: Optional[torch.Tensor] = None
856
- decode_output: Optional[torch.Tensor] = None
857
- if prefill_meta := attn_metadata.prefill_metadata:
858
- # Prompt run.
859
- if (kv_cache.numel() == 0 or prefill_meta.block_tables is None
860
- or prefill_meta.block_tables.numel() == 0):
861
- # normal attention
862
- # When block_tables are not filled, it means q and k are the
863
- # prompt, and they have the same length.
864
- q_seq_start_loc, q_seq_len, k_seq_start_loc, k_seq_len = \
865
- _get_query_key_seq_metadata(prefill_meta, True, attn_type)
866
-
867
- key = key[:num_prefill_kv_tokens]
868
- value = value[:num_prefill_kv_tokens]
869
-
870
- prefill_output = flash_attn_varlen_func(
871
- q=query,
872
- k=key,
873
- v=value,
874
- cu_seqlens_q=q_seq_start_loc,
875
- cu_seqlens_k=k_seq_start_loc,
876
- max_seqlen_q=q_seq_len,
877
- max_seqlen_k=k_seq_len,
878
- softmax_scale=softmax_scale,
879
- causal=_get_causal_option(attn_type),
880
- window_size=window_size,
881
- alibi_slopes=alibi_slopes,
882
- softcap=logits_soft_cap,
883
- )
884
- else:
885
- # prefix-enabled attention
886
- assert attn_type == AttentionType.DECODER, (
887
- "Only decoder-only models support prefix caching")
888
- assert prefill_meta.seq_lens is not None
889
- max_seq_len = max(prefill_meta.seq_lens)
890
- prefill_output = flash_attn_varlen_func( # noqa
891
- q=query,
892
- k=key_cache,
893
- v=value_cache,
894
- cu_seqlens_q=prefill_meta.query_start_loc,
895
- max_seqlen_q=prefill_meta.max_query_len,
896
- cu_seqlens_k=prefill_meta.seq_start_loc,
897
- max_seqlen_k=max_seq_len,
898
- softmax_scale=softmax_scale,
899
- causal=True,
900
- window_size=window_size,
901
- alibi_slopes=alibi_slopes,
902
- block_table=prefill_meta.block_tables,
903
- softcap=logits_soft_cap,
904
- )
905
-
906
- if decode_meta := attn_metadata.decode_metadata:
907
- # Decoding run.
908
- # Use flash_attn_varlen_func kernel for speculative decoding
909
- # because different queries might have different lengths.
910
-
911
- assert decode_meta.max_decode_query_len is not None
912
- # use only for actual varlen decoding
913
- if decode_meta.max_decode_query_len > 1:
914
- assert attn_type == AttentionType.DECODER, (
915
- "Only decoder-only models support max_decode_query_len > 1")
916
- decode_output = flash_attn_varlen_func(
917
- q=decode_query,
918
- k=key_cache,
919
- v=value_cache,
920
- cu_seqlens_q=decode_meta.query_start_loc,
921
- max_seqlen_q=decode_meta.max_decode_query_len,
922
- cu_seqlens_k=decode_meta.seq_start_loc,
923
- max_seqlen_k=decode_meta.max_decode_seq_len,
924
- softmax_scale=softmax_scale,
925
- causal=True,
926
- window_size=window_size,
927
- alibi_slopes=alibi_slopes,
928
- softcap=logits_soft_cap,
929
- block_table=decode_meta.block_tables,
930
- )
931
- else:
932
- # Use flash_attn_with_kvcache for normal decoding.
933
- (
934
- seq_lens_arg,
935
- _,
936
- block_tables_arg,
937
- ) = get_seq_len_block_table_args(decode_meta, False, attn_type)
938
- decode_output = flash_attn_with_kvcache(
939
- q=decode_query.unsqueeze(1),
940
- k_cache=key_cache,
941
- v_cache=value_cache,
942
- block_table=block_tables_arg,
943
- cache_seqlens=seq_lens_arg,
944
- softmax_scale=softmax_scale,
945
- causal=True,
946
- window_size=window_size,
947
- alibi_slopes=alibi_slopes,
948
- softcap=logits_soft_cap,
949
- ).squeeze(1)
950
-
951
- if prefill_output is None:
952
- assert decode_output is not None
953
- return decode_output.view(num_decode_query_tokens, hidden_size)
954
- if decode_output is None:
955
- assert prefill_output is not None
956
- return prefill_output.view(num_prefill_query_tokens, hidden_size)
957
-
958
- assert decode_meta is not None
959
- decode_output = decode_output.squeeze(1)
960
- output = torch.cat([prefill_output, decode_output], dim=0)
961
- return output.view(num_tokens, hidden_size)
962
-
963
-
964
-def unified_flash_attention_fake(
965
- query: torch.Tensor,
966
- key: torch.Tensor,
967
- value: torch.Tensor,
968
- num_heads: int,
969
- head_size: int,
970
- num_kv_heads: int,
971
- kv_cache: torch.Tensor,
972
- kv_cache_dtype: str,
973
- k_scale: float,
974
- v_scale: float,
975
- softmax_scale: float,
976
- attn_type_int_val: int,
977
- window_size: Optional[List[int]] = None,
978
- alibi_slopes: Optional[torch.Tensor] = None,
979
- logits_soft_cap: Optional[float] = None,
980
-) -> torch.Tensor:
981
- return torch.empty_like(query)
982
-
983
-
984
-direct_register_custom_op(
985
- op_name="unified_flash_attention",
986
- op_func=unified_flash_attention,
987
- mutates_args=["kv_cache"],
988
- fake_impl=unified_flash_attention_fake,
989
-)
vllm/attention/backends/flashinfer.py
+111
-169
@@ -30,9 +30,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
30
compute_slot_mapping_start_idx,
31
is_block_tables_empty)
32
from vllm.attention.ops.paged_attn import PagedAttention
33
-from vllm.forward_context import get_forward_context
34
-from vllm.utils import (async_tensor_h2d, direct_register_custom_op,
35
- get_kv_cache_torch_dtype, make_tensor_with_pad)
33
+from vllm.utils import (async_tensor_h2d, get_kv_cache_torch_dtype,
34
+ make_tensor_with_pad)
35
36
if TYPE_CHECKING:
37
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
@@ -774,7 +773,7 @@ class FlashInferImpl(AttentionImpl):
773
attn_metadata: FlashInferMetadata,
774
k_scale: float = 1.0,
775
v_scale: float = 1.0,
777
- attn_type: AttentionType = AttentionType.DECODER,
776
+ attn_type: str = AttentionType.DECODER,
777
) -> torch.Tensor:
778
if attn_type != AttentionType.DECODER:
779
raise NotImplementedError("Encoder self-attention and "
@@ -782,174 +781,117 @@ class FlashInferImpl(AttentionImpl):
781
"are not implemented for "
782
"FlashInferImpl")
783
785
- return torch.ops.vllm.unified_flash_infer(
786
- query,
787
- key,
788
- value,
789
- self.num_heads,
790
- self.head_size,
791
- self.num_kv_heads,
792
- kv_cache,
793
- self.kv_cache_dtype,
794
- k_scale,
795
- v_scale,
796
- self.scale,
797
- self.sliding_window,
798
- self.alibi_slopes,
799
- self.logits_soft_cap,
800
- )
801
-
802
-
803
-def unified_flash_infer(
804
- query: torch.Tensor,
805
- key: torch.Tensor,
806
- value: torch.Tensor,
807
- num_heads: int,
808
- head_size: int,
809
- num_kv_heads: int,
810
- kv_cache: torch.Tensor,
811
- kv_cache_dtype: str,
812
- k_scale: float,
813
- v_scale: float,
814
- softmax_scale: float,
815
- window_size: Optional[List[int]] = None,
816
- alibi_slopes: Optional[torch.Tensor] = None,
817
- logits_soft_cap: Optional[float] = None,
818
-) -> torch.Tensor:
819
-
820
- current_metadata = get_forward_context()
821
- assert current_metadata is not None
822
- assert isinstance(current_metadata, FlashInferMetadata)
823
- attn_metadata: FlashInferMetadata = current_metadata
824
-
825
- num_tokens, hidden_size = query.shape
826
- query = query.view(-1, num_heads, head_size)
827
- key = key.view(-1, num_kv_heads, head_size)
828
- value = value.view(-1, num_kv_heads, head_size)
829
-
830
- if kv_cache.numel() > 0:
831
- # Use the same reshape and cache kernel as flash attention.
832
- ops.reshape_and_cache_flash(
833
- key,
834
- value,
835
- kv_cache[:, 0],
836
- kv_cache[:, 1],
837
- attn_metadata.slot_mapping.flatten(),
838
- kv_cache_dtype,
839
- k_scale,
840
- v_scale,
841
- )
842
- # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
843
- # to process the cache when the kv_cache_dtype is fp8
844
- if kv_cache_dtype.startswith("fp8"):
845
- torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
846
- kv_cache_dtype)
847
- kv_cache = kv_cache.view(torch_dtype)
848
-
849
- num_prefill_tokens = attn_metadata.num_prefill_tokens
850
- num_decode_tokens = attn_metadata.num_decode_tokens
851
- assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
852
- f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
853
- assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
854
- f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
855
- query = query.contiguous() # Flashinfer requires query to be contiguous
856
- # Query for decode. KV is not needed because it is already cached.
857
- # QKV for prefill.
858
- decode_query = query[num_prefill_tokens:]
859
- query = query[:num_prefill_tokens]
860
-
861
- key = key[:num_prefill_tokens]
862
- value = value[:num_prefill_tokens]
863
-
864
- assert query.shape[0] == num_prefill_tokens
865
- assert decode_query.shape[0] == num_decode_tokens
866
-
867
- window_left = window_size[0] if window_size is not None else -1
868
-
869
- prefill_output: Optional[torch.Tensor] = None
870
- decode_output: Optional[torch.Tensor] = None
871
- if prefill_meta := attn_metadata.prefill_metadata:
872
- # We will use flash attention for prefill
873
- # when kv_cache is not provided.
874
- # This happens when vllm runs the profiling to
875
- # determine the number of blocks.
876
- if kv_cache.numel() == 0:
877
- prefill_output = flash_attn_varlen_func(
878
- q=query,
879
- k=key,
880
- v=value,
881
- cu_seqlens_q=prefill_meta.seq_start_loc,
882
- cu_seqlens_k=prefill_meta.seq_start_loc,
883
- max_seqlen_q=prefill_meta.max_prefill_seq_len,
884
- max_seqlen_k=prefill_meta.max_prefill_seq_len,
885
- softmax_scale=softmax_scale,
886
- causal=True,
887
- window_size=window_size,
888
- alibi_slopes=alibi_slopes,
784
+ num_heads: int = self.num_heads
785
+ head_size: int = self.head_size
786
+ num_kv_heads: int = self.num_kv_heads
787
+ kv_cache_dtype: str = self.kv_cache_dtype
788
+ softmax_scale: float = self.scale
789
+ window_size = self.sliding_window
790
+ alibi_slopes = self.alibi_slopes
791
+ logits_soft_cap = self.logits_soft_cap
792
+
793
+ num_tokens, hidden_size = query.shape
794
+ query = query.view(-1, num_heads, head_size)
795
+ key = key.view(-1, num_kv_heads, head_size)
796
+ value = value.view(-1, num_kv_heads, head_size)
797
+
798
+ if kv_cache.numel() > 0:
799
+ # Use the same reshape and cache kernel as flash attention.
800
+ ops.reshape_and_cache_flash(
801
+ key,
802
+ value,
803
+ kv_cache[:, 0],
804
+ kv_cache[:, 1],
805
+ attn_metadata.slot_mapping.flatten(),
806
+ kv_cache_dtype,
807
+ k_scale,
808
+ v_scale,
809
)
890
- else:
891
- assert prefill_meta is not None
892
- assert prefill_meta.prefill_wrapper is not None
893
- prefill_output = prefill_meta.prefill_wrapper.forward(
894
- query,
810
+ # The FlashInfer api requires data to be in fp8_e4m3 or fp8_e5m2
811
+ # to process the cache when the kv_cache_dtype is fp8
812
+ if kv_cache_dtype.startswith("fp8"):
813
+ torch_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
814
+ kv_cache_dtype)
815
+ kv_cache = kv_cache.view(torch_dtype)
816
+
817
+ num_prefill_tokens = attn_metadata.num_prefill_tokens
818
+ num_decode_tokens = attn_metadata.num_decode_tokens
819
+ assert key.shape[0] == num_prefill_tokens + num_decode_tokens, \
820
+ f"key : {key.shape} : #prefill tokens {num_prefill_tokens} : #decode tokens {num_decode_tokens}" # noqa
821
+ assert value.shape[0] == num_prefill_tokens + num_decode_tokens, \
822
+ f"value : {value.shape} : #prefill toks {num_prefill_tokens} : #decode toks {num_decode_tokens}" # noqa
823
+ query = query.contiguous(
824
+ ) # Flashinfer requires query to be contiguous
825
+ # Query for decode. KV is not needed because it is already cached.
826
+ # QKV for prefill.
827
+ decode_query = query[num_prefill_tokens:]
828
+ query = query[:num_prefill_tokens]
829
+
830
+ key = key[:num_prefill_tokens]
831
+ value = value[:num_prefill_tokens]
832
+
833
+ assert query.shape[0] == num_prefill_tokens
834
+ assert decode_query.shape[0] == num_decode_tokens
835
+
836
+ window_left = window_size[0] if window_size is not None else -1
837
+
838
+ prefill_output: Optional[torch.Tensor] = None
839
+ decode_output: Optional[torch.Tensor] = None
840
+ if prefill_meta := attn_metadata.prefill_metadata:
841
+ # We will use flash attention for prefill
842
+ # when kv_cache is not provided.
843
+ # This happens when vllm runs the profiling to
844
+ # determine the number of blocks.
845
+ if kv_cache.numel() == 0:
846
+ prefill_output = flash_attn_varlen_func(
847
+ q=query,
848
+ k=key,
849
+ v=value,
850
+ cu_seqlens_q=prefill_meta.seq_start_loc,
851
+ cu_seqlens_k=prefill_meta.seq_start_loc,
852
+ max_seqlen_q=prefill_meta.max_prefill_seq_len,
853
+ max_seqlen_k=prefill_meta.max_prefill_seq_len,
854
+ softmax_scale=softmax_scale,
855
+ causal=True,
856
+ window_size=window_size,
857
+ alibi_slopes=alibi_slopes,
858
+ )
859
+ else:
860
+ assert prefill_meta is not None
861
+ assert prefill_meta.prefill_wrapper is not None
862
+ prefill_output = prefill_meta.prefill_wrapper.forward(
863
+ query,
864
+ kv_cache,
865
+ logits_soft_cap=logits_soft_cap,
866
+ causal=True,
867
+ k_scale=k_scale,
868
+ v_scale=v_scale,
869
+ window_left=window_left)
870
+ if decode_meta := attn_metadata.decode_metadata:
871
+ assert decode_meta is not None
872
+ assert decode_meta.decode_wrapper is not None
873
+ decode_output = decode_meta.decode_wrapper.forward(
874
+ decode_query,
875
kv_cache,
876
+ sm_scale=softmax_scale,
877
logits_soft_cap=logits_soft_cap,
897
- causal=True,
878
k_scale=k_scale,
879
v_scale=v_scale,
880
window_left=window_left)
901
- if decode_meta := attn_metadata.decode_metadata:
902
- assert attn_metadata.decode_metadata is not None
903
- assert attn_metadata.decode_metadata.decode_wrapper is not None
904
- decode_output = attn_metadata.decode_metadata.decode_wrapper.forward(
905
- decode_query,
906
- kv_cache,
907
- sm_scale=softmax_scale,
908
- logits_soft_cap=logits_soft_cap,
909
- k_scale=k_scale,
910
- v_scale=v_scale,
911
- window_left=window_left)
912
-
913
- if prefill_output is None and decode_output is not None:
914
- # Decode only batch.
915
- output, num_tokens = decode_output, num_decode_tokens
916
- elif decode_output is None and prefill_output is not None:
917
- # Prefill only batch.
918
- output, num_tokens = prefill_output, num_prefill_tokens
919
- else:
920
- # Chunked prefill batch does not work with speculative decoding in
921
- # FlashInfer backend, so the query length for decode should be 1.
922
- assert prefill_output is not None
923
- assert decode_output is not None
924
- assert decode_meta is not None
925
- assert decode_meta.decode_query_len == 1
926
- decode_output = decode_output.squeeze(1)
927
- output = torch.cat([prefill_output, decode_output], dim=0)
928
- return output.view(num_tokens, hidden_size)
929
-
930
-
931
-def unified_flash_infer_fake(
932
- query: torch.Tensor,
933
- key: torch.Tensor,
934
- value: torch.Tensor,
935
- num_heads: int,
936
- head_size: int,
937
- num_kv_heads: int,
938
- kv_cache: torch.Tensor,
939
- kv_cache_dtype: str,
940
- k_scale: float,
941
- v_scale: float,
942
- softmax_scale: float,
943
- window_size: Optional[List[int]] = None,
944
- alibi_slopes: Optional[torch.Tensor] = None,
945
- logits_soft_cap: Optional[float] = None,
946
-) -> torch.Tensor:
947
- return torch.empty_like(query).contiguous()
948
-
949
-
950
-direct_register_custom_op(
951
- op_name="unified_flash_infer",
952
- op_func=unified_flash_infer,
953
- mutates_args=["kv_cache"],
954
- fake_impl=unified_flash_infer_fake,
955
-)
881
+
882
+ if prefill_output is None and decode_output is not None:
883
+ # Decode only batch.
884
+ output, num_tokens = decode_output, num_decode_tokens
885
+ elif decode_output is None and prefill_output is not None:
886
+ # Prefill only batch.
887
+ output, num_tokens = prefill_output, num_prefill_tokens
888
+ else:
889
+ # Chunked prefill batch does not work with speculative decoding in
890
+ # FlashInfer backend, so the query length for decode should be 1.
891
+ assert prefill_output is not None
892
+ assert decode_output is not None
893
+ assert decode_meta is not None
894
+ assert decode_meta.decode_query_len == 1
895
+ decode_output = decode_output.squeeze(1)
896
+ output = torch.cat([prefill_output, decode_output], dim=0)
897
+ return output.view(num_tokens, hidden_size)
vllm/attention/backends/hpu_attn.py
+1
-1
@@ -140,7 +140,7 @@ class HPUAttentionImpl(AttentionImpl, torch.nn.Module):
140
attn_metadata: HPUAttentionMetadata,
141
k_scale: float = 1.0,
142
v_scale: float = 1.0,
143
- attn_type: AttentionType = AttentionType.DECODER,
143
+ attn_type: str = AttentionType.DECODER,
144
) -> torch.Tensor:
145
"""Forward pass with xFormers and PagedAttention.
146
vllm/attention/backends/ipex_attn.py
+1
-1
@@ -172,7 +172,7 @@ class IpexAttnBackendImpl(AttentionImpl[IpexAttnMetadata]):
172
attn_metadata: IpexAttnMetadata, # type: ignore
173
k_scale: float = 1.0,
174
v_scale: float = 1.0,
175
- attn_type: AttentionType = AttentionType.DECODER,
175
+ attn_type: str = AttentionType.DECODER,
176
) -> torch.Tensor:
177
"""Forward pass with IPEX varlen_attention and PagedAttention.
178
vllm/attention/backends/pallas.py
+1
-1
@@ -150,7 +150,7 @@ class PallasAttentionBackendImpl(AttentionImpl):
150
attn_metadata: PallasMetadata,
151
k_scale: float = 1.0,
152
v_scale: float = 1.0,
153
- attn_type: AttentionType = AttentionType.DECODER,
153
+ attn_type: str = AttentionType.DECODER,
154
) -> torch.Tensor:
155
"""Forward pass with Pallas attention.
156
vllm/attention/backends/rocm_flash_attn.py
+1
-1
@@ -414,7 +414,7 @@ class ROCmFlashAttentionImpl(AttentionImpl):
414
attn_metadata: ROCmFlashAttentionMetadata,
415
k_scale: float = 1.0,
416
v_scale: float = 1.0,
417
- attn_type: AttentionType = AttentionType.DECODER,
417
+ attn_type: str = AttentionType.DECODER,
418
) -> torch.Tensor:
419
"""Forward pass with FlashAttention and PagedAttention.
420
vllm/attention/backends/torch_sdpa.py
+6
-6
@@ -141,7 +141,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
141
142
def get_seq_lens(
143
self,
144
- attn_type: AttentionType,
144
+ attn_type: str,
145
):
146
'''
147
Extract appropriate sequence lengths from attention metadata
@@ -174,7 +174,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
174
175
def get_attn_bias(
176
self,
177
- attn_type: AttentionType,
177
+ attn_type: str,
178
) -> Optional[List[torch.Tensor]]:
179
'''
180
Extract appropriate attention bias from attention metadata
@@ -203,7 +203,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
203
def set_attn_bias(
204
self,
205
attn_bias: List[torch.Tensor],
206
- attn_type: AttentionType,
206
+ attn_type: str,
207
) -> None:
208
'''
209
Update appropriate attention bias field of attention metadata,
@@ -229,7 +229,7 @@ class TorchSDPAMetadata(AttentionMetadata, PagedAttentionMetadata):
229
230
def get_seq_len_block_table_args(
231
self,
232
- attn_type: AttentionType,
232
+ attn_type: str,
233
) -> tuple:
234
'''
235
The particular choice of sequence-length- and block-table-related
@@ -426,7 +426,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
426
attn_metadata: TorchSDPAMetadata, # type: ignore
427
k_scale: float = 1.0,
428
v_scale: float = 1.0,
429
- attn_type: AttentionType = AttentionType.DECODER,
429
+ attn_type: str = AttentionType.DECODER,
430
) -> torch.Tensor:
431
"""Forward pass with torch SDPA and PagedAttention.
432
@@ -574,7 +574,7 @@ class TorchSDPABackendImpl(AttentionImpl[TorchSDPAMetadata]):
574
key: torch.Tensor,
575
value: torch.Tensor,
576
attn_metadata: TorchSDPAMetadata,
577
- attn_type: AttentionType = AttentionType.DECODER,
577
+ attn_type: str = AttentionType.DECODER,
578
) -> None:
579
if self.num_kv_heads != self.num_heads:
580
key = key.repeat_interleave(self.num_queries_per_kv, dim=1)
vllm/attention/backends/utils.py
+2
-2
@@ -478,7 +478,7 @@ def is_all_cross_attn_metadata_set(attn_metadata):
478
def get_seq_len_block_table_args(
479
attn_metadata,
480
is_prompt: bool,
481
- attn_type: AttentionType,
481
+ attn_type: str,
482
) -> tuple:
483
'''
484
The particular choice of sequence-length- and block-table-related
@@ -529,7 +529,7 @@ def get_seq_len_block_table_args(
529
530
def get_num_prefill_decode_query_kv_tokens(
531
attn_metadata,
532
- attn_type: AttentionType,
532
+ attn_type: str,
533
) -> Tuple[int, int, int]:
534
"""
535
Calculate the number of prefill and decode tokens for query, key/value
vllm/attention/backends/xformers.py
+4
-4
@@ -284,7 +284,7 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
284
285
def _get_attn_bias(
286
attn_metadata: XFormersMetadata,
287
- attn_type: AttentionType,
287
+ attn_type: str,
288
) -> Optional[AttentionBias]:
289
'''
290
Extract appropriate attention bias from attention metadata
@@ -314,7 +314,7 @@ def _get_attn_bias(
314
def _set_attn_bias(
315
attn_metadata: XFormersMetadata,
316
attn_bias: List[Optional[AttentionBias]],
317
- attn_type: AttentionType,
317
+ attn_type: str,
318
) -> None:
319
'''
320
Update appropriate attention bias field of attention metadata,
@@ -416,7 +416,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
416
attn_metadata: "XFormersMetadata",
417
k_scale: float = 1.0,
418
v_scale: float = 1.0,
419
- attn_type: AttentionType = AttentionType.DECODER,
419
+ attn_type: str = AttentionType.DECODER,
420
) -> torch.Tensor:
421
"""Forward pass with xFormers and PagedAttention.
422
@@ -617,7 +617,7 @@ class XFormersImpl(AttentionImpl[XFormersMetadata]):
617
key: torch.Tensor,
618
value: torch.Tensor,
619
attn_metadata: XFormersMetadata,
620
- attn_type: AttentionType = AttentionType.DECODER,
620
+ attn_type: str = AttentionType.DECODER,
621
) -> torch.Tensor:
622
"""Attention for 1D query of multiple prompts. Multiple prompt
623
tokens are flattened in to `query` input.
vllm/attention/layer.py
+72
-9
@@ -4,12 +4,17 @@ from typing import Any, Dict, List, Optional
4
import torch
5
import torch.nn as nn
6
7
+import vllm.envs as envs
8
from vllm.attention import AttentionMetadata, AttentionType
9
from vllm.attention.selector import get_attn_backend
10
from vllm.config import CacheConfig
11
+from vllm.forward_context import ForwardContext, get_forward_context
12
from vllm.model_executor.layers.quantization.base_config import (
13
QuantizationConfig)
14
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
15
+from vllm.platforms import current_platform
16
+from vllm.plugins import get_current_vllm_config
17
+from vllm.utils import direct_register_custom_op
18
19
20
class Attention(nn.Module):
@@ -86,6 +91,18 @@ class Attention(nn.Module):
91
alibi_slopes, sliding_window, kv_cache_dtype,
92
blocksparse_params, logits_soft_cap)
93
94
+ # For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
95
+ # torch.compile works by registering the attention as one giant
96
+ # opaque custom op. For other platforms, we directly call them
97
+ # and let torch.compile handle them.
98
+ self.use_direct_call = envs.VLLM_USE_V1 or not (
99
+ current_platform.is_cuda_alike() or current_platform.is_cpu())
100
+ compilation_config = get_current_vllm_config().compilation_config
101
+ if prefix in compilation_config.static_forward_context:
102
+ raise ValueError(f"Duplicate layer name: {prefix}")
103
+ compilation_config.static_forward_context[prefix] = self
104
+ self.layer_name = prefix
105
+
106
def forward(
107
self,
108
query: torch.Tensor,
@@ -93,17 +110,22 @@ class Attention(nn.Module):
110
value: torch.Tensor,
111
kv_cache: torch.Tensor,
112
attn_metadata: AttentionMetadata,
96
- attn_type: AttentionType = AttentionType.DECODER,
113
+ attn_type: str = AttentionType.DECODER,
114
) -> torch.Tensor:
115
99
- return self.impl.forward(query,
100
- key,
101
- value,
102
- kv_cache,
103
- attn_metadata,
104
- self._k_scale,
105
- self._v_scale,
106
- attn_type=attn_type)
116
+ if self.use_direct_call:
117
+ return self.impl.forward(query,
118
+ key,
119
+ value,
120
+ kv_cache,
121
+ attn_metadata,
122
+ self._k_scale,
123
+ self._v_scale,
124
+ attn_type=attn_type)
125
+ else:
126
+ return torch.ops.vllm.unified_attention(query, key, value,
127
+ kv_cache, attn_type,
128
+ self.layer_name)
129
130
def extra_repr(self) -> str:
131
s = f"head_size={self.impl.head_size}" # type: ignore
@@ -112,3 +134,44 @@ class Attention(nn.Module):
134
s += f", scale={self.impl.scale}" # type: ignore
135
s += f", backend={self.impl.__class__.__name__}"
136
return s
137
+
138
+
139
+def unified_attention(
140
+ query: torch.Tensor,
141
+ key: torch.Tensor,
142
+ value: torch.Tensor,
143
+ kv_cache: torch.Tensor,
144
+ attn_type: str,
145
+ layer_name: str,
146
+) -> torch.Tensor:
147
+ forward_context: ForwardContext = get_forward_context()
148
+ attn_metadata = forward_context.dynamic_forward_context
149
+ self = forward_context.static_forward_context[layer_name]
150
+ return self.impl.forward(query,
151
+ key,
152
+ value,
153
+ kv_cache,
154
+ attn_metadata,
155
+ self._k_scale,
156
+ self._v_scale,
157
+ attn_type=attn_type)
158
+
159
+
160
+def unified_attention_fake(
161
+ query: torch.Tensor,
162
+ key: torch.Tensor,
163
+ value: torch.Tensor,
164
+ kv_cache: torch.Tensor,
165
+ attn_type: str,
166
+ layer_name: str,
167
+) -> torch.Tensor:
168
+ return torch.empty_like(query).contiguous()
169
+
170
+
171
+direct_register_custom_op(
172
+ op_name="unified_attention",
173
+ op_func=unified_attention,
174
+ mutates_args=["kv_cache"],
175
+ fake_impl=unified_attention_fake,
176
+ dispatch_key=current_platform.dispatch_key,
177
+)
vllm/config.py
+7
-2
@@ -2135,8 +2135,7 @@ class CompilationConfig(BaseModel):
2135
backend: str = ""
2136
custom_ops: List[str] = Field(default_factory=list)
2137
splitting_ops: List[str] = Field(default_factory=lambda: [
2138
- "vllm.unified_flash_attention",
2139
- "vllm.unified_flash_infer",
2138
+ "vllm.unified_attention",
2139
"vllm.unified_v1_flash_attention",
2140
])
2141
@@ -2197,6 +2196,11 @@ class CompilationConfig(BaseModel):
2196
enabled_custom_ops: Counter[str] = PrivateAttr
2197
disabled_custom_ops: Counter[str] = PrivateAttr
2198
2199
+ # Per-model forward context
2200
+ # Mainly used to store attention cls
2201
+ # Map from layer name to the attention cls
2202
+ static_forward_context: Dict[str, Any] = PrivateAttr
2203
+
2204
@classmethod
2205
def from_cli(cls, cli_value: str) -> "CompilationConfig":
2206
"""Parse the CLI value for the compilation config."""
@@ -2228,6 +2232,7 @@ class CompilationConfig(BaseModel):
2232
2233
self.enabled_custom_ops = Counter()
2234
self.disabled_custom_ops = Counter()
2235
+ self.static_forward_context = {}
2236
2237
def init_backend(self) -> Union[str, Callable]:
2238
if self.level == CompilationLevel.NO_COMPILATION:
vllm/forward_context.py
+22
-5
@@ -1,21 +1,38 @@
1
from contextlib import contextmanager
2
-from typing import Any
2
+from dataclasses import dataclass
3
+from typing import Any, Dict, Optional
4
4
-_forward_context: Any = None
5
+from vllm.config import VllmConfig
6
7
7
-def get_forward_context() -> Any:
8
+@dataclass
9
+class ForwardContext:
10
+ static_forward_context: Dict[str, Any]
11
+ # TODO: extend to support per-layer dynamic forward context
12
+ dynamic_forward_context: Any
13
+
14
+
15
+_forward_context: Optional[ForwardContext] = None
16
+
17
+
18
+def get_forward_context() -> ForwardContext:
19
"""Get the current forward context."""
20
+ assert _forward_context is not None, (
21
+ "Forward context is not set. "
22
+ "Please use `set_forward_context` to set the forward context.")
23
return _forward_context
24
25
26
@contextmanager
13
-def set_forward_context(context: Any):
27
+def set_forward_context(context: Any, vllm_config: VllmConfig):
28
"""A context manager that stores the current forward context,
29
can be attention metadata, etc."""
30
global _forward_context
31
prev_context = _forward_context
18
- _forward_context = context
32
+ _forward_context = ForwardContext(
33
+ static_forward_context=vllm_config.compilation_config.
34
+ static_forward_context,
35
+ dynamic_forward_context=context)
36
try:
37
yield
38
finally:
vllm/model_executor/models/arctic.py
+11
-4
@@ -223,6 +223,7 @@ class ArcticAttention(nn.Module):
223
layer_idx: Optional[int] = None,
224
cache_config: Optional[CacheConfig] = None,
225
quant_config: Optional[QuantizationConfig] = None,
226
+ prefix: str = "",
227
):
228
super().__init__()
229
self.config = config
@@ -274,7 +275,8 @@ class ArcticAttention(nn.Module):
275
self.scaling,
276
num_kv_heads=self.num_kv_heads,
277
cache_config=cache_config,
277
- quant_config=quant_config)
278
+ quant_config=quant_config,
279
+ prefix=f"{prefix}.attn")
280
281
def forward(
282
self,
@@ -299,6 +301,7 @@ class ArcticDecoderLayer(nn.Module):
301
layer_idx: int,
302
cache_config: Optional[CacheConfig] = None,
303
quant_config: Optional[QuantizationConfig] = None,
304
+ prefix: str = "",
305
) -> None:
306
super().__init__()
307
self.layer_idx = layer_idx
@@ -308,7 +311,8 @@ class ArcticDecoderLayer(nn.Module):
311
self.self_attn = ArcticAttention(config,
312
layer_idx,
313
cache_config,
311
- quant_config=quant_config)
314
+ quant_config=quant_config,
315
+ prefix=f"{prefix}.self_attn")
316
self.block_sparse_moe = ArcticMoE(
317
config,
318
layer_id=layer_idx,
@@ -380,8 +384,11 @@ class ArcticModel(nn.Module):
384
org_num_embeddings=self.vocab_size)
385
self.start_layer, self.end_layer, self.layers = make_layers(
386
config.num_hidden_layers,
383
- lambda prefix: ArcticDecoderLayer(config, int(
384
- prefix.split(".")[-1]), cache_config, quant_config),
387
+ lambda prefix: ArcticDecoderLayer(config,
388
+ int(prefix.split(".")[-1]),
389
+ cache_config,
390
+ quant_config,
391
+ prefix=prefix),
392
prefix=f"{prefix}.layers")
393
self._attn_implementation = config._attn_implementation
394
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/baichuan.py
+13
-5
@@ -116,6 +116,7 @@ class BaiChuanAttention(nn.Module):
116
max_position_embeddings: int = 8192,
117
cache_config: Optional[CacheConfig] = None,
118
quant_config: Optional[QuantizationConfig] = None,
119
+ prefix: str = "",
120
):
121
super().__init__()
122
self.hidden_size = hidden_size
@@ -158,7 +159,8 @@ class BaiChuanAttention(nn.Module):
159
self.head_dim,
160
scaling,
161
alibi_slopes=alibi_slopes,
161
- quant_config=quant_config)
162
+ quant_config=quant_config,
163
+ prefix=f"{prefix}.attn")
164
else:
165
self.rotary_emb = get_rope(
166
self.head_dim,
@@ -171,7 +173,8 @@ class BaiChuanAttention(nn.Module):
173
self.head_dim,
174
self.scaling,
175
cache_config=cache_config,
174
- quant_config=quant_config)
176
+ quant_config=quant_config,
177
+ prefix=f"{prefix}.attn")
178
179
def forward(
180
self,
@@ -195,7 +198,8 @@ class BaiChuanDecoderLayer(nn.Module):
198
config: PretrainedConfig,
199
position_embedding: str,
200
cache_config: Optional[CacheConfig] = None,
198
- quant_config: Optional[QuantizationConfig] = None):
201
+ quant_config: Optional[QuantizationConfig] = None,
202
+ prefix: str = ""):
203
super().__init__()
204
self.hidden_size = config.hidden_size
205
rope_theta = getattr(config, "rope_theta", 10000)
@@ -209,6 +213,7 @@ class BaiChuanDecoderLayer(nn.Module):
213
max_position_embeddings=max_position_embeddings,
214
cache_config=cache_config,
215
quant_config=quant_config,
216
+ prefix=f"{prefix}.self_attn",
217
)
218
self.mlp = BaiChuanMLP(
219
hidden_size=self.hidden_size,
@@ -275,8 +280,11 @@ class BaiChuanModel(nn.Module):
280
)
281
self.start_layer, self.end_layer, self.layers = make_layers(
282
config.num_hidden_layers,
278
- lambda prefix: BaiChuanDecoderLayer(config, position_embedding,
279
- cache_config, quant_config),
283
+ lambda prefix: BaiChuanDecoderLayer(config,
284
+ position_embedding,
285
+ cache_config,
286
+ quant_config,
287
+ prefix=prefix),
288
prefix=f"{prefix}.layers",
289
)
290
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/bart.py
+35
-13
@@ -126,6 +126,7 @@ class BartEncoderAttention(nn.Module):
126
config: Optional[BartConfig] = None,
127
cache_config: Optional[CacheConfig] = None,
128
quant_config: Optional[QuantizationConfig] = None,
129
+ prefix: str = "",
130
):
131
super().__init__()
132
self.d_model = config.d_model
@@ -178,7 +179,8 @@ class BartEncoderAttention(nn.Module):
179
self.scaling,
180
num_kv_heads=self.num_kv_heads,
181
cache_config=cache_config,
181
- quant_config=quant_config)
182
+ quant_config=quant_config,
183
+ prefix=f"{prefix}.attn")
184
185
def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
186
attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -208,6 +210,7 @@ class BartDecoderSelfAttention(nn.Module):
210
config: Optional[BartConfig] = None,
211
cache_config: Optional[CacheConfig] = None,
212
quant_config: Optional[QuantizationConfig] = None,
213
+ prefix: str = "",
214
):
215
super().__init__()
216
self.d_model = config.d_model
@@ -260,7 +263,8 @@ class BartDecoderSelfAttention(nn.Module):
263
self.scaling,
264
num_kv_heads=self.num_kv_heads,
265
cache_config=cache_config,
263
- quant_config=quant_config)
266
+ quant_config=quant_config,
267
+ prefix=f"{prefix}.attn")
268
269
def forward(self, hidden_states: torch.Tensor, kv_cache: torch.Tensor,
270
attn_metadata: AttentionMetadata) -> torch.Tensor:
@@ -290,6 +294,7 @@ class BartCrossAttention(nn.Module):
294
config: Optional[BartConfig] = None,
295
cache_config: Optional[CacheConfig] = None,
296
quant_config: Optional[QuantizationConfig] = None,
297
+ prefix: str = "",
298
):
299
super().__init__()
300
self.d_model = config.d_model
@@ -342,7 +347,8 @@ class BartCrossAttention(nn.Module):
347
self.scaling,
348
num_kv_heads=self.num_kv_heads,
349
cache_config=cache_config,
345
- quant_config=quant_config)
350
+ quant_config=quant_config,
351
+ prefix=f"{prefix}.attn")
352
353
def forward(
354
self,
@@ -384,6 +390,7 @@ class BartEncoderLayer(nn.Module):
390
config: BartConfig,
391
cache_config: Optional[CacheConfig] = None,
392
quant_config: Optional[QuantizationConfig] = None,
393
+ prefix: str = "",
394
):
395
super().__init__()
396
self.embed_dim = config.d_model
@@ -393,7 +400,9 @@ class BartEncoderLayer(nn.Module):
400
num_heads=config.encoder_attention_heads,
401
config=config,
402
cache_config=cache_config,
396
- quant_config=quant_config)
403
+ quant_config=quant_config,
404
+ prefix=f"{prefix}.self_attn",
405
+ )
406
self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
407
self.activation_fn = get_act_fn(config.activation_function)
408
@@ -464,6 +473,7 @@ class BartDecoderLayer(nn.Module):
473
config: BartConfig,
474
cache_config: Optional[CacheConfig] = None,
475
quant_config: Optional[QuantizationConfig] = None,
476
+ prefix: str = "",
477
):
478
super().__init__()
479
self.embed_dim = config.d_model
@@ -473,7 +483,9 @@ class BartDecoderLayer(nn.Module):
483
num_heads=config.decoder_attention_heads,
484
config=config,
485
cache_config=cache_config,
476
- quant_config=quant_config)
486
+ quant_config=quant_config,
487
+ prefix=f"{prefix}.self_attn",
488
+ )
489
self.activation_fn = get_act_fn(config.activation_function)
490
491
self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
@@ -486,6 +498,7 @@ class BartDecoderLayer(nn.Module):
498
self.embed_dim,
499
config.decoder_attention_heads,
500
config=config,
501
+ prefix=f"{prefix}.encoder_attn",
502
)
503
self.encoder_attn_layer_norm = nn.LayerNorm(self.embed_dim)
504
@@ -578,7 +591,8 @@ class BartEncoder(nn.Module):
591
cache_config: Optional[CacheConfig] = None,
592
quant_config: Optional[QuantizationConfig] = None,
593
lora_config: Optional[LoRAConfig] = None,
581
- embed_tokens: Optional[nn.Embedding] = None):
594
+ embed_tokens: Optional[nn.Embedding] = None,
595
+ prefix: str = ""):
596
super().__init__()
597
598
self.cache_config = cache_config
@@ -599,9 +613,13 @@ class BartEncoder(nn.Module):
613
config.max_position_embeddings,
614
embed_dim,
615
)
602
- self.layers = nn.ModuleList(
603
- [BartEncoderLayer(config,cache_config,quant_config) \
604
- for _ in range(config.encoder_layers)])
616
+ self.layers = nn.ModuleList([
617
+ BartEncoderLayer(config,
618
+ cache_config,
619
+ quant_config,
620
+ prefix=f"{prefix}.layers.{layer_idx}")
621
+ for layer_idx in range(config.encoder_layers)
622
+ ])
623
624
self.layernorm_embedding = nn.LayerNorm(embed_dim)
625
@@ -661,6 +679,7 @@ class BartDecoder(nn.Module):
679
quant_config: Optional[QuantizationConfig] = None,
680
lora_config: Optional[LoRAConfig] = None,
681
embed_tokens: Optional[nn.Embedding] = None,
682
+ prefix: str = "",
683
):
684
super().__init__()
685
self.cache_config = cache_config
@@ -683,8 +702,9 @@ class BartDecoder(nn.Module):
702
)
703
704
self.layers = nn.ModuleList(
686
- [BartDecoderLayer(config,cache_config,quant_config) \
687
- for _ in range(config.decoder_layers)])
705
+ [BartDecoderLayer(config,cache_config,quant_config,
706
+ prefix=f"{prefix}.layers.{layer_idx}") \
707
+ for layer_idx in range(config.decoder_layers)])
708
709
self.layernorm_embedding = nn.LayerNorm(config.d_model)
710
@@ -759,10 +779,12 @@ class BartModel(nn.Module):
779
780
self.encoder = BartEncoder(config,
781
cache_config,
762
- quant_config=quant_config)
782
+ quant_config=quant_config,
783
+ prefix=f"{prefix}.encoder")
784
self.decoder = BartDecoder(config,
785
cache_config,
765
- quant_config=quant_config)
786
+ quant_config=quant_config,
787
+ prefix=f"{prefix}.decoder")
788
789
def forward(self, input_ids: torch.Tensor, positions: torch.Tensor,
790
encoder_input_ids: torch.Tensor,
vllm/model_executor/models/bloom.py
+10
-4
@@ -78,6 +78,7 @@ class BloomAttention(nn.Module):
78
config: BloomConfig,
79
cache_config: Optional[CacheConfig] = None,
80
quant_config: Optional[QuantizationConfig] = None,
81
+ prefix: str = "",
82
):
83
super().__init__()
84
self.hidden_size = config.hidden_size
@@ -116,7 +117,8 @@ class BloomAttention(nn.Module):
117
scaling,
118
alibi_slopes=alibi_slopes,
119
cache_config=cache_config,
119
- quant_config=quant_config)
120
+ quant_config=quant_config,
121
+ prefix=f"{prefix}.attn")
122
123
def forward(
124
self,
@@ -168,14 +170,17 @@ class BloomBlock(nn.Module):
170
config: BloomConfig,
171
cache_config: Optional[CacheConfig] = None,
172
quant_config: Optional[QuantizationConfig] = None,
173
+ prefix: str = "",
174
):
175
super().__init__()
176
hidden_size = config.hidden_size
177
178
self.input_layernorm = nn.LayerNorm(hidden_size,
179
eps=config.layer_norm_epsilon)
177
- self.self_attention = BloomAttention(config, cache_config,
178
- quant_config)
180
+ self.self_attention = BloomAttention(config,
181
+ cache_config,
182
+ quant_config,
183
+ prefix=f"{prefix}.self_attention")
184
self.post_attention_layernorm = nn.LayerNorm(
185
hidden_size, eps=config.layer_norm_epsilon)
186
self.mlp = BloomMLP(config, quant_config)
@@ -242,7 +247,8 @@ class BloomModel(nn.Module):
247
# Transformer blocks
248
self.start_layer, self.end_layer, self.h = make_layers(
249
config.num_hidden_layers,
245
- lambda prefix: BloomBlock(config, cache_config, quant_config),
250
+ lambda prefix: BloomBlock(
251
+ config, cache_config, quant_config, prefix=prefix),
252
prefix=f"{prefix}.h")
253
254
# Final Layer Norm
vllm/model_executor/models/chameleon.py
+9
-2
@@ -223,6 +223,7 @@ class ChameleonAttention(nn.Module):
223
quant_config: Optional[QuantizationConfig] = None,
224
bias: bool = False,
225
cache_config: Optional[CacheConfig] = None,
226
+ prefix: str = "",
227
) -> None:
228
super().__init__()
229
self.hidden_size = hidden_size
@@ -276,7 +277,8 @@ class ChameleonAttention(nn.Module):
277
self.scaling,
278
num_kv_heads=self.num_kv_heads,
279
cache_config=cache_config,
279
- quant_config=quant_config)
280
+ quant_config=quant_config,
281
+ prefix=f"{prefix}.attn")
282
283
def _apply_qk_norm(self, q: torch.Tensor,
284
k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
@@ -313,6 +315,7 @@ class ChameleonDecoderLayer(nn.Module):
315
config: ChameleonConfig,
316
cache_config: Optional[CacheConfig] = None,
317
quant_config: Optional[QuantizationConfig] = None,
318
+ prefix: str = "",
319
) -> None:
320
super().__init__()
321
self.hidden_size = config.hidden_size
@@ -336,6 +339,7 @@ class ChameleonDecoderLayer(nn.Module):
339
quant_config=quant_config,
340
bias=False,
341
cache_config=cache_config,
342
+ prefix=f"{prefix}.self_attn",
343
)
344
self.mlp = ChameleonMLP(
345
hidden_size=self.hidden_size,
@@ -386,6 +390,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
390
config: ChameleonConfig,
391
cache_config: Optional[CacheConfig] = None,
392
quant_config: Optional[QuantizationConfig] = None,
393
+ prefix: str = "",
394
) -> None:
395
super().__init__()
396
self.hidden_size = config.hidden_size
@@ -409,6 +414,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
414
quant_config=quant_config,
415
bias=False,
416
cache_config=cache_config,
417
+ prefix=f"{prefix}.self_attn",
418
)
419
self.mlp = ChameleonMLP(
420
hidden_size=self.hidden_size,
@@ -855,7 +861,8 @@ class ChameleonModel(nn.Module):
861
config.num_hidden_layers,
862
lambda prefix: decoder_layer(config=config,
863
cache_config=cache_config,
858
- quant_config=quant_config),
864
+ quant_config=quant_config,
865
+ prefix=prefix),
866
prefix=f"{prefix}.layers",
867
)
868
vllm/model_executor/models/chatglm.py
+19
-6
@@ -230,6 +230,7 @@ class GLMAttention(nn.Module):
230
config: ChatGLMConfig,
231
cache_config: Optional[CacheConfig] = None,
232
quant_config: Optional[QuantizationConfig] = None,
233
+ prefix: str = "",
234
):
235
super().__init__()
236
self.hidden_size = config.hidden_size
@@ -285,7 +286,8 @@ class GLMAttention(nn.Module):
286
self.scaling,
287
num_kv_heads=self.num_kv_heads,
288
cache_config=cache_config,
288
- quant_config=quant_config)
289
+ quant_config=quant_config,
290
+ prefix=f"{prefix}.attn")
291
292
def forward(
293
self,
@@ -364,6 +366,7 @@ class GLMBlock(nn.Module):
366
config: ChatGLMConfig,
367
cache_config: Optional[CacheConfig] = None,
368
quant_config: Optional[QuantizationConfig] = None,
369
+ prefix: str = "",
370
):
371
super().__init__()
372
self.apply_residual_connection_post_layernorm = (
@@ -377,7 +380,10 @@ class GLMBlock(nn.Module):
380
eps=config.layernorm_epsilon)
381
382
# Self attention.
380
- self.self_attention = GLMAttention(config, cache_config, quant_config)
383
+ self.self_attention = GLMAttention(config,
384
+ cache_config,
385
+ quant_config,
386
+ prefix=f"{prefix}.self_attention")
387
self.hidden_dropout = config.hidden_dropout
388
389
# Layernorm on the attention output
@@ -446,7 +452,8 @@ class GLMTransformer(nn.Module):
452
# Transformer layers.
453
self.start_layer, self.end_layer, self.layers = make_layers(
454
self.num_layers,
449
- lambda prefix: GLMBlock(config, cache_config, quant_config),
455
+ lambda prefix: GLMBlock(
456
+ config, cache_config, quant_config, prefix=prefix),
457
prefix=f"{prefix}.layers",
458
)
459
@@ -500,16 +507,22 @@ class ChatGLMModel(nn.Module):
507
self.num_layers = config.num_layers
508
self.multi_query_group_num = config.multi_query_group_num
509
self.kv_channels = config.kv_channels
503
- self.encoder = GLMTransformer(config, cache_config, quant_config)
510
+ self.encoder = GLMTransformer(config,
511
+ cache_config,
512
+ quant_config,
513
+ prefix=f"{prefix}.encoder")
514
515
self.output_layer = ParallelLMHead(config.padded_vocab_size,
516
config.hidden_size,
507
- quant_config=quant_config)
517
+ quant_config=quant_config,
518
+ prefix=f"{prefix}.output_layer")
519
520
vision_config_flag = getattr(config, 'vision_config', None)
521
if vision_config_flag is not None:
522
self.vision_config = Namespace(**config.vision_config)
512
- self.vision = EVA2CLIPModel(self.config, quant_config)
523
+ self.vision = EVA2CLIPModel(self.config,
524
+ quant_config,
525
+ prefix=f"{prefix}.vision")
526
else:
527
self.vision = None
528
vllm/model_executor/models/commandr.py
+9
-5
@@ -120,6 +120,7 @@ class CohereAttention(nn.Module):
120
config: CohereConfig,
121
cache_config: Optional[CacheConfig] = None,
122
quant_config: Optional[QuantizationConfig] = None,
123
+ prefix: str = "",
124
):
125
super().__init__()
126
tp_size = get_tensor_model_parallel_world_size()
@@ -175,7 +176,8 @@ class CohereAttention(nn.Module):
176
self.scaling,
177
num_kv_heads=self.num_kv_heads,
178
cache_config=cache_config,
178
- quant_config=quant_config)
179
+ quant_config=quant_config,
180
+ prefix=f"{prefix}.attn")
181
if self.use_qk_norm:
182
self.q_norm = LayerNorm(param_shape=(self.num_heads,
183
self.head_dim),
@@ -215,13 +217,15 @@ class CohereDecoderLayer(nn.Module):
217
def __init__(self,
218
config: CohereConfig,
219
cache_config: Optional[CacheConfig] = None,
218
- quant_config: Optional[QuantizationConfig] = None):
220
+ quant_config: Optional[QuantizationConfig] = None,
221
+ prefix: str = ""):
222
super().__init__()
223
self.hidden_size = config.hidden_size
224
225
self.self_attn = CohereAttention(config,
226
cache_config,
224
- quant_config=quant_config)
227
+ quant_config=quant_config,
228
+ prefix=f"{prefix}.self_attn")
229
230
self.mlp = CohereMLP(config, quant_config=quant_config)
231
self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
@@ -271,8 +275,8 @@ class CohereModel(nn.Module):
275
config.hidden_size)
276
self.start_layer, self.end_layer, self.layers = make_layers(
277
config.num_hidden_layers,
274
- lambda prefix: CohereDecoderLayer(config, cache_config,
275
- quant_config),
278
+ lambda prefix: CohereDecoderLayer(
279
+ config, cache_config, quant_config, prefix=prefix),
280
prefix=f"{prefix}.layers")
281
self.norm = LayerNorm(param_shape=(config.hidden_size),
282
eps=config.layer_norm_eps)
vllm/model_executor/models/dbrx.py
+16
-5
@@ -154,6 +154,7 @@ class DbrxAttention(nn.Module):
154
config: DbrxConfig,
155
cache_config: Optional[CacheConfig] = None,
156
quant_config: Optional[QuantizationConfig] = None,
157
+ prefix: str = "",
158
):
159
super().__init__()
160
self.d_model = config.d_model
@@ -208,7 +209,8 @@ class DbrxAttention(nn.Module):
209
self.scaling,
210
num_kv_heads=self.num_kv_heads,
211
cache_config=cache_config,
211
- quant_config=quant_config)
212
+ quant_config=quant_config,
213
+ prefix=f"{prefix}.attn")
214
215
def forward(
216
self,
@@ -234,10 +236,14 @@ class DbrxFusedNormAttention(nn.Module):
236
config: DbrxConfig,
237
cache_config: Optional[CacheConfig] = None,
238
quant_config: Optional[QuantizationConfig] = None,
239
+ prefix: str = "",
240
):
241
super().__init__()
242
self.d_model = config.d_model
240
- self.attn = DbrxAttention(config, cache_config, quant_config)
243
+ self.attn = DbrxAttention(config,
244
+ cache_config,
245
+ quant_config,
246
+ prefix=f"{prefix}.attn")
247
self.norm_1 = nn.LayerNorm(self.d_model)
248
self.norm_2 = nn.LayerNorm(self.d_model)
249
@@ -269,10 +275,14 @@ class DbrxBlock(nn.Module):
275
config: DbrxConfig,
276
cache_config: Optional[CacheConfig] = None,
277
quant_config: Optional[QuantizationConfig] = None,
278
+ prefix: str = "",
279
):
280
super().__init__()
274
- self.norm_attn_norm = DbrxFusedNormAttention(config, cache_config,
275
- quant_config)
281
+ self.norm_attn_norm = DbrxFusedNormAttention(
282
+ config,
283
+ cache_config,
284
+ quant_config,
285
+ prefix=f"{prefix}.norm_attn_norm")
286
self.ffn = DbrxMoE(config, quant_config)
287
288
def forward(
@@ -308,7 +318,8 @@ class DbrxModel(nn.Module):
318
)
319
self.start_layer, self.end_layer, self.blocks = make_layers(
320
config.n_layers,
311
- lambda prefix: DbrxBlock(config, cache_config, quant_config),
321
+ lambda prefix: DbrxBlock(
322
+ config, cache_config, quant_config, prefix=prefix),
323
prefix=f"{prefix}.blocks",
324
)
325
self.norm_f = nn.LayerNorm(config.d_model, eps=1e-5)
vllm/model_executor/models/deepseek.py
+7
-2
@@ -184,6 +184,7 @@ class DeepseekAttention(nn.Module):
184
max_position_embeddings: int = 8192,
185
cache_config: Optional[CacheConfig] = None,
186
quant_config: Optional[QuantizationConfig] = None,
187
+ prefix: str = "",
188
) -> None:
189
super().__init__()
190
self.hidden_size = hidden_size
@@ -236,7 +237,8 @@ class DeepseekAttention(nn.Module):
237
self.scaling,
238
num_kv_heads=self.num_kv_heads,
239
cache_config=cache_config,
239
- quant_config=quant_config)
240
+ quant_config=quant_config,
241
+ prefix=f"{prefix}.attn")
242
243
def forward(
244
self,
@@ -261,6 +263,7 @@ class DeepseekDecoderLayer(nn.Module):
263
layer_idx: int,
264
cache_config: Optional[CacheConfig] = None,
265
quant_config: Optional[QuantizationConfig] = None,
266
+ prefix: str = "",
267
) -> None:
268
super().__init__()
269
self.hidden_size = config.hidden_size
@@ -277,6 +280,7 @@ class DeepseekDecoderLayer(nn.Module):
280
max_position_embeddings=max_position_embeddings,
281
cache_config=cache_config,
282
quant_config=quant_config,
283
+ prefix=f"{prefix}.self_attn",
284
)
285
if (config.n_routed_experts is not None
286
and layer_idx >= config.first_k_dense_replace
@@ -346,7 +350,8 @@ class DeepseekModel(nn.Module):
350
lambda prefix: DeepseekDecoderLayer(config,
351
int(prefix.split(".")[-1]),
352
cache_config,
349
- quant_config=quant_config),
353
+ quant_config=quant_config,
354
+ prefix=prefix),
355
prefix=f"{prefix}.layers")
356
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
357
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/deepseek_v2.py
+2
-1
@@ -268,7 +268,8 @@ class DeepseekV2Attention(nn.Module):
268
self.scaling,
269
num_kv_heads=self.num_local_heads,
270
cache_config=cache_config,
271
- quant_config=quant_config)
271
+ quant_config=quant_config,
272
+ prefix=f"{prefix}.attn")
273
274
def forward(
275
self,
vllm/model_executor/models/exaone.py
+2
-1
@@ -174,6 +174,7 @@ class ExaoneAttention(nn.Module):
174
num_kv_heads=self.num_kv_heads,
175
cache_config=cache_config,
176
quant_config=quant_config,
177
+ prefix=f"{prefix}.attn",
178
)
179
180
def forward(
@@ -219,7 +220,7 @@ class ExaoneBlockAttention(nn.Module):
220
quant_config=quant_config,
221
bias=bias,
222
cache_config=cache_config,
222
- prefix=prefix,
223
+ prefix=f"{prefix}.attention",
224
)
225
226
def forward(
vllm/model_executor/models/falcon.py
+15
-7
@@ -84,6 +84,7 @@ class FalconAttention(nn.Module):
84
config: FalconConfig,
85
cache_config: Optional[CacheConfig] = None,
86
quant_config: Optional[QuantizationConfig] = None,
87
+ prefix: str = "",
88
):
89
super().__init__()
90
@@ -158,7 +159,8 @@ class FalconAttention(nn.Module):
159
self.head_dim,
160
self.inv_norm_factor,
161
num_kv_heads=self.num_kv_heads,
161
- quant_config=quant_config)
162
+ quant_config=quant_config,
163
+ prefix=f"{prefix}.attn")
164
elif self.use_alibi:
165
tp_rank = get_tensor_model_parallel_rank()
166
head_start = tp_rank * self.num_heads
@@ -171,14 +173,16 @@ class FalconAttention(nn.Module):
173
self.inv_norm_factor,
174
num_kv_heads=self.num_kv_heads,
175
alibi_slopes=alibi_slopes,
174
- quant_config=quant_config)
176
+ quant_config=quant_config,
177
+ prefix=f"{prefix}.attn")
178
else:
179
self.attn = Attention(self.num_heads,
180
self.head_dim,
181
scale=self.inv_norm_factor,
182
num_kv_heads=self.num_kv_heads,
183
cache_config=cache_config,
181
- quant_config=quant_config)
184
+ quant_config=quant_config,
185
+ prefix=f"{prefix}.attn")
186
187
def forward(
188
self,
@@ -241,12 +245,16 @@ class FalconDecoderLayer(nn.Module):
245
config: FalconConfig,
246
cache_config: Optional[CacheConfig] = None,
247
quant_config: Optional[QuantizationConfig] = None,
248
+ prefix: str = "",
249
):
250
super().__init__()
251
hidden_size = config.hidden_size
252
self.num_heads = config.num_attention_heads
248
- self.self_attention = FalconAttention(config, cache_config,
249
- quant_config)
253
+ self.self_attention = FalconAttention(
254
+ config,
255
+ cache_config,
256
+ quant_config,
257
+ prefix=f"{prefix}.self_attention")
258
self.mlp = FalconMLP(config, quant_config)
259
self.config = config
260
@@ -357,8 +365,8 @@ class FalconModel(nn.Module):
365
# Transformer blocks
366
self.start_layer, self.end_layer, self.h = make_layers(
367
config.num_hidden_layers,
360
- lambda prefix: FalconDecoderLayer(config, cache_config,
361
- quant_config),
368
+ lambda prefix: FalconDecoderLayer(
369
+ config, cache_config, quant_config, prefix=prefix),
370
prefix=f"{prefix}.h")
371
372
# Final Layer Norm
vllm/model_executor/models/florence2.py
+6
-4
@@ -35,10 +35,12 @@ class Florence2LanguageModel(nn.Module):
35
self.shared = BartScaledWordEmbedding(self.vocab_size, config.d_model)
36
self.encoder = BartEncoder(config,
37
cache_config=cache_config,
38
- quant_config=quant_config)
38
+ quant_config=quant_config,
39
+ prefix=f"{prefix}.encoder")
40
self.decoder = BartDecoder(config,
41
cache_config=cache_config,
41
- quant_config=quant_config)
42
+ quant_config=quant_config,
43
+ prefix=f"{prefix}.decoder")
44
45
if self.config.tie_word_embeddings:
46
self.encoder.embed_tokens.weight = self.shared.weight
@@ -99,7 +101,7 @@ class Florence2LanguageForConditionalGeneration(nn.Module):
101
102
self.config = config
103
self.model = Florence2LanguageModel(vllm_config=vllm_config,
102
- prefix=prefix)
104
+ prefix=f"{prefix}.model")
105
embed_scale = math.sqrt(
106
config.d_model) if config.scale_embedding else 1.0
107
@@ -198,7 +200,7 @@ class Florence2ForConditionalGeneration(nn.Module):
200
# TODO(Isotr0py): Add vision backbone
201
self.language_model = Florence2LanguageForConditionalGeneration(
202
vllm_config=vllm_config.with_hf_config(config.text_config),
201
- prefix=prefix,
203
+ prefix=f"{prefix}.language_model",
204
)
205
206
@property
vllm/model_executor/models/gemma.py
+2
-1
@@ -174,7 +174,8 @@ class GemmaAttention(nn.Module):
174
self.scaling,
175
num_kv_heads=self.num_kv_heads,
176
cache_config=cache_config,
177
- quant_config=quant_config)
177
+ quant_config=quant_config,
178
+ prefix=f"{prefix}.attn")
179
180
def forward(
181
self,
vllm/model_executor/models/gemma2.py
+11
-4
@@ -95,7 +95,8 @@ class Gemma2Attention(nn.Module):
95
rope_theta: float,
96
cache_config: Optional[CacheConfig] = None,
97
quant_config: Optional[QuantizationConfig] = None,
98
- attn_logits_soft_cap: Optional[float] = None) -> None:
98
+ attn_logits_soft_cap: Optional[float] = None,
99
+ prefix: str = "") -> None:
100
super().__init__()
101
self.layer_idx = layer_idx
102
self.config = config
@@ -154,7 +155,8 @@ class Gemma2Attention(nn.Module):
155
num_kv_heads=self.num_kv_heads,
156
cache_config=cache_config,
157
quant_config=quant_config,
157
- logits_soft_cap=attn_logits_soft_cap)
158
+ logits_soft_cap=attn_logits_soft_cap,
159
+ prefix=f"{prefix}.attn")
160
161
def forward(
162
self,
@@ -179,6 +181,7 @@ class Gemma2DecoderLayer(nn.Module):
181
config: Gemma2Config,
182
cache_config: Optional[CacheConfig] = None,
183
quant_config: Optional[QuantizationConfig] = None,
184
+ prefix: str = "",
185
) -> None:
186
super().__init__()
187
self.hidden_size = config.hidden_size
@@ -194,6 +197,7 @@ class Gemma2DecoderLayer(nn.Module):
197
cache_config=cache_config,
198
quant_config=quant_config,
199
attn_logits_soft_cap=config.attn_logit_softcapping,
200
+ prefix=f"{prefix}.self_attn",
201
)
202
self.hidden_size = config.hidden_size
203
self.mlp = Gemma2MLP(
@@ -257,8 +261,11 @@ class Gemma2Model(nn.Module):
261
)
262
self.start_layer, self.end_layer, self.layers = make_layers(
263
config.num_hidden_layers,
260
- lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[
261
- -1]), config, cache_config, quant_config),
264
+ lambda prefix: Gemma2DecoderLayer(int(prefix.split(".")[-1]),
265
+ config,
266
+ cache_config,
267
+ quant_config,
268
+ prefix=prefix),
269
prefix=f"{prefix}.layers")
270
self.norm = GemmaRMSNorm(config.hidden_size, eps=config.rms_norm_eps)
271
vllm/model_executor/models/glm4_vision_encoder.py
+13
-4
@@ -56,6 +56,7 @@ class Attention(nn.Module):
56
self,
57
config,
58
quant_config: Optional[QuantizationConfig] = None,
59
+ prefix: str = '',
60
):
61
super().__init__()
62
self.hidden_size = config.hidden_size
@@ -135,11 +136,14 @@ class TransformerLayer(nn.Module):
136
self,
137
config,
138
quant_config: Optional[QuantizationConfig] = None,
139
+ prefix: str = '',
140
):
141
super().__init__()
142
self.input_layernorm = LayerNorm(config.hidden_size,
143
eps=config.layer_norm_eps)
142
- self.attention = Attention(config, quant_config=quant_config)
144
+ self.attention = Attention(config,
145
+ quant_config=quant_config,
146
+ prefix=f"{prefix}.attention")
147
self.mlp = MLP(config, quant_config=quant_config)
148
self.post_attention_layernorm = LayerNorm(config.hidden_size,
149
eps=config.layer_norm_eps)
@@ -161,11 +165,14 @@ class Transformer(nn.Module):
165
self,
166
config,
167
quant_config: Optional[QuantizationConfig] = None,
168
+ prefix: str = '',
169
):
170
super().__init__()
171
self.layers = nn.ModuleList([
167
- TransformerLayer(config, quant_config=quant_config)
168
- for _ in range(config.num_hidden_layers)
172
+ TransformerLayer(config,
173
+ quant_config=quant_config,
174
+ prefix=f"{prefix}.layer.{layer_idx}")
175
+ for layer_idx in range(config.num_hidden_layers)
176
])
177
178
def forward(self, hidden_states):
@@ -252,12 +259,14 @@ class EVA2CLIPModel(nn.Module):
259
self,
260
config,
261
quant_config: Optional[QuantizationConfig] = None,
262
+ prefix: str = '',
263
):
264
super().__init__()
265
vision_config = Namespace(**config.vision_config)
266
self.patch_embedding = PatchEmbedding(vision_config)
267
self.transformer = Transformer(vision_config,
260
- quant_config=quant_config)
268
+ quant_config=quant_config,
269
+ prefix=f"{prefix}.transformer")
270
self.linear_proj = GLU(config,
271
in_features=config.hidden_size,
272
quant_config=quant_config)
vllm/model_executor/models/gpt2.py
+2
-1
@@ -84,7 +84,8 @@ class GPT2Attention(nn.Module):
84
self.head_dim,
85
scale=self.scale,
86
cache_config=cache_config,
87
- quant_config=quant_config)
87
+ quant_config=quant_config,
88
+ prefix=f"{prefix}.attn")
89
90
def forward(
91
self,
vllm/model_executor/models/gpt_bigcode.py
+10
-3
@@ -52,6 +52,7 @@ class GPTBigCodeAttention(nn.Module):
52
config: GPTBigCodeConfig,
53
cache_config: Optional[CacheConfig] = None,
54
quant_config: Optional[QuantizationConfig] = None,
55
+ prefix: str = "",
56
):
57
super().__init__()
58
self.hidden_size = config.hidden_size
@@ -92,7 +93,8 @@ class GPTBigCodeAttention(nn.Module):
93
scale=self.scale,
94
num_kv_heads=self.num_kv_heads,
95
cache_config=cache_config,
95
- quant_config=quant_config)
96
+ quant_config=quant_config,
97
+ prefix=f"{prefix}.attn")
98
99
def forward(
100
self,
@@ -151,6 +153,7 @@ class GPTBigCodeBlock(nn.Module):
153
config: GPTBigCodeConfig,
154
cache_config: Optional[CacheConfig] = None,
155
quant_config: Optional[QuantizationConfig] = None,
156
+ prefix: str = "",
157
):
158
super().__init__()
159
hidden_size = config.hidden_size
@@ -158,7 +161,10 @@ class GPTBigCodeBlock(nn.Module):
161
hidden_size)
162
163
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
161
- self.attn = GPTBigCodeAttention(config, cache_config, quant_config)
164
+ self.attn = GPTBigCodeAttention(config,
165
+ cache_config,
166
+ quant_config,
167
+ prefix=f"{prefix}.attn")
168
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
169
self.mlp = GPTBigMLP(inner_dim, config, quant_config)
170
@@ -210,7 +216,8 @@ class GPTBigCodeModel(nn.Module):
216
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
217
self.start_layer, self.end_layer, self.h = make_layers(
218
config.num_hidden_layers,
213
- lambda prefix: GPTBigCodeBlock(config, cache_config, quant_config),
219
+ lambda prefix: GPTBigCodeBlock(
220
+ config, cache_config, quant_config, prefix=prefix),
221
prefix=f"{prefix}.h",
222
)
223
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
vllm/model_executor/models/gpt_j.py
+10
-3
@@ -53,6 +53,7 @@ class GPTJAttention(nn.Module):
53
config: GPTJConfig,
54
cache_config: Optional[CacheConfig] = None,
55
quant_config: Optional[QuantizationConfig] = None,
56
+ prefix: str = "",
57
):
58
super().__init__()
59
self.total_num_heads = config.num_attention_heads
@@ -94,7 +95,8 @@ class GPTJAttention(nn.Module):
95
self.head_size,
96
scaling,
97
cache_config=cache_config,
97
- quant_config=quant_config)
98
+ quant_config=quant_config,
99
+ prefix=f"{prefix}.attn")
100
101
def forward(
102
self,
@@ -147,12 +149,16 @@ class GPTJBlock(nn.Module):
149
config: GPTJConfig,
150
cache_config: Optional[CacheConfig] = None,
151
quant_config: Optional[QuantizationConfig] = None,
152
+ prefix: str = "",
153
):
154
super().__init__()
155
inner_dim = (4 * config.n_embd
156
if config.n_inner is None else config.n_inner)
157
self.ln_1 = nn.LayerNorm(config.n_embd, eps=config.layer_norm_epsilon)
155
- self.attn = GPTJAttention(config, cache_config, quant_config)
158
+ self.attn = GPTJAttention(config,
159
+ cache_config,
160
+ quant_config,
161
+ prefix=f"{prefix}.attn")
162
self.mlp = GPTJMLP(inner_dim, config, quant_config)
163
164
def forward(
@@ -193,7 +199,8 @@ class GPTJModel(nn.Module):
199
)
200
self.start_layer, self.end_layer, self.h = make_layers(
201
config.n_layer,
196
- lambda prefix: GPTJBlock(config, cache_config, quant_config),
202
+ lambda prefix: GPTJBlock(
203
+ config, cache_config, quant_config, prefix=prefix),
204
prefix=f"{prefix}.h",
205
)
206
self.ln_f = nn.LayerNorm(self.embed_dim, eps=config.layer_norm_epsilon)
vllm/model_executor/models/gpt_neox.py
+10
-3
@@ -52,6 +52,7 @@ class GPTNeoXAttention(nn.Module):
52
config: GPTNeoXConfig,
53
cache_config: Optional[CacheConfig] = None,
54
quant_config: Optional[QuantizationConfig] = None,
55
+ prefix: str = "",
56
):
57
super().__init__()
58
self.total_num_heads = config.num_attention_heads
@@ -94,7 +95,8 @@ class GPTNeoXAttention(nn.Module):
95
self.head_size,
96
scaling,
97
cache_config=cache_config,
97
- quant_config=quant_config)
98
+ quant_config=quant_config,
99
+ prefix=f"{prefix}.attn")
100
101
def forward(
102
self,
@@ -145,6 +147,7 @@ class GPTNeoXLayer(nn.Module):
147
config: GPTNeoXConfig,
148
cache_config: Optional[CacheConfig] = None,
149
quant_config: Optional[QuantizationConfig] = None,
150
+ prefix: str = "",
151
):
152
super().__init__()
153
self.use_parallel_residual = config.use_parallel_residual
@@ -152,7 +155,10 @@ class GPTNeoXLayer(nn.Module):
155
eps=config.layer_norm_eps)
156
self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
157
eps=config.layer_norm_eps)
155
- self.attention = GPTNeoXAttention(config, cache_config, quant_config)
158
+ self.attention = GPTNeoXAttention(config,
159
+ cache_config,
160
+ quant_config,
161
+ prefix=f"{prefix}.attention")
162
self.mlp = GPTNeoXMLP(config, quant_config)
163
164
def forward(
@@ -205,7 +211,8 @@ class GPTNeoXModel(nn.Module):
211
)
212
self.start_layer, self.end_layer, self.layers = make_layers(
213
config.num_hidden_layers,
208
- lambda prefix: GPTNeoXLayer(config, cache_config, quant_config),
214
+ lambda prefix: GPTNeoXLayer(
215
+ config, cache_config, quant_config, prefix=prefix),
216
prefix=f"{prefix}.layers",
217
)
218
self.final_layer_norm = nn.LayerNorm(config.hidden_size,
vllm/model_executor/models/granite.py
+2
-1
@@ -161,7 +161,8 @@ class GraniteAttention(nn.Module):
161
self.scaling,
162
num_kv_heads=self.num_kv_heads,
163
cache_config=cache_config,
164
- quant_config=quant_config)
164
+ quant_config=quant_config,
165
+ prefix=f"{prefix}.attn")
166
167
def forward(
168
self,
vllm/model_executor/models/granitemoe.py
+2
-1
@@ -164,7 +164,8 @@ class GraniteMoeAttention(nn.Module):
164
self.scaling,
165
num_kv_heads=self.num_kv_heads,
166
cache_config=cache_config,
167
- quant_config=quant_config)
167
+ quant_config=quant_config,
168
+ prefix=f"{prefix}.attn")
169
170
def forward(
171
self,
vllm/model_executor/models/internlm2.py
+15
-6
@@ -1,5 +1,5 @@
1
from functools import partial
2
-from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
2
+from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Type, Union
3
4
import torch
5
from torch import nn
@@ -250,7 +250,12 @@ class InternLMDecoderLayer(nn.Module):
250
@support_torch_compile
251
class InternLM2Model(nn.Module):
252
253
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
253
+ def __init__(
254
+ self,
255
+ *,
256
+ vllm_config: VllmConfig,
257
+ prefix: str = "",
258
+ layer_type: Type[InternLMDecoderLayer] = InternLMDecoderLayer):
259
super().__init__()
260
261
config = vllm_config.model_config.hf_config
@@ -266,7 +271,7 @@ class InternLM2Model(nn.Module):
271
)
272
self.start_layer, self.end_layer, self.layers = make_layers(
273
config.num_hidden_layers,
269
- lambda prefix: InternLMDecoderLayer(
274
+ lambda prefix: layer_type(
275
config, cache_config, quant_config, prefix=prefix),
276
prefix=f"{prefix}.layers")
277
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -316,14 +321,18 @@ class InternLM2Model(nn.Module):
321
322
class InternLM2ForCausalLM(nn.Module, SupportsPP):
323
319
- def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
324
+ def __init__(self,
325
+ *,
326
+ vllm_config: VllmConfig,
327
+ prefix: str = "",
328
+ model_type: Type[InternLM2Model] = InternLM2Model):
329
super().__init__()
330
config = vllm_config.model_config.hf_config
331
quant_config = vllm_config.quant_config
332
self.config = config
333
self.quant_config = quant_config
325
- self.model = InternLM2Model(vllm_config=vllm_config,
326
- prefix=maybe_prefix(prefix, "model"))
334
+ self.model = model_type(vllm_config=vllm_config,
335
+ prefix=maybe_prefix(prefix, "model"))
336
self.output = ParallelLMHead(config.vocab_size,
337
config.hidden_size,
338
quant_config=quant_config,
vllm/model_executor/models/internlm2_ve.py
+6
-17
@@ -14,8 +14,6 @@ from vllm.model_executor.models.internlm2 import (InternLM2Attention,
14
InternLM2MLP, InternLM2Model)
15
from vllm.sequence import IntermediateTensors
16
17
-from .utils import make_layers, maybe_prefix
18
-
17
18
class InternLM2VEDecoderLayer(nn.Module):
19
@@ -105,17 +103,9 @@ class InternLM2VEDecoderLayer(nn.Module):
103
class InternLM2VEModel(InternLM2Model):
104
105
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
108
- super().__init__(vllm_config=vllm_config, prefix=prefix)
109
-
110
- config = vllm_config.model_config.hf_config
111
- cache_config = vllm_config.cache_config
112
- quant_config = vllm_config.quant_config
113
-
114
- self.start_layer, self.end_layer, self.layers = make_layers(
115
- config.num_hidden_layers,
116
- lambda prefix: InternLM2VEDecoderLayer(
117
- config, cache_config, quant_config, prefix=prefix),
118
- prefix=f"{prefix}.layers")
106
+ super().__init__(vllm_config=vllm_config,
107
+ prefix=prefix,
108
+ layer_type=InternLM2VEDecoderLayer)
109
110
def forward(
111
self,
@@ -159,7 +149,6 @@ class InternLM2VEModel(InternLM2Model):
149
class InternLM2VEForCausalLM(InternLM2ForCausalLM):
150
151
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
162
- super().__init__(vllm_config=vllm_config, prefix=prefix)
163
-
164
- self.model = InternLM2VEModel(vllm_config=vllm_config,
165
- prefix=maybe_prefix(prefix, "model"))
152
+ super().__init__(vllm_config=vllm_config,
153
+ prefix=prefix,
154
+ model_type=InternLM2VEModel)
vllm/model_executor/models/jais.py
+10
-3
@@ -76,6 +76,7 @@ class JAISAttention(nn.Module):
76
config: JAISConfig,
77
cache_config: Optional[CacheConfig] = None,
78
quant_config: Optional[QuantizationConfig] = None,
79
+ prefix: str = "",
80
):
81
super().__init__()
82
self.hidden_size = config.hidden_size
@@ -114,7 +115,8 @@ class JAISAttention(nn.Module):
115
scale=self.scale,
116
alibi_slopes=alibi_slopes,
117
cache_config=cache_config,
117
- quant_config=quant_config)
118
+ quant_config=quant_config,
119
+ prefix=f"{prefix}.attn")
120
121
def forward(
122
self,
@@ -178,6 +180,7 @@ class JAISBlock(nn.Module):
180
config: JAISConfig,
181
cache_config: Optional[CacheConfig] = None,
182
quant_config: Optional[QuantizationConfig] = None,
183
+ prefix: str = "",
184
):
185
super().__init__()
186
hidden_size = config.hidden_size
@@ -185,7 +188,10 @@ class JAISBlock(nn.Module):
188
hidden_size)
189
190
self.ln_1 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
188
- self.attn = JAISAttention(config, cache_config, quant_config)
191
+ self.attn = JAISAttention(config,
192
+ cache_config,
193
+ quant_config,
194
+ prefix=f"{prefix}.attn")
195
self.ln_2 = nn.LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
196
self.mlp = JAISMLP(inner_dim, config, quant_config)
197
@@ -241,7 +247,8 @@ class JAISModel(nn.Module):
247
config.num_hidden_layers,
248
lambda prefix: JAISBlock(config=config,
249
cache_config=cache_config,
244
- quant_config=quant_config),
250
+ quant_config=quant_config,
251
+ prefix=prefix),
252
prefix=f"{prefix}.h",
253
)
254
vllm/model_executor/models/jamba.py
+6
-2
@@ -102,7 +102,8 @@ class JambaMambaDecoderLayer(nn.Module):
102
config: JambaConfig,
103
layer_idx: int,
104
cache_config: Optional[CacheConfig] = None,
105
- quant_config: Optional[QuantizationConfig] = None) -> None:
105
+ quant_config: Optional[QuantizationConfig] = None,
106
+ prefix: str = "") -> None:
107
super().__init__()
108
self.config = config
109
self.mamba = MambaMixer(hidden_size= config.hidden_size,
@@ -157,6 +158,7 @@ class JambaAttentionDecoderLayer(nn.Module):
158
layer_idx: int,
159
cache_config: Optional[CacheConfig] = None,
160
quant_config: Optional[QuantizationConfig] = None,
161
+ prefix: str = "",
162
) -> None:
163
super().__init__()
164
self.hidden_size = config.hidden_size
@@ -198,6 +200,7 @@ class JambaAttentionDecoderLayer(nn.Module):
200
self.scaling,
201
num_kv_heads=self.num_kv_heads,
202
cache_config=cache_config,
203
+ prefix=f"{prefix}.attn",
204
)
205
206
num_experts = config.layers_num_experts[layer_idx]
@@ -287,7 +290,8 @@ class JambaModel(nn.Module):
290
layer_class(config,
291
layer_idx=i,
292
cache_config=cache_config,
290
- quant_config=quant_config))
293
+ quant_config=quant_config,
294
+ prefix=f"{prefix}.layers.{i}"))
295
self.layers = nn.ModuleList(decoder_layers)
296
self.final_layernorm = RMSNorm(config.hidden_size,
297
eps=config.rms_norm_eps)
vllm/model_executor/models/llama.py
+1
@@ -174,6 +174,7 @@ class LlamaAttention(nn.Module):
174
num_kv_heads=self.num_kv_heads,
175
cache_config=cache_config,
176
quant_config=quant_config,
177
+ prefix=f"{prefix}.attn",
178
)
179
180
def forward(
vllm/model_executor/models/minicpm.py
+8
-3
@@ -192,6 +192,7 @@ class MiniCPMAttention(nn.Module):
192
max_position_embeddings: int = 8192,
193
cache_config: Optional[CacheConfig] = None,
194
quant_config: Optional[QuantizationConfig] = None,
195
+ prefix: str = "",
196
) -> None:
197
super().__init__()
198
self.hidden_size = hidden_size
@@ -246,7 +247,8 @@ class MiniCPMAttention(nn.Module):
247
self.scaling,
248
num_kv_heads=self.num_kv_heads,
249
cache_config=cache_config,
249
- quant_config=quant_config)
250
+ quant_config=quant_config,
251
+ prefix=f"{prefix}.attn")
252
253
def forward(
254
self,
@@ -273,6 +275,7 @@ class MiniCPMDecoderLayer(nn.Module):
275
config: PretrainedConfig,
276
cache_config: Optional[CacheConfig] = None,
277
quant_config: Optional[QuantizationConfig] = None,
278
+ prefix: str = "",
279
) -> None:
280
super().__init__()
281
self.config = config
@@ -283,6 +286,7 @@ class MiniCPMDecoderLayer(nn.Module):
286
self.rope_scaling = getattr(config, "rope_scaling", None)
287
self.max_position_embeddings = getattr(config,
288
"max_position_embeddings", 8192)
289
+ self.prefix = prefix
290
self._init_attn_block()
291
self._init_ffn_block()
292
@@ -298,6 +302,7 @@ class MiniCPMDecoderLayer(nn.Module):
302
max_position_embeddings=self.max_position_embeddings,
303
cache_config=self.cache_config,
304
quant_config=self.quant_config,
305
+ prefix=f"{self.prefix}.self_attn",
306
)
307
308
def _init_ffn_block(self):
@@ -388,8 +393,8 @@ class MiniCPMModel(nn.Module):
393
):
394
self.start_layer, self.end_layer, self.layers = make_layers(
395
config.num_hidden_layers,
391
- lambda prefix: MiniCPMDecoderLayer(config, cache_config,
392
- quant_config),
396
+ lambda prefix: MiniCPMDecoderLayer(
397
+ config, cache_config, quant_config, prefix=prefix),
398
prefix=f"{prefix}.layers")
399
400
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/minicpm3.py
+6
-3
@@ -60,6 +60,7 @@ class MiniCPM3Attention(nn.Module):
60
max_position_embeddings: int = 8192,
61
cache_config: Optional[CacheConfig] = None,
62
quant_config: Optional[QuantizationConfig] = None,
63
+ prefix: str = "",
64
) -> None:
65
super().__init__()
66
self.hidden_size = hidden_size
@@ -119,7 +120,8 @@ class MiniCPM3Attention(nn.Module):
120
self.scaling,
121
num_kv_heads=self.num_local_heads,
122
cache_config=cache_config,
122
- quant_config=quant_config)
123
+ quant_config=quant_config,
124
+ prefix=f"{prefix}.attn")
125
126
def forward(
127
self,
@@ -195,6 +197,7 @@ class MiniCPM3DecoderLayer(MiniCPMDecoderLayer):
197
max_position_embeddings=self.max_position_embeddings,
198
cache_config=self.cache_config,
199
quant_config=self.quant_config,
200
+ prefix=f"{self.prefix}.self_attn",
201
)
202
203
@@ -209,8 +212,8 @@ class MiniCPM3Model(MiniCPMModel):
212
):
213
self.start_layer, self.end_layer, self.layers = make_layers(
214
config.num_hidden_layers,
212
- lambda prefix: MiniCPM3DecoderLayer(config, cache_config,
213
- quant_config),
215
+ lambda prefix: MiniCPM3DecoderLayer(
216
+ config, cache_config, quant_config, prefix=prefix),
217
prefix=f"{prefix}.layers")
218
219
vllm/model_executor/models/mixtral.py
+2
-1
@@ -166,7 +166,8 @@ class MixtralAttention(nn.Module):
166
self.scaling,
167
num_kv_heads=self.num_kv_heads,
168
cache_config=cache_config,
169
- quant_config=quant_config)
169
+ quant_config=quant_config,
170
+ prefix=f"{prefix}.attn")
171
172
def forward(
173
self,
vllm/model_executor/models/mixtral_quant.py
+9
-3
@@ -170,6 +170,7 @@ class MixtralAttention(nn.Module):
170
rope_theta: float = 10000,
171
quant_config: Optional[QuantizationConfig] = None,
172
cache_config: Optional[CacheConfig] = None,
173
+ prefix: str = "",
174
) -> None:
175
super().__init__()
176
self.hidden_size = hidden_size
@@ -219,7 +220,8 @@ class MixtralAttention(nn.Module):
220
self.scaling,
221
num_kv_heads=self.num_kv_heads,
222
cache_config=cache_config,
222
- quant_config=quant_config)
223
+ quant_config=quant_config,
224
+ prefix=f"{prefix}.attn")
225
226
def forward(
227
self,
@@ -243,6 +245,7 @@ class MixtralDecoderLayer(nn.Module):
245
config: MixtralConfig,
246
cache_config: Optional[CacheConfig] = None,
247
quant_config: Optional[QuantizationConfig] = None,
248
+ prefix: str = "",
249
) -> None:
250
super().__init__()
251
self.hidden_size = config.hidden_size
@@ -255,7 +258,9 @@ class MixtralDecoderLayer(nn.Module):
258
num_kv_heads=config.num_key_value_heads,
259
rope_theta=rope_theta,
260
cache_config=cache_config,
258
- quant_config=quant_config)
261
+ quant_config=quant_config,
262
+ prefix=f"{prefix}.self_attn",
263
+ )
264
self.block_sparse_moe = MixtralMoE(config=config,
265
quant_config=quant_config)
266
self.input_layernorm = RMSNorm(config.hidden_size,
@@ -311,7 +316,8 @@ class MixtralModel(nn.Module):
316
self.start_layer, self.end_layer, self.layers = make_layers(
317
config.num_hidden_layers,
318
lambda prefix: MixtralDecoderLayer(
314
- config, cache_config, quant_config=quant_config),
319
+ config, cache_config, quant_config=quant_config, prefix=prefix
320
+ ),
321
prefix=f"{prefix}.layers")
322
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
323
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/molmo.py
+10
-3
@@ -370,6 +370,7 @@ class MolmoAttention(nn.Module):
370
config: PretrainedConfig,
371
cache_config: Optional[CacheConfig] = None,
372
quant_config: Optional[QuantizationConfig] = None,
373
+ prefix: str = "",
374
) -> None:
375
super().__init__()
376
self.hidden_size = config.hidden_size
@@ -427,7 +428,8 @@ class MolmoAttention(nn.Module):
428
self.scaling,
429
num_kv_heads=self.num_kv_heads,
430
cache_config=cache_config,
430
- quant_config=quant_config)
431
+ quant_config=quant_config,
432
+ prefix=f"{prefix}.attn")
433
434
# Attention output projection.
435
self.o_proj = RowParallelLinear(
@@ -517,10 +519,14 @@ class MolmoDecoderLayer(nn.Module):
519
config: PretrainedConfig,
520
cache_config: Optional[CacheConfig] = None,
521
quant_config: Optional[QuantizationConfig] = None,
522
+ prefix: str = "",
523
) -> None:
524
super().__init__()
525
# Attention block.
523
- self.self_attn = MolmoAttention(config, cache_config, quant_config)
526
+ self.self_attn = MolmoAttention(config,
527
+ cache_config,
528
+ quant_config,
529
+ prefix=f"{prefix}.self_attn")
530
531
# MLP block.
532
self.mlp = MolmoMLP(config, quant_config=quant_config)
@@ -738,7 +744,8 @@ class MolmoModel(nn.Module):
744
else MolmoDecoderLayer
745
self.start_layer, self.end_layer, self.layers = make_layers(
746
config.num_hidden_layers,
741
- lambda prefix: decoder_layer(config, cache_config, quant_config),
747
+ lambda prefix: decoder_layer(
748
+ config, cache_config, quant_config, prefix=prefix),
749
prefix=f"{prefix}.layers",
750
)
751
vllm/model_executor/models/mpt.py
+10
-3
@@ -50,6 +50,7 @@ class MPTAttention(nn.Module):
50
config: MPTConfig,
51
cache_config: Optional[CacheConfig] = None,
52
quant_config: Optional[QuantizationConfig] = None,
53
+ prefix: str = "",
54
):
55
super().__init__()
56
self.d_model = config.d_model
@@ -115,7 +116,8 @@ class MPTAttention(nn.Module):
116
alibi_slopes=alibi_slopes,
117
num_kv_heads=self.num_kv_heads,
118
cache_config=cache_config,
118
- quant_config=quant_config)
119
+ quant_config=quant_config,
120
+ prefix=f"{prefix}.attn")
121
122
def forward(
123
self,
@@ -176,11 +178,15 @@ class MPTBlock(nn.Module):
178
config: MPTConfig,
179
cache_config: Optional[CacheConfig] = None,
180
quant_config: Optional[QuantizationConfig] = None,
181
+ prefix: str = "",
182
):
183
super().__init__()
184
hidden_size = config.d_model
185
self.norm_1 = nn.LayerNorm(hidden_size)
183
- self.attn = MPTAttention(config, cache_config, quant_config)
186
+ self.attn = MPTAttention(config,
187
+ cache_config,
188
+ quant_config,
189
+ prefix=f"{prefix}.attn")
190
self.norm_2 = nn.LayerNorm(hidden_size)
191
self.ffn = MPTMLP(config, quant_config)
192
@@ -224,7 +230,8 @@ class MPTModel(nn.Module):
230
)
231
self.start_layer, self.end_layer, self.blocks = make_layers(
232
config.n_layers,
227
- lambda prefix: MPTBlock(config, cache_config, quant_config),
233
+ lambda prefix: MPTBlock(
234
+ config, cache_config, quant_config, prefix=prefix),
235
prefix=f"{prefix}.blocks")
236
self.norm_f = nn.LayerNorm(config.d_model)
237
if config.no_bias:
vllm/model_executor/models/nemotron.py
+2
-1
@@ -195,7 +195,8 @@ class NemotronAttention(nn.Module):
195
self.scaling,
196
num_kv_heads=self.num_kv_heads,
197
cache_config=cache_config,
198
- quant_config=quant_config)
198
+ quant_config=quant_config,
199
+ prefix=f"{prefix}.attn")
200
201
def forward(
202
self,
vllm/model_executor/models/olmo.py
+11
-5
@@ -62,6 +62,7 @@ class OlmoAttention(nn.Module):
62
config: OlmoConfig,
63
cache_config: Optional[CacheConfig] = None,
64
quant_config: Optional[QuantizationConfig] = None,
65
+ prefix: str = "",
66
):
67
super().__init__()
68
self.config = config
@@ -101,7 +102,8 @@ class OlmoAttention(nn.Module):
102
self.head_dim,
103
scale=self.scaling,
104
cache_config=cache_config,
104
- quant_config=quant_config)
105
+ quant_config=quant_config,
106
+ prefix=f"{prefix}.attn")
107
108
# Attention output projection.
109
self.o_proj = RowParallelLinear(
@@ -184,10 +186,14 @@ class OlmoDecoderLayer(nn.Module):
186
def __init__(self,
187
config: OlmoConfig,
188
cache_config: Optional[CacheConfig] = None,
187
- quant_config: Optional[QuantizationConfig] = None):
189
+ quant_config: Optional[QuantizationConfig] = None,
190
+ prefix: str = ""):
191
super().__init__()
192
# Attention block.
190
- self.self_attn = OlmoAttention(config, cache_config, quant_config)
193
+ self.self_attn = OlmoAttention(config,
194
+ cache_config,
195
+ quant_config,
196
+ prefix=f"{prefix}.self_attn")
197
198
# MLP block.
199
self.mlp = OlmoMLP(config, quant_config)
@@ -238,8 +244,8 @@ class OlmoModel(nn.Module):
244
config.hidden_size)
245
self.start_layer, self.end_layer, self.layers = make_layers(
246
config.num_hidden_layers,
241
- lambda prefix: OlmoDecoderLayer(config, cache_config, quant_config
242
- ),
247
+ lambda prefix: OlmoDecoderLayer(
248
+ config, cache_config, quant_config, prefix=prefix),
249
prefix=f"{prefix}.layers")
250
self.norm = nn.LayerNorm(config.hidden_size,
251
elementwise_affine=False,
vllm/model_executor/models/olmoe.py
+10
-3
@@ -102,6 +102,7 @@ class OlmoeAttention(nn.Module):
102
max_position_embeddings: int = 4096,
103
cache_config: Optional[CacheConfig] = None,
104
quant_config: Optional[QuantizationConfig] = None,
105
+ prefix: str = "",
106
) -> None:
107
super().__init__()
108
self.hidden_size = hidden_size
@@ -156,7 +157,8 @@ class OlmoeAttention(nn.Module):
157
self.scaling,
158
num_kv_heads=self.num_kv_heads,
159
cache_config=cache_config,
159
- quant_config=quant_config)
160
+ quant_config=quant_config,
161
+ prefix=f"{prefix}.attn")
162
163
def forward(
164
self,
@@ -182,6 +184,7 @@ class OlmoeDecoderLayer(nn.Module):
184
layer_idx: int,
185
cache_config: Optional[CacheConfig] = None,
186
quant_config: Optional[QuantizationConfig] = None,
187
+ prefix: str = "",
188
) -> None:
189
super().__init__()
190
self.hidden_size = config.hidden_size
@@ -199,6 +202,7 @@ class OlmoeDecoderLayer(nn.Module):
202
max_position_embeddings=max_position_embeddings,
203
cache_config=cache_config,
204
quant_config=quant_config,
205
+ prefix=f"{prefix}.self_attn",
206
)
207
208
self.mlp = OlmoeMoE(
@@ -260,8 +264,11 @@ class OlmoeModel(nn.Module):
264
)
265
self.start_layer, self.end_layer, self.layers = make_layers(
266
config.num_hidden_layers,
263
- lambda prefix: OlmoeDecoderLayer(config, int(
264
- prefix.split(".")[-1]), cache_config, quant_config),
267
+ lambda prefix: OlmoeDecoderLayer(config,
268
+ int(prefix.split(".")[-1]),
269
+ cache_config,
270
+ quant_config,
271
+ prefix=prefix),
272
prefix=f"{prefix}.layers")
273
self.norm = RMSNorm(config.hidden_size, eps=1e-5)
274
vllm/model_executor/models/orion.py
+6
-5
@@ -75,6 +75,7 @@ class OrionAttention(nn.Module):
75
max_position_embeddings: int = 8192,
76
cache_config: Optional[CacheConfig] = None,
77
quant_config: Optional[QuantizationConfig] = None,
78
+ prefix: str = "",
79
) -> None:
80
super().__init__()
81
self.hidden_size = hidden_size
@@ -126,7 +127,8 @@ class OrionAttention(nn.Module):
127
self.scaling,
128
num_kv_heads=self.num_kv_heads,
129
cache_config=cache_config,
129
- quant_config=quant_config)
130
+ quant_config=quant_config,
131
+ prefix=f"{prefix}.attn")
132
133
def forward(
134
self,
@@ -150,6 +152,7 @@ class OrionDecoderLayer(nn.Module):
152
config: PretrainedConfig,
153
cache_config: Optional[CacheConfig] = None,
154
quant_config: Optional[QuantizationConfig] = None,
155
+ prefix: str = "",
156
) -> None:
157
super().__init__()
158
self.hidden_size = config.hidden_size
@@ -166,6 +169,7 @@ class OrionDecoderLayer(nn.Module):
169
max_position_embeddings=max_position_embeddings,
170
cache_config=cache_config,
171
quant_config=quant_config,
172
+ prefix=f"{prefix}.self_attn",
173
)
174
self.mlp = OrionMLP(
175
hidden_size=self.hidden_size,
@@ -226,10 +230,7 @@ class OrionModel(nn.Module):
230
self.start_layer, self.end_layer, self.layers = make_layers(
231
config.num_hidden_layers,
232
lambda prefix: OrionDecoderLayer(
229
- config,
230
- cache_config,
231
- quant_config,
232
- ),
233
+ config, cache_config, quant_config, prefix=prefix),
234
prefix=f"{prefix}.layers")
235
self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
236
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/persimmon.py
+10
-6
@@ -75,7 +75,8 @@ class PersimmonAttention(nn.Module):
75
def __init__(self,
76
config: PersimmonConfig,
77
cache_config: Optional[CacheConfig] = None,
78
- quant_config: Optional[QuantizationConfig] = None):
78
+ quant_config: Optional[QuantizationConfig] = None,
79
+ prefix: str = ""):
80
super().__init__()
81
self.config = config
82
tensor_parallel_world_size = get_tensor_model_parallel_world_size()
@@ -122,7 +123,8 @@ class PersimmonAttention(nn.Module):
123
self.head_dim,
124
scale=self.scaling,
125
cache_config=cache_config,
125
- quant_config=quant_config)
126
+ quant_config=quant_config,
127
+ prefix=f"{prefix}.attn")
128
129
def _split_heads(self, x: torch.Tensor) -> torch.Tensor:
130
# [seq_length, hidden_size] -> [seq_length, num_heads, head_dim]
@@ -167,12 +169,14 @@ class PersimmonDecoderLayer(nn.Module):
169
def __init__(self,
170
config: PersimmonConfig,
171
cache_config: Optional[CacheConfig] = None,
170
- quant_config: Optional[QuantizationConfig] = None):
172
+ quant_config: Optional[QuantizationConfig] = None,
173
+ prefix: str = ""):
174
super().__init__()
175
self.hidden_size = config.hidden_size
176
self.self_attn = PersimmonAttention(config=config,
177
cache_config=cache_config,
175
- quant_config=quant_config)
178
+ quant_config=quant_config,
179
+ prefix=f"{prefix}.self_attn")
180
self.mlp = PersimmonMLP(config, quant_config=quant_config)
181
self.input_layernorm = nn.LayerNorm(config.hidden_size,
182
eps=config.layer_norm_eps)
@@ -226,8 +230,8 @@ class PersimmonModel(nn.Module):
230
config.hidden_size)
231
self.start_layer, self.end_layer, self.layers = make_layers(
232
config.num_hidden_layers,
229
- lambda prefix: PersimmonDecoderLayer(config, cache_config,
230
- quant_config),
233
+ lambda prefix: PersimmonDecoderLayer(
234
+ config, cache_config, quant_config, prefix=prefix),
235
prefix=f"{prefix}.layers")
236
self.final_layernorm = nn.LayerNorm(config.hidden_size,
237
eps=config.layer_norm_eps)
vllm/model_executor/models/phi.py
+12
-5
@@ -69,7 +69,8 @@ class PhiAttention(nn.Module):
69
def __init__(self,
70
config: PhiConfig,
71
cache_config: Optional[CacheConfig] = None,
72
- quant_config: Optional[QuantizationConfig] = None):
72
+ quant_config: Optional[QuantizationConfig] = None,
73
+ prefix: str = ""):
74
super().__init__()
75
self.total_num_heads = config.num_attention_heads
76
self.hidden_size = config.hidden_size
@@ -116,7 +117,8 @@ class PhiAttention(nn.Module):
117
self.head_size,
118
scaling,
119
cache_config=cache_config,
119
- quant_config=quant_config)
120
+ quant_config=quant_config,
121
+ prefix=f"{prefix}.attn")
122
123
def forward(
124
self,
@@ -167,11 +169,15 @@ class PhiLayer(nn.Module):
169
def __init__(self,
170
config: PhiConfig,
171
cache_config: Optional[CacheConfig] = None,
170
- quant_config: Optional[QuantizationConfig] = None):
172
+ quant_config: Optional[QuantizationConfig] = None,
173
+ prefix: str = ""):
174
super().__init__()
175
self.input_layernorm = nn.LayerNorm(config.hidden_size,
176
eps=config.layer_norm_eps)
174
- self.self_attn = PhiAttention(config, cache_config, quant_config)
177
+ self.self_attn = PhiAttention(config,
178
+ cache_config,
179
+ quant_config,
180
+ prefix=f"{prefix}.self_attn")
181
self.mlp = PhiMLP(config, quant_config)
182
183
def forward(
@@ -210,7 +216,8 @@ class PhiModel(nn.Module):
216
config.hidden_size)
217
self.start_layer, self.end_layer, self.layers = make_layers(
218
config.num_hidden_layers,
213
- lambda prefix: PhiLayer(config, cache_config, quant_config),
219
+ lambda prefix: PhiLayer(
220
+ config, cache_config, quant_config, prefix=prefix),
221
prefix=f"{prefix}.layers")
222
self.final_layernorm = nn.LayerNorm(config.hidden_size,
223
eps=config.layer_norm_eps)
vllm/model_executor/models/phi3_small.py
+15
-11
@@ -117,6 +117,7 @@ class Phi3SmallSelfAttention(nn.Module):
117
layer_idx: int,
118
cache_config: Optional[CacheConfig] = None,
119
quant_config: Optional[QuantizationConfig] = None,
120
+ prefix: str = "",
121
) -> None:
122
super().__init__()
123
self.layer_idx = layer_idx
@@ -214,15 +215,14 @@ class Phi3SmallSelfAttention(nn.Module):
215
"homo_head": self.homo_heads
216
}
217
217
- self.attn = Attention(
218
- self.num_heads_per_partition,
219
- self.head_dim,
220
- self.scale,
221
- num_kv_heads=self.num_kv_heads_per_partion,
222
- cache_config=cache_config,
223
- quant_config=quant_config,
224
- blocksparse_params=bs_params,
225
- )
218
+ self.attn = Attention(self.num_heads_per_partition,
219
+ self.head_dim,
220
+ self.scale,
221
+ num_kv_heads=self.num_kv_heads_per_partion,
222
+ cache_config=cache_config,
223
+ quant_config=quant_config,
224
+ blocksparse_params=bs_params,
225
+ prefix=f"{prefix}.attn")
226
227
def forward(
228
self,
@@ -259,13 +259,15 @@ class Phi3SmallDecoderLayer(nn.Module):
259
layer_idx: int,
260
cache_config: Optional[CacheConfig] = None,
261
quant_config: Optional[QuantizationConfig] = None,
262
+ prefix: str = "",
263
):
264
super().__init__()
265
self.hidden_size = config.hidden_size
266
self.self_attn = Phi3SmallSelfAttention(config,
267
layer_idx,
268
cache_config=cache_config,
268
- quant_config=quant_config)
269
+ quant_config=quant_config,
270
+ prefix=f"{prefix}.self_attn")
271
self.mlp = Phi3SmallMLP(config, quant_config)
272
273
self.input_layernorm = nn.LayerNorm(config.hidden_size,
@@ -315,7 +317,9 @@ class Phi3SmallModel(nn.Module):
317
config.num_hidden_layers,
318
lambda prefix: Phi3SmallDecoderLayer(config,
319
int(prefix.split('.')[-1]),
318
- cache_config, quant_config),
320
+ cache_config,
321
+ quant_config,
322
+ prefix=prefix),
323
prefix=f"{prefix}.layers")
324
325
self.final_layernorm = nn.LayerNorm(config.hidden_size,
vllm/model_executor/models/phimoe.py
+6
-2
@@ -294,6 +294,7 @@ class PhiMoEAttention(nn.Module):
294
cache_config: Optional[CacheConfig] = None,
295
quant_config: Optional[QuantizationConfig] = None,
296
rope_scaling: Optional[dict] = None,
297
+ prefix: str = "",
298
) -> None:
299
super().__init__()
300
self.hidden_size = hidden_size
@@ -347,6 +348,7 @@ class PhiMoEAttention(nn.Module):
348
num_kv_heads=self.num_kv_heads,
349
cache_config=cache_config,
350
quant_config=quant_config,
351
+ prefix=f"{prefix}.attn",
352
)
353
354
def forward(
@@ -371,6 +373,7 @@ class PhiMoEDecoderLayer(nn.Module):
373
config: PhiMoEConfig,
374
cache_config: Optional[CacheConfig] = None,
375
quant_config: Optional[QuantizationConfig] = None,
376
+ prefix: str = "",
377
) -> None:
378
super().__init__()
379
self.hidden_size = config.hidden_size
@@ -385,6 +388,7 @@ class PhiMoEDecoderLayer(nn.Module):
388
cache_config=cache_config,
389
quant_config=quant_config,
390
rope_scaling=config.rope_scaling,
391
+ prefix=f"{prefix}.self_attn",
392
)
393
self.block_sparse_moe = PhiMoE(
394
num_experts=config.num_local_experts,
@@ -454,8 +458,8 @@ class PhiMoEModel(nn.Module):
458
)
459
self.start_layer, self.end_layer, self.layers = make_layers(
460
config.num_hidden_layers,
457
- lambda prefix: PhiMoEDecoderLayer(config, cache_config,
458
- quant_config),
461
+ lambda prefix: PhiMoEDecoderLayer(
462
+ config, cache_config, quant_config, prefix=prefix),
463
prefix=f"{prefix}.layers")
464
self.norm = nn.LayerNorm(config.hidden_size,
465
eps=config.rms_norm_eps,
vllm/model_executor/models/qwen.py
+8
-3
@@ -442,6 +442,7 @@ class QWenAttention(nn.Module):
442
rope_scaling: Optional[Dict[str, Any]] = None,
443
cache_config: Optional[CacheConfig] = None,
444
quant_config: Optional[QuantizationConfig] = None,
445
+ prefix: str = "",
446
):
447
super().__init__()
448
self.hidden_size = hidden_size
@@ -478,7 +479,8 @@ class QWenAttention(nn.Module):
479
self.head_dim,
480
self.scaling,
481
cache_config=cache_config,
481
- quant_config=quant_config)
482
+ quant_config=quant_config,
483
+ prefix=f"{prefix}.attn")
484
485
def forward(
486
self,
@@ -502,6 +504,7 @@ class QWenBlock(nn.Module):
504
config: PretrainedConfig,
505
cache_config: Optional[CacheConfig] = None,
506
quant_config: Optional[QuantizationConfig] = None,
507
+ prefix: str = "",
508
):
509
super().__init__()
510
self.ln_1 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -514,7 +517,8 @@ class QWenBlock(nn.Module):
517
rope_theta=rope_theta,
518
rope_scaling=rope_scaling,
519
cache_config=cache_config,
517
- quant_config=quant_config)
520
+ quant_config=quant_config,
521
+ prefix=f"{prefix}.attn")
522
523
self.ln_2 = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
524
@@ -568,7 +572,8 @@ class QWenModel(nn.Module):
572
)
573
self.start_layer, self.end_layer, self.h = make_layers(
574
config.num_hidden_layers,
571
- lambda prefix: QWenBlock(config, cache_config, quant_config),
575
+ lambda prefix: QWenBlock(
576
+ config, cache_config, quant_config, prefix=prefix),
577
prefix=f"{prefix}.h")
578
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
579
self.make_empty_intermediate_tensors = (
vllm/model_executor/models/qwen2_moe.py
+7
-2
@@ -168,6 +168,7 @@ class Qwen2MoeAttention(nn.Module):
168
max_position_embeddings: int = 8192,
169
cache_config: Optional[CacheConfig] = None,
170
quant_config: Optional[QuantizationConfig] = None,
171
+ prefix: str = "",
172
) -> None:
173
super().__init__()
174
self.hidden_size = hidden_size
@@ -220,7 +221,8 @@ class Qwen2MoeAttention(nn.Module):
221
self.scaling,
222
num_kv_heads=self.num_kv_heads,
223
cache_config=cache_config,
223
- quant_config=quant_config)
224
+ quant_config=quant_config,
225
+ prefix=f"{prefix}.attn")
226
227
def forward(
228
self,
@@ -245,6 +247,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
247
layer_idx: int,
248
cache_config: Optional[CacheConfig] = None,
249
quant_config: Optional[QuantizationConfig] = None,
250
+ prefix: str = "",
251
) -> None:
252
super().__init__()
253
self.hidden_size = config.hidden_size
@@ -261,6 +264,7 @@ class Qwen2MoeDecoderLayer(nn.Module):
264
max_position_embeddings=max_position_embeddings,
265
cache_config=cache_config,
266
quant_config=quant_config,
267
+ prefix=f"{prefix}.self_attn",
268
)
269
270
# Note: Qwen/Qwen2-57B-A14B-Instruct does not have
@@ -336,7 +340,8 @@ class Qwen2MoeModel(nn.Module):
340
layer_idx=int(
341
prefix.split(".")[-1]),
342
cache_config=cache_config,
339
- quant_config=quant_config),
343
+ quant_config=quant_config,
344
+ prefix=prefix),
345
prefix=f"{prefix}.layers",
346
)
347
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/model_executor/models/solar.py
+1
@@ -167,6 +167,7 @@ class SolarAttention(nn.Module):
167
num_kv_heads=self.num_kv_heads,
168
cache_config=cache_config,
169
quant_config=quant_config,
170
+ prefix=f"{prefix}.attn",
171
)
172
173
def forward(
vllm/model_executor/models/stablelm.py
+11
-5
@@ -77,7 +77,8 @@ class StablelmAttention(nn.Module):
77
def __init__(self,
78
config: PretrainedConfig,
79
cache_config: Optional[CacheConfig] = None,
80
- quant_config: Optional[QuantizationConfig] = None) -> None:
80
+ quant_config: Optional[QuantizationConfig] = None,
81
+ prefix: str = "") -> None:
82
super().__init__()
83
self.config = config
84
self.hidden_size = config.hidden_size
@@ -131,7 +132,8 @@ class StablelmAttention(nn.Module):
132
self.scaling,
133
num_kv_heads=self.num_key_value_heads,
134
cache_config=cache_config,
134
- quant_config=quant_config)
135
+ quant_config=quant_config,
136
+ prefix=f"{prefix}.attn")
137
138
def forward(
139
self,
@@ -155,9 +157,13 @@ class StablelmDecoderLayer(nn.Module):
157
config: PretrainedConfig,
158
cache_config: Optional[CacheConfig] = None,
159
quant_config: Optional[QuantizationConfig] = None,
160
+ prefix: str = "",
161
) -> None:
162
super().__init__()
160
- self.self_attn = StablelmAttention(config, cache_config, quant_config)
163
+ self.self_attn = StablelmAttention(config,
164
+ cache_config,
165
+ quant_config,
166
+ prefix=f"{prefix}.self_attn")
167
self.mlp = StablelmMLP(config, quant_config)
168
norm_eps = getattr(config, "norm_eps",
169
getattr(config, "layer_norm_eps", 1e-05))
@@ -207,8 +213,8 @@ class StableLMEpochModel(nn.Module):
213
)
214
self.start_layer, self.end_layer, self.layers = make_layers(
215
config.num_hidden_layers,
210
- lambda prefix: StablelmDecoderLayer(config, cache_config,
211
- quant_config),
216
+ lambda prefix: StablelmDecoderLayer(
217
+ config, cache_config, quant_config, prefix=prefix),
218
prefix=f"{prefix}.layers",
219
)
220
norm_eps = getattr(config, "norm_eps",
vllm/model_executor/models/starcoder2.py
+10
-5
@@ -52,7 +52,8 @@ class Starcoder2Attention(nn.Module):
52
def __init__(self,
53
config: Starcoder2Config,
54
cache_config: Optional[CacheConfig] = None,
55
- quant_config: Optional[QuantizationConfig] = None):
55
+ quant_config: Optional[QuantizationConfig] = None,
56
+ prefix: str = ""):
57
super().__init__()
58
self.config = config
59
@@ -105,7 +106,8 @@ class Starcoder2Attention(nn.Module):
106
self.scaling,
107
num_kv_heads=self.num_kv_heads,
108
cache_config=cache_config,
108
- quant_config=quant_config)
109
+ quant_config=quant_config,
110
+ prefix=f"{prefix}.attn")
111
112
def forward(
113
self,
@@ -154,12 +156,14 @@ class Starcoder2DecoderLayer(nn.Module):
156
def __init__(self,
157
config: Starcoder2Config,
158
cache_config: Optional[CacheConfig] = None,
157
- quant_config: Optional[QuantizationConfig] = None):
159
+ quant_config: Optional[QuantizationConfig] = None,
160
+ prefix: str = ""):
161
super().__init__()
162
self.hidden_size = config.hidden_size
163
self.self_attn = Starcoder2Attention(config,
164
cache_config,
162
- quant_config=quant_config)
165
+ quant_config=quant_config,
166
+ prefix=f"{prefix}.self_attn")
167
self.mlp = Starcoder2MLP(config, quant_config=quant_config)
168
self.input_layernorm = nn.LayerNorm(config.hidden_size,
169
eps=config.norm_epsilon)
@@ -213,7 +217,8 @@ class Starcoder2Model(nn.Module):
217
self.start_layer, self.end_layer, self.layers = make_layers(
218
config.num_hidden_layers,
219
lambda prefix: Starcoder2DecoderLayer(
216
- config, cache_config, quant_config=quant_config),
220
+ config, cache_config, quant_config=quant_config, prefix=prefix
221
+ ),
222
prefix=f"{prefix}.layers",
223
)
224
self.norm = nn.LayerNorm(config.hidden_size, eps=config.norm_epsilon)
vllm/model_executor/models/xverse.py
+7
-3
@@ -93,6 +93,7 @@ class XverseAttention(nn.Module):
93
quant_config: Optional[QuantizationConfig] = None,
94
bias: bool = False,
95
cache_config: Optional[CacheConfig] = None,
96
+ prefix: str = "",
97
) -> None:
98
super().__init__()
99
self.hidden_size = hidden_size
@@ -138,7 +139,8 @@ class XverseAttention(nn.Module):
139
self.scaling,
140
num_kv_heads=self.num_kv_heads,
141
cache_config=cache_config,
141
- quant_config=quant_config)
142
+ quant_config=quant_config,
143
+ prefix=f"{prefix}.attn")
144
145
def forward(
146
self,
@@ -162,6 +164,7 @@ class XverseDecoderLayer(nn.Module):
164
config: PretrainedConfig,
165
cache_config: Optional[CacheConfig] = None,
166
quant_config: Optional[QuantizationConfig] = None,
167
+ prefix: str = "",
168
) -> None:
169
super().__init__()
170
self.hidden_size = config.hidden_size
@@ -180,6 +183,7 @@ class XverseDecoderLayer(nn.Module):
183
quant_config=quant_config,
184
bias=getattr(config, "bias", False),
185
cache_config=cache_config,
186
+ prefix=f"{prefix}.self_attn",
187
)
188
self.mlp = XverseMLP(
189
hidden_size=self.hidden_size,
@@ -243,8 +247,8 @@ class XverseModel(nn.Module):
247
)
248
self.start_layer, self.end_layer, self.layers = make_layers(
249
config.num_hidden_layers,
246
- lambda prefix: XverseDecoderLayer(config, cache_config,
247
- quant_config),
250
+ lambda prefix: XverseDecoderLayer(
251
+ config, cache_config, quant_config, prefix=prefix),
252
prefix=f"{prefix}.layers",
253
)
254
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
vllm/platforms/cpu.py
+1
@@ -20,6 +20,7 @@ logger = init_logger(__name__)
20
class CpuPlatform(Platform):
21
_enum = PlatformEnum.CPU
22
device_type: str = "cpu"
23
+ dispatch_key: str = "CPU"
24
25
@classmethod
26
def get_device_name(cls, device_id: int = 0) -> str:
vllm/platforms/cuda.py
+1
@@ -121,6 +121,7 @@ def device_id_to_physical_device_id(device_id: int) -> int:
121
class CudaPlatform(Platform):
122
_enum = PlatformEnum.CUDA
123
device_type: str = "cuda"
124
+ dispatch_key: str = "CUDA"
125
126
@classmethod
127
def get_device_capability(cls, device_id: int = 0) -> DeviceCapability:
vllm/platforms/hpu.py
+1
@@ -13,6 +13,7 @@ else:
13
class HpuPlatform(Platform):
14
_enum = PlatformEnum.HPU
15
device_type: str = "hpu"
16
+ dispatch_key: str = "HPU"
17
18
@classmethod
19
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/interface.py
+4
@@ -57,6 +57,10 @@ class DeviceCapability(NamedTuple):
57
class Platform:
58
_enum: PlatformEnum
59
device_type: str
60
+ # available dispatch keys:
61
+ # check https://github.com/pytorch/pytorch/blob/313dac6c1ca0fa0cde32477509cce32089f8532a/torchgen/model.py#L134 # noqa
62
+ # use "CPU" as a fallback for platforms not registered in PyTorch
63
+ dispatch_key: str = "CPU"
64
65
def is_cuda(self) -> bool:
66
return self._enum == PlatformEnum.CUDA
vllm/platforms/openvino.py
+1
@@ -18,6 +18,7 @@ logger = init_logger(__name__)
18
class OpenVinoPlatform(Platform):
19
_enum = PlatformEnum.OPENVINO
20
device_type: str = "openvino"
21
+ dispatch_key: str = "CPU"
22
23
@classmethod
24
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/rocm.py
+1
@@ -36,6 +36,7 @@ if os.environ.get("VLLM_WORKER_MULTIPROC_METHOD", None) in ["fork", None]:
36
class RocmPlatform(Platform):
37
_enum = PlatformEnum.ROCM
38
device_type: str = "cuda"
39
+ dispatch_key: str = "CUDA"
40
41
@classmethod
42
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/tpu.py
+1
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
17
class TpuPlatform(Platform):
18
_enum = PlatformEnum.TPU
19
device_type: str = "tpu"
20
+ dispatch_key: str = "XLA"
21
22
@classmethod
23
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/platforms/xpu.py
+1
@@ -17,6 +17,7 @@ logger = init_logger(__name__)
17
class XPUPlatform(Platform):
18
_enum = PlatformEnum.XPU
19
device_type: str = "xpu"
20
+ dispatch_key: str = "XPU"
21
22
@classmethod
23
def get_default_attn_backend(cls, selected_backend: _Backend) -> _Backend:
vllm/spec_decode/draft_model_runner.py
+2
-1
@@ -273,7 +273,8 @@ class TP1DraftModelRunner(ModelRunner):
273
if previous_hidden_states is not None else {}
274
275
# Run model
276
- with set_forward_context(model_input.attn_metadata):
276
+ with set_forward_context(model_input.attn_metadata,
277
+ self.vllm_config):
278
hidden_states = model_executable(
279
input_ids=model_input.input_tokens,
280
positions=model_input.input_positions,
vllm/utils.py
+2
-1
@@ -1573,6 +1573,7 @@ def direct_register_custom_op(
1573
mutates_args: List[str],
1574
fake_impl: Optional[Callable] = None,
1575
target_lib: Optional[Library] = None,
1576
+ dispatch_key: str = "CUDA",
1577
):
1578
"""
1579
`torch.library.custom_op` can have significant overhead because it
@@ -1601,7 +1602,7 @@ def direct_register_custom_op(
1602
schema_str = torch._custom_op.impl.infer_schema(op_func, mutates_args)
1603
my_lib = target_lib or vllm_lib
1604
my_lib.define(op_name + schema_str)
1604
- my_lib.impl(op_name, op_func, "CUDA")
1605
+ my_lib.impl(op_name, op_func, dispatch_key=dispatch_key)
1606
if fake_impl is not None:
1607
my_lib._register_fake(op_name, fake_impl)
1608
vllm/v1/attention/backends/flash_attn.py
+2
-1
@@ -173,7 +173,8 @@ def unified_v1_flash_attention(
173
alibi_slopes: Optional[torch.Tensor] = None,
174
logits_soft_cap: Optional[float] = None,
175
) -> None:
176
- current_metadata = get_forward_context()
176
+ context = get_forward_context()
177
+ current_metadata = context.dynamic_forward_context
178
if current_metadata is None:
179
# Profiling run.
180
return
vllm/v1/worker/gpu_model_runner.py
+2
-2
@@ -447,7 +447,7 @@ class GPUModelRunner:
447
448
# Run the decoder.
449
# Use persistent buffers for CUDA graphs.
450
- with set_forward_context(attn_metadata):
450
+ with set_forward_context(attn_metadata, self.vllm_config):
451
hidden_states = self.model(
452
input_ids=None,
453
positions=self.positions[:num_input_tokens],
@@ -523,7 +523,7 @@ class GPUModelRunner:
523
num_tokens: int,
524
kv_caches: List[torch.Tensor],
525
) -> torch.Tensor:
526
- with set_forward_context(None):
526
+ with set_forward_context(None, self.vllm_config):
527
hidden_states = model(
528
input_ids=None,
529
positions=self.positions[:num_tokens],
vllm/worker/embedding_model_runner.py
+1
-1
@@ -97,7 +97,7 @@ class EmbeddingModelRunner(
97
model_forward_end = torch.cuda.Event(enable_timing=True)
98
model_forward_start.record()
99
100
- with set_forward_context(model_input.attn_metadata):
100
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
101
hidden_or_intermediate_states = model_executable(
102
input_ids=model_input.input_tokens,
103
positions=model_input.input_positions,
vllm/worker/enc_dec_model_runner.py
+1
-1
@@ -176,7 +176,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
176
} if self.has_inner_state else {}
177
178
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
179
- with set_forward_context(model_input.attn_metadata):
179
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
180
hidden_or_intermediate_states = model_executable(
181
input_ids=model_input.input_tokens,
182
positions=model_input.input_positions,
vllm/worker/model_runner.py
+2
-2
@@ -1503,7 +1503,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1503
self._update_inputs_to_capture_for_enc_dec_model(
1504
capture_inputs)
1505
1506
- with set_forward_context(attn_metadata):
1506
+ with set_forward_context(attn_metadata, self.vllm_config):
1507
graph_runner.capture(**capture_inputs)
1508
self.graph_memory_pool = graph_runner.graph.pool()
1509
self.graph_runners[virtual_engine][batch_size] = (
@@ -1649,7 +1649,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1649
model_forward_end = torch.cuda.Event(enable_timing=True)
1650
model_forward_start.record()
1651
1652
- with set_forward_context(model_input.attn_metadata):
1652
+ with set_forward_context(model_input.attn_metadata, self.vllm_config):
1653
hidden_or_intermediate_states = model_executable(
1654
input_ids=model_input.input_tokens,
1655
positions=model_input.input_positions,