@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e8697faf03

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 13, 14:30 UTC

[V0 deprecation] Remove no longer used `get_metadata_cls` (#28370)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>

Lucas Wilkinson committed Nov 10, 2025 at 01:32 UTC e8697faf037d3eb90b3817bee5d7ea596ab08b12

20 files changed +9 -332

tests/kernels/utils.py

+3 -241

@@ -4,24 +4,21 @@

5 import itertools

6 import random

7 -import unittest

7 from collections.abc import Sequence

8 from numbers import Number

9 from typing import Any, NamedTuple

10 +from unittest.mock import patch

12 import pytest

13 import torch

14 from torch._prims_common import TensorLikeType

16 from tests.kernels.quant_utils import native_w8a8_block_matmul

17 -from vllm.attention import AttentionBackend, AttentionMetadata, AttentionType

18 -from vllm.attention.backends.registry import _Backend

17 +from vllm.attention import AttentionType

18 from vllm.model_executor.layers.activation import SiluAndMul

19 from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input

20 from vllm.utils import (

21 STR_BACKEND_ENV_VAR,

23 - STR_FLASH_ATTN_VAL,

24 - STR_XFORMERS_ATTN_VAL,

22 )

23 from vllm.utils.torch_utils import make_tensor_with_pad

@@ -512,50 +509,6 @@ def pack_qkv(qkv: QKVInputs, device: torch.device | str) -> PackedQKVInputs:

509 )

510

511

515 -def make_backend(backend_name: str) -> AttentionBackend:

516 - """

517 - Construct the backend instance determined by the backend_name string

518 - argument.

519 -

520 - Note: at time of writing the Attention wrapper automatically selects

521 - its own backend for Attention.forward(); so the backend instance which

522 - you generate with this function is not meant to be used for *running*

523 - inference, but rather for generating compatible metadata structures

524 - using backend.make_metadata()

525 -

526 -

527 - Returns:

528 -

529 - * Backend instance

530 - """

531 - if backend_name == STR_XFORMERS_ATTN_VAL:

532 - from vllm.v1.attention.backends.xformers import XFormersAttentionBackend

533 -

534 - return XFormersAttentionBackend()

535 - if backend_name == STR_FLASH_ATTN_VAL:

536 - from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend

537 -

538 - return FlashAttentionBackend()

539 - if backend_name == "TRITON_ATTN":

540 - from vllm.v1.attention.backends.triton_attn import TritonAttentionBackend

541 -

542 - return TritonAttentionBackend()

543 - if backend_name == "FLEX_ATTENTION":

544 - from vllm.v1.attention.backends.flex_attention import FlexAttentionBackend

545 -

546 - return FlexAttentionBackend()

547 - if backend_name == "TORCH_SDPA":

548 - from vllm.v1.attention.backends.cpu_attn import TorchSDPABackend

549 -

550 - return TorchSDPABackend()

551 - if backend_name == "FLASHINFER":

552 - from vllm.v1.attention.backends.flashinfer import FlashInferBackend

553 -

554 - return FlashInferBackend()

555 -

556 - raise AssertionError(f"Unrecognized backend_name {backend_name} for unit test")

557 -

558 -

512 def make_alibi_bias(

513 alibi_slopes: torch.Tensor,

514 num_kv_heads: int,

@@ -877,197 +830,6 @@ def make_block_tables_slot_mapping(

830 return (block_tables_tensor, slot_mapping_list, max_block_idx)

831

832

880 -def make_test_metadata(

881 - attn_backend: _Backend,

882 - is_prompt: bool,

883 - seq_lens: list[int] | None,

884 - decoder_test_params: PhaseTestParameters | None,

885 - device: torch.device | str,

886 - encoder_test_params: PhaseTestParameters | None = None,

887 - cross_test_params: PhaseTestParameters | None = None,

888 -) -> AttentionMetadata:

889 - """

890 - Construct fake attention metadata for a given test phase

891 - (prefill-phase or decode-phase).

892 -

893 - encoder_test_params and cross_test_params arguments allow encoder

894 - attention and enc/dec cross-attention (respectively) to use distinct

895 - metadata values from decoder self-attention (decoder_test_params.)

896 -

897 - if encoder_test_params and cross_test_params are None, the attention

898 - metadata will support decoder-only scenario.

899 -

900 - Assumptions:

901 -

902 - * No chunked prefill -> a batch is 100% prefill or 100% decode, never both

903 -

904 - Arguments:

905 -

906 - * attn_backend_name: Backend for sourcing attention kernels

907 - * is_prompt: prefill if True, o/w decode

908 - * seq_lens: list of token counts for each sequence

909 - * decoder_test_params: decoder self-attention test params;

910 - this function requires

911 - kv_mmap (memory mapping) field

912 - * device: CPU or CUDA device

913 - * encoder_test_params: encoder attention test params;

914 - this function requires encoder query

915 - sequence lengths field. If None,

916 - encoder query sequence lengths are

917 - treated as None

918 - * cross_test_params: enc/dec cross-attention test params;

919 - this function requires kv_mmap field.

920 - If None, KV cache memory map data

921 - structures are treated as None

922 -

923 - Return:

924 -

925 - * AttentionMetadata structure

926 - """

927 -

928 - # Decoder self-attention memory mapping

929 - # decoder_test_params is None signals encoder-only

930 - # scenario, so kv_mmap is None

931 - kv_mmap = None if decoder_test_params is None else decoder_test_params.kv_mmap

932 -

933 - # This function constructs metadata assuming no chunked prefill,

934 - # i.e. 100% prefill tokens or 100% decode tokens

935 - #

936 - # - If is_prompt, num_prefills_or_decodes is the number of prefills

937 - # and num_prefill_or_decode_tokens is the number of prefill tokens

938 - # - If not is_prompt, num_prefills_or_decodes is the number of decodes

939 - # and num_prefill_or_decode_tokens is the number of decode tokens

940 - #

941 - # seq_lens is None signals encoder-only

942 - # scenario, in which case num_prefills_or_decodes and

943 - # num_prefill_or_decode_tokens are unused

944 - num_prefills_or_decodes = None if seq_lens is None else len(seq_lens)

945 -

946 - num_prefill_or_decode_tokens = (

947 - None if seq_lens is None else (sum(seq_lens) if is_prompt else len(seq_lens))

948 - )

949 -

950 - # Seems for non-prefix-caching scenarios context_lens

951 - # is never needed

952 - context_lens = None

953 -

954 - if encoder_test_params is None:

955 - encoder_seq_lens = None

956 - num_encoder_tokens = None

957 - else:

958 - # Encoder/decoder or encoder-only models only:

959 - # * Extract encoder input sequence lengths

960 - assert encoder_test_params.packed_qkvo.packed_qkv is not None

961 - encoder_seq_lens = encoder_test_params.packed_qkvo.packed_qkv.q_seq_lens

962 - num_encoder_tokens = (

963 - None if encoder_seq_lens is None else (sum(encoder_seq_lens))

964 - )

965 -

966 - # For encoder/decoder or encoder-only models only, extract *cross-attention*

967 - # slot_mapping and block table (kv_mmap)

968 - cross_kv_mmap = None if cross_test_params is None else cross_test_params.kv_mmap

969 -

970 - attn_backend_obj = make_backend(attn_backend.name)

971 -

972 - if is_prompt:

973 - # Prefill-phase scenario

974 -

975 - num_prefills = num_prefills_or_decodes

976 - num_prefill_tokens = num_prefill_or_decode_tokens

977 - num_decode_tokens = 0

978 -

979 - (

980 - seq_lens_tensor,

981 - context_lens_tensor,

982 - _,

983 - _,

984 - seq_start_loc,

985 - encoder_seq_lens_tensor,

986 - encoder_seq_start_loc,

987 - max_encoder_seq_len,

988 - ) = _make_metadata_tensors(

989 - seq_lens, context_lens, encoder_seq_lens, device=device

990 - )

991 - return attn_backend_obj.make_metadata(

992 - num_prefills=num_prefills,

993 - slot_mapping=(None if kv_mmap is None else kv_mmap.slot_mapping),

994 - enable_kv_scales_calculation=True,

995 - num_prefill_tokens=num_prefill_tokens,

996 - num_decode_tokens=num_decode_tokens,

997 - seq_lens=seq_lens,

998 - seq_lens_tensor=seq_lens_tensor,

999 - seq_start_loc=seq_start_loc,

1000 - max_prefill_seq_len=None if seq_lens is None else max(seq_lens),

1001 - max_decode_seq_len=0,

1002 - context_lens_tensor=context_lens_tensor,

1003 - block_tables=(None if kv_mmap is None else kv_mmap.block_tables),

1004 - use_cuda_graph=False,

1005 - num_encoder_tokens=num_encoder_tokens,

1006 - encoder_seq_lens=encoder_seq_lens,

1007 - encoder_seq_lens_tensor=encoder_seq_lens_tensor,

1008 - encoder_seq_start_loc=encoder_seq_start_loc,

1009 - max_encoder_seq_len=max_encoder_seq_len,

1010 - cross_slot_mapping=(

1011 - None if cross_kv_mmap is None else cross_kv_mmap.slot_mapping

1012 - ),

1013 - cross_block_tables=(

1014 - None if cross_kv_mmap is None else cross_kv_mmap.block_tables

1015 - ),

1016 - )

1017 -

1018 - else: # not is_prompt

1019 - # Decode-phase scenario

1020 -

1021 - assert kv_mmap is not None

1022 - assert num_prefill_or_decode_tokens is not None

1023 - assert seq_lens is not None

1024 -

1025 - num_prefills = 0

1026 - num_prefill_tokens = 0

1027 - num_decode_tokens = num_prefill_or_decode_tokens

1028 -

1029 - (

1030 - seq_lens_tensor,

1031 - context_lens_tensor,

1032 - _,

1033 - _,

1034 - seq_start_loc,

1035 - encoder_seq_lens_tensor,

1036 - encoder_seq_start_loc,

1037 - max_encoder_seq_len,

1038 - ) = _make_metadata_tensors(

1039 - seq_lens, context_lens, encoder_seq_lens, device=device

1040 - )

1041 -

1042 - return attn_backend_obj.make_metadata(

1043 - num_prefills=num_prefills,

1044 - slot_mapping=kv_mmap.slot_mapping,

1045 - enable_kv_scales_calculation=True,

1046 - num_prefill_tokens=num_prefill_tokens,

1047 - num_decode_tokens=num_decode_tokens,

1048 - seq_lens=seq_lens,

1049 - seq_lens_tensor=seq_lens_tensor,

1050 - seq_start_loc=seq_start_loc,

1051 - max_prefill_seq_len=0,

1052 - max_decode_seq_len=max(seq_lens),

1053 - max_decode_query_len=1,

1054 - context_lens_tensor=context_lens_tensor,

1055 - block_tables=kv_mmap.block_tables,

1056 - use_cuda_graph=False,

1057 - num_encoder_tokens=num_encoder_tokens,

1058 - encoder_seq_lens=encoder_seq_lens,

1059 - encoder_seq_lens_tensor=encoder_seq_lens_tensor,

1060 - encoder_seq_start_loc=encoder_seq_start_loc,

1061 - max_encoder_seq_len=max_encoder_seq_len,

1062 - cross_slot_mapping=(

1063 - None if cross_kv_mmap is None else cross_kv_mmap.slot_mapping

1064 - ),

1065 - cross_block_tables=(

1066 - None if cross_kv_mmap is None else cross_kv_mmap.block_tables

1067 - ),

1068 - )

1069 -

1070 -

833 def assert_actual_matches_ideal(

834 test_params: PhaseTestParameters, output_under_test: torch.Tensor, backend: str

835 ) -> None:

@@ -1308,7 +1070,7 @@ def opcheck(

1070 raise_exception: bool = True,

1071 cond: bool = True,

1072 ) -> dict[str, str]:

1311 - with unittest.mock.patch("torch.allclose", new=fp8_allclose):

1073 + with patch("torch.allclose", new=fp8_allclose):

1074 return (

1075 torch.library.opcheck(

1076 op, args, kwargs, test_utils=test_utils, raise_exception=raise_exception

vllm/attention/backends/abstract.py

-9

@@ -51,19 +51,10 @@ class AttentionBackend(ABC):

51 def get_impl_cls() -> type["AttentionImpl"]:

52 raise NotImplementedError

54 - @staticmethod

55 - @abstractmethod

56 - def get_metadata_cls() -> type["AttentionMetadata"]:

57 - raise NotImplementedError

58 -

54 @classmethod

55 def get_supported_kernel_block_size(cls) -> list[int | MultipleOf]:

56 return cls.get_impl_cls().get_supported_kernel_block_size()

63 - @classmethod

64 - def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":

65 - return cls.get_metadata_cls()(*args, **kwargs)

66 -

58 @staticmethod

59 @abstractmethod

60 def get_builder_cls(): # -> Type["AttentionMetadataBuilder"]:

vllm/v1/attention/backends/cpu_attn.py

-4

@@ -66,10 +66,6 @@ class TorchSDPABackend(AttentionBackend):

66 def get_impl_cls() -> type["TorchSDPABackendImpl"]:

67 return TorchSDPABackendImpl

69 - @staticmethod

70 - def get_metadata_cls() -> type["AttentionMetadata"]:

71 - return TorchSDPAMetadata

72 -

69 @staticmethod

70 def get_builder_cls() -> type["TorchSDPAMetadataBuilderV1"]:

71 return TorchSDPAMetadataBuilderV1

vllm/v1/attention/backends/flash_attn.py

-5

@@ -11,7 +11,6 @@ from vllm import envs

11 from vllm.attention.backends.abstract import (

12 AttentionBackend,

13 AttentionImpl,

14 - AttentionMetadata,

14 AttentionType,

15 MultipleOf,

16 is_quantized_kv_cache,

@@ -90,10 +89,6 @@ class FlashAttentionBackend(AttentionBackend):

89 def get_impl_cls() -> type["FlashAttentionImpl"]:

90 return FlashAttentionImpl

93 - @staticmethod

94 - def get_metadata_cls() -> type["AttentionMetadata"]:

95 - return FlashAttentionMetadata

96 -

92 @staticmethod

93 def get_builder_cls() -> type["FlashAttentionMetadataBuilder"]:

94 return FlashAttentionMetadataBuilder

vllm/v1/attention/backends/flashinfer.py

-4

@@ -195,10 +195,6 @@ class FlashInferBackend(AttentionBackend):

195 def get_impl_cls() -> type["FlashInferImpl"]:

196 return FlashInferImpl

197

198 - @staticmethod

199 - def get_metadata_cls() -> type["FlashInferMetadata"]:

200 - return FlashInferMetadata

201 -

198 @staticmethod

199 def get_builder_cls() -> type["FlashInferMetadataBuilder"]:

200 return FlashInferMetadataBuilder

vllm/v1/attention/backends/flex_attention.py

-5

@@ -20,7 +20,6 @@ from torch.nn.attention.flex_attention import (

20 from vllm.attention.backends.abstract import (

21 AttentionBackend,

22 AttentionImpl,

23 - AttentionMetadata,

23 AttentionType,

24 is_quantized_kv_cache,

25 )

@@ -89,10 +88,6 @@ class FlexAttentionBackend(AttentionBackend):

88 def get_impl_cls() -> type["FlexAttentionImpl"]:

89 return FlexAttentionImpl

92 - @staticmethod

93 - def get_metadata_cls() -> type["AttentionMetadata"]:

94 - return FlexAttentionMetadata

95 -

91 @staticmethod

92 def get_kv_cache_shape(

93 num_blocks: int,

vllm/v1/attention/backends/mla/common.py

-5

@@ -201,7 +201,6 @@ from vllm import _custom_ops as ops

201 from vllm.attention.backends.abstract import (

202 AttentionBackend,

203 AttentionLayer,

204 - AttentionMetadata,

204 MLAAttentionImpl,

205 )

206 from vllm.attention.backends.utils import get_mla_dims

@@ -307,10 +306,6 @@ class MLACommonBackend(AttentionBackend):

306 def get_name() -> str:

307 return "TRITON_MLA"

308

310 - @staticmethod

311 - def get_metadata_cls() -> type["AttentionMetadata"]:

312 - return MLACommonMetadata

313 -

309 @staticmethod

310 def get_builder_cls() -> type["MLACommonMetadataBuilder"]:

311 return MLACommonMetadataBuilder

vllm/v1/attention/backends/mla/flashattn_mla.py

-4

@@ -41,10 +41,6 @@ class FlashAttnMLABackend(MLACommonBackend):

41 def get_name() -> str:

42 return "FLASH_ATTN_MLA"

44 - @staticmethod

45 - def get_metadata_cls() -> type["FlashAttnMLAMetadata"]:

46 - return FlashAttnMLAMetadata

47 -

44 @staticmethod

45 def get_builder_cls() -> type["FlashAttnMLAMetadataBuilder"]:

46 return FlashAttnMLAMetadataBuilder

vllm/v1/attention/backends/mla/flashmla.py

-4

@@ -40,10 +40,6 @@ class FlashMLABackend(MLACommonBackend):

40 def get_name() -> str:

41 return "FLASHMLA"

43 - @staticmethod

44 - def get_metadata_cls() -> type["FlashMLAMetadata"]:

45 - return FlashMLAMetadata

46 -

43 @staticmethod

44 def get_builder_cls() -> type["FlashMLAMetadataBuilder"]:

45 return FlashMLAMetadataBuilder

vllm/v1/attention/backends/mla/flashmla_sparse.py

-5

@@ -10,7 +10,6 @@ from vllm import _custom_ops as ops

10 from vllm.attention.backends.abstract import (

11 AttentionBackend,

12 AttentionLayer,

13 - AttentionMetadata,

13 )

14 from vllm.attention.backends.utils import get_mla_dims

15 from vllm.attention.ops.flashmla import (

@@ -57,10 +56,6 @@ class FlashMLASparseBackend(AttentionBackend):

56 def get_name() -> str:

57 return "FLASHMLA_SPARSE"

60 - @staticmethod

61 - def get_metadata_cls() -> type[AttentionMetadata]:

62 - return FlashMLASparseMetadata

63 -

59 @staticmethod

60 def get_builder_cls() -> type["FlashMLASparseMetadataBuilder"]:

61 return FlashMLASparseMetadataBuilder

vllm/v1/attention/backends/mla/indexer.py

-5

@@ -7,7 +7,6 @@ import torch

8 from vllm.attention.backends.abstract import (

9 AttentionBackend,

10 - AttentionMetadata,

10 MultipleOf,

11 )

12 from vllm.config import VllmConfig

@@ -24,10 +23,6 @@ logger = init_logger(__name__)

25 class DeepseekV32IndexerBackend(AttentionBackend):

27 - @staticmethod

28 - def get_metadata_cls() -> type["AttentionMetadata"]:

29 - return DeepseekV32IndexerMetadata

30 -

26 @classmethod

27 def get_supported_head_sizes(cls) -> list[int]:

28 return [32, 64, 128]

vllm/v1/attention/backends/mla/rocm_aiter_mla.py

-4

@@ -35,10 +35,6 @@ class AiterMLABackend(MLACommonBackend):

35 def get_impl_cls() -> type["AiterMLAImpl"]:

36 return AiterMLAImpl

38 - @staticmethod

39 - def get_metadata_cls() -> type["AiterMLAMetadata"]:

40 - return AiterMLAMetadata

41 -

38 @staticmethod

39 def get_builder_cls() -> type["AiterMLAMetadataBuilder"]:

40 return AiterMLAMetadataBuilder

vllm/v1/attention/backends/pallas.py

-4

@@ -108,10 +108,6 @@ class PallasAttentionBackend(AttentionBackend):

108 def get_impl_cls() -> type["PallasAttentionBackendImpl"]:

109 return PallasAttentionBackendImpl

110

111 - @staticmethod

112 - def get_metadata_cls() -> type["PallasMetadata"]:

113 - return PallasMetadata

114 -

111 @staticmethod

112 def get_kv_cache_shape(

113 num_blocks: int,

vllm/v1/attention/backends/rocm_aiter_fa.py

-5

@@ -9,7 +9,6 @@ import torch

9 from vllm.attention.backends.abstract import (

10 AttentionBackend,

11 AttentionImpl,

12 - AttentionMetadata,

12 AttentionType,

13 MultipleOf,

14 )

@@ -479,10 +478,6 @@ class AiterFlashAttentionBackend(AttentionBackend):

478 def get_impl_cls() -> type["AiterFlashAttentionImpl"]:

479 return AiterFlashAttentionImpl

480

482 - @staticmethod

483 - def get_metadata_cls() -> type["AttentionMetadata"]:

484 - return AiterFlashAttentionMetadata

485 -

481 @staticmethod

482 def get_builder_cls() -> type["AiterFlashAttentionMetadataBuilder"]:

483 return AiterFlashAttentionMetadataBuilder

vllm/v1/attention/backends/rocm_aiter_unified_attn.py

+1 -6

@@ -5,7 +5,7 @@

5 import torch

7 from vllm import _custom_ops as ops

8 -from vllm.attention.backends.abstract import AttentionMetadata, AttentionType

8 +from vllm.attention.backends.abstract import AttentionType

9 from vllm.logger import init_logger

10 from vllm.model_executor.layers.quantization.utils.quant_utils import (

11 QuantKey,

@@ -15,7 +15,6 @@ from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata

15 from vllm.v1.attention.backends.rocm_attn import (

16 RocmAttentionBackend,

17 RocmAttentionImpl,

18 - RocmAttentionMetadata,

18 RocmAttentionMetadataBuilder,

19 )

@@ -33,10 +32,6 @@ class RocmAiterUnifiedAttentionBackend(RocmAttentionBackend):

32 def get_impl_cls() -> type["RocmAiterUnifiedAttentionImpl"]:

33 return RocmAiterUnifiedAttentionImpl

36 - @staticmethod

37 - def get_metadata_cls() -> type["AttentionMetadata"]:

38 - return RocmAttentionMetadata

39 -

35 @staticmethod

36 def get_kv_cache_shape(

37 num_blocks: int,

vllm/v1/attention/backends/rocm_attn.py

-5

@@ -10,7 +10,6 @@ import torch

10 from vllm.attention.backends.abstract import (

11 AttentionBackend,

12 AttentionImpl,

13 - AttentionMetadata,

13 AttentionType,

14 )

15 from vllm.attention.ops.chunked_prefill_paged_decode import chunked_prefill_paged_decode

@@ -182,10 +181,6 @@ class RocmAttentionBackend(AttentionBackend):

181 def get_impl_cls() -> type["RocmAttentionImpl"]:

182 return RocmAttentionImpl

183

185 - @staticmethod

186 - def get_metadata_cls() -> type["AttentionMetadata"]:

187 - return RocmAttentionMetadata

188 -

184 @staticmethod

185 def get_kv_cache_shape(

186 num_blocks: int,

vllm/v1/attention/backends/tree_attn.py

-5

@@ -12,7 +12,6 @@ from vllm import _custom_ops as ops

12 from vllm.attention.backends.abstract import (

13 AttentionBackend,

14 AttentionImpl,

15 - AttentionMetadata,

15 AttentionType,

16 MultipleOf,

17 )

@@ -64,10 +63,6 @@ class TreeAttentionBackend(AttentionBackend):

63 def get_impl_cls() -> type["TreeAttentionImpl"]:

64 return TreeAttentionImpl

67 - @staticmethod

68 - def get_metadata_cls() -> type["AttentionMetadata"]:

69 - return TreeAttentionMetadata

70 -

66 @staticmethod

67 def get_kv_cache_shape(

68 num_blocks: int,

vllm/v1/attention/backends/triton_attn.py

-5

@@ -10,7 +10,6 @@ import torch

10 from vllm.attention.backends.abstract import (

11 AttentionBackend,

12 AttentionImpl,

13 - AttentionMetadata,

13 AttentionType,

14 MultipleOf,

15 )

@@ -176,10 +175,6 @@ class TritonAttentionBackend(AttentionBackend):

175 def get_impl_cls() -> type["TritonAttentionImpl"]:

176 return TritonAttentionImpl

177

179 - @staticmethod

180 - def get_metadata_cls() -> type["AttentionMetadata"]:

181 - return TritonAttentionMetadata

182 -

178 @staticmethod

179 def get_kv_cache_shape(

180 num_blocks: int,

vllm/v1/attention/backends/xformers.py

-5

@@ -10,7 +10,6 @@ import torch

10 from vllm.attention.backends.abstract import (

11 AttentionBackend,

12 AttentionImpl,

13 - AttentionMetadata,

13 AttentionType,

14 MultipleOf,

15 )

@@ -105,10 +104,6 @@ class XFormersAttentionBackend(AttentionBackend):

104 def get_impl_cls() -> type["XFormersAttentionImpl"]:

105 return XFormersAttentionImpl

106

108 - @staticmethod

109 - def get_metadata_cls() -> type["AttentionMetadata"]:

110 - return XFormersAttentionMetadata

111 -

107 @staticmethod

108 def get_kv_cache_shape(

109 num_blocks: int,

vllm/v1/worker/gpu_model_runner.py

+5 -2

@@ -20,7 +20,11 @@ from tqdm import tqdm

21 import vllm.envs as envs

22 from vllm.attention import Attention, AttentionType

23 -from vllm.attention.backends.abstract import AttentionBackend, MultipleOf

23 +from vllm.attention.backends.abstract import (

24 + AttentionBackend,

25 + AttentionMetadata,

26 + MultipleOf,

27 +)

28 from vllm.compilation.counter import compilation_counter

29 from vllm.compilation.cuda_graph import CUDAGraphWrapper

30 from vllm.compilation.monitor import set_cudagraph_capturing_enabled

@@ -82,7 +86,6 @@ from vllm.utils.torch_utils import (

86 kv_cache_dtype_str_to_dtype,

87 supports_dynamo,

88 )

85 -from vllm.v1.attention.backends.flash_attn import AttentionMetadata

89 from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadataBuilder

90 from vllm.v1.attention.backends.utils import (

91 AttentionCGSupport,