A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Bugfix] Avoid import AttentionMetadata explicitly in Mllama (#10593)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 24, 2024 at 02:12 UTC
04668ebe7a35b69f1d2f8b04ef255bb16c8d2a01
5 files changed
+21
-11
vllm/attention/backends/blocksparse_attn.py
+5
@@ -87,6 +87,11 @@ class BlocksparseParams:
87
88
class BlocksparseFlashAttentionBackend(AttentionBackend):
89
90
+ @staticmethod
91
+ def get_name() -> str:
92
+ # For attention layer compatibility
93
+ return "FLASH_ATTN"
94
+
95
@staticmethod
96
def get_impl_cls() -> Type["BlocksparseFlashAttentionImpl"]:
97
return BlocksparseFlashAttentionImpl
vllm/attention/layer.py
+2
-1
@@ -6,7 +6,7 @@ import torch.nn as nn
6
7
import vllm.envs as envs
8
from vllm.attention import AttentionMetadata, AttentionType
9
-from vllm.attention.selector import get_attn_backend
9
+from vllm.attention.selector import backend_name_to_enum, get_attn_backend
10
from vllm.config import CacheConfig
11
from vllm.forward_context import ForwardContext, get_forward_context
12
from vllm.model_executor.layers.quantization.base_config import (
@@ -98,6 +98,7 @@ class Attention(nn.Module):
98
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
99
alibi_slopes, sliding_window, kv_cache_dtype,
100
blocksparse_params, logits_soft_cap)
101
+ self.backend = backend_name_to_enum(attn_backend.get_name())
102
103
# For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
104
# torch.compile works by registering the attention as one giant
vllm/model_executor/models/mllama.py
+7
-7
@@ -32,9 +32,8 @@ from transformers.models.mllama.processing_mllama import (
32
33
import vllm.distributed.parallel_state as ps
34
from vllm.attention import Attention, AttentionMetadata, AttentionType
35
-from vllm.attention.backends.flash_attn import FlashAttentionMetadata
36
-from vllm.attention.backends.xformers import XFormersMetadata
35
from vllm.attention.ops.paged_attn import PagedAttention
36
+from vllm.attention.selector import _Backend
37
from vllm.config import VllmConfig
38
from vllm.distributed import get_tensor_model_parallel_world_size
39
from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
@@ -828,7 +827,8 @@ class MllamaTextCrossAttention(nn.Module):
827
) -> torch.Tensor:
828
# Skip writing kv-cache for the initial profiling run.
829
if len(kv_cache.shape) > 1:
831
- if isinstance(attn_metadata, FlashAttentionMetadata):
830
+ if self.attn.backend in (_Backend.FLASH_ATTN,
831
+ _Backend.FLASH_ATTN_VLLM_V1):
832
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
833
cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
834
torch.ops._C_cache_ops.reshape_and_cache_flash(
@@ -842,7 +842,7 @@ class MllamaTextCrossAttention(nn.Module):
842
1.0,
843
1.0,
844
)
845
- elif isinstance(attn_metadata, XFormersMetadata):
845
+ elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
846
key_cache, value_cache = PagedAttention.split_kv_cache(
847
kv_cache, self.num_local_key_value_heads, self.head_dim)
848
cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -852,9 +852,9 @@ class MllamaTextCrossAttention(nn.Module):
852
attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
853
else:
854
raise ValueError(
855
- f"Unsupported AttentionMetadata {type(attn_metadata)} "
856
- f"class found. Expected the AttentionMetadata to "
857
- f"be either XFormersMetadata or FlashAttentionMetadata.")
855
+ f"Unsupported Attention backend {self.attn.backend} "
856
+ "enum found. Expected the Attention backend to be "
857
+ "FLASH_ATTN, FLASH_ATTN_VLLM_V1, XFORMERS or TORCH_SDPA.")
858
859
# We have to call torch.sdpa for prefill when using a
860
# custom cross-attention mask. Because the mask is not a
vllm/platforms/openvino.py
+6
-2
@@ -1,7 +1,5 @@
1
from typing import TYPE_CHECKING
2
3
-import openvino as ov
4
-import openvino.properties.hint as hints
3
import torch
4
5
import vllm.envs as envs
@@ -16,6 +14,12 @@ else:
14
15
logger = init_logger(__name__)
16
17
+try:
18
+ import openvino as ov
19
+ import openvino.properties.hint as hints
20
+except ImportError as e:
21
+ logger.warning("Failed to import OpenVINO with %r", e)
22
+
23
24
class OpenVinoPlatform(Platform):
25
_enum = PlatformEnum.OPENVINO
vllm/v1/attention/backends/flash_attn.py
+1
-1
@@ -19,7 +19,7 @@ class FlashAttentionBackend(AttentionBackend):
19
20
@staticmethod
21
def get_name() -> str:
22
- return "flash-attn-vllm-v1"
22
+ return "FLASH_ATTN_VLLM_V1"
23
24
@staticmethod
25
def get_impl_cls() -> Type["FlashAttentionImpl"]: