@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 04668ebe7a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Avoid import AttentionMetadata explicitly in Mllama (#10593)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 24, 2024 at 02:12 UTC 04668ebe7a35b69f1d2f8b04ef255bb16c8d2a01
5 files changed +21 -11
vllm/attention/backends/blocksparse_attn.py
+5
@@ -87,6 +87,11 @@ class BlocksparseParams:
87
88 class BlocksparseFlashAttentionBackend(AttentionBackend):
89
90 + @staticmethod
91 + def get_name() -> str:
92 + # For attention layer compatibility
93 + return "FLASH_ATTN"
94 +
95 @staticmethod
96 def get_impl_cls() -> Type["BlocksparseFlashAttentionImpl"]:
97 return BlocksparseFlashAttentionImpl
vllm/attention/layer.py
+2 -1
@@ -6,7 +6,7 @@ import torch.nn as nn
6
7 import vllm.envs as envs
8 from vllm.attention import AttentionMetadata, AttentionType
9 -from vllm.attention.selector import get_attn_backend
9 +from vllm.attention.selector import backend_name_to_enum, get_attn_backend
10 from vllm.config import CacheConfig
11 from vllm.forward_context import ForwardContext, get_forward_context
12 from vllm.model_executor.layers.quantization.base_config import (
@@ -98,6 +98,7 @@ class Attention(nn.Module):
98 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
99 alibi_slopes, sliding_window, kv_cache_dtype,
100 blocksparse_params, logits_soft_cap)
101 + self.backend = backend_name_to_enum(attn_backend.get_name())
102
103 # For cuda-alike (CUDA and ROCM) and cpu platforms, we control how
104 # torch.compile works by registering the attention as one giant
vllm/model_executor/models/mllama.py
+7 -7
@@ -32,9 +32,8 @@ from transformers.models.mllama.processing_mllama import (
32
33 import vllm.distributed.parallel_state as ps
34 from vllm.attention import Attention, AttentionMetadata, AttentionType
35 -from vllm.attention.backends.flash_attn import FlashAttentionMetadata
36 -from vllm.attention.backends.xformers import XFormersMetadata
35 from vllm.attention.ops.paged_attn import PagedAttention
36 +from vllm.attention.selector import _Backend
37 from vllm.config import VllmConfig
38 from vllm.distributed import get_tensor_model_parallel_world_size
39 from vllm.inputs import (INPUT_REGISTRY, DummyData, EncoderDecoderInputs,
@@ -828,7 +827,8 @@ class MllamaTextCrossAttention(nn.Module):
827 ) -> torch.Tensor:
828 # Skip writing kv-cache for the initial profiling run.
829 if len(kv_cache.shape) > 1:
831 - if isinstance(attn_metadata, FlashAttentionMetadata):
830 + if self.attn.backend in (_Backend.FLASH_ATTN,
831 + _Backend.FLASH_ATTN_VLLM_V1):
832 cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
833 cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
834 torch.ops._C_cache_ops.reshape_and_cache_flash(
@@ -842,7 +842,7 @@ class MllamaTextCrossAttention(nn.Module):
842 1.0,
843 1.0,
844 )
845 - elif isinstance(attn_metadata, XFormersMetadata):
845 + elif self.attn.backend in (_Backend.XFORMERS, _Backend.TORCH_SDPA):
846 key_cache, value_cache = PagedAttention.split_kv_cache(
847 kv_cache, self.num_local_key_value_heads, self.head_dim)
848 cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
@@ -852,9 +852,9 @@ class MllamaTextCrossAttention(nn.Module):
852 attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
853 else:
854 raise ValueError(
855 - f"Unsupported AttentionMetadata {type(attn_metadata)} "
856 - f"class found. Expected the AttentionMetadata to "
857 - f"be either XFormersMetadata or FlashAttentionMetadata.")
855 + f"Unsupported Attention backend {self.attn.backend} "
856 + "enum found. Expected the Attention backend to be "
857 + "FLASH_ATTN, FLASH_ATTN_VLLM_V1, XFORMERS or TORCH_SDPA.")
858
859 # We have to call torch.sdpa for prefill when using a
860 # custom cross-attention mask. Because the mask is not a
vllm/platforms/openvino.py
+6 -2
@@ -1,7 +1,5 @@
1 from typing import TYPE_CHECKING
2
3 -import openvino as ov
4 -import openvino.properties.hint as hints
3 import torch
4
5 import vllm.envs as envs
@@ -16,6 +14,12 @@ else:
14
15 logger = init_logger(__name__)
16
17 +try:
18 + import openvino as ov
19 + import openvino.properties.hint as hints
20 +except ImportError as e:
21 + logger.warning("Failed to import OpenVINO with %r", e)
22 +
23
24 class OpenVinoPlatform(Platform):
25 _enum = PlatformEnum.OPENVINO
vllm/v1/attention/backends/flash_attn.py
+1 -1
@@ -19,7 +19,7 @@ class FlashAttentionBackend(AttentionBackend):
19
20 @staticmethod
21 def get_name() -> str:
22 - return "flash-attn-vllm-v1"
22 + return "FLASH_ATTN_VLLM_V1"
23
24 @staticmethod
25 def get_impl_cls() -> Type["FlashAttentionImpl"]: