@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf1d62a644
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Support SDPA attention for Molmo vision backbone (#9410)

Isotr0py committed Oct 16, 2024 at 19:52 UTC cf1d62a644d2539d2fd7af9beac0f3363d288d87
3 files changed +61 -78
vllm/model_executor/models/molmo.py
+15 -37
@@ -1,4 +1,3 @@
1 -import logging
1 import math
2 import re
3 from array import array
@@ -14,10 +13,8 @@ from torch import nn
13 from torch.nn import functional as F
14 from transformers import PretrainedConfig
15
17 -import vllm.envs as envs
16 from vllm.attention import Attention, AttentionMetadata
19 -from vllm.attention.selector import (_Backend, backend_name_to_enum,
20 - get_global_forced_attn_backend)
17 +from vllm.attention.selector import _Backend
18 from vllm.config import CacheConfig, MultiModalConfig
19 from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
20 get_tensor_model_parallel_world_size,
@@ -43,12 +40,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40 from vllm.model_executor.models.interfaces import SupportsMultiModal
41 from vllm.model_executor.models.utils import make_layers
42 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
46 -from vllm.platforms import current_platform
43 from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44 SequenceData)
45 from vllm.transformers_utils.processor import get_processor
46
51 -log = logging.getLogger(__name__)
47 +from .utils import get_vit_attn_backend
48
49 # TODO: hard-coded for now. Consider making it configurable.
50 VIT_LAYERS = [-2, -9]
@@ -190,35 +186,12 @@ class MultiHeadDotProductAttention(nn.Module):
186 )
187
188 # Detect attention implementation.
193 - selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
194 - if selected_backend is None:
195 - backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
196 - if backend_by_env_var is not None:
197 - selected_backend = backend_name_to_enum(backend_by_env_var)
198 - if selected_backend is None:
199 - # For Volta and Turing GPUs, use xformers instead.
200 - device_available = current_platform.get_device_capability()[0] >= 8
201 - if device_available:
202 - from transformers.utils import is_flash_attn_2_available
203 - if is_flash_attn_2_available():
204 - self._use_flash_attn = True
205 - else:
206 - log.warning(
207 - "Current Molmo implementation has a bug with "
208 - "`vllm-flash-attn` inside vision module, so we use "
209 - "xformers backend instead. You can run `pip install "
210 - "flash-attn to use flash-attention backend.")
211 - self._use_flash_attn = False
212 - else:
213 - self._use_flash_attn = False
214 - else:
215 - if selected_backend == _Backend.FLASH_ATTN:
216 - self._use_flash_attn = True
217 - elif selected_backend == _Backend.XFORMERS:
218 - self._use_flash_attn = False
219 - else:
220 - raise RuntimeError(
221 - f"Molmo does not support {selected_backend} backend now.")
189 + self.attn_backend: _Backend = get_vit_attn_backend()
190 + if self.attn_backend not in {
191 + _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
192 + }:
193 + raise RuntimeError(
194 + f"Molmo does not support {self.attn_backend} backend now.")
195
196 def forward(self,
197 inputs_q: torch.Tensor,
@@ -240,10 +213,15 @@ class MultiHeadDotProductAttention(nn.Module):
213 xk = xk.view(*kv_shape)
214 xv = xv.view(*kv_shape)
215
243 - if self._use_flash_attn:
216 + if self.attn_backend == _Backend.FLASH_ATTN:
217 from flash_attn import flash_attn_func
218 output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
246 - else:
219 + elif self.attn_backend == _Backend.TORCH_SDPA:
220 + xq, xk, xv = (rearrange(x, "b s h d -> b h s d")
221 + for x in (xq, xk, xv))
222 + output = F.scaled_dot_product_attention(xq, xk, xv)
223 + output = rearrange(output, "b h s d -> b s h d ")
224 + elif self.attn_backend == _Backend.XFORMERS:
225 from xformers import ops as xops
226 output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
227
vllm/model_executor/models/qwen2_vl.py
+12 -40
@@ -39,10 +39,8 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
39 from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40 make_batched_images, make_batched_videos, smart_resize)
41
42 -import vllm.envs as envs
42 from vllm.attention import AttentionMetadata
44 -from vllm.attention.selector import (_Backend, backend_name_to_enum,
45 - get_global_forced_attn_backend)
43 +from vllm.attention.selector import _Backend
44 from vllm.config import CacheConfig, MultiModalConfig
45 from vllm.distributed import get_pp_group, parallel_state
46 from vllm.distributed import utils as dist_utils
@@ -63,14 +61,13 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
61 MultiModalInputs)
62 from vllm.multimodal.base import MultiModalData
63 from vllm.multimodal.image import cached_get_image_processor
66 -from vllm.platforms import current_platform
64 from vllm.sequence import IntermediateTensors, SequenceData
65 from vllm.transformers_utils.config import uses_mrope
66 from vllm.transformers_utils.processor import get_processor
70 -from vllm.utils import is_cpu
67
68 from .interfaces import SupportsMultiModal, SupportsPP
73 -from .utils import (PPMissingLayer, is_pp_missing_parameter,
69 +from .utils import (PPMissingLayer, get_vit_attn_backend,
70 + is_pp_missing_parameter,
71 make_empty_intermediate_tensors_factory)
72
73 logger = init_logger(__name__)
@@ -215,37 +212,12 @@ class Qwen2VisionAttention(nn.Module):
212 quant_config=quant_config)
213
214 # Detect attention implementation.
218 - selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
219 - if selected_backend is None:
220 - backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
221 - if backend_by_env_var is not None:
222 - selected_backend = backend_name_to_enum(backend_by_env_var)
223 - if selected_backend is None:
224 - # For Volta and Turing GPUs, use xformers instead.
225 - device_available = current_platform.has_device_capability(80)
226 - if device_available:
227 - from transformers.utils import is_flash_attn_2_available
228 -
229 - if is_flash_attn_2_available():
230 - self._use_flash_attn = True
231 - else:
232 - logger.warning(
233 - "Current Qwen2-VL implementation has a bug with "
234 - "`vllm-flash-attn` inside vision module, so we use "
235 - "xformers backend instead. You can run `pip install "
236 - "flash-attn to use flash-attention backend.")
237 - self._use_flash_attn = False
238 - else:
239 - self._use_flash_attn = False
240 - else:
241 - if selected_backend == _Backend.FLASH_ATTN:
242 - self._use_flash_attn = True
243 - elif selected_backend == _Backend.XFORMERS:
244 - self._use_flash_attn = False
245 - else:
246 - raise RuntimeError(
247 - f"Qwen2-VL does not support {selected_backend} backend now."
248 - )
215 + self.attn_backend: _Backend = get_vit_attn_backend()
216 + if self.attn_backend not in {
217 + _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
218 + }:
219 + raise RuntimeError(
220 + f"Qwen2-VL does not support {self.attn_backend} backend now.")
221
222 def forward(
223 self,
@@ -274,7 +246,7 @@ class Qwen2VisionAttention(nn.Module):
246 q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
247 k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
248
277 - if self._use_flash_attn:
249 + if self.attn_backend == _Backend.FLASH_ATTN:
250 # from vllm_flash_attn.flash_attn_interface import (
251 # flash_attn_varlen_func)
252 from flash_attn import flash_attn_varlen_func
@@ -295,7 +267,7 @@ class Qwen2VisionAttention(nn.Module):
267 context_layer = rearrange(output,
268 "(b s) ... -> b s ...",
269 b=batch_size)
298 - elif is_cpu():
270 + elif self.attn_backend == _Backend.TORCH_SDPA:
271 seq_length = q.size(1)
272 q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
273 attention_mask = torch.zeros([1, seq_length, seq_length],
@@ -310,7 +282,7 @@ class Qwen2VisionAttention(nn.Module):
282 attention_mask,
283 dropout_p=0.0)
284 context_layer = rearrange(output, "b h s d -> b s h d ")
313 - else:
285 + elif self.attn_backend == _Backend.XFORMERS:
286 from xformers import ops as xops
287 from xformers.ops.fmha.attn_bias import BlockDiagonalMask
288
vllm/model_executor/models/utils.py
+34 -1
@@ -8,15 +8,22 @@ import torch.nn as nn
8 from torch.func import functional_call
9 from transformers import PretrainedConfig
10
11 +import vllm.envs as envs
12 +from vllm.attention.selector import (_Backend, backend_name_to_enum,
13 + get_global_forced_attn_backend)
14 from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
15 SchedulerConfig)
16 +from vllm.logger import init_logger
17 from vllm.model_executor.layers.quantization import QuantizationConfig
18 from vllm.model_executor.model_loader.loader import build_model
19 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20 from vllm.model_executor.models import ModelRegistry
21 from vllm.multimodal.base import NestedTensors
22 +from vllm.platforms import current_platform
23 from vllm.sequence import IntermediateTensors
19 -from vllm.utils import is_pin_memory_available
24 +from vllm.utils import is_cpu, is_pin_memory_available
25 +
26 +logger = init_logger(__name__)
27
28 WeightsMapping = Mapping[str, Optional[str]]
29 """If a key maps to a value of `None`, the corresponding weight is ignored."""
@@ -487,3 +494,29 @@ class LLMWrapper(nn.Module):
494 def __call__(self, *args: Any, **kwargs: Any) -> Any:
495 llm = super().__getattr__(self.model_name)
496 return llm(*args, **kwargs)
497 +
498 +
499 +def get_vit_attn_backend() -> _Backend:
500 + selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
501 + if selected_backend is None:
502 + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
503 + if backend_by_env_var is not None:
504 + selected_backend = backend_name_to_enum(backend_by_env_var)
505 + if selected_backend is None:
506 + # For Volta and Turing GPUs, use xformers instead.
507 + device_available = current_platform.has_device_capability(80)
508 + if device_available:
509 + from transformers.utils import is_flash_attn_2_available
510 + if is_flash_attn_2_available():
511 + selected_backend = _Backend.FLASH_ATTN
512 + else:
513 + logger.warning(
514 + "Current `vllm-flash-attn` has a bug inside vision module, "
515 + "so we use xformers backend instead. You can run "
516 + "`pip install flash-attn` to use flash-attention backend.")
517 + selected_backend = _Backend.XFORMERS
518 + elif is_cpu():
519 + selected_backend = _Backend.TORCH_SDPA
520 + else:
521 + selected_backend = _Backend.XFORMERS
522 + return selected_backend