A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Support SDPA attention for Molmo vision backbone (#9410)
Isotr0py committed
Oct 16, 2024 at 19:52 UTC
cf1d62a644d2539d2fd7af9beac0f3363d288d87
3 files changed
+61
-78
vllm/model_executor/models/molmo.py
+15
-37
@@ -1,4 +1,3 @@
1
-import logging
1
import math
2
import re
3
from array import array
@@ -14,10 +13,8 @@ from torch import nn
13
from torch.nn import functional as F
14
from transformers import PretrainedConfig
15
17
-import vllm.envs as envs
16
from vllm.attention import Attention, AttentionMetadata
19
-from vllm.attention.selector import (_Backend, backend_name_to_enum,
20
- get_global_forced_attn_backend)
17
+from vllm.attention.selector import _Backend
18
from vllm.config import CacheConfig, MultiModalConfig
19
from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
20
get_tensor_model_parallel_world_size,
@@ -43,12 +40,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
40
from vllm.model_executor.models.interfaces import SupportsMultiModal
41
from vllm.model_executor.models.utils import make_layers
42
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
46
-from vllm.platforms import current_platform
43
from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
44
SequenceData)
45
from vllm.transformers_utils.processor import get_processor
46
51
-log = logging.getLogger(__name__)
47
+from .utils import get_vit_attn_backend
48
49
# TODO: hard-coded for now. Consider making it configurable.
50
VIT_LAYERS = [-2, -9]
@@ -190,35 +186,12 @@ class MultiHeadDotProductAttention(nn.Module):
186
)
187
188
# Detect attention implementation.
193
- selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
194
- if selected_backend is None:
195
- backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
196
- if backend_by_env_var is not None:
197
- selected_backend = backend_name_to_enum(backend_by_env_var)
198
- if selected_backend is None:
199
- # For Volta and Turing GPUs, use xformers instead.
200
- device_available = current_platform.get_device_capability()[0] >= 8
201
- if device_available:
202
- from transformers.utils import is_flash_attn_2_available
203
- if is_flash_attn_2_available():
204
- self._use_flash_attn = True
205
- else:
206
- log.warning(
207
- "Current Molmo implementation has a bug with "
208
- "`vllm-flash-attn` inside vision module, so we use "
209
- "xformers backend instead. You can run `pip install "
210
- "flash-attn to use flash-attention backend.")
211
- self._use_flash_attn = False
212
- else:
213
- self._use_flash_attn = False
214
- else:
215
- if selected_backend == _Backend.FLASH_ATTN:
216
- self._use_flash_attn = True
217
- elif selected_backend == _Backend.XFORMERS:
218
- self._use_flash_attn = False
219
- else:
220
- raise RuntimeError(
221
- f"Molmo does not support {selected_backend} backend now.")
189
+ self.attn_backend: _Backend = get_vit_attn_backend()
190
+ if self.attn_backend not in {
191
+ _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
192
+ }:
193
+ raise RuntimeError(
194
+ f"Molmo does not support {self.attn_backend} backend now.")
195
196
def forward(self,
197
inputs_q: torch.Tensor,
@@ -240,10 +213,15 @@ class MultiHeadDotProductAttention(nn.Module):
213
xk = xk.view(*kv_shape)
214
xv = xv.view(*kv_shape)
215
243
- if self._use_flash_attn:
216
+ if self.attn_backend == _Backend.FLASH_ATTN:
217
from flash_attn import flash_attn_func
218
output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
246
- else:
219
+ elif self.attn_backend == _Backend.TORCH_SDPA:
220
+ xq, xk, xv = (rearrange(x, "b s h d -> b h s d")
221
+ for x in (xq, xk, xv))
222
+ output = F.scaled_dot_product_attention(xq, xk, xv)
223
+ output = rearrange(output, "b h s d -> b s h d ")
224
+ elif self.attn_backend == _Backend.XFORMERS:
225
from xformers import ops as xops
226
output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
227
vllm/model_executor/models/qwen2_vl.py
+12
-40
@@ -39,10 +39,8 @@ from transformers.models.qwen2_vl.configuration_qwen2_vl import (
39
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
40
make_batched_images, make_batched_videos, smart_resize)
41
42
-import vllm.envs as envs
42
from vllm.attention import AttentionMetadata
44
-from vllm.attention.selector import (_Backend, backend_name_to_enum,
45
- get_global_forced_attn_backend)
43
+from vllm.attention.selector import _Backend
44
from vllm.config import CacheConfig, MultiModalConfig
45
from vllm.distributed import get_pp_group, parallel_state
46
from vllm.distributed import utils as dist_utils
@@ -63,14 +61,13 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
61
MultiModalInputs)
62
from vllm.multimodal.base import MultiModalData
63
from vllm.multimodal.image import cached_get_image_processor
66
-from vllm.platforms import current_platform
64
from vllm.sequence import IntermediateTensors, SequenceData
65
from vllm.transformers_utils.config import uses_mrope
66
from vllm.transformers_utils.processor import get_processor
70
-from vllm.utils import is_cpu
67
68
from .interfaces import SupportsMultiModal, SupportsPP
73
-from .utils import (PPMissingLayer, is_pp_missing_parameter,
69
+from .utils import (PPMissingLayer, get_vit_attn_backend,
70
+ is_pp_missing_parameter,
71
make_empty_intermediate_tensors_factory)
72
73
logger = init_logger(__name__)
@@ -215,37 +212,12 @@ class Qwen2VisionAttention(nn.Module):
212
quant_config=quant_config)
213
214
# Detect attention implementation.
218
- selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
219
- if selected_backend is None:
220
- backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
221
- if backend_by_env_var is not None:
222
- selected_backend = backend_name_to_enum(backend_by_env_var)
223
- if selected_backend is None:
224
- # For Volta and Turing GPUs, use xformers instead.
225
- device_available = current_platform.has_device_capability(80)
226
- if device_available:
227
- from transformers.utils import is_flash_attn_2_available
228
-
229
- if is_flash_attn_2_available():
230
- self._use_flash_attn = True
231
- else:
232
- logger.warning(
233
- "Current Qwen2-VL implementation has a bug with "
234
- "`vllm-flash-attn` inside vision module, so we use "
235
- "xformers backend instead. You can run `pip install "
236
- "flash-attn to use flash-attention backend.")
237
- self._use_flash_attn = False
238
- else:
239
- self._use_flash_attn = False
240
- else:
241
- if selected_backend == _Backend.FLASH_ATTN:
242
- self._use_flash_attn = True
243
- elif selected_backend == _Backend.XFORMERS:
244
- self._use_flash_attn = False
245
- else:
246
- raise RuntimeError(
247
- f"Qwen2-VL does not support {selected_backend} backend now."
248
- )
215
+ self.attn_backend: _Backend = get_vit_attn_backend()
216
+ if self.attn_backend not in {
217
+ _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
218
+ }:
219
+ raise RuntimeError(
220
+ f"Qwen2-VL does not support {self.attn_backend} backend now.")
221
222
def forward(
223
self,
@@ -274,7 +246,7 @@ class Qwen2VisionAttention(nn.Module):
246
q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
247
k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
248
277
- if self._use_flash_attn:
249
+ if self.attn_backend == _Backend.FLASH_ATTN:
250
# from vllm_flash_attn.flash_attn_interface import (
251
# flash_attn_varlen_func)
252
from flash_attn import flash_attn_varlen_func
@@ -295,7 +267,7 @@ class Qwen2VisionAttention(nn.Module):
267
context_layer = rearrange(output,
268
"(b s) ... -> b s ...",
269
b=batch_size)
298
- elif is_cpu():
270
+ elif self.attn_backend == _Backend.TORCH_SDPA:
271
seq_length = q.size(1)
272
q, k, v = [rearrange(x, "b s h d -> b h s d") for x in [q, k, v]]
273
attention_mask = torch.zeros([1, seq_length, seq_length],
@@ -310,7 +282,7 @@ class Qwen2VisionAttention(nn.Module):
282
attention_mask,
283
dropout_p=0.0)
284
context_layer = rearrange(output, "b h s d -> b s h d ")
313
- else:
285
+ elif self.attn_backend == _Backend.XFORMERS:
286
from xformers import ops as xops
287
from xformers.ops.fmha.attn_bias import BlockDiagonalMask
288
vllm/model_executor/models/utils.py
+34
-1
@@ -8,15 +8,22 @@ import torch.nn as nn
8
from torch.func import functional_call
9
from transformers import PretrainedConfig
10
11
+import vllm.envs as envs
12
+from vllm.attention.selector import (_Backend, backend_name_to_enum,
13
+ get_global_forced_attn_backend)
14
from vllm.config import (CacheConfig, LoRAConfig, MultiModalConfig,
15
SchedulerConfig)
16
+from vllm.logger import init_logger
17
from vllm.model_executor.layers.quantization import QuantizationConfig
18
from vllm.model_executor.model_loader.loader import build_model
19
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
20
from vllm.model_executor.models import ModelRegistry
21
from vllm.multimodal.base import NestedTensors
22
+from vllm.platforms import current_platform
23
from vllm.sequence import IntermediateTensors
19
-from vllm.utils import is_pin_memory_available
24
+from vllm.utils import is_cpu, is_pin_memory_available
25
+
26
+logger = init_logger(__name__)
27
28
WeightsMapping = Mapping[str, Optional[str]]
29
"""If a key maps to a value of `None`, the corresponding weight is ignored."""
@@ -487,3 +494,29 @@ class LLMWrapper(nn.Module):
494
def __call__(self, *args: Any, **kwargs: Any) -> Any:
495
llm = super().__getattr__(self.model_name)
496
return llm(*args, **kwargs)
497
+
498
+
499
+def get_vit_attn_backend() -> _Backend:
500
+ selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
501
+ if selected_backend is None:
502
+ backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
503
+ if backend_by_env_var is not None:
504
+ selected_backend = backend_name_to_enum(backend_by_env_var)
505
+ if selected_backend is None:
506
+ # For Volta and Turing GPUs, use xformers instead.
507
+ device_available = current_platform.has_device_capability(80)
508
+ if device_available:
509
+ from transformers.utils import is_flash_attn_2_available
510
+ if is_flash_attn_2_available():
511
+ selected_backend = _Backend.FLASH_ATTN
512
+ else:
513
+ logger.warning(
514
+ "Current `vllm-flash-attn` has a bug inside vision module, "
515
+ "so we use xformers backend instead. You can run "
516
+ "`pip install flash-attn` to use flash-attention backend.")
517
+ selected_backend = _Backend.XFORMERS
518
+ elif is_cpu():
519
+ selected_backend = _Backend.TORCH_SDPA
520
+ else:
521
+ selected_backend = _Backend.XFORMERS
522
+ return selected_backend