@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e7ebb662d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Remove transformers attention porting in VITs (#10414)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 18, 2024 at 21:45 UTC e7ebb662d777a9617644428031c1cf80c38939ba
7 files changed +139 -102
vllm/model_executor/models/blip.py
+36 -30
@@ -4,10 +4,11 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5 import torch
6 import torch.nn as nn
7 +import torch.nn.functional as F
8 from PIL import Image
9 from transformers import Blip2VisionConfig, BlipVisionConfig
9 -from transformers.models.blip.modeling_blip import BlipAttention
10
11 +from vllm.attention.selector import _Backend
12 from vllm.config import ModelConfig
13 from vllm.distributed import divide, get_tensor_model_parallel_world_size
14 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -21,11 +22,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
22 repeat_and_pad_placeholder_tokens)
23 from vllm.sequence import SequenceData
24
24 -try:
25 - from xformers import ops as xops
26 - USE_XFORMERS_OPS = True
27 -except ImportError:
28 - USE_XFORMERS_OPS = False
25 +from .utils import get_vit_attn_backend
26
27
28 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -168,7 +165,7 @@ class BlipVisionEmbeddings(nn.Module):
165 return embeddings
166
167
171 -class BlipParallelAttention(nn.Module):
168 +class BlipAttention(nn.Module):
169 """Multi-headed attention from 'Attention Is All You Need' paper"""
170
171 def __init__(
@@ -208,6 +205,12 @@ class BlipParallelAttention(nn.Module):
205 self.tp_size = get_tensor_model_parallel_world_size()
206 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
207
208 + # Detect attention implementation.
209 + self.attn_backend = get_vit_attn_backend(support_fa=False)
210 + if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
211 + raise RuntimeError(
212 + f"BLIP does not support {self.attn_backend} backend now.")
213 +
214 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
215 return tensor.view(bsz, seq_len, self.num_heads,
216 self.head_dim).transpose(1, 2).contiguous()
@@ -231,11 +234,26 @@ class BlipParallelAttention(nn.Module):
234 self.num_heads_per_partition,
235 self.head_dim)
236
234 - out = xops.memory_efficient_attention_forward(query_states,
235 - key_states,
236 - value_states,
237 - p=self.dropout,
238 - scale=self.scale)
237 + if self.attn_backend == _Backend.XFORMERS:
238 + from xformers import ops as xops
239 +
240 + out = xops.memory_efficient_attention_forward(query_states,
241 + key_states,
242 + value_states,
243 + p=self.dropout,
244 + scale=self.scale)
245 + elif self.attn_backend == _Backend.TORCH_SDPA:
246 + query_states, key_states, value_states = (x.transpose(1, 2)
247 + for x in (query_states,
248 + key_states,
249 + value_states))
250 + out = F.scaled_dot_product_attention(query_states,
251 + key_states,
252 + value_states,
253 + dropout_p=self.dropout,
254 + scale=self.scale)
255 + out = out.transpose(1, 2)
256 +
257 out = out.view(bsz, tgt_len, -1)
258 attn_output, _ = self.projection(out)
259
@@ -285,18 +303,11 @@ class BlipEncoderLayer(nn.Module):
303 super().__init__()
304
305 # fallback to sdpa attention if tp unavailable
288 - num_heads = config.num_attention_heads
289 - tp_size = get_tensor_model_parallel_world_size()
290 - if USE_XFORMERS_OPS and num_heads % tp_size == 0:
291 - self.self_attn = BlipParallelAttention(
292 - config,
293 - quant_config=quant_config,
294 - prefix=f"{prefix}.self_attn",
295 - )
296 - else:
297 - # Blip doesn't have SDPA attention implemented in transformers
298 - # use eager attention instead for cpu backend
299 - self.self_attn = BlipAttention(config)
306 + self.self_attn = BlipAttention(
307 + config,
308 + quant_config=quant_config,
309 + prefix=f"{prefix}.self_attn",
310 + )
311 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
312 eps=config.layer_norm_eps)
313 self.mlp = BlipMLP(config,
@@ -374,11 +385,6 @@ class BlipVisionModel(nn.Module):
385 prefix: str = "",
386 ) -> None:
387 super().__init__()
377 -
378 - tp_size = get_tensor_model_parallel_world_size()
379 - num_heads = config.num_attention_heads
380 - self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
381 -
388 self.config = config
389
390 self.embeddings = BlipVisionEmbeddings(config)
@@ -422,7 +428,7 @@ class BlipVisionModel(nn.Module):
428 ("qkv_proj", "q_proj", "q"),
429 ("qkv_proj", "k_proj", "k"),
430 ("qkv_proj", "v_proj", "v"),
425 - ] if self.shard_weight else []
431 + ]
432 params_dict = dict(self.named_parameters())
433 loaded_params: Set[str] = set()
434 layer_count = len(self.encoder.layers)
vllm/model_executor/models/clip.py
+36 -29
@@ -5,10 +5,11 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
5 import numpy as np
6 import torch
7 import torch.nn as nn
8 +import torch.nn.functional as F
9 from PIL import Image
10 from transformers import CLIPVisionConfig
10 -from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12 +from vllm.attention.selector import _Backend
13 from vllm.config import ModelConfig
14 from vllm.distributed import divide, get_tensor_model_parallel_world_size
15 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -23,11 +24,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24 repeat_and_pad_placeholder_tokens)
25 from vllm.sequence import SequenceData
26
26 -try:
27 - from xformers import ops as xops
28 - USE_XFORMERS_OPS = True
29 -except ImportError:
30 - USE_XFORMERS_OPS = False
27 +from .utils import get_vit_attn_backend
28
29
30 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -197,7 +194,7 @@ class CLIPVisionEmbeddings(nn.Module):
194 return embeddings
195
196
200 -class CLIPParallelAttention(nn.Module):
197 +class CLIPAttention(nn.Module):
198 """Multi-headed attention from 'Attention Is All You Need' paper"""
199
200 def __init__(
@@ -237,6 +234,12 @@ class CLIPParallelAttention(nn.Module):
234 self.tp_size = get_tensor_model_parallel_world_size()
235 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
236
237 + # Detect attention implementation.
238 + self.attn_backend = get_vit_attn_backend(support_fa=False)
239 + if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
240 + raise RuntimeError(
241 + f"CLIP does not support {self.attn_backend} backend now.")
242 +
243 def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
244 return tensor.view(bsz, seq_len, self.num_heads,
245 self.head_dim).transpose(1, 2).contiguous()
@@ -261,11 +264,26 @@ class CLIPParallelAttention(nn.Module):
264 self.num_heads_per_partition,
265 self.head_dim)
266
264 - out = xops.memory_efficient_attention_forward(query_states,
265 - key_states,
266 - value_states,
267 - p=self.dropout,
268 - scale=self.scale)
267 + if self.attn_backend == _Backend.XFORMERS:
268 + from xformers import ops as xops
269 +
270 + out = xops.memory_efficient_attention_forward(query_states,
271 + key_states,
272 + value_states,
273 + p=self.dropout,
274 + scale=self.scale)
275 + elif self.attn_backend == _Backend.TORCH_SDPA:
276 + query_states, key_states, value_states = (x.transpose(1, 2)
277 + for x in (query_states,
278 + key_states,
279 + value_states))
280 + out = F.scaled_dot_product_attention(query_states,
281 + key_states,
282 + value_states,
283 + dropout_p=self.dropout,
284 + scale=self.scale)
285 + out = out.transpose(1, 2)
286 +
287 out = out.view(bsz, tgt_len, -1)
288 attn_output, _ = self.out_proj(out)
289
@@ -311,17 +329,11 @@ class CLIPEncoderLayer(nn.Module):
329 prefix: str = "",
330 ) -> None:
331 super().__init__()
314 -
315 - num_heads = config.num_attention_heads
316 - tp_size = get_tensor_model_parallel_world_size()
317 - if USE_XFORMERS_OPS and num_heads % tp_size == 0:
318 - self.self_attn = CLIPParallelAttention(
319 - config,
320 - quant_config=quant_config,
321 - prefix=f"{prefix}.self_attn",
322 - )
323 - else:
324 - self.self_attn = CLIPSdpaAttention(config)
332 + self.self_attn = CLIPAttention(
333 + config,
334 + quant_config=quant_config,
335 + prefix=f"{prefix}.self_attn",
336 + )
337 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
338 eps=config.layer_norm_eps)
339 self.mlp = CLIPMLP(config,
@@ -461,11 +473,6 @@ class CLIPVisionModel(nn.Module):
473 prefix: str = "",
474 ) -> None:
475 super().__init__()
464 -
465 - tp_size = get_tensor_model_parallel_world_size()
466 - num_heads = config.num_attention_heads
467 - self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
468 -
476 self.vision_model = CLIPVisionTransformer(
477 config=config,
478 quant_config=quant_config,
@@ -490,7 +497,7 @@ class CLIPVisionModel(nn.Module):
497 ("qkv_proj", "q_proj", "q"),
498 ("qkv_proj", "k_proj", "k"),
499 ("qkv_proj", "v_proj", "v"),
493 - ] if self.shard_weight else []
500 + ]
501 params_dict = dict(self.named_parameters())
502 loaded_params: Set[str] = set()
503 layer_count = len(self.vision_model.encoder.layers)
vllm/model_executor/models/intern_vit.py
+22 -10
@@ -12,6 +12,7 @@ import torch.nn as nn
12 import torch.nn.functional as F
13 from transformers import PretrainedConfig
14
15 +from vllm.attention.selector import _Backend
16 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
17 get_tensor_model_parallel_world_size,
18 split_tensor_along_last_dim,
@@ -24,11 +25,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25 from vllm.model_executor.layers.quantization import QuantizationConfig
26 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
27 -try:
28 - from xformers import ops as xops
29 - USE_XFORMERS_OPS = True
30 -except ImportError:
31 - USE_XFORMERS_OPS = False
28 +from .utils import get_vit_attn_backend
29
30 NORM2FN = {
31 'rms_norm': RMSNorm,
@@ -186,6 +183,11 @@ class InternParallelAttention(nn.Module):
183 prefix=f"{prefix}.proj",
184 )
185
186 + self.attn_backend = get_vit_attn_backend(support_fa=False)
187 + if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
188 + raise RuntimeError(
189 + f"InternViT does not support {self.attn_backend} backend now.")
190 +
191 def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
192 if self.tp_size > 1:
193 q = tensor_model_parallel_all_gather(q.contiguous())
@@ -211,11 +213,21 @@ class InternParallelAttention(nn.Module):
213 k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
214 v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
215
214 - x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
215 - x = x.view(B, N, -1)
216 + if self.attn_backend == _Backend.XFORMERS:
217 + from xformers import ops as xops
218
217 - x, _ = self.proj(x)
218 - return x
219 + out = xops.memory_efficient_attention_forward(q,
220 + k,
221 + v,
222 + scale=self.scale)
223 + elif self.attn_backend == _Backend.TORCH_SDPA:
224 + q, k, v = (x.transpose(1, 2) for x in (q, k, v))
225 + out = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
226 + out = out.transpose(1, 2)
227 +
228 + out = out.view(B, N, -1)
229 + out, _ = self.proj(out)
230 + return out
231
232
233 class InternSdpaAttention(nn.Module):
@@ -362,7 +374,7 @@ class InternVisionEncoderLayer(nn.Module):
374 tp_size = get_tensor_model_parallel_world_size()
375 num_heads = config.num_attention_heads
376
365 - if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
377 + if (num_heads + num_dummy_heads) % tp_size == 0:
378 return InternParallelAttention(config,
379 quant_config=quant_config,
380 num_dummy_heads=num_dummy_heads,
vllm/model_executor/models/molmo.py
+1 -1
@@ -187,7 +187,7 @@ class MultiHeadDotProductAttention(nn.Module):
187 )
188
189 # Detect attention implementation.
190 - self.attn_backend: _Backend = get_vit_attn_backend()
190 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
191 if self.attn_backend not in {
192 _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
193 }:
vllm/model_executor/models/qwen2_vl.py
+1 -1
@@ -260,7 +260,7 @@ class Qwen2VisionAttention(nn.Module):
260 prefix=f"{prefix}.proj")
261
262 # Detect attention implementation.
263 - self.attn_backend: _Backend = get_vit_attn_backend()
263 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
264 if self.attn_backend not in {
265 _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
266 }:
vllm/model_executor/models/siglip.py
+35 -28
@@ -6,11 +6,12 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
6
7 import numpy as np
8 import torch
9 +import torch.nn.functional as F
10 from PIL import Image
11 from torch import nn
12 from transformers import SiglipVisionConfig
12 -from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14 +from vllm.attention.selector import _Backend
15 from vllm.config import ModelConfig
16 from vllm.distributed import divide, get_tensor_model_parallel_world_size
17 from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -27,11 +28,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28 repeat_and_pad_placeholder_tokens)
29 from vllm.sequence import SequenceData
30
30 -try:
31 - from xformers import ops as xops
32 - USE_XFORMERS_OPS = True
33 -except ImportError:
34 - USE_XFORMERS_OPS = False
31 +from .utils import get_vit_attn_backend
32
33
34 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -254,7 +251,7 @@ class SiglipVisionEmbeddings(nn.Module):
251 return embeddings
252
253
257 -class SiglipParallelAttention(nn.Module):
254 +class SiglipAttention(nn.Module):
255
256 def __init__(
257 self,
@@ -293,6 +290,11 @@ class SiglipParallelAttention(nn.Module):
290 self.tp_size = get_tensor_model_parallel_world_size()
291 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
292
293 + self.attn_backend = get_vit_attn_backend(support_fa=False)
294 + if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
295 + raise RuntimeError(
296 + f"SIGLIP does not support {self.attn_backend} backend now.")
297 +
298 def forward(
299 self,
300 hidden_states: torch.Tensor,
@@ -313,11 +315,26 @@ class SiglipParallelAttention(nn.Module):
315 self.num_heads_per_partition,
316 self.head_dim)
317
316 - out = xops.memory_efficient_attention_forward(query_states,
317 - key_states,
318 - value_states,
319 - p=self.dropout,
320 - scale=self.scale)
318 + if self.attn_backend == _Backend.XFORMERS:
319 + from xformers import ops as xops
320 +
321 + out = xops.memory_efficient_attention_forward(query_states,
322 + key_states,
323 + value_states,
324 + p=self.dropout,
325 + scale=self.scale)
326 + elif self.attn_backend == _Backend.TORCH_SDPA:
327 + query_states, key_states, value_states = (x.transpose(1, 2)
328 + for x in (query_states,
329 + key_states,
330 + value_states))
331 + out = F.scaled_dot_product_attention(query_states,
332 + key_states,
333 + value_states,
334 + dropout_p=self.dropout,
335 + scale=self.scale)
336 + out = out.transpose(1, 2)
337 +
338 out = out.view(batch_size, q_len, -1)
339 attn_output, _ = self.out_proj(out)
340
@@ -372,17 +389,11 @@ class SiglipEncoderLayer(nn.Module):
389
390 self.embed_dim = config.hidden_size
391
375 - num_heads = config.num_attention_heads
376 - tp_size = get_tensor_model_parallel_world_size()
377 - if USE_XFORMERS_OPS and num_heads % tp_size == 0:
378 - self.self_attn = SiglipParallelAttention(
379 - config,
380 - quant_config=quant_config,
381 - prefix=f"{prefix}.self_attn",
382 - )
383 - else:
384 - self.self_attn = SiglipSdpaAttention(config)
385 -
392 + self.self_attn = SiglipAttention(
393 + config,
394 + quant_config=quant_config,
395 + prefix=f"{prefix}.self_attn",
396 + )
397 self.layer_norm1 = nn.LayerNorm(self.embed_dim,
398 eps=config.layer_norm_eps)
399 self.mlp = SiglipMLP(
@@ -569,10 +580,6 @@ class SiglipVisionModel(nn.Module):
580 ) -> None:
581 super().__init__()
582
572 - num_heads = config.num_attention_heads
573 - tp_size = get_tensor_model_parallel_world_size()
574 - self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
575 -
583 self.vision_model = SiglipVisionTransformer(
584 config,
585 quant_config,
@@ -601,7 +608,7 @@ class SiglipVisionModel(nn.Module):
608 ("qkv_proj", "q_proj", "q"),
609 ("qkv_proj", "k_proj", "k"),
610 ("qkv_proj", "v_proj", "v"),
604 - ] if self.shard_weight else []
611 + ]
612 params_dict = dict(self.named_parameters())
613 loaded_params: Set[str] = set()
614 layer_count = len(self.vision_model.encoder.layers)
vllm/model_executor/models/utils.py
+8 -3
@@ -587,7 +587,11 @@ class LLMWrapper(nn.Module):
587 return llm(*args, **kwargs)
588
589
590 -def get_vit_attn_backend() -> _Backend:
590 +def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
591 + """
592 + Get the available attention backend for Vision Transformer.
593 + """
594 + # TODO(Isotr0py): Remove `support_fa` after support FA for all ViTs attn.
595 selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
596 if selected_backend is None:
597 backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
@@ -596,7 +600,7 @@ def get_vit_attn_backend() -> _Backend:
600 if selected_backend is None:
601 # For Volta and Turing GPUs, use xformers instead.
602 device_available = current_platform.has_device_capability(80)
599 - if device_available:
603 + if device_available and support_fa:
604 from transformers.utils import is_flash_attn_2_available
605 if is_flash_attn_2_available():
606 selected_backend = _Backend.FLASH_ATTN
@@ -606,7 +610,8 @@ def get_vit_attn_backend() -> _Backend:
610 "so we use xformers backend instead. You can run "
611 "`pip install flash-attn` to use flash-attention backend.")
612 selected_backend = _Backend.XFORMERS
609 - elif current_platform.is_cpu():
613 + elif current_platform.is_cpu() or current_platform.is_rocm():
614 + # ROCM doesn't support xformers
615 selected_backend = _Backend.TORCH_SDPA
616 else:
617 selected_backend = _Backend.XFORMERS