A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Remove transformers attention porting in VITs (#10414)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 18, 2024 at 21:45 UTC
e7ebb662d777a9617644428031c1cf80c38939ba
7 files changed
+139
-102
vllm/model_executor/models/blip.py
+36
-30
@@ -4,10 +4,11 @@ from typing import Iterable, Optional, Set, Tuple, Union
4
5
import torch
6
import torch.nn as nn
7
+import torch.nn.functional as F
8
from PIL import Image
9
from transformers import Blip2VisionConfig, BlipVisionConfig
9
-from transformers.models.blip.modeling_blip import BlipAttention
10
11
+from vllm.attention.selector import _Backend
12
from vllm.config import ModelConfig
13
from vllm.distributed import divide, get_tensor_model_parallel_world_size
14
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -21,11 +22,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
22
repeat_and_pad_placeholder_tokens)
23
from vllm.sequence import SequenceData
24
24
-try:
25
- from xformers import ops as xops
26
- USE_XFORMERS_OPS = True
27
-except ImportError:
28
- USE_XFORMERS_OPS = False
25
+from .utils import get_vit_attn_backend
26
27
28
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -168,7 +165,7 @@ class BlipVisionEmbeddings(nn.Module):
165
return embeddings
166
167
171
-class BlipParallelAttention(nn.Module):
168
+class BlipAttention(nn.Module):
169
"""Multi-headed attention from 'Attention Is All You Need' paper"""
170
171
def __init__(
@@ -208,6 +205,12 @@ class BlipParallelAttention(nn.Module):
205
self.tp_size = get_tensor_model_parallel_world_size()
206
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
207
208
+ # Detect attention implementation.
209
+ self.attn_backend = get_vit_attn_backend(support_fa=False)
210
+ if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
211
+ raise RuntimeError(
212
+ f"BLIP does not support {self.attn_backend} backend now.")
213
+
214
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
215
return tensor.view(bsz, seq_len, self.num_heads,
216
self.head_dim).transpose(1, 2).contiguous()
@@ -231,11 +234,26 @@ class BlipParallelAttention(nn.Module):
234
self.num_heads_per_partition,
235
self.head_dim)
236
234
- out = xops.memory_efficient_attention_forward(query_states,
235
- key_states,
236
- value_states,
237
- p=self.dropout,
238
- scale=self.scale)
237
+ if self.attn_backend == _Backend.XFORMERS:
238
+ from xformers import ops as xops
239
+
240
+ out = xops.memory_efficient_attention_forward(query_states,
241
+ key_states,
242
+ value_states,
243
+ p=self.dropout,
244
+ scale=self.scale)
245
+ elif self.attn_backend == _Backend.TORCH_SDPA:
246
+ query_states, key_states, value_states = (x.transpose(1, 2)
247
+ for x in (query_states,
248
+ key_states,
249
+ value_states))
250
+ out = F.scaled_dot_product_attention(query_states,
251
+ key_states,
252
+ value_states,
253
+ dropout_p=self.dropout,
254
+ scale=self.scale)
255
+ out = out.transpose(1, 2)
256
+
257
out = out.view(bsz, tgt_len, -1)
258
attn_output, _ = self.projection(out)
259
@@ -285,18 +303,11 @@ class BlipEncoderLayer(nn.Module):
303
super().__init__()
304
305
# fallback to sdpa attention if tp unavailable
288
- num_heads = config.num_attention_heads
289
- tp_size = get_tensor_model_parallel_world_size()
290
- if USE_XFORMERS_OPS and num_heads % tp_size == 0:
291
- self.self_attn = BlipParallelAttention(
292
- config,
293
- quant_config=quant_config,
294
- prefix=f"{prefix}.self_attn",
295
- )
296
- else:
297
- # Blip doesn't have SDPA attention implemented in transformers
298
- # use eager attention instead for cpu backend
299
- self.self_attn = BlipAttention(config)
306
+ self.self_attn = BlipAttention(
307
+ config,
308
+ quant_config=quant_config,
309
+ prefix=f"{prefix}.self_attn",
310
+ )
311
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
312
eps=config.layer_norm_eps)
313
self.mlp = BlipMLP(config,
@@ -374,11 +385,6 @@ class BlipVisionModel(nn.Module):
385
prefix: str = "",
386
) -> None:
387
super().__init__()
377
-
378
- tp_size = get_tensor_model_parallel_world_size()
379
- num_heads = config.num_attention_heads
380
- self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
381
-
388
self.config = config
389
390
self.embeddings = BlipVisionEmbeddings(config)
@@ -422,7 +428,7 @@ class BlipVisionModel(nn.Module):
428
("qkv_proj", "q_proj", "q"),
429
("qkv_proj", "k_proj", "k"),
430
("qkv_proj", "v_proj", "v"),
425
- ] if self.shard_weight else []
431
+ ]
432
params_dict = dict(self.named_parameters())
433
loaded_params: Set[str] = set()
434
layer_count = len(self.encoder.layers)
vllm/model_executor/models/clip.py
+36
-29
@@ -5,10 +5,11 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
5
import numpy as np
6
import torch
7
import torch.nn as nn
8
+import torch.nn.functional as F
9
from PIL import Image
10
from transformers import CLIPVisionConfig
10
-from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12
+from vllm.attention.selector import _Backend
13
from vllm.config import ModelConfig
14
from vllm.distributed import divide, get_tensor_model_parallel_world_size
15
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -23,11 +24,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
24
repeat_and_pad_placeholder_tokens)
25
from vllm.sequence import SequenceData
26
26
-try:
27
- from xformers import ops as xops
28
- USE_XFORMERS_OPS = True
29
-except ImportError:
30
- USE_XFORMERS_OPS = False
27
+from .utils import get_vit_attn_backend
28
29
30
def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -197,7 +194,7 @@ class CLIPVisionEmbeddings(nn.Module):
194
return embeddings
195
196
200
-class CLIPParallelAttention(nn.Module):
197
+class CLIPAttention(nn.Module):
198
"""Multi-headed attention from 'Attention Is All You Need' paper"""
199
200
def __init__(
@@ -237,6 +234,12 @@ class CLIPParallelAttention(nn.Module):
234
self.tp_size = get_tensor_model_parallel_world_size()
235
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
236
237
+ # Detect attention implementation.
238
+ self.attn_backend = get_vit_attn_backend(support_fa=False)
239
+ if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
240
+ raise RuntimeError(
241
+ f"CLIP does not support {self.attn_backend} backend now.")
242
+
243
def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
244
return tensor.view(bsz, seq_len, self.num_heads,
245
self.head_dim).transpose(1, 2).contiguous()
@@ -261,11 +264,26 @@ class CLIPParallelAttention(nn.Module):
264
self.num_heads_per_partition,
265
self.head_dim)
266
264
- out = xops.memory_efficient_attention_forward(query_states,
265
- key_states,
266
- value_states,
267
- p=self.dropout,
268
- scale=self.scale)
267
+ if self.attn_backend == _Backend.XFORMERS:
268
+ from xformers import ops as xops
269
+
270
+ out = xops.memory_efficient_attention_forward(query_states,
271
+ key_states,
272
+ value_states,
273
+ p=self.dropout,
274
+ scale=self.scale)
275
+ elif self.attn_backend == _Backend.TORCH_SDPA:
276
+ query_states, key_states, value_states = (x.transpose(1, 2)
277
+ for x in (query_states,
278
+ key_states,
279
+ value_states))
280
+ out = F.scaled_dot_product_attention(query_states,
281
+ key_states,
282
+ value_states,
283
+ dropout_p=self.dropout,
284
+ scale=self.scale)
285
+ out = out.transpose(1, 2)
286
+
287
out = out.view(bsz, tgt_len, -1)
288
attn_output, _ = self.out_proj(out)
289
@@ -311,17 +329,11 @@ class CLIPEncoderLayer(nn.Module):
329
prefix: str = "",
330
) -> None:
331
super().__init__()
314
-
315
- num_heads = config.num_attention_heads
316
- tp_size = get_tensor_model_parallel_world_size()
317
- if USE_XFORMERS_OPS and num_heads % tp_size == 0:
318
- self.self_attn = CLIPParallelAttention(
319
- config,
320
- quant_config=quant_config,
321
- prefix=f"{prefix}.self_attn",
322
- )
323
- else:
324
- self.self_attn = CLIPSdpaAttention(config)
332
+ self.self_attn = CLIPAttention(
333
+ config,
334
+ quant_config=quant_config,
335
+ prefix=f"{prefix}.self_attn",
336
+ )
337
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
338
eps=config.layer_norm_eps)
339
self.mlp = CLIPMLP(config,
@@ -461,11 +473,6 @@ class CLIPVisionModel(nn.Module):
473
prefix: str = "",
474
) -> None:
475
super().__init__()
464
-
465
- tp_size = get_tensor_model_parallel_world_size()
466
- num_heads = config.num_attention_heads
467
- self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
468
-
476
self.vision_model = CLIPVisionTransformer(
477
config=config,
478
quant_config=quant_config,
@@ -490,7 +497,7 @@ class CLIPVisionModel(nn.Module):
497
("qkv_proj", "q_proj", "q"),
498
("qkv_proj", "k_proj", "k"),
499
("qkv_proj", "v_proj", "v"),
493
- ] if self.shard_weight else []
500
+ ]
501
params_dict = dict(self.named_parameters())
502
loaded_params: Set[str] = set()
503
layer_count = len(self.vision_model.encoder.layers)
vllm/model_executor/models/intern_vit.py
+22
-10
@@ -12,6 +12,7 @@ import torch.nn as nn
12
import torch.nn.functional as F
13
from transformers import PretrainedConfig
14
15
+from vllm.attention.selector import _Backend
16
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
17
get_tensor_model_parallel_world_size,
18
split_tensor_along_last_dim,
@@ -24,11 +25,7 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
25
from vllm.model_executor.layers.quantization import QuantizationConfig
26
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
27
-try:
28
- from xformers import ops as xops
29
- USE_XFORMERS_OPS = True
30
-except ImportError:
31
- USE_XFORMERS_OPS = False
28
+from .utils import get_vit_attn_backend
29
30
NORM2FN = {
31
'rms_norm': RMSNorm,
@@ -186,6 +183,11 @@ class InternParallelAttention(nn.Module):
183
prefix=f"{prefix}.proj",
184
)
185
186
+ self.attn_backend = get_vit_attn_backend(support_fa=False)
187
+ if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
188
+ raise RuntimeError(
189
+ f"InternViT does not support {self.attn_backend} backend now.")
190
+
191
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
192
if self.tp_size > 1:
193
q = tensor_model_parallel_all_gather(q.contiguous())
@@ -211,11 +213,21 @@ class InternParallelAttention(nn.Module):
213
k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
214
v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
215
214
- x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
215
- x = x.view(B, N, -1)
216
+ if self.attn_backend == _Backend.XFORMERS:
217
+ from xformers import ops as xops
218
217
- x, _ = self.proj(x)
218
- return x
219
+ out = xops.memory_efficient_attention_forward(q,
220
+ k,
221
+ v,
222
+ scale=self.scale)
223
+ elif self.attn_backend == _Backend.TORCH_SDPA:
224
+ q, k, v = (x.transpose(1, 2) for x in (q, k, v))
225
+ out = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
226
+ out = out.transpose(1, 2)
227
+
228
+ out = out.view(B, N, -1)
229
+ out, _ = self.proj(out)
230
+ return out
231
232
233
class InternSdpaAttention(nn.Module):
@@ -362,7 +374,7 @@ class InternVisionEncoderLayer(nn.Module):
374
tp_size = get_tensor_model_parallel_world_size()
375
num_heads = config.num_attention_heads
376
365
- if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
377
+ if (num_heads + num_dummy_heads) % tp_size == 0:
378
return InternParallelAttention(config,
379
quant_config=quant_config,
380
num_dummy_heads=num_dummy_heads,
vllm/model_executor/models/molmo.py
+1
-1
@@ -187,7 +187,7 @@ class MultiHeadDotProductAttention(nn.Module):
187
)
188
189
# Detect attention implementation.
190
- self.attn_backend: _Backend = get_vit_attn_backend()
190
+ self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
191
if self.attn_backend not in {
192
_Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
193
}:
vllm/model_executor/models/qwen2_vl.py
+1
-1
@@ -260,7 +260,7 @@ class Qwen2VisionAttention(nn.Module):
260
prefix=f"{prefix}.proj")
261
262
# Detect attention implementation.
263
- self.attn_backend: _Backend = get_vit_attn_backend()
263
+ self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
264
if self.attn_backend not in {
265
_Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
266
}:
vllm/model_executor/models/siglip.py
+35
-28
@@ -6,11 +6,12 @@ from typing import Iterable, List, Optional, Set, Tuple, Union
6
7
import numpy as np
8
import torch
9
+import torch.nn.functional as F
10
from PIL import Image
11
from torch import nn
12
from transformers import SiglipVisionConfig
12
-from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14
+from vllm.attention.selector import _Backend
15
from vllm.config import ModelConfig
16
from vllm.distributed import divide, get_tensor_model_parallel_world_size
17
from vllm.inputs import DecoderOnlyInputs, token_inputs
@@ -27,11 +28,7 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
28
repeat_and_pad_placeholder_tokens)
29
from vllm.sequence import SequenceData
30
30
-try:
31
- from xformers import ops as xops
32
- USE_XFORMERS_OPS = True
33
-except ImportError:
34
- USE_XFORMERS_OPS = False
31
+from .utils import get_vit_attn_backend
32
33
34
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
@@ -254,7 +251,7 @@ class SiglipVisionEmbeddings(nn.Module):
251
return embeddings
252
253
257
-class SiglipParallelAttention(nn.Module):
254
+class SiglipAttention(nn.Module):
255
256
def __init__(
257
self,
@@ -293,6 +290,11 @@ class SiglipParallelAttention(nn.Module):
290
self.tp_size = get_tensor_model_parallel_world_size()
291
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
292
293
+ self.attn_backend = get_vit_attn_backend(support_fa=False)
294
+ if self.attn_backend not in {_Backend.TORCH_SDPA, _Backend.XFORMERS}:
295
+ raise RuntimeError(
296
+ f"SIGLIP does not support {self.attn_backend} backend now.")
297
+
298
def forward(
299
self,
300
hidden_states: torch.Tensor,
@@ -313,11 +315,26 @@ class SiglipParallelAttention(nn.Module):
315
self.num_heads_per_partition,
316
self.head_dim)
317
316
- out = xops.memory_efficient_attention_forward(query_states,
317
- key_states,
318
- value_states,
319
- p=self.dropout,
320
- scale=self.scale)
318
+ if self.attn_backend == _Backend.XFORMERS:
319
+ from xformers import ops as xops
320
+
321
+ out = xops.memory_efficient_attention_forward(query_states,
322
+ key_states,
323
+ value_states,
324
+ p=self.dropout,
325
+ scale=self.scale)
326
+ elif self.attn_backend == _Backend.TORCH_SDPA:
327
+ query_states, key_states, value_states = (x.transpose(1, 2)
328
+ for x in (query_states,
329
+ key_states,
330
+ value_states))
331
+ out = F.scaled_dot_product_attention(query_states,
332
+ key_states,
333
+ value_states,
334
+ dropout_p=self.dropout,
335
+ scale=self.scale)
336
+ out = out.transpose(1, 2)
337
+
338
out = out.view(batch_size, q_len, -1)
339
attn_output, _ = self.out_proj(out)
340
@@ -372,17 +389,11 @@ class SiglipEncoderLayer(nn.Module):
389
390
self.embed_dim = config.hidden_size
391
375
- num_heads = config.num_attention_heads
376
- tp_size = get_tensor_model_parallel_world_size()
377
- if USE_XFORMERS_OPS and num_heads % tp_size == 0:
378
- self.self_attn = SiglipParallelAttention(
379
- config,
380
- quant_config=quant_config,
381
- prefix=f"{prefix}.self_attn",
382
- )
383
- else:
384
- self.self_attn = SiglipSdpaAttention(config)
385
-
392
+ self.self_attn = SiglipAttention(
393
+ config,
394
+ quant_config=quant_config,
395
+ prefix=f"{prefix}.self_attn",
396
+ )
397
self.layer_norm1 = nn.LayerNorm(self.embed_dim,
398
eps=config.layer_norm_eps)
399
self.mlp = SiglipMLP(
@@ -569,10 +580,6 @@ class SiglipVisionModel(nn.Module):
580
) -> None:
581
super().__init__()
582
572
- num_heads = config.num_attention_heads
573
- tp_size = get_tensor_model_parallel_world_size()
574
- self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
575
-
583
self.vision_model = SiglipVisionTransformer(
584
config,
585
quant_config,
@@ -601,7 +608,7 @@ class SiglipVisionModel(nn.Module):
608
("qkv_proj", "q_proj", "q"),
609
("qkv_proj", "k_proj", "k"),
610
("qkv_proj", "v_proj", "v"),
604
- ] if self.shard_weight else []
611
+ ]
612
params_dict = dict(self.named_parameters())
613
loaded_params: Set[str] = set()
614
layer_count = len(self.vision_model.encoder.layers)
vllm/model_executor/models/utils.py
+8
-3
@@ -587,7 +587,11 @@ class LLMWrapper(nn.Module):
587
return llm(*args, **kwargs)
588
589
590
-def get_vit_attn_backend() -> _Backend:
590
+def get_vit_attn_backend(support_fa: bool = False) -> _Backend:
591
+ """
592
+ Get the available attention backend for Vision Transformer.
593
+ """
594
+ # TODO(Isotr0py): Remove `support_fa` after support FA for all ViTs attn.
595
selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
596
if selected_backend is None:
597
backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
@@ -596,7 +600,7 @@ def get_vit_attn_backend() -> _Backend:
600
if selected_backend is None:
601
# For Volta and Turing GPUs, use xformers instead.
602
device_available = current_platform.has_device_capability(80)
599
- if device_available:
603
+ if device_available and support_fa:
604
from transformers.utils import is_flash_attn_2_available
605
if is_flash_attn_2_available():
606
selected_backend = _Backend.FLASH_ATTN
@@ -606,7 +610,8 @@ def get_vit_attn_backend() -> _Backend:
610
"so we use xformers backend instead. You can run "
611
"`pip install flash-attn` to use flash-attention backend.")
612
selected_backend = _Backend.XFORMERS
609
- elif current_platform.is_cpu():
613
+ elif current_platform.is_cpu() or current_platform.is_rocm():
614
+ # ROCM doesn't support xformers
615
selected_backend = _Backend.TORCH_SDPA
616
else:
617
selected_backend = _Backend.XFORMERS