A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Bugfix][VLM] Add fallback to SDPA for ViT model running on CPU backend (#8061)
Isotr0py committed
Sep 3, 2024 at 21:37 UTC
ec266536b7c4d4d308566ac928a69fcb9ef94462
5 files changed
+157
-44
vllm/model_executor/models/blip.py
+20
-5
@@ -7,7 +7,7 @@ import torch
7
import torch.nn as nn
8
from PIL import Image
9
from transformers import Blip2VisionConfig, BlipVisionConfig
10
-from xformers import ops as xops
10
+from transformers.models.blip.modeling_blip import BlipAttention
11
12
from vllm.config import ModelConfig
13
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -21,6 +21,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
21
repeat_and_pad_placeholder_tokens)
22
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
23
24
+try:
25
+ from xformers import ops as xops
26
+ USE_XFORMERS_OPS = True
27
+except ImportError:
28
+ USE_XFORMERS_OPS = False
29
+
30
31
def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
32
assert image_size % patch_size == 0
@@ -156,7 +162,7 @@ class BlipVisionEmbeddings(nn.Module):
162
return embeddings
163
164
159
-class BlipAttention(nn.Module):
165
+class BlipParallelAttention(nn.Module):
166
"""Multi-headed attention from 'Attention Is All You Need' paper"""
167
168
def __init__(
@@ -224,7 +230,7 @@ class BlipAttention(nn.Module):
230
out = out.view(bsz, tgt_len, -1)
231
attn_output, _ = self.projection(out)
232
227
- return attn_output
233
+ return attn_output, None
234
235
236
class BlipMLP(nn.Module):
@@ -261,7 +267,16 @@ class BlipEncoderLayer(nn.Module):
267
quant_config: Optional[QuantizationConfig] = None):
268
super().__init__()
269
264
- self.self_attn = BlipAttention(config, quant_config=quant_config)
270
+ # fallback to sdpa attention if tp unavailable
271
+ num_heads = config.num_attention_heads
272
+ tp_size = get_tensor_model_parallel_world_size()
273
+ if USE_XFORMERS_OPS and num_heads % tp_size == 0:
274
+ self.self_attn = BlipParallelAttention(config,
275
+ quant_config=quant_config)
276
+ else:
277
+ # Blip doesn't have SDPA attention implemented in transformers
278
+ # use eager attention instead for cpu backend
279
+ self.self_attn = BlipAttention(config)
280
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
281
eps=config.layer_norm_eps)
282
self.mlp = BlipMLP(config, quant_config=quant_config)
@@ -272,7 +287,7 @@ class BlipEncoderLayer(nn.Module):
287
residual = hidden_states
288
289
hidden_states = self.layer_norm1(hidden_states)
275
- hidden_states = self.self_attn(hidden_states=hidden_states)
290
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
291
hidden_states = residual + hidden_states
292
293
residual = hidden_states
vllm/model_executor/models/clip.py
+22
-6
@@ -7,7 +7,7 @@ import torch
7
import torch.nn as nn
8
from PIL import Image
9
from transformers import CLIPVisionConfig
10
-from xformers import ops as xops
10
+from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12
from vllm.config import ModelConfig
13
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -22,6 +22,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
22
repeat_and_pad_placeholder_tokens)
23
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
24
25
+try:
26
+ from xformers import ops as xops
27
+ USE_XFORMERS_OPS = True
28
+except ImportError:
29
+ USE_XFORMERS_OPS = False
30
+
31
32
def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
33
assert image_size % patch_size == 0
@@ -162,7 +168,7 @@ class CLIPVisionEmbeddings(nn.Module):
168
return embeddings
169
170
165
-class CLIPAttention(nn.Module):
171
+class CLIPParallelAttention(nn.Module):
172
"""Multi-headed attention from 'Attention Is All You Need' paper"""
173
174
def __init__(
@@ -231,7 +237,7 @@ class CLIPAttention(nn.Module):
237
out = out.view(bsz, tgt_len, -1)
238
attn_output, _ = self.out_proj(out)
239
234
- return attn_output
240
+ return attn_output, None
241
242
243
class CLIPMLP(nn.Module):
@@ -266,7 +272,13 @@ class CLIPEncoderLayer(nn.Module):
272
quant_config: Optional[QuantizationConfig] = None):
273
super().__init__()
274
269
- self.self_attn = CLIPAttention(config, quant_config=quant_config)
275
+ num_heads = config.num_attention_heads
276
+ tp_size = get_tensor_model_parallel_world_size()
277
+ if USE_XFORMERS_OPS and num_heads % tp_size == 0:
278
+ self.self_attn = CLIPParallelAttention(config,
279
+ quant_config=quant_config)
280
+ else:
281
+ self.self_attn = CLIPSdpaAttention(config)
282
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
283
eps=config.layer_norm_eps)
284
self.mlp = CLIPMLP(config, quant_config=quant_config)
@@ -278,7 +290,7 @@ class CLIPEncoderLayer(nn.Module):
290
residual = hidden_states
291
292
hidden_states = self.layer_norm1(hidden_states)
281
- hidden_states = self.self_attn(hidden_states=hidden_states)
293
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
294
hidden_states = residual + hidden_states
295
296
residual = hidden_states
@@ -365,6 +377,10 @@ class CLIPVisionModel(nn.Module):
377
quant_config: Optional[QuantizationConfig] = None,
378
num_hidden_layers_override: Optional[int] = None):
379
super().__init__()
380
+ tp_size = get_tensor_model_parallel_world_size()
381
+ num_heads = config.num_attention_heads
382
+ self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
383
+
384
self.vision_model = CLIPVisionTransformer(
385
config=config,
386
quant_config=quant_config,
@@ -386,7 +402,7 @@ class CLIPVisionModel(nn.Module):
402
("qkv_proj", "q_proj", "q"),
403
("qkv_proj", "k_proj", "k"),
404
("qkv_proj", "v_proj", "v"),
389
- ]
405
+ ] if self.shard_weight else []
406
params_dict = dict(self.named_parameters())
407
layer_count = len(self.vision_model.encoder.layers)
408
vllm/model_executor/models/intern_vit.py
+70
-9
@@ -10,7 +10,6 @@ import torch
10
import torch.nn as nn
11
import torch.nn.functional as F
12
from transformers import PretrainedConfig
13
-from xformers import ops as xops
13
14
from vllm.distributed import divide, get_tensor_model_parallel_world_size
15
from vllm.model_executor.layers.activation import get_act_fn
@@ -21,6 +20,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
23
+try:
24
+ from xformers import ops as xops
25
+ USE_XFORMERS_OPS = True
26
+except ImportError:
27
+ USE_XFORMERS_OPS = False
28
+
29
NORM2FN = {
30
'rms_norm': RMSNorm,
31
'layer_norm': nn.LayerNorm,
@@ -81,7 +86,7 @@ class InternVisionEmbeddings(nn.Module):
86
return embeddings
87
88
84
-class InternAttention(nn.Module):
89
+class InternParallelAttention(nn.Module):
90
"""Multi-headed attention from 'Attention Is All You Need' paper"""
91
92
def __init__(
@@ -140,18 +145,67 @@ class InternAttention(nn.Module):
145
k = self.k_norm.forward_native(k.flatten(-2,
146
-1)).view(B_, N_, H_, D_)
147
143
- x = xops.memory_efficient_attention_forward(
144
- q,
145
- k,
146
- v,
147
- scale=self.scale,
148
- )
148
+ x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
149
x = x.view(B, N, -1)
150
151
x, _ = self.proj(x)
152
return x
153
154
155
+class InternSdpaAttention(nn.Module):
156
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
157
+
158
+ def __init__(self, config: PretrainedConfig):
159
+ super().__init__()
160
+ self.config = config
161
+ self.embed_dim = config.hidden_size
162
+ self.num_heads = config.num_attention_heads
163
+ self.head_dim = self.embed_dim // self.num_heads
164
+ if self.head_dim * self.num_heads != self.embed_dim:
165
+ raise ValueError(
166
+ f'embed_dim must be divisible by num_heads '
167
+ f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
168
+ f' {self.num_heads}).')
169
+
170
+ self.scale = self.head_dim**-0.5
171
+ self.qkv = nn.Linear(self.embed_dim,
172
+ 3 * self.embed_dim,
173
+ bias=config.qkv_bias)
174
+
175
+ self.qk_normalization = config.qk_normalization
176
+
177
+ if self.qk_normalization:
178
+ self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
179
+ self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
180
+
181
+ self.proj = nn.Linear(self.embed_dim, self.embed_dim)
182
+
183
+ def forward(self, x):
184
+ B, N, C = x.shape
185
+ qkv = self.qkv(x)
186
+ q, k, v = qkv.chunk(3, dim=-1)
187
+
188
+ q = q.view(B, N, self.num_heads, self.head_dim)
189
+ k = k.view(B, N, self.num_heads, self.head_dim)
190
+ v = v.view(B, N, self.num_heads, self.head_dim)
191
+
192
+ if self.qk_normalization:
193
+ B_, N_, H_, D_ = q.shape
194
+ q = self.q_norm.forward_native(q.flatten(-2,
195
+ -1)).view(B_, N_, H_, D_)
196
+ k = self.k_norm.forward_native(k.flatten(-2,
197
+ -1)).view(B_, N_, H_, D_)
198
+ q = q.transpose(1, 2)
199
+ k = k.transpose(1, 2)
200
+ v = v.transpose(1, 2)
201
+
202
+ x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
203
+ x = x.transpose(1, 2).view(B, N, -1)
204
+
205
+ x = self.proj(x)
206
+ return x
207
+
208
+
209
class InternMLP(nn.Module):
210
211
def __init__(self,
@@ -187,7 +241,14 @@ class InternVisionEncoderLayer(nn.Module):
241
self.intermediate_size = config.intermediate_size
242
self.norm_type = config.norm_type
243
190
- self.attn = InternAttention(config, quant_config=quant_config)
244
+ # fallback to sdpa attention if tp unavailable
245
+ tp_size = get_tensor_model_parallel_world_size()
246
+ num_heads = config.num_attention_heads
247
+ if USE_XFORMERS_OPS and num_heads % tp_size == 0:
248
+ self.attn = InternParallelAttention(config,
249
+ quant_config=quant_config)
250
+ else:
251
+ self.attn = InternSdpaAttention(config)
252
self.mlp = InternMLP(config, quant_config=quant_config)
253
self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
254
eps=config.layer_norm_eps)
vllm/model_executor/models/paligemma.py
+23
-19
@@ -307,26 +307,30 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
307
if key_to_modify in name:
308
name = name.replace(key_to_modify, new_key)
309
use_default_weight_loading = False
310
- for (param_name, shard_name, shard_id) in stacked_params_mapping:
311
- if shard_name not in name:
312
- continue
313
- name = name.replace(shard_name, param_name)
314
- # Skip loading extra bias for GPTQ models.
315
- if name.endswith(".bias") and name not in params_dict:
316
- continue
317
- param = params_dict[name]
318
- weight_loader = param.weight_loader
319
- weight_loader(param, loaded_weight, shard_id)
320
- break
310
+ if "vision" not in name or self.vision_tower.shard_weight:
311
+ for (param_name, shard_name,
312
+ shard_id) in stacked_params_mapping:
313
+ if shard_name not in name:
314
+ continue
315
+ name = name.replace(shard_name, param_name)
316
+ # Skip loading extra bias for GPTQ models.
317
+ if name.endswith(".bias") and name not in params_dict:
318
+ continue
319
+ param = params_dict[name]
320
+ weight_loader = param.weight_loader
321
+ weight_loader(param, loaded_weight, shard_id)
322
+ break
323
+ else:
324
+ # lm_head is not used in vllm as it is tied with
325
+ # embed_token. To prevent errors, skip loading
326
+ # lm_head.weight.
327
+ if "lm_head.weight" in name:
328
+ continue
329
+ # Skip loading extra bias for GPTQ models.
330
+ if name.endswith(".bias") and name not in params_dict:
331
+ continue
332
+ use_default_weight_loading = True
333
else:
322
- # lm_head is not used in vllm as it is tied with
323
- # embed_token. To prevent errors, skip loading
324
- # lm_head.weight.
325
- if "lm_head.weight" in name:
326
- continue
327
- # Skip loading extra bias for GPTQ models.
328
- if name.endswith(".bias") and name not in params_dict:
329
- continue
334
use_default_weight_loading = True
335
336
if use_default_weight_loading:
vllm/model_executor/models/siglip.py
+22
-5
@@ -9,7 +9,7 @@ import torch
9
from PIL import Image
10
from torch import nn
11
from transformers import SiglipVisionConfig
12
-from xformers import ops as xops
12
+from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14
from vllm.config import ModelConfig
15
from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -26,6 +26,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
26
repeat_and_pad_placeholder_tokens)
27
from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
28
29
+try:
30
+ from xformers import ops as xops
31
+ USE_XFORMERS_OPS = True
32
+except ImportError:
33
+ USE_XFORMERS_OPS = False
34
+
35
36
def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
37
# Since interpolation is applied, the image size need not be divisible
@@ -219,7 +225,7 @@ class SiglipVisionEmbeddings(nn.Module):
225
return embeddings
226
227
222
-class SiglipAttention(nn.Module):
228
+class SiglipParallelAttention(nn.Module):
229
230
def __init__(
231
self,
@@ -282,7 +288,7 @@ class SiglipAttention(nn.Module):
288
out = out.view(batch_size, q_len, -1)
289
attn_output, _ = self.out_proj(out)
290
285
- return attn_output
291
+ return attn_output, None
292
293
294
class SiglipMLP(nn.Module):
@@ -327,7 +333,14 @@ class SiglipEncoderLayer(nn.Module):
333
super().__init__()
334
self.embed_dim = config.hidden_size
335
330
- self.self_attn = SiglipAttention(config, quant_config=quant_config)
336
+ num_heads = config.num_attention_heads
337
+ tp_size = get_tensor_model_parallel_world_size()
338
+ if USE_XFORMERS_OPS and num_heads % tp_size == 0:
339
+ self.self_attn = SiglipParallelAttention(config,
340
+ quant_config=quant_config)
341
+ else:
342
+ self.self_attn = SiglipSdpaAttention(config)
343
+
344
self.layer_norm1 = nn.LayerNorm(self.embed_dim,
345
eps=config.layer_norm_eps)
346
self.mlp = SiglipMLP(
@@ -344,7 +357,7 @@ class SiglipEncoderLayer(nn.Module):
357
residual = hidden_states
358
359
hidden_states = self.layer_norm1(hidden_states)
347
- hidden_states = self.self_attn(hidden_states=hidden_states)
360
+ hidden_states, _ = self.self_attn(hidden_states=hidden_states)
361
hidden_states = residual + hidden_states
362
363
residual = hidden_states
@@ -476,6 +489,10 @@ class SiglipVisionModel(nn.Module):
489
num_hidden_layers_override: Optional[int] = None,
490
):
491
super().__init__()
492
+ num_heads = config.num_attention_heads
493
+ tp_size = get_tensor_model_parallel_world_size()
494
+ self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
495
+
496
self.vision_model = SiglipVisionTransformer(
497
config,
498
quant_config,