@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec266536b7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Bugfix][VLM] Add fallback to SDPA for ViT model running on CPU backend (#8061)

Isotr0py committed Sep 3, 2024 at 21:37 UTC ec266536b7c4d4d308566ac928a69fcb9ef94462
5 files changed +157 -44
vllm/model_executor/models/blip.py
+20 -5
@@ -7,7 +7,7 @@ import torch
7 import torch.nn as nn
8 from PIL import Image
9 from transformers import Blip2VisionConfig, BlipVisionConfig
10 -from xformers import ops as xops
10 +from transformers.models.blip.modeling_blip import BlipAttention
11
12 from vllm.config import ModelConfig
13 from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -21,6 +21,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
21 repeat_and_pad_placeholder_tokens)
22 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
23
24 +try:
25 + from xformers import ops as xops
26 + USE_XFORMERS_OPS = True
27 +except ImportError:
28 + USE_XFORMERS_OPS = False
29 +
30
31 def get_blip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
32 assert image_size % patch_size == 0
@@ -156,7 +162,7 @@ class BlipVisionEmbeddings(nn.Module):
162 return embeddings
163
164
159 -class BlipAttention(nn.Module):
165 +class BlipParallelAttention(nn.Module):
166 """Multi-headed attention from 'Attention Is All You Need' paper"""
167
168 def __init__(
@@ -224,7 +230,7 @@ class BlipAttention(nn.Module):
230 out = out.view(bsz, tgt_len, -1)
231 attn_output, _ = self.projection(out)
232
227 - return attn_output
233 + return attn_output, None
234
235
236 class BlipMLP(nn.Module):
@@ -261,7 +267,16 @@ class BlipEncoderLayer(nn.Module):
267 quant_config: Optional[QuantizationConfig] = None):
268 super().__init__()
269
264 - self.self_attn = BlipAttention(config, quant_config=quant_config)
270 + # fallback to sdpa attention if tp unavailable
271 + num_heads = config.num_attention_heads
272 + tp_size = get_tensor_model_parallel_world_size()
273 + if USE_XFORMERS_OPS and num_heads % tp_size == 0:
274 + self.self_attn = BlipParallelAttention(config,
275 + quant_config=quant_config)
276 + else:
277 + # Blip doesn't have SDPA attention implemented in transformers
278 + # use eager attention instead for cpu backend
279 + self.self_attn = BlipAttention(config)
280 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
281 eps=config.layer_norm_eps)
282 self.mlp = BlipMLP(config, quant_config=quant_config)
@@ -272,7 +287,7 @@ class BlipEncoderLayer(nn.Module):
287 residual = hidden_states
288
289 hidden_states = self.layer_norm1(hidden_states)
275 - hidden_states = self.self_attn(hidden_states=hidden_states)
290 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
291 hidden_states = residual + hidden_states
292
293 residual = hidden_states
vllm/model_executor/models/clip.py
+22 -6
@@ -7,7 +7,7 @@ import torch
7 import torch.nn as nn
8 from PIL import Image
9 from transformers import CLIPVisionConfig
10 -from xformers import ops as xops
10 +from transformers.models.clip.modeling_clip import CLIPSdpaAttention
11
12 from vllm.config import ModelConfig
13 from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -22,6 +22,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
22 repeat_and_pad_placeholder_tokens)
23 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
24
25 +try:
26 + from xformers import ops as xops
27 + USE_XFORMERS_OPS = True
28 +except ImportError:
29 + USE_XFORMERS_OPS = False
30 +
31
32 def get_clip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
33 assert image_size % patch_size == 0
@@ -162,7 +168,7 @@ class CLIPVisionEmbeddings(nn.Module):
168 return embeddings
169
170
165 -class CLIPAttention(nn.Module):
171 +class CLIPParallelAttention(nn.Module):
172 """Multi-headed attention from 'Attention Is All You Need' paper"""
173
174 def __init__(
@@ -231,7 +237,7 @@ class CLIPAttention(nn.Module):
237 out = out.view(bsz, tgt_len, -1)
238 attn_output, _ = self.out_proj(out)
239
234 - return attn_output
240 + return attn_output, None
241
242
243 class CLIPMLP(nn.Module):
@@ -266,7 +272,13 @@ class CLIPEncoderLayer(nn.Module):
272 quant_config: Optional[QuantizationConfig] = None):
273 super().__init__()
274
269 - self.self_attn = CLIPAttention(config, quant_config=quant_config)
275 + num_heads = config.num_attention_heads
276 + tp_size = get_tensor_model_parallel_world_size()
277 + if USE_XFORMERS_OPS and num_heads % tp_size == 0:
278 + self.self_attn = CLIPParallelAttention(config,
279 + quant_config=quant_config)
280 + else:
281 + self.self_attn = CLIPSdpaAttention(config)
282 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
283 eps=config.layer_norm_eps)
284 self.mlp = CLIPMLP(config, quant_config=quant_config)
@@ -278,7 +290,7 @@ class CLIPEncoderLayer(nn.Module):
290 residual = hidden_states
291
292 hidden_states = self.layer_norm1(hidden_states)
281 - hidden_states = self.self_attn(hidden_states=hidden_states)
293 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
294 hidden_states = residual + hidden_states
295
296 residual = hidden_states
@@ -365,6 +377,10 @@ class CLIPVisionModel(nn.Module):
377 quant_config: Optional[QuantizationConfig] = None,
378 num_hidden_layers_override: Optional[int] = None):
379 super().__init__()
380 + tp_size = get_tensor_model_parallel_world_size()
381 + num_heads = config.num_attention_heads
382 + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
383 +
384 self.vision_model = CLIPVisionTransformer(
385 config=config,
386 quant_config=quant_config,
@@ -386,7 +402,7 @@ class CLIPVisionModel(nn.Module):
402 ("qkv_proj", "q_proj", "q"),
403 ("qkv_proj", "k_proj", "k"),
404 ("qkv_proj", "v_proj", "v"),
389 - ]
405 + ] if self.shard_weight else []
406 params_dict = dict(self.named_parameters())
407 layer_count = len(self.vision_model.encoder.layers)
408
vllm/model_executor/models/intern_vit.py
+70 -9
@@ -10,7 +10,6 @@ import torch
10 import torch.nn as nn
11 import torch.nn.functional as F
12 from transformers import PretrainedConfig
13 -from xformers import ops as xops
13
14 from vllm.distributed import divide, get_tensor_model_parallel_world_size
15 from vllm.model_executor.layers.activation import get_act_fn
@@ -21,6 +20,12 @@ from vllm.model_executor.layers.linear import (ColumnParallelLinear,
20 from vllm.model_executor.layers.quantization import QuantizationConfig
21 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
22
23 +try:
24 + from xformers import ops as xops
25 + USE_XFORMERS_OPS = True
26 +except ImportError:
27 + USE_XFORMERS_OPS = False
28 +
29 NORM2FN = {
30 'rms_norm': RMSNorm,
31 'layer_norm': nn.LayerNorm,
@@ -81,7 +86,7 @@ class InternVisionEmbeddings(nn.Module):
86 return embeddings
87
88
84 -class InternAttention(nn.Module):
89 +class InternParallelAttention(nn.Module):
90 """Multi-headed attention from 'Attention Is All You Need' paper"""
91
92 def __init__(
@@ -140,18 +145,67 @@ class InternAttention(nn.Module):
145 k = self.k_norm.forward_native(k.flatten(-2,
146 -1)).view(B_, N_, H_, D_)
147
143 - x = xops.memory_efficient_attention_forward(
144 - q,
145 - k,
146 - v,
147 - scale=self.scale,
148 - )
148 + x = xops.memory_efficient_attention_forward(q, k, v, scale=self.scale)
149 x = x.view(B, N, -1)
150
151 x, _ = self.proj(x)
152 return x
153
154
155 +class InternSdpaAttention(nn.Module):
156 + """Multi-headed attention from 'Attention Is All You Need' paper"""
157 +
158 + def __init__(self, config: PretrainedConfig):
159 + super().__init__()
160 + self.config = config
161 + self.embed_dim = config.hidden_size
162 + self.num_heads = config.num_attention_heads
163 + self.head_dim = self.embed_dim // self.num_heads
164 + if self.head_dim * self.num_heads != self.embed_dim:
165 + raise ValueError(
166 + f'embed_dim must be divisible by num_heads '
167 + f'(got `embed_dim`: {self.embed_dim} and `num_heads`:'
168 + f' {self.num_heads}).')
169 +
170 + self.scale = self.head_dim**-0.5
171 + self.qkv = nn.Linear(self.embed_dim,
172 + 3 * self.embed_dim,
173 + bias=config.qkv_bias)
174 +
175 + self.qk_normalization = config.qk_normalization
176 +
177 + if self.qk_normalization:
178 + self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
179 + self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
180 +
181 + self.proj = nn.Linear(self.embed_dim, self.embed_dim)
182 +
183 + def forward(self, x):
184 + B, N, C = x.shape
185 + qkv = self.qkv(x)
186 + q, k, v = qkv.chunk(3, dim=-1)
187 +
188 + q = q.view(B, N, self.num_heads, self.head_dim)
189 + k = k.view(B, N, self.num_heads, self.head_dim)
190 + v = v.view(B, N, self.num_heads, self.head_dim)
191 +
192 + if self.qk_normalization:
193 + B_, N_, H_, D_ = q.shape
194 + q = self.q_norm.forward_native(q.flatten(-2,
195 + -1)).view(B_, N_, H_, D_)
196 + k = self.k_norm.forward_native(k.flatten(-2,
197 + -1)).view(B_, N_, H_, D_)
198 + q = q.transpose(1, 2)
199 + k = k.transpose(1, 2)
200 + v = v.transpose(1, 2)
201 +
202 + x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
203 + x = x.transpose(1, 2).view(B, N, -1)
204 +
205 + x = self.proj(x)
206 + return x
207 +
208 +
209 class InternMLP(nn.Module):
210
211 def __init__(self,
@@ -187,7 +241,14 @@ class InternVisionEncoderLayer(nn.Module):
241 self.intermediate_size = config.intermediate_size
242 self.norm_type = config.norm_type
243
190 - self.attn = InternAttention(config, quant_config=quant_config)
244 + # fallback to sdpa attention if tp unavailable
245 + tp_size = get_tensor_model_parallel_world_size()
246 + num_heads = config.num_attention_heads
247 + if USE_XFORMERS_OPS and num_heads % tp_size == 0:
248 + self.attn = InternParallelAttention(config,
249 + quant_config=quant_config)
250 + else:
251 + self.attn = InternSdpaAttention(config)
252 self.mlp = InternMLP(config, quant_config=quant_config)
253 self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
254 eps=config.layer_norm_eps)
vllm/model_executor/models/paligemma.py
+23 -19
@@ -307,26 +307,30 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
307 if key_to_modify in name:
308 name = name.replace(key_to_modify, new_key)
309 use_default_weight_loading = False
310 - for (param_name, shard_name, shard_id) in stacked_params_mapping:
311 - if shard_name not in name:
312 - continue
313 - name = name.replace(shard_name, param_name)
314 - # Skip loading extra bias for GPTQ models.
315 - if name.endswith(".bias") and name not in params_dict:
316 - continue
317 - param = params_dict[name]
318 - weight_loader = param.weight_loader
319 - weight_loader(param, loaded_weight, shard_id)
320 - break
310 + if "vision" not in name or self.vision_tower.shard_weight:
311 + for (param_name, shard_name,
312 + shard_id) in stacked_params_mapping:
313 + if shard_name not in name:
314 + continue
315 + name = name.replace(shard_name, param_name)
316 + # Skip loading extra bias for GPTQ models.
317 + if name.endswith(".bias") and name not in params_dict:
318 + continue
319 + param = params_dict[name]
320 + weight_loader = param.weight_loader
321 + weight_loader(param, loaded_weight, shard_id)
322 + break
323 + else:
324 + # lm_head is not used in vllm as it is tied with
325 + # embed_token. To prevent errors, skip loading
326 + # lm_head.weight.
327 + if "lm_head.weight" in name:
328 + continue
329 + # Skip loading extra bias for GPTQ models.
330 + if name.endswith(".bias") and name not in params_dict:
331 + continue
332 + use_default_weight_loading = True
333 else:
322 - # lm_head is not used in vllm as it is tied with
323 - # embed_token. To prevent errors, skip loading
324 - # lm_head.weight.
325 - if "lm_head.weight" in name:
326 - continue
327 - # Skip loading extra bias for GPTQ models.
328 - if name.endswith(".bias") and name not in params_dict:
329 - continue
334 use_default_weight_loading = True
335
336 if use_default_weight_loading:
vllm/model_executor/models/siglip.py
+22 -5
@@ -9,7 +9,7 @@ import torch
9 from PIL import Image
10 from torch import nn
11 from transformers import SiglipVisionConfig
12 -from xformers import ops as xops
12 +from transformers.models.siglip.modeling_siglip import SiglipSdpaAttention
13
14 from vllm.config import ModelConfig
15 from vllm.distributed import divide, get_tensor_model_parallel_world_size
@@ -26,6 +26,12 @@ from vllm.multimodal.utils import (cached_get_tokenizer,
26 repeat_and_pad_placeholder_tokens)
27 from vllm.sequence import VLLM_TOKEN_ID_ARRAY_TYPE, SequenceData
28
29 +try:
30 + from xformers import ops as xops
31 + USE_XFORMERS_OPS = True
32 +except ImportError:
33 + USE_XFORMERS_OPS = False
34 +
35
36 def get_siglip_patch_grid_length(*, image_size: int, patch_size: int) -> int:
37 # Since interpolation is applied, the image size need not be divisible
@@ -219,7 +225,7 @@ class SiglipVisionEmbeddings(nn.Module):
225 return embeddings
226
227
222 -class SiglipAttention(nn.Module):
228 +class SiglipParallelAttention(nn.Module):
229
230 def __init__(
231 self,
@@ -282,7 +288,7 @@ class SiglipAttention(nn.Module):
288 out = out.view(batch_size, q_len, -1)
289 attn_output, _ = self.out_proj(out)
290
285 - return attn_output
291 + return attn_output, None
292
293
294 class SiglipMLP(nn.Module):
@@ -327,7 +333,14 @@ class SiglipEncoderLayer(nn.Module):
333 super().__init__()
334 self.embed_dim = config.hidden_size
335
330 - self.self_attn = SiglipAttention(config, quant_config=quant_config)
336 + num_heads = config.num_attention_heads
337 + tp_size = get_tensor_model_parallel_world_size()
338 + if USE_XFORMERS_OPS and num_heads % tp_size == 0:
339 + self.self_attn = SiglipParallelAttention(config,
340 + quant_config=quant_config)
341 + else:
342 + self.self_attn = SiglipSdpaAttention(config)
343 +
344 self.layer_norm1 = nn.LayerNorm(self.embed_dim,
345 eps=config.layer_norm_eps)
346 self.mlp = SiglipMLP(
@@ -344,7 +357,7 @@ class SiglipEncoderLayer(nn.Module):
357 residual = hidden_states
358
359 hidden_states = self.layer_norm1(hidden_states)
347 - hidden_states = self.self_attn(hidden_states=hidden_states)
360 + hidden_states, _ = self.self_attn(hidden_states=hidden_states)
361 hidden_states = residual + hidden_states
362
363 residual = hidden_states
@@ -476,6 +489,10 @@ class SiglipVisionModel(nn.Module):
489 num_hidden_layers_override: Optional[int] = None,
490 ):
491 super().__init__()
492 + num_heads = config.num_attention_heads
493 + tp_size = get_tensor_model_parallel_world_size()
494 + self.shard_weight = USE_XFORMERS_OPS and num_heads % tp_size == 0
495 +
496 self.vision_model = SiglipVisionTransformer(
497 config,
498 quant_config,