@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f97be32d1d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[VLM][Model] TP support for ViTs (#7186)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Jungho Christopher Cho committed Aug 31, 2024 at 00:19 UTC f97be32d1da4cfda933a0dbfbc681861f96390d9
9 files changed +340 -289
tests/models/test_intern_vit.py
+1 -2
@@ -6,8 +6,6 @@ import torch.nn as nn
6 from huggingface_hub import snapshot_download
7 from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
9 -from vllm.model_executor.models.intern_vit import InternVisionModel
10 -
9 from ..conftest import _ImageAssets, cleanup
10
11 pytestmark = pytest.mark.vlm
@@ -49,6 +47,7 @@ def run_intern_vit_test(
47 for pixel_value in pixel_values
48 ]
49
50 + from vllm.model_executor.models.intern_vit import InternVisionModel
51 vllm_model = InternVisionModel(config)
52 vllm_model.load_weights(hf_model.state_dict().items())
53
tests/models/test_internvl.py
+31 -32
@@ -6,9 +6,6 @@ import torch
6 from PIL.Image import Image
7 from transformers import AutoConfig
8
9 -from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
10 - IMG_START,
11 - image_to_pixel_values)
9 from vllm.multimodal.utils import rescale_image_size
10 from vllm.utils import is_cpu
11
@@ -33,35 +30,6 @@ models = [
30 ]
31
32
36 -class InternVLProcessor:
37 - """A simple processor for InternVL2 HF model which misses a processor."""
38 -
39 - def __init__(self, hf_runner: HfRunner):
40 - self.num_image_token = hf_runner.model.num_image_token
41 - self.tokenizer = hf_runner.tokenizer
42 - self.dtype = hf_runner.model.dtype
43 -
44 - self.config = AutoConfig.from_pretrained(hf_runner.model_name)
45 - self.vision_config = self.config.vision_config
46 - self.use_thumbnail = self.config.use_thumbnail
47 - self.min_num = self.config.min_dynamic_patch
48 - self.max_num = self.config.max_dynamic_patch
49 - self.image_size = self.vision_config.image_size
50 -
51 - def __call__(self, text: str, images: Image, **kwargs):
52 - pixel_values = image_to_pixel_values(images, self.image_size,
53 - self.min_num, self.max_num,
54 - self.use_thumbnail).to(self.dtype)
55 - num_patches_list = [pixel_values.shape[0]]
56 - for num_patches in num_patches_list:
57 - context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
58 - image_tokens = IMG_START + context_tokens + IMG_END
59 - text = text.replace('<image>', image_tokens, 1)
60 - prompt = self.tokenizer(text, return_tensors="pt")
61 - prompt.update({"pixel_values": pixel_values})
62 - return prompt
63 -
64 -
33 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
34 def generate(
35 self,
@@ -127,6 +95,37 @@ def run_test(
95 # if we run HF first, the cuda initialization will be done and it
96 # will hurt multiprocessing backend with fork method (the default method).
97
98 + class InternVLProcessor:
99 + """A simple processor for InternVL2 which misses a processor."""
100 +
101 + def __init__(self, hf_runner: HfRunner):
102 + self.num_image_token = hf_runner.model.num_image_token
103 + self.tokenizer = hf_runner.tokenizer
104 + self.dtype = hf_runner.model.dtype
105 +
106 + self.config = AutoConfig.from_pretrained(hf_runner.model_name)
107 + self.vision_config = self.config.vision_config
108 + self.use_thumbnail = self.config.use_thumbnail
109 + self.min_num = self.config.min_dynamic_patch
110 + self.max_num = self.config.max_dynamic_patch
111 + self.image_size = self.vision_config.image_size
112 +
113 + def __call__(self, text: str, images: Image, **kwargs):
114 + from vllm.model_executor.models.internvl import (
115 + IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
116 + pixel_values = image_to_pixel_values(
117 + images, self.image_size, self.min_num, self.max_num,
118 + self.use_thumbnail).to(self.dtype)
119 + num_patches_list = [pixel_values.shape[0]]
120 + for num_patches in num_patches_list:
121 + context_tokens = IMG_CONTEXT * self.num_image_token \
122 + * num_patches
123 + image_tokens = IMG_START + context_tokens + IMG_END
124 + text = text.replace('<image>', image_tokens, 1)
125 + prompt = self.tokenizer(text, return_tensors="pt")
126 + prompt.update({"pixel_values": pixel_values})
127 + return prompt
128 +
129 # max_model_len should be greater than image_feature_size
130 with vllm_runner(model,
131 max_model_len=4096,
vllm/model_executor/models/blip.py
+76 -3
@@ -7,12 +7,14 @@ import torch
7 import torch.nn as nn
8 from PIL import Image
9 from transformers import Blip2VisionConfig, BlipVisionConfig
10 -from transformers.models.blip.modeling_blip import BlipAttention
10 +from xformers import ops as xops
11
12 from vllm.config import ModelConfig
13 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
14 from vllm.inputs import LLMInputs
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 + QKVParallelLinear,
18 RowParallelLinear)
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -154,6 +156,77 @@ class BlipVisionEmbeddings(nn.Module):
156 return embeddings
157
158
159 +class BlipAttention(nn.Module):
160 + """Multi-headed attention from 'Attention Is All You Need' paper"""
161 +
162 + def __init__(
163 + self,
164 + config: BlipVisionConfig,
165 + quant_config: Optional[QuantizationConfig] = None,
166 + ):
167 + super().__init__()
168 + self.config = config
169 + self.embed_dim = config.hidden_size
170 + self.num_heads = config.num_attention_heads
171 + self.head_dim = self.embed_dim // self.num_heads
172 + if self.head_dim * self.num_heads != self.embed_dim:
173 + raise ValueError(
174 + "embed_dim must be divisible by num_heads "
175 + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
176 + f" {self.num_heads}).")
177 + self.scale = self.head_dim**-0.5
178 + self.dropout = config.attention_dropout
179 +
180 + self.qkv = QKVParallelLinear(
181 + self.embed_dim,
182 + self.head_dim,
183 + self.num_heads,
184 + bias=config.qkv_bias,
185 + quant_config=quant_config,
186 + )
187 + self.projection = RowParallelLinear(
188 + self.embed_dim,
189 + self.embed_dim,
190 + quant_config=quant_config,
191 + )
192 +
193 + self.tp_size = get_tensor_model_parallel_world_size()
194 + self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
195 +
196 + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
197 + return tensor.view(bsz, seq_len, self.num_heads,
198 + self.head_dim).transpose(1, 2).contiguous()
199 +
200 + def forward(
201 + self,
202 + hidden_states: torch.Tensor,
203 + ):
204 + """Input shape: Batch x Time x Channel"""
205 + bsz, tgt_len, _ = hidden_states.size()
206 +
207 + qkv_states, _ = self.qkv(hidden_states)
208 + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
209 + query_states = query_states.view(bsz, tgt_len,
210 + self.num_heads_per_partition,
211 + self.head_dim)
212 + key_states = key_states.view(bsz, tgt_len,
213 + self.num_heads_per_partition,
214 + self.head_dim)
215 + value_states = value_states.view(bsz, tgt_len,
216 + self.num_heads_per_partition,
217 + self.head_dim)
218 +
219 + out = xops.memory_efficient_attention_forward(query_states,
220 + key_states,
221 + value_states,
222 + p=self.dropout,
223 + scale=self.scale)
224 + out = out.view(bsz, tgt_len, -1)
225 + attn_output, _ = self.projection(out)
226 +
227 + return attn_output
228 +
229 +
230 class BlipMLP(nn.Module):
231
232 def __init__(self,
@@ -188,7 +261,7 @@ class BlipEncoderLayer(nn.Module):
261 quant_config: Optional[QuantizationConfig] = None):
262 super().__init__()
263
191 - self.self_attn = BlipAttention(config)
264 + self.self_attn = BlipAttention(config, quant_config=quant_config)
265 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
266 eps=config.layer_norm_eps)
267 self.mlp = BlipMLP(config, quant_config=quant_config)
@@ -199,7 +272,7 @@ class BlipEncoderLayer(nn.Module):
272 residual = hidden_states
273
274 hidden_states = self.layer_norm1(hidden_states)
202 - hidden_states, _ = self.self_attn(hidden_states=hidden_states)
275 + hidden_states = self.self_attn(hidden_states=hidden_states)
276 hidden_states = residual + hidden_states
277
278 residual = hidden_states
vllm/model_executor/models/blip2.py
+1 -2
@@ -714,8 +714,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
714 use_default_weight_loading = False
715 if "vision" in name:
716 if self.vision_model is not None:
717 - # We only do sharding for language model and
718 - # not vision model for now.
717 + # BlipVisionModel does not need sharding
718 use_default_weight_loading = True
719 else:
720 for (param_name, weight_name,
vllm/model_executor/models/clip.py
+98 -7
@@ -7,12 +7,14 @@ import torch
7 import torch.nn as nn
8 from PIL import Image
9 from transformers import CLIPVisionConfig
10 -from transformers.models.clip.modeling_clip import CLIPAttention
10 +from xformers import ops as xops
11
12 from vllm.config import ModelConfig
13 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
14 from vllm.inputs import LLMInputs
15 from vllm.model_executor.layers.activation import get_act_fn
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17 + QKVParallelLinear,
18 RowParallelLinear)
19 from vllm.model_executor.layers.quantization import QuantizationConfig
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -160,6 +162,78 @@ class CLIPVisionEmbeddings(nn.Module):
162 return embeddings
163
164
165 +class CLIPAttention(nn.Module):
166 + """Multi-headed attention from 'Attention Is All You Need' paper"""
167 +
168 + def __init__(
169 + self,
170 + config: CLIPVisionConfig,
171 + quant_config: Optional[QuantizationConfig] = None,
172 + ):
173 + super().__init__()
174 + self.config = config
175 + self.embed_dim = config.hidden_size
176 + self.num_heads = config.num_attention_heads
177 + self.head_dim = self.embed_dim // self.num_heads
178 + if self.head_dim * self.num_heads != self.embed_dim:
179 + raise ValueError(
180 + "embed_dim must be divisible by num_heads "
181 + f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
182 + f" {self.num_heads}).")
183 + self.scale = self.head_dim**-0.5
184 + self.dropout = config.attention_dropout
185 +
186 + self.qkv_proj = QKVParallelLinear(
187 + hidden_size=self.embed_dim,
188 + head_size=self.head_dim,
189 + total_num_heads=self.num_heads,
190 + quant_config=quant_config,
191 + )
192 +
193 + self.out_proj = RowParallelLinear(
194 + input_size=self.embed_dim,
195 + output_size=self.embed_dim,
196 + quant_config=quant_config,
197 + )
198 +
199 + self.tp_size = get_tensor_model_parallel_world_size()
200 + self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
201 +
202 + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
203 + return tensor.view(bsz, seq_len, self.num_heads,
204 + self.head_dim).transpose(1, 2).contiguous()
205 +
206 + def forward(
207 + self,
208 + hidden_states: torch.Tensor,
209 + ):
210 + """Input shape: Batch x Time x Channel"""
211 + bsz, tgt_len, _ = hidden_states.size()
212 +
213 + qkv_states, _ = self.qkv_proj(hidden_states)
214 + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
215 +
216 + query_states = query_states.view(bsz, tgt_len,
217 + self.num_heads_per_partition,
218 + self.head_dim)
219 + key_states = key_states.view(bsz, tgt_len,
220 + self.num_heads_per_partition,
221 + self.head_dim)
222 + value_states = value_states.view(bsz, tgt_len,
223 + self.num_heads_per_partition,
224 + self.head_dim)
225 +
226 + out = xops.memory_efficient_attention_forward(query_states,
227 + key_states,
228 + value_states,
229 + p=self.dropout,
230 + scale=self.scale)
231 + out = out.view(bsz, tgt_len, -1)
232 + attn_output, _ = self.out_proj(out)
233 +
234 + return attn_output
235 +
236 +
237 class CLIPMLP(nn.Module):
238
239 def __init__(self,
@@ -192,7 +266,7 @@ class CLIPEncoderLayer(nn.Module):
266 quant_config: Optional[QuantizationConfig] = None):
267 super().__init__()
268
195 - self.self_attn = CLIPAttention(config)
269 + self.self_attn = CLIPAttention(config, quant_config=quant_config)
270 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
271 eps=config.layer_norm_eps)
272 self.mlp = CLIPMLP(config, quant_config=quant_config)
@@ -204,7 +278,7 @@ class CLIPEncoderLayer(nn.Module):
278 residual = hidden_states
279
280 hidden_states = self.layer_norm1(hidden_states)
207 - hidden_states, _ = self.self_attn(hidden_states=hidden_states)
281 + hidden_states = self.self_attn(hidden_states=hidden_states)
282 hidden_states = residual + hidden_states
283
284 residual = hidden_states
@@ -304,7 +378,15 @@ class CLIPVisionModel(nn.Module):
378 def device(self):
379 return next(self.parameters()).device
380
381 + # (TODO) Add prefix argument for filtering out weights to be loaded
382 + # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
383 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
384 + stacked_params_mapping = [
385 + # (param_name, shard_name, shard_id)
386 + ("qkv_proj", "q_proj", "q"),
387 + ("qkv_proj", "k_proj", "k"),
388 + ("qkv_proj", "v_proj", "v"),
389 + ]
390 params_dict = dict(self.named_parameters())
391 layer_count = len(self.vision_model.encoder.layers)
392
@@ -318,7 +400,16 @@ class CLIPVisionModel(nn.Module):
400 if layer_idx >= layer_count:
401 continue
402
321 - param = params_dict[name]
322 - weight_loader = getattr(param, "weight_loader",
323 - default_weight_loader)
324 - weight_loader(param, loaded_weight)
403 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
404 + if weight_name not in name:
405 + continue
406 +
407 + param = params_dict[name.replace(weight_name, param_name)]
408 + weight_loader = param.weight_loader
409 + weight_loader(param, loaded_weight, shard_id)
410 + break
411 + else:
412 + param = params_dict[name]
413 + weight_loader = getattr(param, "weight_loader",
414 + default_weight_loader)
415 + weight_loader(param, loaded_weight)
vllm/model_executor/models/intern_vit.py
+45 -19
@@ -10,10 +10,13 @@ import torch
10 import torch.nn as nn
11 import torch.nn.functional as F
12 from transformers import PretrainedConfig
13 +from xformers import ops as xops
14
15 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.layernorm import RMSNorm
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19 + QKVParallelLinear,
20 RowParallelLinear)
21 from vllm.model_executor.layers.quantization import QuantizationConfig
22 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -81,7 +84,11 @@ class InternVisionEmbeddings(nn.Module):
84 class InternAttention(nn.Module):
85 """Multi-headed attention from 'Attention Is All You Need' paper"""
86
84 - def __init__(self, config: PretrainedConfig):
87 + def __init__(
88 + self,
89 + config: PretrainedConfig,
90 + quant_config: Optional[QuantizationConfig] = None,
91 + ):
92 super().__init__()
93 self.config = config
94 self.embed_dim = config.hidden_size
@@ -94,9 +101,13 @@ class InternAttention(nn.Module):
101 f' {self.num_heads}).')
102
103 self.scale = self.head_dim**-0.5
97 - self.qkv = nn.Linear(self.embed_dim,
98 - 3 * self.embed_dim,
99 - bias=config.qkv_bias)
104 + self.qkv = QKVParallelLinear(
105 + self.embed_dim,
106 + self.head_dim,
107 + self.num_heads,
108 + bias=config.qkv_bias,
109 + quant_config=quant_config,
110 + )
111
112 self.qk_normalization = config.qk_normalization
113
@@ -104,25 +115,40 @@ class InternAttention(nn.Module):
115 self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
116 self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
117
107 - self.proj = nn.Linear(self.embed_dim, self.embed_dim)
118 + self.proj = RowParallelLinear(
119 + self.embed_dim,
120 + self.embed_dim,
121 + quant_config=quant_config,
122 + )
123 +
124 + self.tp_size = get_tensor_model_parallel_world_size()
125 + self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
126
127 def forward(self, x):
128 B, N, C = x.shape
111 - qkv = self.qkv(x).reshape(B, N, 3, self.num_heads,
112 - C // self.num_heads).permute(2, 0, 3, 1, 4)
113 - q, k, v = qkv.unbind(0)
114 -
115 - if self.qk_normalization:
116 - B_, H_, N_, D_ = q.shape
117 - q = self.q_norm.forward_native(q.transpose(1, 2).flatten(
118 - -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
119 - k = self.k_norm.forward_native(k.transpose(1, 2).flatten(
120 - -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
129 + qkv, _ = self.qkv(x)
130 + q, k, v = qkv.chunk(3, dim=-1)
131
122 - x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
123 - x = x.transpose(1, 2).reshape(B, N, C)
132 + q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
133 + k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
134 + v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
135
125 - x = self.proj(x)
136 + if self.qk_normalization:
137 + B_, N_, H_, D_ = q.shape
138 + q = self.q_norm.forward_native(q.flatten(-2,
139 + -1)).view(B_, N_, H_, D_)
140 + k = self.k_norm.forward_native(k.flatten(-2,
141 + -1)).view(B_, N_, H_, D_)
142 +
143 + x = xops.memory_efficient_attention_forward(
144 + q,
145 + k,
146 + v,
147 + scale=self.scale,
148 + )
149 + x = x.view(B, N, -1)
150 +
151 + x, _ = self.proj(x)
152 return x
153
154
@@ -161,7 +187,7 @@ class InternVisionEncoderLayer(nn.Module):
187 self.intermediate_size = config.intermediate_size
188 self.norm_type = config.norm_type
189
164 - self.attn = InternAttention(config)
190 + self.attn = InternAttention(config, quant_config=quant_config)
191 self.mlp = InternMLP(config, quant_config=quant_config)
192 self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
193 eps=config.layer_norm_eps)
vllm/model_executor/models/paligemma.py
+20 -28
@@ -145,7 +145,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
145 self.config = config
146 self.multimodal_config = multimodal_config
147
148 - # TODO(ywang96): Port over SiglipVisionModel & TP
148 self.vision_tower = SiglipVisionModel(config.vision_config)
149 self.multi_modal_projector = PaliGemmaMultiModalProjector(
150 vision_hidden_size=config.vision_config.hidden_size,
@@ -308,34 +307,27 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
307 if key_to_modify in name:
308 name = name.replace(key_to_modify, new_key)
309 use_default_weight_loading = False
311 - if "vision" in name:
312 - if self.vision_tower is not None:
313 - # We only do sharding for language model and
314 - # not vision model for now.
315 - use_default_weight_loading = True
310 + for (param_name, shard_name, shard_id) in stacked_params_mapping:
311 + if shard_name not in name:
312 + continue
313 + name = name.replace(shard_name, param_name)
314 + # Skip loading extra bias for GPTQ models.
315 + if name.endswith(".bias") and name not in params_dict:
316 + continue
317 + param = params_dict[name]
318 + weight_loader = param.weight_loader
319 + weight_loader(param, loaded_weight, shard_id)
320 + break
321 else:
317 - for (param_name, shard_name,
318 - shard_id) in stacked_params_mapping:
319 - if shard_name not in name:
320 - continue
321 - name = name.replace(shard_name, param_name)
322 - # Skip loading extra bias for GPTQ models.
323 - if name.endswith(".bias") and name not in params_dict:
324 - continue
325 - param = params_dict[name]
326 - weight_loader = param.weight_loader
327 - weight_loader(param, loaded_weight, shard_id)
328 - break
329 - else:
330 - # lm_head is not used in vllm as it is tied with
331 - # embed_token. To prevent errors, skip loading
332 - # lm_head.weight.
333 - if "lm_head.weight" in name:
334 - continue
335 - # Skip loading extra bias for GPTQ models.
336 - if name.endswith(".bias") and name not in params_dict:
337 - continue
338 - use_default_weight_loading = True
322 + # lm_head is not used in vllm as it is tied with
323 + # embed_token. To prevent errors, skip loading
324 + # lm_head.weight.
325 + if "lm_head.weight" in name:
326 + continue
327 + # Skip loading extra bias for GPTQ models.
328 + if name.endswith(".bias") and name not in params_dict:
329 + continue
330 + use_default_weight_loading = True
331
332 if use_default_weight_loading:
333 param = params_dict[name]
vllm/model_executor/models/phi3v.py
+36 -17
@@ -71,6 +71,23 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
71 projection_dim=768)
72
73
74 +def _init_img_processor(hf_config: PretrainedConfig):
75 + clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
76 + layer_idx = hf_config.img_processor.get('layer_idx', -2)
77 +
78 + # Initialize the CLIP only up to the required feature layer
79 + if layer_idx < 0:
80 + num_hidden_layers = clip_config.num_hidden_layers + \
81 + layer_idx + 1
82 + else:
83 + num_hidden_layers = layer_idx + 1
84 +
85 + img_processor = CLIPVisionModel(
86 + clip_config, num_hidden_layers_override=num_hidden_layers)
87 +
88 + return img_processor
89 +
90 +
91 class Phi3VImagePixelInputs(TypedDict):
92 type: Literal["pixel_values"]
93 data: Union[torch.Tensor, List[torch.Tensor]]
@@ -139,18 +156,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
156 hidden_size = config.n_embd if hasattr(
157 config, 'n_embd') else config.hidden_size
158
142 - clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
143 - self.layer_idx = config.img_processor.get('layer_idx', -2)
144 -
145 - # Initialize the CLIP only up to the required feature layer
146 - if self.layer_idx < 0:
147 - num_hidden_layers = clip_config.num_hidden_layers + \
148 - self.layer_idx + 1
149 - else:
150 - num_hidden_layers = self.layer_idx + 1
159 + self.img_processor = _init_img_processor(config)
160
152 - self.img_processor = CLIPVisionModel(
153 - clip_config, num_hidden_layers_override=num_hidden_layers)
161 image_dim_out = config.img_processor['image_dim_out']
162 self.num_img_tokens = config.img_processor['num_img_tokens']
163
@@ -656,23 +663,27 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
663 (".gate_up_proj", ".gate_proj", 0),
664 (".gate_up_proj", ".up_proj", 1),
665 ]
666 +
667 + # TODO(ChristopherCho): This is a temporary fix to load
668 + # the vision weights with CLIPVisionModel.load_weights()
669 + vision_weights = []
670 params_dict = dict(self.named_parameters())
671 for name, loaded_weight in weights:
672 if "rotary_emb.inv_freq" in name:
673 continue
663 - # post_layernorm is not needed in CLIPVisionModel
664 - if "vision_model.post_layernorm" in name:
674 + # Skip loading the img_processor weights since they are
675 + # loaded separately.
676 + if "vision_embed_tokens.img_processor" in name:
677 + vision_weights.append((name, loaded_weight))
678 continue
679 +
680 for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
681 if key_to_modify in name:
682 name = name.replace(key_to_modify, new_key)
683 for (param_name, weight_name, shard_id) in stacked_params_mapping:
670 - # We only do sharding for language model
671 - # and not vision model for now.
672 - if "vision_embed_tokens" in name and self.vision_embed_tokens:
673 - continue
684 if weight_name not in name:
685 continue
686 +
687 param = params_dict[name.replace(weight_name, param_name)]
688 weight_loader = param.weight_loader
689 weight_loader(param, loaded_weight, shard_id)
@@ -686,3 +697,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
697 weight_loader = getattr(param, "weight_loader",
698 default_weight_loader)
699 weight_loader(param, loaded_weight)
700 +
701 + # We use regex to extract the sub-module name
702 + # from "model.vision_embed_tokens.img_processor.*"
703 + vision_weights = [
704 + (re.search(r"vision_embed_tokens\.img_processor\.(.*)",
705 + n).group(1), w) for n, w in vision_weights
706 + ]
707 + self.vision_embed_tokens.img_processor.load_weights(vision_weights)
vllm/model_executor/models/siglip.py
+32 -179
@@ -9,12 +9,10 @@ import torch
9 from PIL import Image
10 from torch import nn
11 from transformers import SiglipVisionConfig
12 -from transformers.models.siglip.modeling_siglip import SiglipAttention
13 -from vllm_flash_attn import flash_attn_func
14 -from xformers.ops import memory_efficient_attention
12 +from xformers import ops as xops
13
14 from vllm.config import ModelConfig
17 -from vllm.distributed import get_tensor_model_parallel_world_size
15 +from vllm.distributed import divide, get_tensor_model_parallel_world_size
16 from vllm.inputs import LLMInputs
17 from vllm.model_executor.layers.activation import get_act_fn
18 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -221,9 +219,7 @@ class SiglipVisionEmbeddings(nn.Module):
219 return embeddings
220
221
224 -# NOTE: Not used - kept for later when we TP the ViT
225 -# TODO(ChristopherCho): Implement TP version of Attention
226 -class SiglipTPAttention(nn.Module):
222 +class SiglipAttention(nn.Module):
223
224 def __init__(
225 self,
@@ -233,38 +229,30 @@ class SiglipTPAttention(nn.Module):
229 super().__init__()
230 self.config = config
231 self.embed_dim = config.hidden_size
236 -
237 - tp_size = get_tensor_model_parallel_world_size()
238 - self.total_num_heads = config.num_attention_heads
239 - if self.total_num_heads % tp_size != 0:
240 - raise ValueError(
241 - f"Number of attention heads ({self.total_num_heads}) "
242 - "must be divisible by the tensor model parallel size"
243 - f" ({tp_size}).")
244 -
245 - self.num_heads = self.total_num_heads // tp_size
246 - self.head_dim = self.embed_dim // self.total_num_heads
247 - if self.head_dim * self.total_num_heads != self.embed_dim:
232 + self.num_heads = config.num_attention_heads
233 + self.head_dim = self.embed_dim // self.num_heads
234 + if self.head_dim * self.num_heads != self.embed_dim:
235 raise ValueError(f"embed_dim must be divisible by num_heads (got "
236 "`embed_dim`: {self.embed_dim} and `num_heads`:"
237 f" {self.num_heads}).")
251 - self.qkv_size = self.num_heads * self.head_dim
238 +
239 self.scale = self.head_dim**-0.5
240 self.dropout = config.attention_dropout
254 -
241 self.qkv_proj = QKVParallelLinear(
242 hidden_size=self.embed_dim,
243 head_size=self.head_dim,
258 - total_num_heads=self.total_num_heads,
244 + total_num_heads=self.num_heads,
245 quant_config=quant_config,
246 )
247 +
248 self.out_proj = RowParallelLinear(
249 input_size=self.embed_dim,
250 output_size=self.embed_dim,
251 quant_config=quant_config,
252 )
253
267 - self.attn_fn = self._basic_attention_forward
254 + self.tp_size = get_tensor_model_parallel_world_size()
255 + self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
256
257 def forward(
258 self,
@@ -274,163 +262,29 @@ class SiglipTPAttention(nn.Module):
262 batch_size, q_len, _ = hidden_states.size()
263
264 qkv_states, _ = self.qkv_proj(hidden_states)
277 - query_states, key_states, value_states = qkv_states.split(
278 - [self.qkv_size] * 3, dim=-1)
279 -
280 - attn_output = self.attn_fn(
281 - q=query_states,
282 - k=key_states,
283 - v=value_states,
284 - batch_size=batch_size,
285 - q_len=q_len,
286 - )
287 -
288 - attn_output, _ = self.out_proj(attn_output)
289 - return attn_output
290 -
291 - def _basic_attention_forward(self, q, k, v, batch_size, q_len):
292 - q = q.view(batch_size, q_len, self.num_heads,
293 - self.head_dim).transpose(1, 2)
294 - k = k.view(batch_size, q_len, self.num_heads,
295 - self.head_dim).transpose(1, 2)
296 - v = v.view(batch_size, q_len, self.num_heads,
297 - self.head_dim).transpose(1, 2)
298 -
299 - k_v_seq_len = k.shape[-2]
300 - attn_weights = torch.matmul(q, k.transpose(2, 3)) * self.scale
301 -
302 - if attn_weights.size() != (
303 - batch_size,
304 - self.num_heads,
305 - q_len,
306 - k_v_seq_len,
307 - ):
308 - raise ValueError(
309 - "Attention weights should be of size "
310 - f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
311 - f" {attn_weights.size()}")
312 -
313 - # upcast attention to fp32
314 - attn_weights = nn.functional.softmax(attn_weights,
315 - dim=-1,
316 - dtype=torch.float32).to(q.dtype)
317 - attn_weights = nn.functional.dropout(attn_weights,
318 - p=self.dropout,
319 - training=self.training)
320 - attn_output = torch.matmul(attn_weights, v)
321 -
322 - if attn_output.size() != (
323 - batch_size,
324 - self.num_heads,
325 - q_len,
326 - self.head_dim,
327 - ):
328 - raise ValueError(
329 - "`attn_output` should be of size "
330 - f"{(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
331 - f" {attn_output.size()}")
332 -
333 - attn_output = attn_output.transpose(1, 2).contiguous()
334 - attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
335 -
336 - return attn_output
337 -
338 -
339 -# NOTE: Not used - kept for later when we TP the ViT
340 -# TODO(ChristopherCho): flash_attn_func is not working properly.
341 -# It constantly throws a CUDA error.
342 -class SiglipFlashAttention2(SiglipTPAttention):
343 -
344 - def __init__(self, *args, **kwargs):
345 - super().__init__(*args, **kwargs)
346 - self.attn_fn = self._flash_attention_forward
347 -
348 - # Ported from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L449
349 - # and https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/modeling_flash_attention_utils.py#L133
350 - def _flash_attention_forward(self, q, k, v, batch_size, q_len, *args,
351 - **kwargs):
352 - """Implements the multihead softmax attention.
353 - Arguments
354 - ---------
355 - q, k, v: The tensor containing the
356 - query, key, and value. (B, S, H, D)
357 - """
358 -
359 - q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
360 - k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
361 - v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
362 -
363 - attn_output = flash_attn_func(
364 - q,
365 - k,
366 - v,
367 - dropout_p=self.dropout,
368 - causal=False,
369 - )
370 -
371 - attn_output = attn_output.reshape(batch_size, q_len,
372 - self.embed_dim).contiguous()
265 + query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
266 +
267 + query_states = query_states.view(batch_size, q_len,
268 + self.num_heads_per_partition,
269 + self.head_dim)
270 + key_states = key_states.view(batch_size, q_len,
271 + self.num_heads_per_partition,
272 + self.head_dim)
273 + value_states = value_states.view(batch_size, q_len,
274 + self.num_heads_per_partition,
275 + self.head_dim)
276 +
277 + out = xops.memory_efficient_attention_forward(query_states,
278 + key_states,
279 + value_states,
280 + p=self.dropout,
281 + scale=self.scale)
282 + out = out.view(batch_size, q_len, -1)
283 + attn_output, _ = self.out_proj(out)
284
285 return attn_output
286
287
377 -# NOTE: Not used - kept for later when we TP the ViT
378 -class SiglipSdpaAttention(SiglipTPAttention):
379 -
380 - def __init__(self, *args, **kwargs):
381 - super().__init__(*args, **kwargs)
382 - self.is_causal = False
383 - self.attn_fn = self._sdpa_attention_forward
384 -
385 - def _sdpa_attention_forward(self, q, k, v, batch_size, q_len):
386 - q = q.view(batch_size, q_len, self.num_heads,
387 - self.head_dim).transpose(1, 2)
388 - k = k.view(batch_size, q_len, self.num_heads,
389 - self.head_dim).transpose(1, 2)
390 - v = v.view(batch_size, q_len, self.num_heads,
391 - self.head_dim).transpose(1, 2)
392 -
393 - attn_output = torch.nn.functional.scaled_dot_product_attention(
394 - q, k, v, dropout_p=self.dropout, is_causal=False, scale=self.scale)
395 -
396 - attn_output = attn_output.transpose(1, 2).contiguous()
397 - attn_output = attn_output.view(batch_size, q_len, self.embed_dim)
398 -
399 - return attn_output
400 -
401 -
402 -# NOTE: Not used - kept for later when we TP the ViT
403 -class SiglipxFormersAttention(SiglipTPAttention):
404 -
405 - def __init__(self, *args, **kwargs):
406 - super().__init__(*args, **kwargs)
407 - self.attn_fn = self._xformers_attention_forward
408 -
409 - def _xformers_attention_forward(self, q, k, v, batch_size, q_len):
410 - q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
411 - k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
412 - v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
413 -
414 - attn_output = memory_efficient_attention(q,
415 - k,
416 - v,
417 - p=0.0,
418 - scale=self.scale)
419 - attn_output = attn_output.reshape(batch_size, q_len,
420 - self.embed_dim).contiguous()
421 -
422 - return attn_output
423 -
424 -
425 -# NOTE: Not used - kept for later when we TP the ViT
426 -SIGLIP_ATTENTION_CLASSES = {
427 - "eager": SiglipTPAttention,
428 - "flash_attention_2": SiglipFlashAttention2,
429 - "sdpa": SiglipSdpaAttention,
430 - "xformers": SiglipxFormersAttention,
431 -}
432 -
433 -
288 class SiglipMLP(nn.Module):
289
290 def __init__(
@@ -473,8 +327,7 @@ class SiglipEncoderLayer(nn.Module):
327 super().__init__()
328 self.embed_dim = config.hidden_size
329
476 - # TODO(ChristopherCho): use TP'ed Attention block
477 - self.self_attn = SiglipAttention(config)
330 + self.self_attn = SiglipAttention(config, quant_config=quant_config)
331 self.layer_norm1 = nn.LayerNorm(self.embed_dim,
332 eps=config.layer_norm_eps)
333 self.mlp = SiglipMLP(
@@ -491,7 +344,7 @@ class SiglipEncoderLayer(nn.Module):
344 residual = hidden_states
345
346 hidden_states = self.layer_norm1(hidden_states)
494 - hidden_states, _ = self.self_attn(hidden_states=hidden_states)
347 + hidden_states = self.self_attn(hidden_states=hidden_states)
348 hidden_states = residual + hidden_states
349
350 residual = hidden_states