A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[VLM][Model] TP support for ViTs (#7186)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Jungho Christopher Cho committed
Aug 31, 2024 at 00:19 UTC
f97be32d1da4cfda933a0dbfbc681861f96390d9
9 files changed
+340
-289
tests/models/test_intern_vit.py
+1
-2
@@ -6,8 +6,6 @@ import torch.nn as nn
6
from huggingface_hub import snapshot_download
7
from transformers import AutoConfig, AutoModel, CLIPImageProcessor
8
9
-from vllm.model_executor.models.intern_vit import InternVisionModel
10
-
9
from ..conftest import _ImageAssets, cleanup
10
11
pytestmark = pytest.mark.vlm
@@ -49,6 +47,7 @@ def run_intern_vit_test(
47
for pixel_value in pixel_values
48
]
49
50
+ from vllm.model_executor.models.intern_vit import InternVisionModel
51
vllm_model = InternVisionModel(config)
52
vllm_model.load_weights(hf_model.state_dict().items())
53
tests/models/test_internvl.py
+31
-32
@@ -6,9 +6,6 @@ import torch
6
from PIL.Image import Image
7
from transformers import AutoConfig
8
9
-from vllm.model_executor.models.internvl import (IMG_CONTEXT, IMG_END,
10
- IMG_START,
11
- image_to_pixel_values)
9
from vllm.multimodal.utils import rescale_image_size
10
from vllm.utils import is_cpu
11
@@ -33,35 +30,6 @@ models = [
30
]
31
32
36
-class InternVLProcessor:
37
- """A simple processor for InternVL2 HF model which misses a processor."""
38
-
39
- def __init__(self, hf_runner: HfRunner):
40
- self.num_image_token = hf_runner.model.num_image_token
41
- self.tokenizer = hf_runner.tokenizer
42
- self.dtype = hf_runner.model.dtype
43
-
44
- self.config = AutoConfig.from_pretrained(hf_runner.model_name)
45
- self.vision_config = self.config.vision_config
46
- self.use_thumbnail = self.config.use_thumbnail
47
- self.min_num = self.config.min_dynamic_patch
48
- self.max_num = self.config.max_dynamic_patch
49
- self.image_size = self.vision_config.image_size
50
-
51
- def __call__(self, text: str, images: Image, **kwargs):
52
- pixel_values = image_to_pixel_values(images, self.image_size,
53
- self.min_num, self.max_num,
54
- self.use_thumbnail).to(self.dtype)
55
- num_patches_list = [pixel_values.shape[0]]
56
- for num_patches in num_patches_list:
57
- context_tokens = IMG_CONTEXT * self.num_image_token * num_patches
58
- image_tokens = IMG_START + context_tokens + IMG_END
59
- text = text.replace('<image>', image_tokens, 1)
60
- prompt = self.tokenizer(text, return_tensors="pt")
61
- prompt.update({"pixel_values": pixel_values})
62
- return prompt
63
-
64
-
33
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
34
def generate(
35
self,
@@ -127,6 +95,37 @@ def run_test(
95
# if we run HF first, the cuda initialization will be done and it
96
# will hurt multiprocessing backend with fork method (the default method).
97
98
+ class InternVLProcessor:
99
+ """A simple processor for InternVL2 which misses a processor."""
100
+
101
+ def __init__(self, hf_runner: HfRunner):
102
+ self.num_image_token = hf_runner.model.num_image_token
103
+ self.tokenizer = hf_runner.tokenizer
104
+ self.dtype = hf_runner.model.dtype
105
+
106
+ self.config = AutoConfig.from_pretrained(hf_runner.model_name)
107
+ self.vision_config = self.config.vision_config
108
+ self.use_thumbnail = self.config.use_thumbnail
109
+ self.min_num = self.config.min_dynamic_patch
110
+ self.max_num = self.config.max_dynamic_patch
111
+ self.image_size = self.vision_config.image_size
112
+
113
+ def __call__(self, text: str, images: Image, **kwargs):
114
+ from vllm.model_executor.models.internvl import (
115
+ IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
116
+ pixel_values = image_to_pixel_values(
117
+ images, self.image_size, self.min_num, self.max_num,
118
+ self.use_thumbnail).to(self.dtype)
119
+ num_patches_list = [pixel_values.shape[0]]
120
+ for num_patches in num_patches_list:
121
+ context_tokens = IMG_CONTEXT * self.num_image_token \
122
+ * num_patches
123
+ image_tokens = IMG_START + context_tokens + IMG_END
124
+ text = text.replace('<image>', image_tokens, 1)
125
+ prompt = self.tokenizer(text, return_tensors="pt")
126
+ prompt.update({"pixel_values": pixel_values})
127
+ return prompt
128
+
129
# max_model_len should be greater than image_feature_size
130
with vllm_runner(model,
131
max_model_len=4096,
vllm/model_executor/models/blip.py
+76
-3
@@ -7,12 +7,14 @@ import torch
7
import torch.nn as nn
8
from PIL import Image
9
from transformers import Blip2VisionConfig, BlipVisionConfig
10
-from transformers.models.blip.modeling_blip import BlipAttention
10
+from xformers import ops as xops
11
12
from vllm.config import ModelConfig
13
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
14
from vllm.inputs import LLMInputs
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
+ QKVParallelLinear,
18
RowParallelLinear)
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.multimodal.utils import (cached_get_tokenizer,
@@ -154,6 +156,77 @@ class BlipVisionEmbeddings(nn.Module):
156
return embeddings
157
158
159
+class BlipAttention(nn.Module):
160
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
161
+
162
+ def __init__(
163
+ self,
164
+ config: BlipVisionConfig,
165
+ quant_config: Optional[QuantizationConfig] = None,
166
+ ):
167
+ super().__init__()
168
+ self.config = config
169
+ self.embed_dim = config.hidden_size
170
+ self.num_heads = config.num_attention_heads
171
+ self.head_dim = self.embed_dim // self.num_heads
172
+ if self.head_dim * self.num_heads != self.embed_dim:
173
+ raise ValueError(
174
+ "embed_dim must be divisible by num_heads "
175
+ f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
176
+ f" {self.num_heads}).")
177
+ self.scale = self.head_dim**-0.5
178
+ self.dropout = config.attention_dropout
179
+
180
+ self.qkv = QKVParallelLinear(
181
+ self.embed_dim,
182
+ self.head_dim,
183
+ self.num_heads,
184
+ bias=config.qkv_bias,
185
+ quant_config=quant_config,
186
+ )
187
+ self.projection = RowParallelLinear(
188
+ self.embed_dim,
189
+ self.embed_dim,
190
+ quant_config=quant_config,
191
+ )
192
+
193
+ self.tp_size = get_tensor_model_parallel_world_size()
194
+ self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
195
+
196
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
197
+ return tensor.view(bsz, seq_len, self.num_heads,
198
+ self.head_dim).transpose(1, 2).contiguous()
199
+
200
+ def forward(
201
+ self,
202
+ hidden_states: torch.Tensor,
203
+ ):
204
+ """Input shape: Batch x Time x Channel"""
205
+ bsz, tgt_len, _ = hidden_states.size()
206
+
207
+ qkv_states, _ = self.qkv(hidden_states)
208
+ query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
209
+ query_states = query_states.view(bsz, tgt_len,
210
+ self.num_heads_per_partition,
211
+ self.head_dim)
212
+ key_states = key_states.view(bsz, tgt_len,
213
+ self.num_heads_per_partition,
214
+ self.head_dim)
215
+ value_states = value_states.view(bsz, tgt_len,
216
+ self.num_heads_per_partition,
217
+ self.head_dim)
218
+
219
+ out = xops.memory_efficient_attention_forward(query_states,
220
+ key_states,
221
+ value_states,
222
+ p=self.dropout,
223
+ scale=self.scale)
224
+ out = out.view(bsz, tgt_len, -1)
225
+ attn_output, _ = self.projection(out)
226
+
227
+ return attn_output
228
+
229
+
230
class BlipMLP(nn.Module):
231
232
def __init__(self,
@@ -188,7 +261,7 @@ class BlipEncoderLayer(nn.Module):
261
quant_config: Optional[QuantizationConfig] = None):
262
super().__init__()
263
191
- self.self_attn = BlipAttention(config)
264
+ self.self_attn = BlipAttention(config, quant_config=quant_config)
265
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
266
eps=config.layer_norm_eps)
267
self.mlp = BlipMLP(config, quant_config=quant_config)
@@ -199,7 +272,7 @@ class BlipEncoderLayer(nn.Module):
272
residual = hidden_states
273
274
hidden_states = self.layer_norm1(hidden_states)
202
- hidden_states, _ = self.self_attn(hidden_states=hidden_states)
275
+ hidden_states = self.self_attn(hidden_states=hidden_states)
276
hidden_states = residual + hidden_states
277
278
residual = hidden_states
vllm/model_executor/models/blip2.py
+1
-2
@@ -714,8 +714,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal):
714
use_default_weight_loading = False
715
if "vision" in name:
716
if self.vision_model is not None:
717
- # We only do sharding for language model and
718
- # not vision model for now.
717
+ # BlipVisionModel does not need sharding
718
use_default_weight_loading = True
719
else:
720
for (param_name, weight_name,
vllm/model_executor/models/clip.py
+98
-7
@@ -7,12 +7,14 @@ import torch
7
import torch.nn as nn
8
from PIL import Image
9
from transformers import CLIPVisionConfig
10
-from transformers.models.clip.modeling_clip import CLIPAttention
10
+from xformers import ops as xops
11
12
from vllm.config import ModelConfig
13
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
14
from vllm.inputs import LLMInputs
15
from vllm.model_executor.layers.activation import get_act_fn
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
17
+ QKVParallelLinear,
18
RowParallelLinear)
19
from vllm.model_executor.layers.quantization import QuantizationConfig
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -160,6 +162,78 @@ class CLIPVisionEmbeddings(nn.Module):
162
return embeddings
163
164
165
+class CLIPAttention(nn.Module):
166
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
167
+
168
+ def __init__(
169
+ self,
170
+ config: CLIPVisionConfig,
171
+ quant_config: Optional[QuantizationConfig] = None,
172
+ ):
173
+ super().__init__()
174
+ self.config = config
175
+ self.embed_dim = config.hidden_size
176
+ self.num_heads = config.num_attention_heads
177
+ self.head_dim = self.embed_dim // self.num_heads
178
+ if self.head_dim * self.num_heads != self.embed_dim:
179
+ raise ValueError(
180
+ "embed_dim must be divisible by num_heads "
181
+ f"(got `embed_dim`: {self.embed_dim} and `num_heads`:"
182
+ f" {self.num_heads}).")
183
+ self.scale = self.head_dim**-0.5
184
+ self.dropout = config.attention_dropout
185
+
186
+ self.qkv_proj = QKVParallelLinear(
187
+ hidden_size=self.embed_dim,
188
+ head_size=self.head_dim,
189
+ total_num_heads=self.num_heads,
190
+ quant_config=quant_config,
191
+ )
192
+
193
+ self.out_proj = RowParallelLinear(
194
+ input_size=self.embed_dim,
195
+ output_size=self.embed_dim,
196
+ quant_config=quant_config,
197
+ )
198
+
199
+ self.tp_size = get_tensor_model_parallel_world_size()
200
+ self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
201
+
202
+ def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int):
203
+ return tensor.view(bsz, seq_len, self.num_heads,
204
+ self.head_dim).transpose(1, 2).contiguous()
205
+
206
+ def forward(
207
+ self,
208
+ hidden_states: torch.Tensor,
209
+ ):
210
+ """Input shape: Batch x Time x Channel"""
211
+ bsz, tgt_len, _ = hidden_states.size()
212
+
213
+ qkv_states, _ = self.qkv_proj(hidden_states)
214
+ query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
215
+
216
+ query_states = query_states.view(bsz, tgt_len,
217
+ self.num_heads_per_partition,
218
+ self.head_dim)
219
+ key_states = key_states.view(bsz, tgt_len,
220
+ self.num_heads_per_partition,
221
+ self.head_dim)
222
+ value_states = value_states.view(bsz, tgt_len,
223
+ self.num_heads_per_partition,
224
+ self.head_dim)
225
+
226
+ out = xops.memory_efficient_attention_forward(query_states,
227
+ key_states,
228
+ value_states,
229
+ p=self.dropout,
230
+ scale=self.scale)
231
+ out = out.view(bsz, tgt_len, -1)
232
+ attn_output, _ = self.out_proj(out)
233
+
234
+ return attn_output
235
+
236
+
237
class CLIPMLP(nn.Module):
238
239
def __init__(self,
@@ -192,7 +266,7 @@ class CLIPEncoderLayer(nn.Module):
266
quant_config: Optional[QuantizationConfig] = None):
267
super().__init__()
268
195
- self.self_attn = CLIPAttention(config)
269
+ self.self_attn = CLIPAttention(config, quant_config=quant_config)
270
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
271
eps=config.layer_norm_eps)
272
self.mlp = CLIPMLP(config, quant_config=quant_config)
@@ -204,7 +278,7 @@ class CLIPEncoderLayer(nn.Module):
278
residual = hidden_states
279
280
hidden_states = self.layer_norm1(hidden_states)
207
- hidden_states, _ = self.self_attn(hidden_states=hidden_states)
281
+ hidden_states = self.self_attn(hidden_states=hidden_states)
282
hidden_states = residual + hidden_states
283
284
residual = hidden_states
@@ -304,7 +378,15 @@ class CLIPVisionModel(nn.Module):
378
def device(self):
379
return next(self.parameters()).device
380
381
+ # (TODO) Add prefix argument for filtering out weights to be loaded
382
+ # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
383
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
384
+ stacked_params_mapping = [
385
+ # (param_name, shard_name, shard_id)
386
+ ("qkv_proj", "q_proj", "q"),
387
+ ("qkv_proj", "k_proj", "k"),
388
+ ("qkv_proj", "v_proj", "v"),
389
+ ]
390
params_dict = dict(self.named_parameters())
391
layer_count = len(self.vision_model.encoder.layers)
392
@@ -318,7 +400,16 @@ class CLIPVisionModel(nn.Module):
400
if layer_idx >= layer_count:
401
continue
402
321
- param = params_dict[name]
322
- weight_loader = getattr(param, "weight_loader",
323
- default_weight_loader)
324
- weight_loader(param, loaded_weight)
403
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
404
+ if weight_name not in name:
405
+ continue
406
+
407
+ param = params_dict[name.replace(weight_name, param_name)]
408
+ weight_loader = param.weight_loader
409
+ weight_loader(param, loaded_weight, shard_id)
410
+ break
411
+ else:
412
+ param = params_dict[name]
413
+ weight_loader = getattr(param, "weight_loader",
414
+ default_weight_loader)
415
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/intern_vit.py
+45
-19
@@ -10,10 +10,13 @@ import torch
10
import torch.nn as nn
11
import torch.nn.functional as F
12
from transformers import PretrainedConfig
13
+from xformers import ops as xops
14
15
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.layernorm import RMSNorm
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
19
+ QKVParallelLinear,
20
RowParallelLinear)
21
from vllm.model_executor.layers.quantization import QuantizationConfig
22
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -81,7 +84,11 @@ class InternVisionEmbeddings(nn.Module):
84
class InternAttention(nn.Module):
85
"""Multi-headed attention from 'Attention Is All You Need' paper"""
86
84
- def __init__(self, config: PretrainedConfig):
87
+ def __init__(
88
+ self,
89
+ config: PretrainedConfig,
90
+ quant_config: Optional[QuantizationConfig] = None,
91
+ ):
92
super().__init__()
93
self.config = config
94
self.embed_dim = config.hidden_size
@@ -94,9 +101,13 @@ class InternAttention(nn.Module):
101
f' {self.num_heads}).')
102
103
self.scale = self.head_dim**-0.5
97
- self.qkv = nn.Linear(self.embed_dim,
98
- 3 * self.embed_dim,
99
- bias=config.qkv_bias)
104
+ self.qkv = QKVParallelLinear(
105
+ self.embed_dim,
106
+ self.head_dim,
107
+ self.num_heads,
108
+ bias=config.qkv_bias,
109
+ quant_config=quant_config,
110
+ )
111
112
self.qk_normalization = config.qk_normalization
113
@@ -104,25 +115,40 @@ class InternAttention(nn.Module):
115
self.q_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
116
self.k_norm = RMSNorm(self.embed_dim, eps=config.layer_norm_eps)
117
107
- self.proj = nn.Linear(self.embed_dim, self.embed_dim)
118
+ self.proj = RowParallelLinear(
119
+ self.embed_dim,
120
+ self.embed_dim,
121
+ quant_config=quant_config,
122
+ )
123
+
124
+ self.tp_size = get_tensor_model_parallel_world_size()
125
+ self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
126
127
def forward(self, x):
128
B, N, C = x.shape
111
- qkv = self.qkv(x).reshape(B, N, 3, self.num_heads,
112
- C // self.num_heads).permute(2, 0, 3, 1, 4)
113
- q, k, v = qkv.unbind(0)
114
-
115
- if self.qk_normalization:
116
- B_, H_, N_, D_ = q.shape
117
- q = self.q_norm.forward_native(q.transpose(1, 2).flatten(
118
- -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
119
- k = self.k_norm.forward_native(k.transpose(1, 2).flatten(
120
- -2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
129
+ qkv, _ = self.qkv(x)
130
+ q, k, v = qkv.chunk(3, dim=-1)
131
122
- x = F.scaled_dot_product_attention(q, k, v, scale=self.scale)
123
- x = x.transpose(1, 2).reshape(B, N, C)
132
+ q = q.view(B, N, self.num_heads_per_partition, self.head_dim)
133
+ k = k.view(B, N, self.num_heads_per_partition, self.head_dim)
134
+ v = v.view(B, N, self.num_heads_per_partition, self.head_dim)
135
125
- x = self.proj(x)
136
+ if self.qk_normalization:
137
+ B_, N_, H_, D_ = q.shape
138
+ q = self.q_norm.forward_native(q.flatten(-2,
139
+ -1)).view(B_, N_, H_, D_)
140
+ k = self.k_norm.forward_native(k.flatten(-2,
141
+ -1)).view(B_, N_, H_, D_)
142
+
143
+ x = xops.memory_efficient_attention_forward(
144
+ q,
145
+ k,
146
+ v,
147
+ scale=self.scale,
148
+ )
149
+ x = x.view(B, N, -1)
150
+
151
+ x, _ = self.proj(x)
152
return x
153
154
@@ -161,7 +187,7 @@ class InternVisionEncoderLayer(nn.Module):
187
self.intermediate_size = config.intermediate_size
188
self.norm_type = config.norm_type
189
164
- self.attn = InternAttention(config)
190
+ self.attn = InternAttention(config, quant_config=quant_config)
191
self.mlp = InternMLP(config, quant_config=quant_config)
192
self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
193
eps=config.layer_norm_eps)
vllm/model_executor/models/paligemma.py
+20
-28
@@ -145,7 +145,6 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
145
self.config = config
146
self.multimodal_config = multimodal_config
147
148
- # TODO(ywang96): Port over SiglipVisionModel & TP
148
self.vision_tower = SiglipVisionModel(config.vision_config)
149
self.multi_modal_projector = PaliGemmaMultiModalProjector(
150
vision_hidden_size=config.vision_config.hidden_size,
@@ -308,34 +307,27 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
307
if key_to_modify in name:
308
name = name.replace(key_to_modify, new_key)
309
use_default_weight_loading = False
311
- if "vision" in name:
312
- if self.vision_tower is not None:
313
- # We only do sharding for language model and
314
- # not vision model for now.
315
- use_default_weight_loading = True
310
+ for (param_name, shard_name, shard_id) in stacked_params_mapping:
311
+ if shard_name not in name:
312
+ continue
313
+ name = name.replace(shard_name, param_name)
314
+ # Skip loading extra bias for GPTQ models.
315
+ if name.endswith(".bias") and name not in params_dict:
316
+ continue
317
+ param = params_dict[name]
318
+ weight_loader = param.weight_loader
319
+ weight_loader(param, loaded_weight, shard_id)
320
+ break
321
else:
317
- for (param_name, shard_name,
318
- shard_id) in stacked_params_mapping:
319
- if shard_name not in name:
320
- continue
321
- name = name.replace(shard_name, param_name)
322
- # Skip loading extra bias for GPTQ models.
323
- if name.endswith(".bias") and name not in params_dict:
324
- continue
325
- param = params_dict[name]
326
- weight_loader = param.weight_loader
327
- weight_loader(param, loaded_weight, shard_id)
328
- break
329
- else:
330
- # lm_head is not used in vllm as it is tied with
331
- # embed_token. To prevent errors, skip loading
332
- # lm_head.weight.
333
- if "lm_head.weight" in name:
334
- continue
335
- # Skip loading extra bias for GPTQ models.
336
- if name.endswith(".bias") and name not in params_dict:
337
- continue
338
- use_default_weight_loading = True
322
+ # lm_head is not used in vllm as it is tied with
323
+ # embed_token. To prevent errors, skip loading
324
+ # lm_head.weight.
325
+ if "lm_head.weight" in name:
326
+ continue
327
+ # Skip loading extra bias for GPTQ models.
328
+ if name.endswith(".bias") and name not in params_dict:
329
+ continue
330
+ use_default_weight_loading = True
331
332
if use_default_weight_loading:
333
param = params_dict[name]
vllm/model_executor/models/phi3v.py
+36
-17
@@ -71,6 +71,23 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
71
projection_dim=768)
72
73
74
+def _init_img_processor(hf_config: PretrainedConfig):
75
+ clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
76
+ layer_idx = hf_config.img_processor.get('layer_idx', -2)
77
+
78
+ # Initialize the CLIP only up to the required feature layer
79
+ if layer_idx < 0:
80
+ num_hidden_layers = clip_config.num_hidden_layers + \
81
+ layer_idx + 1
82
+ else:
83
+ num_hidden_layers = layer_idx + 1
84
+
85
+ img_processor = CLIPVisionModel(
86
+ clip_config, num_hidden_layers_override=num_hidden_layers)
87
+
88
+ return img_processor
89
+
90
+
91
class Phi3VImagePixelInputs(TypedDict):
92
type: Literal["pixel_values"]
93
data: Union[torch.Tensor, List[torch.Tensor]]
@@ -139,18 +156,8 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
156
hidden_size = config.n_embd if hasattr(
157
config, 'n_embd') else config.hidden_size
158
142
- clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
143
- self.layer_idx = config.img_processor.get('layer_idx', -2)
144
-
145
- # Initialize the CLIP only up to the required feature layer
146
- if self.layer_idx < 0:
147
- num_hidden_layers = clip_config.num_hidden_layers + \
148
- self.layer_idx + 1
149
- else:
150
- num_hidden_layers = self.layer_idx + 1
159
+ self.img_processor = _init_img_processor(config)
160
152
- self.img_processor = CLIPVisionModel(
153
- clip_config, num_hidden_layers_override=num_hidden_layers)
161
image_dim_out = config.img_processor['image_dim_out']
162
self.num_img_tokens = config.img_processor['num_img_tokens']
163
@@ -656,23 +663,27 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
663
(".gate_up_proj", ".gate_proj", 0),
664
(".gate_up_proj", ".up_proj", 1),
665
]
666
+
667
+ # TODO(ChristopherCho): This is a temporary fix to load
668
+ # the vision weights with CLIPVisionModel.load_weights()
669
+ vision_weights = []
670
params_dict = dict(self.named_parameters())
671
for name, loaded_weight in weights:
672
if "rotary_emb.inv_freq" in name:
673
continue
663
- # post_layernorm is not needed in CLIPVisionModel
664
- if "vision_model.post_layernorm" in name:
674
+ # Skip loading the img_processor weights since they are
675
+ # loaded separately.
676
+ if "vision_embed_tokens.img_processor" in name:
677
+ vision_weights.append((name, loaded_weight))
678
continue
679
+
680
for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
681
if key_to_modify in name:
682
name = name.replace(key_to_modify, new_key)
683
for (param_name, weight_name, shard_id) in stacked_params_mapping:
670
- # We only do sharding for language model
671
- # and not vision model for now.
672
- if "vision_embed_tokens" in name and self.vision_embed_tokens:
673
- continue
684
if weight_name not in name:
685
continue
686
+
687
param = params_dict[name.replace(weight_name, param_name)]
688
weight_loader = param.weight_loader
689
weight_loader(param, loaded_weight, shard_id)
@@ -686,3 +697,11 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal):
697
weight_loader = getattr(param, "weight_loader",
698
default_weight_loader)
699
weight_loader(param, loaded_weight)
700
+
701
+ # We use regex to extract the sub-module name
702
+ # from "model.vision_embed_tokens.img_processor.*"
703
+ vision_weights = [
704
+ (re.search(r"vision_embed_tokens\.img_processor\.(.*)",
705
+ n).group(1), w) for n, w in vision_weights
706
+ ]
707
+ self.vision_embed_tokens.img_processor.load_weights(vision_weights)
vllm/model_executor/models/siglip.py
+32
-179
@@ -9,12 +9,10 @@ import torch
9
from PIL import Image
10
from torch import nn
11
from transformers import SiglipVisionConfig
12
-from transformers.models.siglip.modeling_siglip import SiglipAttention
13
-from vllm_flash_attn import flash_attn_func
14
-from xformers.ops import memory_efficient_attention
12
+from xformers import ops as xops
13
14
from vllm.config import ModelConfig
17
-from vllm.distributed import get_tensor_model_parallel_world_size
15
+from vllm.distributed import divide, get_tensor_model_parallel_world_size
16
from vllm.inputs import LLMInputs
17
from vllm.model_executor.layers.activation import get_act_fn
18
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -221,9 +219,7 @@ class SiglipVisionEmbeddings(nn.Module):
219
return embeddings
220
221
224
-# NOTE: Not used - kept for later when we TP the ViT
225
-# TODO(ChristopherCho): Implement TP version of Attention
226
-class SiglipTPAttention(nn.Module):
222
+class SiglipAttention(nn.Module):
223
224
def __init__(
225
self,
@@ -233,38 +229,30 @@ class SiglipTPAttention(nn.Module):
229
super().__init__()
230
self.config = config
231
self.embed_dim = config.hidden_size
236
-
237
- tp_size = get_tensor_model_parallel_world_size()
238
- self.total_num_heads = config.num_attention_heads
239
- if self.total_num_heads % tp_size != 0:
240
- raise ValueError(
241
- f"Number of attention heads ({self.total_num_heads}) "
242
- "must be divisible by the tensor model parallel size"
243
- f" ({tp_size}).")
244
-
245
- self.num_heads = self.total_num_heads // tp_size
246
- self.head_dim = self.embed_dim // self.total_num_heads
247
- if self.head_dim * self.total_num_heads != self.embed_dim:
232
+ self.num_heads = config.num_attention_heads
233
+ self.head_dim = self.embed_dim // self.num_heads
234
+ if self.head_dim * self.num_heads != self.embed_dim:
235
raise ValueError(f"embed_dim must be divisible by num_heads (got "
236
"`embed_dim`: {self.embed_dim} and `num_heads`:"
237
f" {self.num_heads}).")
251
- self.qkv_size = self.num_heads * self.head_dim
238
+
239
self.scale = self.head_dim**-0.5
240
self.dropout = config.attention_dropout
254
-
241
self.qkv_proj = QKVParallelLinear(
242
hidden_size=self.embed_dim,
243
head_size=self.head_dim,
258
- total_num_heads=self.total_num_heads,
244
+ total_num_heads=self.num_heads,
245
quant_config=quant_config,
246
)
247
+
248
self.out_proj = RowParallelLinear(
249
input_size=self.embed_dim,
250
output_size=self.embed_dim,
251
quant_config=quant_config,
252
)
253
267
- self.attn_fn = self._basic_attention_forward
254
+ self.tp_size = get_tensor_model_parallel_world_size()
255
+ self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
256
257
def forward(
258
self,
@@ -274,163 +262,29 @@ class SiglipTPAttention(nn.Module):
262
batch_size, q_len, _ = hidden_states.size()
263
264
qkv_states, _ = self.qkv_proj(hidden_states)
277
- query_states, key_states, value_states = qkv_states.split(
278
- [self.qkv_size] * 3, dim=-1)
279
-
280
- attn_output = self.attn_fn(
281
- q=query_states,
282
- k=key_states,
283
- v=value_states,
284
- batch_size=batch_size,
285
- q_len=q_len,
286
- )
287
-
288
- attn_output, _ = self.out_proj(attn_output)
289
- return attn_output
290
-
291
- def _basic_attention_forward(self, q, k, v, batch_size, q_len):
292
- q = q.view(batch_size, q_len, self.num_heads,
293
- self.head_dim).transpose(1, 2)
294
- k = k.view(batch_size, q_len, self.num_heads,
295
- self.head_dim).transpose(1, 2)
296
- v = v.view(batch_size, q_len, self.num_heads,
297
- self.head_dim).transpose(1, 2)
298
-
299
- k_v_seq_len = k.shape[-2]
300
- attn_weights = torch.matmul(q, k.transpose(2, 3)) * self.scale
301
-
302
- if attn_weights.size() != (
303
- batch_size,
304
- self.num_heads,
305
- q_len,
306
- k_v_seq_len,
307
- ):
308
- raise ValueError(
309
- "Attention weights should be of size "
310
- f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
311
- f" {attn_weights.size()}")
312
-
313
- # upcast attention to fp32
314
- attn_weights = nn.functional.softmax(attn_weights,
315
- dim=-1,
316
- dtype=torch.float32).to(q.dtype)
317
- attn_weights = nn.functional.dropout(attn_weights,
318
- p=self.dropout,
319
- training=self.training)
320
- attn_output = torch.matmul(attn_weights, v)
321
-
322
- if attn_output.size() != (
323
- batch_size,
324
- self.num_heads,
325
- q_len,
326
- self.head_dim,
327
- ):
328
- raise ValueError(
329
- "`attn_output` should be of size "
330
- f"{(batch_size, self.num_heads, q_len, self.head_dim)}, but is"
331
- f" {attn_output.size()}")
332
-
333
- attn_output = attn_output.transpose(1, 2).contiguous()
334
- attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
335
-
336
- return attn_output
337
-
338
-
339
-# NOTE: Not used - kept for later when we TP the ViT
340
-# TODO(ChristopherCho): flash_attn_func is not working properly.
341
-# It constantly throws a CUDA error.
342
-class SiglipFlashAttention2(SiglipTPAttention):
343
-
344
- def __init__(self, *args, **kwargs):
345
- super().__init__(*args, **kwargs)
346
- self.attn_fn = self._flash_attention_forward
347
-
348
- # Ported from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L449
349
- # and https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/modeling_flash_attention_utils.py#L133
350
- def _flash_attention_forward(self, q, k, v, batch_size, q_len, *args,
351
- **kwargs):
352
- """Implements the multihead softmax attention.
353
- Arguments
354
- ---------
355
- q, k, v: The tensor containing the
356
- query, key, and value. (B, S, H, D)
357
- """
358
-
359
- q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
360
- k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
361
- v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
362
-
363
- attn_output = flash_attn_func(
364
- q,
365
- k,
366
- v,
367
- dropout_p=self.dropout,
368
- causal=False,
369
- )
370
-
371
- attn_output = attn_output.reshape(batch_size, q_len,
372
- self.embed_dim).contiguous()
265
+ query_states, key_states, value_states = qkv_states.chunk(3, dim=-1)
266
+
267
+ query_states = query_states.view(batch_size, q_len,
268
+ self.num_heads_per_partition,
269
+ self.head_dim)
270
+ key_states = key_states.view(batch_size, q_len,
271
+ self.num_heads_per_partition,
272
+ self.head_dim)
273
+ value_states = value_states.view(batch_size, q_len,
274
+ self.num_heads_per_partition,
275
+ self.head_dim)
276
+
277
+ out = xops.memory_efficient_attention_forward(query_states,
278
+ key_states,
279
+ value_states,
280
+ p=self.dropout,
281
+ scale=self.scale)
282
+ out = out.view(batch_size, q_len, -1)
283
+ attn_output, _ = self.out_proj(out)
284
285
return attn_output
286
287
377
-# NOTE: Not used - kept for later when we TP the ViT
378
-class SiglipSdpaAttention(SiglipTPAttention):
379
-
380
- def __init__(self, *args, **kwargs):
381
- super().__init__(*args, **kwargs)
382
- self.is_causal = False
383
- self.attn_fn = self._sdpa_attention_forward
384
-
385
- def _sdpa_attention_forward(self, q, k, v, batch_size, q_len):
386
- q = q.view(batch_size, q_len, self.num_heads,
387
- self.head_dim).transpose(1, 2)
388
- k = k.view(batch_size, q_len, self.num_heads,
389
- self.head_dim).transpose(1, 2)
390
- v = v.view(batch_size, q_len, self.num_heads,
391
- self.head_dim).transpose(1, 2)
392
-
393
- attn_output = torch.nn.functional.scaled_dot_product_attention(
394
- q, k, v, dropout_p=self.dropout, is_causal=False, scale=self.scale)
395
-
396
- attn_output = attn_output.transpose(1, 2).contiguous()
397
- attn_output = attn_output.view(batch_size, q_len, self.embed_dim)
398
-
399
- return attn_output
400
-
401
-
402
-# NOTE: Not used - kept for later when we TP the ViT
403
-class SiglipxFormersAttention(SiglipTPAttention):
404
-
405
- def __init__(self, *args, **kwargs):
406
- super().__init__(*args, **kwargs)
407
- self.attn_fn = self._xformers_attention_forward
408
-
409
- def _xformers_attention_forward(self, q, k, v, batch_size, q_len):
410
- q = q.view(batch_size, q_len, self.num_heads, self.head_dim)
411
- k = k.view(batch_size, q_len, self.num_heads, self.head_dim)
412
- v = v.view(batch_size, q_len, self.num_heads, self.head_dim)
413
-
414
- attn_output = memory_efficient_attention(q,
415
- k,
416
- v,
417
- p=0.0,
418
- scale=self.scale)
419
- attn_output = attn_output.reshape(batch_size, q_len,
420
- self.embed_dim).contiguous()
421
-
422
- return attn_output
423
-
424
-
425
-# NOTE: Not used - kept for later when we TP the ViT
426
-SIGLIP_ATTENTION_CLASSES = {
427
- "eager": SiglipTPAttention,
428
- "flash_attention_2": SiglipFlashAttention2,
429
- "sdpa": SiglipSdpaAttention,
430
- "xformers": SiglipxFormersAttention,
431
-}
432
-
433
-
288
class SiglipMLP(nn.Module):
289
290
def __init__(
@@ -473,8 +327,7 @@ class SiglipEncoderLayer(nn.Module):
327
super().__init__()
328
self.embed_dim = config.hidden_size
329
476
- # TODO(ChristopherCho): use TP'ed Attention block
477
- self.self_attn = SiglipAttention(config)
330
+ self.self_attn = SiglipAttention(config, quant_config=quant_config)
331
self.layer_norm1 = nn.LayerNorm(self.embed_dim,
332
eps=config.layer_norm_eps)
333
self.mlp = SiglipMLP(
@@ -491,7 +344,7 @@ class SiglipEncoderLayer(nn.Module):
344
residual = hidden_states
345
346
hidden_states = self.layer_norm1(hidden_states)
494
- hidden_states, _ = self.self_attn(hidden_states=hidden_states)
347
+ hidden_states = self.self_attn(hidden_states=hidden_states)
348
hidden_states = residual + hidden_states
349
350
residual = hidden_states