A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[VLM] Enable overriding whether post layernorm is used in vision encoder + fix quant args (#9217)
Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
Oct 23, 2024 at 19:27 UTC
c18e1a34189812af21aa504f9166de5ed4a86675
18 files changed
+551
-253
vllm/model_executor/layers/quantization/awq.py
+16
-4
@@ -3,7 +3,8 @@ from typing import Any, Dict, List, Optional
3
import torch
4
5
from vllm import _custom_ops as ops
6
-from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6
+from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7
+ UnquantizedLinearMethod)
8
from vllm.model_executor.layers.quantization.base_config import (
9
QuantizationConfig)
10
from vllm.model_executor.parameter import (GroupQuantScaleParameter,
@@ -21,10 +22,12 @@ class AWQConfig(QuantizationConfig):
22
weight_bits: int,
23
group_size: int,
24
zero_point: bool,
25
+ modules_to_not_convert: Optional[List[str]] = None,
26
) -> None:
27
self.weight_bits = weight_bits
28
self.group_size = group_size
29
self.zero_point = zero_point
30
+ self.modules_to_not_convert = modules_to_not_convert or []
31
32
if self.weight_bits != 4:
33
raise ValueError(
@@ -35,7 +38,8 @@ class AWQConfig(QuantizationConfig):
38
def __repr__(self) -> str:
39
return (f"AWQConfig(weight_bits={self.weight_bits}, "
40
f"group_size={self.group_size}, "
38
- f"zero_point={self.zero_point})")
41
+ f"zero_point={self.zero_point}, "
42
+ f"modules_to_not_convert={self.modules_to_not_convert})")
43
44
def get_name(self) -> str:
45
return "awq"
@@ -61,11 +65,15 @@ class AWQConfig(QuantizationConfig):
65
weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
66
group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
67
zero_point = cls.get_from_keys(config, ["zero_point"])
64
- return cls(weight_bits, group_size, zero_point)
68
+ modules_to_not_convert = cls.get_from_keys_or(
69
+ config, ["modules_to_not_convert"], None)
70
+ return cls(weight_bits, group_size, zero_point, modules_to_not_convert)
71
72
def get_quant_method(self, layer: torch.nn.Module,
67
- prefix: str) -> Optional["AWQLinearMethod"]:
73
+ prefix: str) -> Optional["LinearMethodBase"]:
74
if isinstance(layer, LinearBase):
75
+ if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
76
+ return UnquantizedLinearMethod()
77
return AWQLinearMethod(self)
78
return None
79
@@ -73,6 +81,10 @@ class AWQConfig(QuantizationConfig):
81
return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
82
83
84
+def is_layer_skipped_awq(prefix: str, modules_to_not_convert: List[str]):
85
+ return any(module_name in prefix for module_name in modules_to_not_convert)
86
+
87
+
88
class AWQLinearMethod(LinearMethodBase):
89
"""Linear method for AWQ.
90
vllm/model_executor/models/blip.py
+59
-28
@@ -122,7 +122,7 @@ def input_processor_for_blip(
122
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
123
class BlipVisionEmbeddings(nn.Module):
124
125
- def __init__(self, config: BlipVisionConfig):
125
+ def __init__(self, config: Union[BlipVisionConfig, Blip2VisionConfig]):
126
super().__init__()
127
128
self.config = config
@@ -167,9 +167,10 @@ class BlipParallelAttention(nn.Module):
167
168
def __init__(
169
self,
170
- config: BlipVisionConfig,
170
+ config: Union[BlipVisionConfig, Blip2VisionConfig],
171
quant_config: Optional[QuantizationConfig] = None,
172
- ):
172
+ prefix: str = "",
173
+ ) -> None:
174
super().__init__()
175
self.config = config
176
self.embed_dim = config.hidden_size
@@ -189,11 +190,13 @@ class BlipParallelAttention(nn.Module):
190
self.num_heads,
191
bias=config.qkv_bias,
192
quant_config=quant_config,
193
+ prefix=f"{prefix}.qkv",
194
)
195
self.projection = RowParallelLinear(
196
self.embed_dim,
197
self.embed_dim,
198
quant_config=quant_config,
199
+ prefix=f"{prefix}.projection",
200
)
201
202
self.tp_size = get_tensor_model_parallel_world_size()
@@ -235,9 +238,12 @@ class BlipParallelAttention(nn.Module):
238
239
class BlipMLP(nn.Module):
240
238
- def __init__(self,
239
- config: BlipVisionConfig,
240
- quant_config: Optional[QuantizationConfig] = None):
241
+ def __init__(
242
+ self,
243
+ config: BlipVisionConfig,
244
+ quant_config: Optional[QuantizationConfig] = None,
245
+ prefix: str = "",
246
+ ) -> None:
247
super().__init__()
248
249
self.config = config
@@ -246,11 +252,13 @@ class BlipMLP(nn.Module):
252
self.fc1 = ColumnParallelLinear(config.hidden_size,
253
config.intermediate_size,
254
bias=True,
249
- quant_config=quant_config)
255
+ quant_config=quant_config,
256
+ prefix=f"{prefix}.fc1")
257
self.fc2 = RowParallelLinear(config.intermediate_size,
258
config.hidden_size,
259
bias=True,
253
- quant_config=quant_config)
260
+ quant_config=quant_config,
261
+ prefix=f"{prefix}.fc2")
262
263
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
264
hidden_states, _ = self.fc1(hidden_states)
@@ -262,24 +270,32 @@ class BlipMLP(nn.Module):
270
271
class BlipEncoderLayer(nn.Module):
272
265
- def __init__(self,
266
- config: BlipVisionConfig,
267
- quant_config: Optional[QuantizationConfig] = None):
273
+ def __init__(
274
+ self,
275
+ config: BlipVisionConfig,
276
+ quant_config: Optional[QuantizationConfig] = None,
277
+ prefix: str = "",
278
+ ) -> None:
279
super().__init__()
280
281
# fallback to sdpa attention if tp unavailable
282
num_heads = config.num_attention_heads
283
tp_size = get_tensor_model_parallel_world_size()
284
if USE_XFORMERS_OPS and num_heads % tp_size == 0:
274
- self.self_attn = BlipParallelAttention(config,
275
- quant_config=quant_config)
285
+ self.self_attn = BlipParallelAttention(
286
+ config,
287
+ quant_config=quant_config,
288
+ prefix=f"{prefix}.self_attn",
289
+ )
290
else:
291
# Blip doesn't have SDPA attention implemented in transformers
292
# use eager attention instead for cpu backend
293
self.self_attn = BlipAttention(config)
294
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
295
eps=config.layer_norm_eps)
282
- self.mlp = BlipMLP(config, quant_config=quant_config)
296
+ self.mlp = BlipMLP(config,
297
+ quant_config=quant_config,
298
+ prefix=f"{prefix}.mlp")
299
self.layer_norm2 = nn.LayerNorm(config.hidden_size,
300
eps=config.layer_norm_eps)
301
@@ -307,10 +323,13 @@ class BlipEncoder(nn.Module):
323
config: BlipConfig
324
"""
325
310
- def __init__(self,
311
- config: BlipVisionConfig,
312
- quant_config: Optional[QuantizationConfig] = None,
313
- num_hidden_layers_override: Optional[int] = None):
326
+ def __init__(
327
+ self,
328
+ config: BlipVisionConfig,
329
+ quant_config: Optional[QuantizationConfig] = None,
330
+ num_hidden_layers_override: Optional[int] = None,
331
+ prefix: str = "",
332
+ ) -> None:
333
super().__init__()
334
335
self.config = config
@@ -321,8 +340,10 @@ class BlipEncoder(nn.Module):
340
num_hidden_layers = num_hidden_layers_override
341
342
self.layers = nn.ModuleList([
324
- BlipEncoderLayer(config=config, quant_config=quant_config)
325
- for _ in range(num_hidden_layers)
343
+ BlipEncoderLayer(config=config,
344
+ quant_config=quant_config,
345
+ prefix=f"{prefix}.layers.{layer_idx}")
346
+ for layer_idx in range(num_hidden_layers)
347
])
348
349
def forward(self, inputs_embeds: torch.Tensor):
@@ -337,10 +358,15 @@ class BlipVisionModel(nn.Module):
358
config_class = BlipVisionConfig
359
main_input_name = "pixel_values"
360
340
- def __init__(self,
341
- config: BlipVisionConfig,
342
- quant_config: Optional[QuantizationConfig] = None,
343
- num_hidden_layers_override: Optional[int] = None):
361
+ def __init__(
362
+ self,
363
+ config: BlipVisionConfig,
364
+ quant_config: Optional[QuantizationConfig] = None,
365
+ *,
366
+ num_hidden_layers_override: Optional[int] = None,
367
+ require_post_norm: Optional[bool] = None,
368
+ prefix: str = "",
369
+ ) -> None:
370
super().__init__()
371
372
tp_size = get_tensor_model_parallel_world_size()
@@ -354,19 +380,24 @@ class BlipVisionModel(nn.Module):
380
config=config,
381
quant_config=quant_config,
382
num_hidden_layers_override=num_hidden_layers_override,
383
+ prefix=f"{prefix}.encoder",
384
)
385
386
+ num_hidden_layers = config.num_hidden_layers
387
if len(self.encoder.layers) > config.num_hidden_layers:
388
raise ValueError(
361
- f"The original encoder only has {config.num_hidden_layers} "
389
+ f"The original encoder only has {num_hidden_layers} "
390
f"layers, but you requested {len(self.encoder.layers)} layers."
391
)
364
- elif len(self.encoder.layers) == config.num_hidden_layers:
392
+
393
+ # If possible, skip post_layernorm to conserve memory
394
+ if require_post_norm is None:
395
+ require_post_norm = len(self.encoder.layers) == num_hidden_layers
396
+
397
+ if require_post_norm:
398
self.post_layernorm = nn.LayerNorm(config.hidden_size,
399
eps=config.layer_norm_eps)
400
else:
368
- # post_layernorm is unused when we extract intermediate features
369
- # In this case, we can skip it to conserve memory
401
self.post_layernorm = None
402
403
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/blip2.py
+1
-1
@@ -490,7 +490,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
490
self.multimodal_config = multimodal_config
491
492
# TODO: Optionally initializes this for supporting embeddings.
493
- self.vision_model = BlipVisionModel(config.vision_config)
493
+ self.vision_model = BlipVisionModel(config.vision_config, quant_config)
494
495
self.query_tokens = nn.Parameter(
496
torch.zeros(1, config.num_query_tokens,
vllm/model_executor/models/clip.py
+73
-31
@@ -192,6 +192,7 @@ class CLIPParallelAttention(nn.Module):
192
self,
193
config: CLIPVisionConfig,
194
quant_config: Optional[QuantizationConfig] = None,
195
+ prefix: str = "",
196
):
197
super().__init__()
198
self.config = config
@@ -211,12 +212,14 @@ class CLIPParallelAttention(nn.Module):
212
head_size=self.head_dim,
213
total_num_heads=self.num_heads,
214
quant_config=quant_config,
215
+ prefix=f"{prefix}.qkv_proj",
216
)
217
218
self.out_proj = RowParallelLinear(
219
input_size=self.embed_dim,
220
output_size=self.embed_dim,
221
quant_config=quant_config,
222
+ prefix=f"{prefix}.out_proj",
223
)
224
225
self.tp_size = get_tensor_model_parallel_world_size()
@@ -259,20 +262,25 @@ class CLIPParallelAttention(nn.Module):
262
263
class CLIPMLP(nn.Module):
264
262
- def __init__(self,
263
- config: CLIPVisionConfig,
264
- quant_config: Optional[QuantizationConfig] = None):
265
+ def __init__(
266
+ self,
267
+ config: CLIPVisionConfig,
268
+ quant_config: Optional[QuantizationConfig] = None,
269
+ prefix: str = "",
270
+ ) -> None:
271
super().__init__()
272
self.config = config
273
self.activation_fn = get_act_fn(config.hidden_act)
274
self.fc1 = ColumnParallelLinear(config.hidden_size,
275
config.intermediate_size,
276
bias=True,
271
- quant_config=quant_config)
277
+ quant_config=quant_config,
278
+ prefix=f"{prefix}.fc1")
279
self.fc2 = RowParallelLinear(config.intermediate_size,
280
config.hidden_size,
281
bias=True,
275
- quant_config=quant_config)
282
+ quant_config=quant_config,
283
+ prefix=f"{prefix}.fc2")
284
285
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
286
hidden_states, _ = self.fc1(hidden_states)
@@ -284,21 +292,29 @@ class CLIPMLP(nn.Module):
292
293
class CLIPEncoderLayer(nn.Module):
294
287
- def __init__(self,
288
- config: CLIPVisionConfig,
289
- quant_config: Optional[QuantizationConfig] = None):
295
+ def __init__(
296
+ self,
297
+ config: CLIPVisionConfig,
298
+ quant_config: Optional[QuantizationConfig] = None,
299
+ prefix: str = "",
300
+ ) -> None:
301
super().__init__()
302
303
num_heads = config.num_attention_heads
304
tp_size = get_tensor_model_parallel_world_size()
305
if USE_XFORMERS_OPS and num_heads % tp_size == 0:
295
- self.self_attn = CLIPParallelAttention(config,
296
- quant_config=quant_config)
306
+ self.self_attn = CLIPParallelAttention(
307
+ config,
308
+ quant_config=quant_config,
309
+ prefix=f"{prefix}.self_attn",
310
+ )
311
else:
312
self.self_attn = CLIPSdpaAttention(config)
313
self.layer_norm1 = nn.LayerNorm(config.hidden_size,
314
eps=config.layer_norm_eps)
301
- self.mlp = CLIPMLP(config, quant_config=quant_config)
315
+ self.mlp = CLIPMLP(config,
316
+ quant_config=quant_config,
317
+ prefix=f"{prefix}.mlp")
318
self.layer_norm2 = nn.LayerNorm(config.hidden_size,
319
eps=config.layer_norm_eps)
320
@@ -327,11 +343,15 @@ class CLIPEncoder(nn.Module):
343
config: CLIPConfig
344
"""
345
330
- def __init__(self,
331
- config: CLIPVisionConfig,
332
- quant_config: Optional[QuantizationConfig] = None,
333
- num_hidden_layers_override: Optional[int] = None):
346
+ def __init__(
347
+ self,
348
+ config: CLIPVisionConfig,
349
+ quant_config: Optional[QuantizationConfig] = None,
350
+ num_hidden_layers_override: Optional[int] = None,
351
+ prefix: str = "",
352
+ ) -> None:
353
super().__init__()
354
+
355
self.config = config
356
357
if num_hidden_layers_override is None:
@@ -339,8 +359,10 @@ class CLIPEncoder(nn.Module):
359
else:
360
num_hidden_layers = num_hidden_layers_override
361
self.layers = nn.ModuleList([
342
- CLIPEncoderLayer(config=config, quant_config=quant_config)
343
- for _ in range(num_hidden_layers)
362
+ CLIPEncoderLayer(config=config,
363
+ quant_config=quant_config,
364
+ prefix=f"{prefix}.layers.{layer_idx}")
365
+ for layer_idx in range(num_hidden_layers)
366
])
367
368
def forward(self, inputs_embeds: torch.Tensor):
@@ -354,11 +376,17 @@ class CLIPEncoder(nn.Module):
376
377
class CLIPVisionTransformer(nn.Module):
378
357
- def __init__(self,
358
- config: CLIPVisionConfig,
359
- quant_config: Optional[QuantizationConfig] = None,
360
- num_hidden_layers_override: Optional[int] = None):
379
+ def __init__(
380
+ self,
381
+ config: CLIPVisionConfig,
382
+ quant_config: Optional[QuantizationConfig] = None,
383
+ *,
384
+ num_hidden_layers_override: Optional[int] = None,
385
+ require_post_norm: Optional[bool] = None,
386
+ prefix: str = "",
387
+ ) -> None:
388
super().__init__()
389
+
390
self.config = config
391
embed_dim = config.hidden_size
392
@@ -370,19 +398,25 @@ class CLIPVisionTransformer(nn.Module):
398
self.encoder = CLIPEncoder(
399
config=config,
400
quant_config=quant_config,
373
- num_hidden_layers_override=num_hidden_layers_override)
401
+ num_hidden_layers_override=num_hidden_layers_override,
402
+ prefix=f"{prefix}.encoder",
403
+ )
404
405
+ num_hidden_layers = config.num_hidden_layers
406
if len(self.encoder.layers) > config.num_hidden_layers:
407
raise ValueError(
377
- f"The original encoder only has {config.num_hidden_layers} "
408
+ f"The original encoder only has {num_hidden_layers} "
409
f"layers, but you requested {len(self.encoder.layers)} layers."
410
)
380
- elif len(self.encoder.layers) == config.num_hidden_layers:
411
+
412
+ # If possible, skip post_layernorm to conserve memory
413
+ if require_post_norm is None:
414
+ require_post_norm = len(self.encoder.layers) == num_hidden_layers
415
+
416
+ if require_post_norm:
417
self.post_layernorm = nn.LayerNorm(embed_dim,
418
eps=config.layer_norm_eps)
419
else:
384
- # post_layernorm is unused when we extract intermediate features
385
- # In this case, we can skip it to conserve memory
420
self.post_layernorm = None
421
422
def forward(
@@ -405,10 +439,15 @@ class CLIPVisionModel(nn.Module):
439
config_class = CLIPVisionConfig
440
main_input_name = "pixel_values"
441
408
- def __init__(self,
409
- config: CLIPVisionConfig,
410
- quant_config: Optional[QuantizationConfig] = None,
411
- num_hidden_layers_override: Optional[int] = None):
442
+ def __init__(
443
+ self,
444
+ config: CLIPVisionConfig,
445
+ quant_config: Optional[QuantizationConfig] = None,
446
+ *,
447
+ num_hidden_layers_override: Optional[int] = None,
448
+ require_post_norm: Optional[bool] = None,
449
+ prefix: str = "",
450
+ ) -> None:
451
super().__init__()
452
453
tp_size = get_tensor_model_parallel_world_size()
@@ -418,7 +457,10 @@ class CLIPVisionModel(nn.Module):
457
self.vision_model = CLIPVisionTransformer(
458
config=config,
459
quant_config=quant_config,
421
- num_hidden_layers_override=num_hidden_layers_override)
460
+ num_hidden_layers_override=num_hidden_layers_override,
461
+ require_post_norm=require_post_norm,
462
+ prefix=f"{prefix}.vision_model",
463
+ )
464
465
def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
466
return self.vision_model(pixel_values)
vllm/model_executor/models/idefics2_vision_model.py
+41
-10
@@ -113,7 +113,8 @@ class Idefics2VisionAttention(nn.Module):
113
self,
114
config: Idefics2Config,
115
quant_config: Optional[QuantizationConfig] = None,
116
- ):
116
+ prefix: str = "",
117
+ ) -> None:
118
super().__init__()
119
self.config = config
120
self.embed_dim = config.hidden_size
@@ -130,12 +131,14 @@ class Idefics2VisionAttention(nn.Module):
131
self.head_dim,
132
self.num_heads,
133
quant_config=quant_config,
134
+ prefix=f"{prefix}.qkv_proj",
135
)
136
self.out_proj = RowParallelLinear(
137
self.embed_dim,
138
self.embed_dim,
139
bias=True,
140
quant_config=quant_config,
141
+ prefix=f"{prefix}.out_proj",
142
)
143
self.tp_size = get_tensor_model_parallel_world_size()
144
self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
@@ -178,7 +181,8 @@ class Idefics2VisionMLP(nn.Module):
181
self,
182
config: Idefics2Config,
183
quant_config: Optional[QuantizationConfig] = None,
181
- ):
184
+ prefix: str = "",
185
+ ) -> None:
186
super().__init__()
187
self.config = config
188
self.activation_fn = get_act_fn(config.hidden_act)
@@ -187,12 +191,14 @@ class Idefics2VisionMLP(nn.Module):
191
config.intermediate_size,
192
bias=True,
193
quant_config=quant_config,
194
+ prefix=f"{prefix}.fc1",
195
)
196
self.fc2 = RowParallelLinear(
197
config.intermediate_size,
198
config.hidden_size,
199
bias=True,
200
quant_config=quant_config,
201
+ prefix=f"{prefix}.fc2",
202
)
203
204
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -204,13 +210,22 @@ class Idefics2VisionMLP(nn.Module):
210
211
class Idefics2EncoderLayer(nn.Module):
212
207
- def __init__(self, config: Idefics2Config):
213
+ def __init__(
214
+ self,
215
+ config: Idefics2Config,
216
+ quant_config: Optional[QuantizationConfig] = None,
217
+ prefix: str = "",
218
+ ) -> None:
219
super().__init__()
220
self.embed_dim = config.hidden_size
210
- self.self_attn = Idefics2VisionAttention(config)
221
+ self.self_attn = Idefics2VisionAttention(config,
222
+ quant_config=quant_config,
223
+ prefix=f"{prefix}.self_attn")
224
self.layer_norm1 = nn.LayerNorm(self.embed_dim,
225
eps=config.layer_norm_eps)
213
- self.mlp = Idefics2VisionMLP(config)
226
+ self.mlp = Idefics2VisionMLP(config,
227
+ quant_config=quant_config,
228
+ prefix=f"{prefix}.mlp")
229
self.layer_norm2 = nn.LayerNorm(self.embed_dim,
230
eps=config.layer_norm_eps)
231
@@ -245,12 +260,20 @@ class Idefics2Encoder(nn.Module):
260
config: Idefics2Config
261
"""
262
248
- def __init__(self, config: Idefics2Config):
263
+ def __init__(
264
+ self,
265
+ config: Idefics2Config,
266
+ quant_config: Optional[QuantizationConfig] = None,
267
+ prefix: str = "",
268
+ ) -> None:
269
super().__init__()
270
+
271
self.config = config
272
self.layers = nn.ModuleList([
252
- Idefics2EncoderLayer(config)
253
- for _ in range(config.num_hidden_layers)
273
+ Idefics2EncoderLayer(config,
274
+ quant_config=quant_config,
275
+ prefix=f"{prefix}.layers.{layer_idx}")
276
+ for layer_idx in range(config.num_hidden_layers)
277
])
278
279
def forward(
@@ -275,12 +298,20 @@ class Idefics2Encoder(nn.Module):
298
299
class Idefics2VisionTransformer(nn.Module):
300
278
- def __init__(self, config: Idefics2VisionConfig):
301
+ def __init__(
302
+ self,
303
+ config: Idefics2VisionConfig,
304
+ quant_config: Optional[QuantizationConfig] = None,
305
+ prefix: str = "",
306
+ ) -> None:
307
super().__init__()
308
+
309
embed_dim = config.hidden_size
310
self.config = config
311
self.embeddings = Idefics2VisionEmbeddings(config)
283
- self.encoder = Idefics2Encoder(config)
312
+ self.encoder = Idefics2Encoder(config,
313
+ quant_config=quant_config,
314
+ prefix=f"{prefix}.encoder")
315
self.post_layernorm = nn.LayerNorm(embed_dim,
316
eps=config.layer_norm_eps)
317
vllm/model_executor/models/intern_vit.py
+30
-11
@@ -137,6 +137,7 @@ class InternParallelAttention(nn.Module):
137
quant_config: Optional[QuantizationConfig] = None,
138
*,
139
num_dummy_heads: int = 0,
140
+ prefix: str = "",
141
) -> None:
142
super().__init__()
143
@@ -165,6 +166,7 @@ class InternParallelAttention(nn.Module):
166
num_dummy_heads + self.num_heads,
167
bias=config.qkv_bias,
168
quant_config=quant_config,
169
+ prefix=f"{prefix}.qkv",
170
)
171
172
self.qk_normalization = config.qk_normalization
@@ -181,6 +183,7 @@ class InternParallelAttention(nn.Module):
183
self.dummy_dim,
184
self.embed_dim,
185
quant_config=quant_config,
186
+ prefix=f"{prefix}.proj",
187
)
188
189
def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
@@ -284,20 +287,26 @@ class InternSdpaAttention(nn.Module):
287
288
class InternMLP(nn.Module):
289
287
- def __init__(self,
288
- config: PretrainedConfig,
289
- quant_config: Optional[QuantizationConfig] = None):
290
+ def __init__(
291
+ self,
292
+ config: PretrainedConfig,
293
+ quant_config: Optional[QuantizationConfig] = None,
294
+ prefix: str = "",
295
+ ) -> None:
296
super().__init__()
297
+
298
self.config = config
299
self.activation_fn = get_act_fn(config.hidden_act)
300
self.fc1 = ColumnParallelLinear(config.hidden_size,
301
config.intermediate_size,
302
bias=True,
296
- quant_config=quant_config)
303
+ quant_config=quant_config,
304
+ prefix=f"{prefix}.fc1")
305
self.fc2 = RowParallelLinear(config.intermediate_size,
306
config.hidden_size,
307
bias=True,
300
- quant_config=quant_config)
308
+ quant_config=quant_config,
309
+ prefix=f"{prefix}.fc2")
310
311
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
312
hidden_states, _ = self.fc1(hidden_states)
@@ -315,6 +324,7 @@ class InternVisionEncoderLayer(nn.Module):
324
quant_config: Optional[QuantizationConfig] = None,
325
*,
326
num_dummy_heads: int = 0,
327
+ prefix: str = "",
328
) -> None:
329
super().__init__()
330
@@ -324,9 +334,12 @@ class InternVisionEncoderLayer(nn.Module):
334
335
self.attn = self._init_attn(config,
336
quant_config,
327
- num_dummy_heads=num_dummy_heads)
337
+ num_dummy_heads=num_dummy_heads,
338
+ prefix=f"{prefix}.attn")
339
329
- self.mlp = InternMLP(config, quant_config=quant_config)
340
+ self.mlp = InternMLP(config,
341
+ quant_config=quant_config,
342
+ prefix=f"{prefix}.mlp")
343
self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
344
eps=config.layer_norm_eps)
345
self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
@@ -343,6 +356,7 @@ class InternVisionEncoderLayer(nn.Module):
356
quant_config: Optional[QuantizationConfig],
357
*,
358
num_dummy_heads: int,
359
+ prefix: str = "",
360
):
361
# fallback to sdpa attention if tp unavailable
362
tp_size = get_tensor_model_parallel_world_size()
@@ -351,7 +365,8 @@ class InternVisionEncoderLayer(nn.Module):
365
if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
366
return InternParallelAttention(config,
367
quant_config=quant_config,
354
- num_dummy_heads=num_dummy_heads)
368
+ num_dummy_heads=num_dummy_heads,
369
+ prefix=prefix)
370
371
return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
372
@@ -377,6 +392,7 @@ class InternVisionEncoder(nn.Module):
392
*,
393
num_hidden_layers_override: Optional[int] = None,
394
num_dummy_heads: int = 0,
395
+ prefix: str = "",
396
):
397
super().__init__()
398
@@ -390,8 +406,9 @@ class InternVisionEncoder(nn.Module):
406
self.layers = nn.ModuleList([
407
InternVisionEncoderLayer(config,
408
quant_config,
393
- num_dummy_heads=num_dummy_heads)
394
- for _ in range(num_hidden_layers)
409
+ num_dummy_heads=num_dummy_heads,
410
+ prefix=f"{prefix}.layers.{layer_idx}")
411
+ for layer_idx in range(num_hidden_layers)
412
])
413
414
def forward(self, inputs_embeds: torch.Tensor):
@@ -412,7 +429,8 @@ class InternVisionModel(nn.Module):
429
*,
430
num_hidden_layers_override: Optional[int] = None,
431
num_dummy_heads: int = 0,
415
- ):
432
+ prefix: str = "",
433
+ ) -> None:
434
super().__init__()
435
436
self.config = config
@@ -423,6 +441,7 @@ class InternVisionModel(nn.Module):
441
quant_config=quant_config,
442
num_hidden_layers_override=num_hidden_layers_override,
443
num_dummy_heads=num_dummy_heads,
444
+ prefix=f"{prefix}.encoder",
445
)
446
447
def get_input_embeddings(self):
vllm/model_executor/models/internvl.py
+35
-6
@@ -19,7 +19,8 @@ from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, MultiModalConfig
20
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21
token_inputs)
22
-from vllm.model_executor.layers.quantization import QuantizationConfig
22
+from vllm.model_executor.layers.quantization import (AWQConfig,
23
+ QuantizationConfig)
24
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25
from vllm.model_executor.models.intern_vit import (InternVisionModel,
26
InternVisionPatchModel)
@@ -418,11 +419,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
419
420
self.config = config
421
self.multimodal_config = multimodal_config
422
+ self._patch_quant_config(config, quant_config)
423
424
image_size = config.force_image_size or config.vision_config.image_size
425
patch_size = config.vision_config.patch_size
426
self.patch_size = patch_size
425
- self.select_layer = config.select_layer
427
self.num_image_token = int(
428
(image_size // patch_size)**2 * (config.downsample_ratio**2))
429
self.downsample_ratio = config.downsample_ratio
@@ -430,7 +431,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
431
432
self.llm_arch_name = config.text_config.architectures[0]
433
self.is_mono = self.llm_arch_name == 'InternLM2VEForCausalLM'
433
- self.vision_model = self._init_vision_model(config, self.is_mono)
434
+ self.vision_model = self._init_vision_model(
435
+ config,
436
+ quant_config=quant_config,
437
+ is_mono=self.is_mono,
438
+ prefix="vision_model",
439
+ )
440
441
self.language_model = init_vllm_registered_model(
442
config.text_config, cache_config, quant_config)
@@ -441,6 +447,18 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
447
self.make_empty_intermediate_tensors = (
448
self.language_model.make_empty_intermediate_tensors)
449
450
+ def _patch_quant_config(self, config: PretrainedConfig,
451
+ quant_config: QuantizationConfig):
452
+ # the awq models from OpenGVLab missing `modules_to_not_convert`
453
+ # patch the quant_config to add `modules_to_not_convert` back
454
+ if isinstance(quant_config, AWQConfig):
455
+ text_config = config.text_config
456
+ llm_quant_config = getattr(text_config, "quantization_config",
457
+ None)
458
+ if (not quant_config.modules_to_not_convert) and \
459
+ (llm_quant_config is not None):
460
+ quant_config.modules_to_not_convert.append("vision_model")
461
+
462
@cached_property
463
def sampler(self):
464
if hasattr(self.language_model, "sampler"):
@@ -448,17 +466,28 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
466
467
return Sampler()
468
451
- def _init_vision_model(self, config: PretrainedConfig, is_mono: bool):
469
+ def _init_vision_model(
470
+ self,
471
+ config: PretrainedConfig,
472
+ quant_config: Optional[QuantizationConfig],
473
+ *,
474
+ is_mono: bool,
475
+ prefix: str,
476
+ ):
477
if not is_mono:
453
- vision_feature_layer = self.select_layer
478
+ vision_feature_layer = config.select_layer
479
if vision_feature_layer < 0:
480
num_hidden_layers = config.vision_config.num_hidden_layers \
481
+ vision_feature_layer + 1
482
else:
483
num_hidden_layers = vision_feature_layer + 1
484
+
485
return InternVisionModel(
486
config.vision_config,
461
- num_hidden_layers_override=num_hidden_layers)
487
+ quant_config=quant_config,
488
+ num_hidden_layers_override=num_hidden_layers,
489
+ prefix=prefix,
490
+ )
491
else:
492
return InternVisionPatchModel(config.vision_config)
493
vllm/model_executor/models/llava.py
+25
-7
@@ -1,12 +1,12 @@
1
from functools import cached_property
2
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3
- TypedDict, Union)
2
+from typing import (Iterable, List, Literal, Mapping, Optional, Protocol,
3
+ Tuple, TypedDict, Union)
4
5
import torch
6
import torch.nn as nn
7
from PIL import Image
8
from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9
- SiglipVisionConfig)
9
+ PretrainedConfig, SiglipVisionConfig)
10
11
from vllm.attention import AttentionMetadata
12
from vllm.config import CacheConfig, MultiModalConfig
@@ -200,7 +200,17 @@ def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
200
raise NotImplementedError(msg)
201
202
203
-def _init_vision_tower(hf_config: LlavaConfig):
203
+class LlavaLikeConfig(Protocol):
204
+ vision_config: PretrainedConfig
205
+ vision_feature_layer: int
206
+
207
+
208
+def init_vision_tower_for_llava(
209
+ hf_config: LlavaLikeConfig,
210
+ quant_config: Optional[QuantizationConfig],
211
+ *,
212
+ require_post_norm: Optional[bool] = None,
213
+):
214
vision_config = hf_config.vision_config
215
216
# Initialize the vision tower only up to the required feature layer
@@ -214,16 +224,24 @@ def _init_vision_tower(hf_config: LlavaConfig):
224
if isinstance(vision_config, CLIPVisionConfig):
225
return CLIPVisionModel(
226
vision_config,
227
+ quant_config,
228
num_hidden_layers_override=num_hidden_layers,
229
+ require_post_norm=require_post_norm,
230
)
231
elif isinstance(vision_config, SiglipVisionConfig):
232
return SiglipVisionModel(
233
vision_config,
234
+ quant_config,
235
num_hidden_layers_override=num_hidden_layers,
236
+ require_post_norm=require_post_norm,
237
)
238
elif isinstance(vision_config, PixtralVisionConfig):
225
- # TODO: allow layer override?
226
- return PixtralHFVisionModel(vision_config)
239
+ return PixtralHFVisionModel(
240
+ vision_config,
241
+ quant_config,
242
+ num_hidden_layers_override=num_hidden_layers,
243
+ require_post_norm=require_post_norm,
244
+ )
245
246
msg = f"Unsupported vision config: {type(vision_config)}"
247
raise NotImplementedError(msg)
@@ -255,7 +273,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
273
config.projector_hidden_act = "gelu"
274
275
# TODO: Optionally initializes this for supporting embeddings.
258
- self.vision_tower = _init_vision_tower(config)
276
+ self.vision_tower = init_vision_tower_for_llava(config, quant_config)
277
self.multi_modal_projector = LlavaMultiModalProjector(
278
vision_hidden_size=config.vision_config.hidden_size,
279
text_hidden_size=config.text_config.hidden_size,
vllm/model_executor/models/llava_next.py
+2
-28
@@ -26,7 +26,7 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
26
dummy_seq_data_for_clip, get_clip_image_feature_size,
27
get_clip_patch_grid_length, input_processor_for_clip)
28
from .interfaces import SupportsMultiModal, SupportsPP
29
-from .llava import LlavaMultiModalProjector
29
+from .llava import LlavaMultiModalProjector, init_vision_tower_for_llava
30
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32
get_siglip_patch_grid_length, input_processor_for_siglip)
@@ -259,32 +259,6 @@ def input_processor_for_llava_next(ctx: InputContext,
259
raise NotImplementedError(msg)
260
261
262
-def _init_vision_tower(hf_config: LlavaNextConfig):
263
- vision_config = hf_config.vision_config
264
-
265
- # Initialize the vision tower only up to the required feature layer
266
- vision_feature_layer = hf_config.vision_feature_layer
267
- if vision_feature_layer < 0:
268
- num_hidden_layers = hf_config.vision_config.num_hidden_layers \
269
- + vision_feature_layer + 1
270
- else:
271
- num_hidden_layers = vision_feature_layer + 1
272
-
273
- if isinstance(vision_config, CLIPVisionConfig):
274
- return CLIPVisionModel(
275
- vision_config,
276
- num_hidden_layers_override=num_hidden_layers,
277
- )
278
- elif isinstance(vision_config, SiglipVisionConfig):
279
- return SiglipVisionModel(
280
- vision_config,
281
- num_hidden_layers_override=num_hidden_layers,
282
- )
283
-
284
- msg = f"Unsupported vision config: {type(vision_config)}"
285
- raise NotImplementedError(msg)
286
-
287
-
262
@MULTIMODAL_REGISTRY.register_image_input_mapper()
263
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
264
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
@@ -303,7 +277,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
277
self.multimodal_config = multimodal_config
278
279
# TODO: Optionally initializes this for supporting embeddings.
306
- self.vision_tower = _init_vision_tower(config)
280
+ self.vision_tower = init_vision_tower_for_llava(config, quant_config)
281
self.image_newline = nn.Parameter(
282
torch.empty(config.text_config.hidden_size))
283
self.multi_modal_projector = LlavaMultiModalProjector(
vllm/model_executor/models/llava_next_video.py
+2
-27
@@ -26,6 +26,7 @@ from vllm.utils import is_list_of
26
27
from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
28
from .interfaces import SupportsMultiModal, SupportsPP
29
+from .llava import init_vision_tower_for_llava
30
from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
dummy_seq_data_for_siglip)
32
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
@@ -179,32 +180,6 @@ def input_processor_for_llava_next_video(ctx: InputContext,
180
raise NotImplementedError(msg)
181
182
182
-def _init_vision_tower(hf_config: LlavaNextVideoConfig):
183
- vision_config = hf_config.vision_config
184
-
185
- # Initialize the vision tower only up to the required feature layer
186
- vision_feature_layer = hf_config.vision_feature_layer
187
- if vision_feature_layer < 0:
188
- num_hidden_layers = hf_config.vision_config.num_hidden_layers \
189
- + vision_feature_layer + 1
190
- else:
191
- num_hidden_layers = vision_feature_layer + 1
192
-
193
- if isinstance(vision_config, CLIPVisionConfig):
194
- return CLIPVisionModel(
195
- vision_config,
196
- num_hidden_layers_override=num_hidden_layers,
197
- )
198
- elif isinstance(vision_config, SiglipVisionConfig):
199
- return SiglipVisionModel(
200
- vision_config,
201
- num_hidden_layers_override=num_hidden_layers,
202
- )
203
-
204
- msg = f"Unsupported vision config: {type(vision_config)}"
205
- raise NotImplementedError(msg)
206
-
207
-
183
# adopted from transformers modeling_llava_next_video.py
184
class LlavaNextVideoPooler(nn.Module):
185
@@ -281,7 +256,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
256
self.multimodal_config = multimodal_config
257
258
# Initialize the vision tower only up to the required feature layer
284
- self.vision_tower = _init_vision_tower(config)
259
+ self.vision_tower = init_vision_tower_for_llava(config, quant_config)
260
self.vision_resampler = LlavaNextVideoPooler(config)
261
self.multi_modal_projector = LlavaNextMultiModalProjector(
262
vision_hidden_size=config.vision_config.hidden_size,
vllm/model_executor/models/llava_onevision.py
+2
-27
@@ -31,6 +31,7 @@ from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
31
dummy_video_for_clip, get_clip_image_feature_size,
32
get_clip_patch_grid_length, input_processor_for_clip)
33
from .interfaces import SupportsMultiModal, SupportsPP
34
+from .llava import init_vision_tower_for_llava
35
from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36
dummy_video_for_siglip, get_siglip_image_feature_size,
37
get_siglip_patch_grid_length, input_processor_for_siglip)
@@ -357,32 +358,6 @@ def input_processor_for_llava_onevision(ctx: InputContext,
358
raise NotImplementedError(msg)
359
360
360
-def _init_vision_tower(hf_config: LlavaOnevisionConfig):
361
- vision_config = hf_config.vision_config
362
-
363
- # Initialize the vision tower only up to the required feature layer
364
- vision_feature_layer = hf_config.vision_feature_layer
365
- if vision_feature_layer < 0:
366
- num_hidden_layers = hf_config.vision_config.num_hidden_layers \
367
- + vision_feature_layer + 1
368
- else:
369
- num_hidden_layers = vision_feature_layer + 1
370
-
371
- if isinstance(vision_config, CLIPVisionConfig):
372
- return CLIPVisionModel(
373
- vision_config,
374
- num_hidden_layers_override=num_hidden_layers,
375
- )
376
- elif isinstance(vision_config, SiglipVisionConfig):
377
- return SiglipVisionModel(
378
- vision_config,
379
- num_hidden_layers_override=num_hidden_layers,
380
- )
381
-
382
- msg = f"Unsupported vision config: {type(vision_config)}"
383
- raise NotImplementedError(msg)
384
-
385
-
361
class LlavaOnevisionMultiModalProjector(nn.Module):
362
363
def __init__(self, config: LlavaOnevisionConfig):
@@ -425,7 +400,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
400
self.multimodal_config = multimodal_config
401
402
# Initialize the vision tower only up to the required feature layer
428
- self.vision_tower = _init_vision_tower(config)
403
+ self.vision_tower = init_vision_tower_for_llava(config, quant_config)
404
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
405
self.language_model = init_vllm_registered_model(
406
config.text_config, cache_config, quant_config)
vllm/model_executor/models/minicpmv.py
+25
-8
@@ -395,7 +395,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
395
396
self.version = get_version_by_config(self.config)
397
self.llm = self.init_llm(config, cache_config, quant_config)
398
- self.vpm = self.init_vision_module()
398
+ self.vpm = self.init_vision_module(config, quant_config)
399
param_dtype = torch.get_default_dtype()
400
self.vpm.to(dtype=param_dtype)
401
self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -647,7 +647,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
647
) -> nn.Module:
648
raise NotImplementedError
649
650
- def init_vision_module(self) -> nn.Module:
650
+ def init_vision_module(
651
+ self,
652
+ config: PretrainedConfig,
653
+ quant_config: Optional[QuantizationConfig],
654
+ ) -> nn.Module:
655
raise NotImplementedError
656
657
def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
@@ -693,7 +697,11 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
697
quant_config=quant_config),
698
name="model")
699
696
- def init_vision_module(self) -> nn.Module:
700
+ def init_vision_module(
701
+ self,
702
+ config: PretrainedConfig,
703
+ quant_config: Optional[QuantizationConfig],
704
+ ) -> nn.Module:
705
# TODO :refactor this vision model
706
try:
707
import timm
@@ -817,8 +825,13 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
825
quant_config=quant_config),
826
name="model")
827
820
- def init_vision_module(self) -> nn.Module:
821
- model = Idefics2VisionTransformer(self.config.vision_config)
828
+ def init_vision_module(
829
+ self,
830
+ config: PretrainedConfig,
831
+ quant_config: Optional[QuantizationConfig],
832
+ ) -> nn.Module:
833
+ model = Idefics2VisionTransformer(config.vision_config,
834
+ quant_config=quant_config)
835
if self.config.drop_vision_last_layer:
836
model.encoder.layers = model.encoder.layers[:-1]
837
return model
@@ -929,9 +942,13 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
942
quant_config=quant_config),
943
name="model")
944
932
- def init_vision_module(self) -> nn.Module:
933
-
934
- model = Idefics2VisionTransformer(self.config.vision_config)
945
+ def init_vision_module(
946
+ self,
947
+ config: PretrainedConfig,
948
+ quant_config: Optional[QuantizationConfig],
949
+ ) -> nn.Module:
950
+ model = Idefics2VisionTransformer(config.vision_config,
951
+ quant_config=quant_config)
952
if self.config.drop_vision_last_layer:
953
model.encoder.layers = model.encoder.layers[:-1]
954
return model
vllm/model_executor/models/mllama.py
+89
-31
@@ -379,9 +379,13 @@ class MllamaVisionSdpaAttention(nn.Module):
379
380
class MllamaVisionEncoderLayer(nn.Module):
381
382
- def __init__(self,
383
- config: config_mllama.MllamaVisionConfig,
384
- is_gated: bool = False):
382
+ def __init__(
383
+ self,
384
+ config: config_mllama.MllamaVisionConfig,
385
+ quant_config: Optional[QuantizationConfig],
386
+ prefix: str = "",
387
+ is_gated: bool = False,
388
+ ) -> None:
389
super().__init__()
390
391
self.hidden_size = config.hidden_size
@@ -390,7 +394,9 @@ class MllamaVisionEncoderLayer(nn.Module):
394
self.intermediate_size = config.intermediate_size
395
396
self.self_attn = MllamaVisionSdpaAttention(config)
393
- self.mlp = CLIPMLP(config)
397
+ self.mlp = CLIPMLP(config,
398
+ quant_config=quant_config,
399
+ prefix=f"{prefix}.mlp")
400
401
self.input_layernorm = nn.LayerNorm(self.hidden_size,
402
eps=config.norm_eps)
@@ -427,16 +433,23 @@ class MllamaVisionEncoderLayer(nn.Module):
433
434
class MllamaVisionEncoder(nn.Module):
435
430
- def __init__(self,
431
- config: config_mllama.MllamaVisionConfig,
432
- num_layers=32,
433
- is_gated=False,
434
- output_hidden_states=None):
436
+ def __init__(
437
+ self,
438
+ config: config_mllama.MllamaVisionConfig,
439
+ quant_config: Optional[QuantizationConfig],
440
+ num_layers: int = 32,
441
+ is_gated: bool = False,
442
+ output_hidden_states=None,
443
+ prefix: str = "",
444
+ ) -> None:
445
super().__init__()
446
self.config = config
447
self.layers = nn.ModuleList([
438
- MllamaVisionEncoderLayer(config, is_gated)
439
- for _ in range(num_layers)
448
+ MllamaVisionEncoderLayer(config,
449
+ quant_config=quant_config,
450
+ is_gated=is_gated,
451
+ prefix=f"{prefix}.layers.{layer_idx}")
452
+ for layer_idx in range(num_layers)
453
])
454
self.output_hidden_states = output_hidden_states or []
455
@@ -463,8 +476,14 @@ class MllamaVisionEncoder(nn.Module):
476
477
class MllamaVisionModel(nn.Module):
478
466
- def __init__(self, config: config_mllama.MllamaVisionConfig):
479
+ def __init__(
480
+ self,
481
+ config: config_mllama.MllamaVisionConfig,
482
+ quant_config: Optional[QuantizationConfig],
483
+ prefix: str = "",
484
+ ) -> None:
485
super().__init__()
486
+
487
self.image_size = config.image_size
488
self.patch_size = config.patch_size
489
self.max_num_tiles = config.max_num_tiles
@@ -500,12 +519,19 @@ class MllamaVisionModel(nn.Module):
519
# encoders
520
self.transformer = MllamaVisionEncoder(
521
config,
522
+ quant_config,
523
config.num_hidden_layers,
524
is_gated=False,
505
- output_hidden_states=config.intermediate_layers_indices)
506
- self.global_transformer = MllamaVisionEncoder(config,
507
- config.num_global_layers,
508
- is_gated=True)
525
+ output_hidden_states=config.intermediate_layers_indices,
526
+ prefix=f"{prefix}.transformer",
527
+ )
528
+ self.global_transformer = MllamaVisionEncoder(
529
+ config,
530
+ quant_config,
531
+ config.num_global_layers,
532
+ is_gated=True,
533
+ prefix=f"{prefix}.global_transformer",
534
+ )
535
536
def apply_class_embedding(self,
537
hidden_state: torch.Tensor) -> torch.Tensor:
@@ -648,6 +674,7 @@ class MllamaTextCrossAttention(nn.Module):
674
config: Optional[config_mllama.MllamaTextConfig] = None,
675
layer_idx: Optional[int] = None,
676
quant_config: Optional[QuantizationConfig] = None,
677
+ prefix: str = "",
678
):
679
super().__init__()
680
self.config = config
@@ -673,6 +700,7 @@ class MllamaTextCrossAttention(nn.Module):
700
self.num_key_value_heads,
701
bias=False,
702
quant_config=quant_config,
703
+ prefix=f"{prefix}.qkv_proj",
704
)
705
self.o_proj = RowParallelLinear(
706
self.num_heads * self.head_dim,
@@ -680,6 +708,7 @@ class MllamaTextCrossAttention(nn.Module):
708
bias=False,
709
input_is_parallel=True,
710
quant_config=quant_config,
711
+ prefix=f"{prefix}.o_proj",
712
)
713
# vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
714
# use huggingface's instead
@@ -692,6 +721,7 @@ class MllamaTextCrossAttention(nn.Module):
721
self.head_dim,
722
self.scaling,
723
self.num_local_key_value_heads,
724
+ prefix=f"{prefix}.attn",
725
)
726
727
def forward(
@@ -791,15 +821,21 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
821
"""Cross-attention transformer block with tanh-gated attention
822
and feedforward."""
823
794
- def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int,
795
- quant_config: Optional[QuantizationConfig]) \
796
- -> None:
824
+ def __init__(
825
+ self,
826
+ config: config_mllama.MllamaTextConfig,
827
+ layer_idx: int,
828
+ quant_config: Optional[QuantizationConfig],
829
+ prefix: str = "",
830
+ ) -> None:
831
super().__init__()
832
+
833
self.layer_idx = layer_idx
834
self.cross_attn = MllamaTextCrossAttention(
835
config=config,
836
layer_idx=layer_idx,
837
quant_config=quant_config,
838
+ prefix=f"{prefix}.cross_attn",
839
)
840
841
self.input_layernorm = RMSNorm(config.hidden_size,
@@ -811,6 +847,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
847
intermediate_size=config.intermediate_size,
848
hidden_act=config.hidden_act,
849
quant_config=quant_config,
850
+ prefix=f"{prefix}.mlp",
851
)
852
self.post_attention_layernorm = RMSNorm(config.hidden_size,
853
eps=config.rms_norm_eps)
@@ -854,10 +891,15 @@ class MllamaTextModel(nn.Module):
891
config_class = config_mllama.MllamaTextConfig
892
base_model_prefix = "model"
893
857
- def __init__(self, config: config_mllama.MllamaTextConfig,
858
- cache_config: Optional[CacheConfig],
859
- quant_config: Optional[QuantizationConfig]):
894
+ def __init__(
895
+ self,
896
+ config: config_mllama.MllamaTextConfig,
897
+ cache_config: Optional[CacheConfig],
898
+ quant_config: Optional[QuantizationConfig],
899
+ prefix: str = "",
900
+ ) -> None:
901
super().__init__()
902
+
903
self.padding_idx = config.pad_token_id
904
self.vocab_size = config.vocab_size
905
self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
@@ -869,13 +911,20 @@ class MllamaTextModel(nn.Module):
911
if layer_idx in self.cross_attention_layers:
912
layers.append(
913
MllamaCrossAttentionDecoderLayer(
872
- config, layer_idx, quant_config=quant_config))
914
+ config,
915
+ layer_idx,
916
+ quant_config=quant_config,
917
+ prefix=f"{prefix}.layers.{layer_idx}",
918
+ ))
919
else:
920
# TODO: force LlamaDecoderLayer to config.attention_bias=False
921
layers.append(
876
- LlamaDecoderLayer(config,
877
- cache_config=cache_config,
878
- quant_config=quant_config))
922
+ LlamaDecoderLayer(
923
+ config,
924
+ cache_config=cache_config,
925
+ quant_config=quant_config,
926
+ prefix=f"{prefix}.layers.{layer_idx}",
927
+ ))
928
929
self.layers = nn.ModuleList(layers)
930
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -932,12 +981,19 @@ class MllamaForCausalLM(nn.Module):
981
"MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
982
]
983
935
- def __init__(self, config: config_mllama.MllamaTextConfig,
936
- cache_config: Optional[CacheConfig],
937
- quant_config: Optional[QuantizationConfig]):
984
+ def __init__(
985
+ self,
986
+ config: config_mllama.MllamaTextConfig,
987
+ cache_config: Optional[CacheConfig],
988
+ quant_config: Optional[QuantizationConfig],
989
+ prefix: str = "",
990
+ ) -> None:
991
super().__init__()
992
self.vocab_size = config.vocab_size
940
- self.model = MllamaTextModel(config, cache_config, quant_config)
993
+ self.model = MllamaTextModel(config,
994
+ cache_config,
995
+ quant_config,
996
+ prefix=f"{prefix}.model")
997
self.lm_head = ParallelLMHead(
998
config.vocab_size,
999
config.hidden_size,
@@ -994,11 +1050,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1050
config.pad_token_id if config.pad_token_id is not None else -1
1051
self.image_size = config.vision_config.image_size
1052
997
- self.vision_model = MllamaVisionModel(config.vision_config)
1053
+ self.vision_model = MllamaVisionModel(config.vision_config,
1054
+ quant_config)
1055
self.language_model = MllamaForCausalLM(
1056
config.text_config,
1057
cache_config=cache_config,
1058
quant_config=quant_config,
1059
+ prefix="language_model",
1060
)
1061
self.multi_modal_projector = nn.Linear(
1062
config.vision_config.vision_output_dim,
vllm/model_executor/models/nvlm_d.py
+5
@@ -4,10 +4,13 @@
4
# Copyright (c) 2024 NVIDIA
5
# Licensed under Apache 2.0 License [see LICENSE for details]
6
# --------------------------------------------------------
7
+from typing import Optional
8
+
9
import torch.nn as nn
10
from transformers import PretrainedConfig
11
12
from vllm.inputs import INPUT_REGISTRY
13
+from vllm.model_executor.layers.quantization import QuantizationConfig
14
from vllm.multimodal import MULTIMODAL_REGISTRY
15
16
from .intern_vit import InternVisionModel
@@ -56,9 +59,11 @@ class NVLM_D_Model(InternVLChatModel):
59
)
60
61
def _init_vision_model(self, config: PretrainedConfig,
62
+ quant_config: Optional[QuantizationConfig],
63
num_hidden_layers: int):
64
# We added additional dummy heads to the original num of heads to make
65
# the number of heads divisible by 8.
66
return InternVisionModel(config.vision_config,
67
+ quant_config=quant_config,
68
num_hidden_layers_override=num_hidden_layers,
69
num_dummy_heads=7)
vllm/model_executor/models/paligemma.py
+2
-1
@@ -142,7 +142,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
142
self.config = config
143
self.multimodal_config = multimodal_config
144
145
- self.vision_tower = SiglipVisionModel(config.vision_config)
145
+ self.vision_tower = SiglipVisionModel(config.vision_config,
146
+ quant_config)
147
self.multi_modal_projector = PaliGemmaMultiModalProjector(
148
vision_hidden_size=config.vision_config.hidden_size,
149
projection_dim=config.vision_config.projection_dim)
vllm/model_executor/models/phi3v.py
+10
-5
@@ -70,7 +70,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
70
projection_dim=768)
71
72
73
-def _init_img_processor(hf_config: PretrainedConfig):
73
+def _init_img_processor(hf_config: PretrainedConfig,
74
+ quant_config: Optional[QuantizationConfig]):
75
clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
76
layer_idx = hf_config.img_processor.get('layer_idx', -2)
77
@@ -82,7 +83,10 @@ def _init_img_processor(hf_config: PretrainedConfig):
83
num_hidden_layers = layer_idx + 1
84
85
img_processor = CLIPVisionModel(
85
- clip_config, num_hidden_layers_override=num_hidden_layers)
86
+ clip_config,
87
+ quant_config,
88
+ num_hidden_layers_override=num_hidden_layers,
89
+ )
90
91
return img_processor
92
@@ -148,14 +152,15 @@ class Phi3ImageEmbeddingBase(nn.Module):
152
class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
153
"""Phi3 Image embedding with HD transform."""
154
151
- def __init__(self, config: PretrainedConfig) -> None:
155
+ def __init__(self, config: PretrainedConfig,
156
+ quant_config: Optional[QuantizationConfig]) -> None:
157
super().__init__()
158
159
# n_embed or hidden_size
160
hidden_size = config.n_embd if hasattr(
161
config, 'n_embd') else config.hidden_size
162
158
- self.img_processor = _init_img_processor(config)
163
+ self.img_processor = _init_img_processor(config, quant_config)
164
165
image_dim_out = config.img_processor['image_dim_out']
166
self.num_img_tokens = config.img_processor['num_img_tokens']
@@ -535,7 +540,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
540
)
541
542
# TODO: Optionally initializes this for supporting input embeddings.
538
- self.vision_embed_tokens = Phi3HDImageEmbedding(config)
543
+ self.vision_embed_tokens = Phi3HDImageEmbedding(config, quant_config)
544
545
self.language_model = LlamaForCausalLM(config, cache_config,
546
quant_config)
vllm/model_executor/models/pixtral.py
+79
-11
@@ -767,9 +767,17 @@ def input_processor_for_pixtral_hf(
767
768
class PixtralHFMLP(nn.Module):
769
770
- def __init__(self, config: PixtralVisionConfig):
770
+ def __init__(
771
+ self,
772
+ config: PixtralVisionConfig,
773
+ quant_config: Optional[QuantizationConfig] = None,
774
+ *,
775
+ prefix: str = "",
776
+ ) -> None:
777
super().__init__()
778
+
779
assert config.intermediate_size is not None
780
+ # TODO: Use quant_config and prefix after optimizing this
781
self.gate_proj = nn.Linear(config.hidden_size,
782
config.intermediate_size,
783
bias=False)
@@ -787,8 +795,15 @@ class PixtralHFMLP(nn.Module):
795
796
class PixtralHFAttention(nn.Module):
797
790
- def __init__(self, config: PixtralVisionConfig):
798
+ def __init__(
799
+ self,
800
+ config: PixtralVisionConfig,
801
+ quant_config: Optional[QuantizationConfig] = None,
802
+ *,
803
+ prefix: str = "",
804
+ ) -> None:
805
super().__init__()
806
+
807
self.config = config
808
assert not config.hidden_size % config.num_attention_heads
809
self.n_heads = config.num_attention_heads
@@ -796,6 +811,7 @@ class PixtralHFAttention(nn.Module):
811
812
self.scale = self.head_dim**-0.5
813
814
+ # TODO: Use quant_config and prefix after optimizing this
815
self.q_proj = nn.Linear(config.hidden_size,
816
config.hidden_size,
817
bias=False)
@@ -840,11 +856,22 @@ class PixtralHFAttention(nn.Module):
856
857
class PixtralHFTransformerBlock(nn.Module):
858
843
- def __init__(self, config: PixtralVisionConfig):
859
+ def __init__(
860
+ self,
861
+ config: PixtralVisionConfig,
862
+ quant_config: Optional[QuantizationConfig] = None,
863
+ *,
864
+ prefix: str = "",
865
+ ) -> None:
866
super().__init__()
867
+
868
self.attention_norm = RMSNorm(config.hidden_size, eps=1e-5)
846
- self.attention = PixtralHFAttention(config)
847
- self.feed_forward = PixtralHFMLP(config)
869
+ self.attention = PixtralHFAttention(config,
870
+ quant_config=quant_config,
871
+ prefix=f"{prefix}.attention")
872
+ self.feed_forward = PixtralHFMLP(config,
873
+ quant_config=quant_config,
874
+ prefix=f"{prefix}.feed_forward")
875
self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)
876
877
def forward(
@@ -864,11 +891,27 @@ class PixtralHFTransformerBlock(nn.Module):
891
892
class PixtralHFTransformer(nn.Module):
893
867
- def __init__(self, config: PixtralVisionConfig):
894
+ def __init__(
895
+ self,
896
+ config: PixtralVisionConfig,
897
+ quant_config: Optional[QuantizationConfig] = None,
898
+ *,
899
+ num_hidden_layers_override: Optional[int] = None,
900
+ prefix: str = "",
901
+ ) -> None:
902
super().__init__()
869
- self.layers = torch.nn.ModuleList()
870
- for _ in range(config.num_hidden_layers):
871
- self.layers.append(PixtralHFTransformerBlock(config))
903
+
904
+ if num_hidden_layers_override is None:
905
+ num_hidden_layers = config.num_hidden_layers
906
+ else:
907
+ num_hidden_layers = num_hidden_layers_override
908
+
909
+ self.layers = nn.ModuleList([
910
+ PixtralHFTransformerBlock(config=config,
911
+ quant_config=quant_config,
912
+ prefix=f"{prefix}.layers.{layer_idx}")
913
+ for layer_idx in range(num_hidden_layers)
914
+ ])
915
916
def forward(
917
self,
@@ -883,7 +926,15 @@ class PixtralHFTransformer(nn.Module):
926
927
class PixtralHFVisionModel(nn.Module):
928
886
- def __init__(self, config: PixtralVisionConfig):
929
+ def __init__(
930
+ self,
931
+ config: PixtralVisionConfig,
932
+ quant_config: Optional[QuantizationConfig] = None,
933
+ *,
934
+ num_hidden_layers_override: Optional[int] = None,
935
+ require_post_norm: Optional[bool] = None,
936
+ prefix: str = "",
937
+ ) -> None:
938
super().__init__()
939
940
self.config = config
@@ -895,7 +946,24 @@ class PixtralHFVisionModel(nn.Module):
946
bias=False,
947
)
948
self.ln_pre = RMSNorm(config.hidden_size, eps=1e-5)
898
- self.transformer = PixtralHFTransformer(config)
949
+ self.transformer = PixtralHFTransformer(
950
+ config,
951
+ quant_config,
952
+ num_hidden_layers_override=num_hidden_layers_override,
953
+ prefix=f"{prefix}.transformer",
954
+ )
955
+
956
+ num_hidden_layers = config.num_hidden_layers
957
+ if len(self.transformer.layers) > config.num_hidden_layers:
958
+ raise ValueError(
959
+ f"The original encoder only has {num_hidden_layers} "
960
+ f"layers, but you requested {len(self.transformer.layers)} "
961
+ "layers.")
962
+
963
+ if require_post_norm is True:
964
+ msg = "PixtralHFVisionModel does not have post-layernorm"
965
+ raise ValueError(msg)
966
+
967
self.dtype = next(self.parameters()).dtype
968
self.device = next(self.parameters()).device
969
self.patch_positional_embedding = PixtralRotaryEmbedding(
vllm/model_executor/models/siglip.py
+55
-17
@@ -248,8 +248,10 @@ class SiglipParallelAttention(nn.Module):
248
self,
249
config: SiglipVisionConfig,
250
quant_config: Optional[QuantizationConfig] = None,
251
- ):
251
+ prefix: str = "",
252
+ ) -> None:
253
super().__init__()
254
+
255
self.config = config
256
self.embed_dim = config.hidden_size
257
self.num_heads = config.num_attention_heads
@@ -266,12 +268,14 @@ class SiglipParallelAttention(nn.Module):
268
head_size=self.head_dim,
269
total_num_heads=self.num_heads,
270
quant_config=quant_config,
271
+ prefix=f"{prefix}.qkv_proj",
272
)
273
274
self.out_proj = RowParallelLinear(
275
input_size=self.embed_dim,
276
output_size=self.embed_dim,
277
quant_config=quant_config,
278
+ prefix=f"{prefix}.out_proj",
279
)
280
281
self.tp_size = get_tensor_model_parallel_world_size()
@@ -314,8 +318,10 @@ class SiglipMLP(nn.Module):
318
self,
319
config: SiglipVisionConfig,
320
quant_config: Optional[QuantizationConfig] = None,
317
- ):
321
+ prefix: str = "",
322
+ ) -> None:
323
super().__init__()
324
+
325
self.config = config
326
self.activation_fn = get_act_fn(config.hidden_act)
327
@@ -326,11 +332,13 @@ class SiglipMLP(nn.Module):
332
config.hidden_size,
333
config.intermediate_size,
334
quant_config=quant_config if quantizable else None,
335
+ prefix=f"{prefix}.fc1",
336
)
337
self.fc2 = RowParallelLinear(
338
config.intermediate_size,
339
config.hidden_size,
340
quant_config=quant_config if quantizable else None,
341
+ prefix=f"{prefix}.fc2",
342
)
343
344
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -346,15 +354,20 @@ class SiglipEncoderLayer(nn.Module):
354
self,
355
config: SiglipVisionConfig,
356
quant_config: Optional[QuantizationConfig] = None,
349
- ):
357
+ prefix: str = "",
358
+ ) -> None:
359
super().__init__()
360
+
361
self.embed_dim = config.hidden_size
362
363
num_heads = config.num_attention_heads
364
tp_size = get_tensor_model_parallel_world_size()
365
if USE_XFORMERS_OPS and num_heads % tp_size == 0:
356
- self.self_attn = SiglipParallelAttention(config,
357
- quant_config=quant_config)
366
+ self.self_attn = SiglipParallelAttention(
367
+ config,
368
+ quant_config=quant_config,
369
+ prefix=f"{prefix}.self_attn",
370
+ )
371
else:
372
self.self_attn = SiglipSdpaAttention(config)
373
@@ -363,6 +376,7 @@ class SiglipEncoderLayer(nn.Module):
376
self.mlp = SiglipMLP(
377
config,
378
quant_config=quant_config,
379
+ prefix=f"{prefix}.mlp",
380
)
381
self.layer_norm2 = nn.LayerNorm(self.embed_dim,
382
eps=config.layer_norm_eps)
@@ -392,8 +406,10 @@ class SiglipEncoder(nn.Module):
406
config: SiglipVisionConfig,
407
quant_config: Optional[QuantizationConfig] = None,
408
num_hidden_layers_override: Optional[int] = None,
395
- ):
409
+ prefix: str = "",
410
+ ) -> None:
411
super().__init__()
412
+
413
self.config = config
414
415
if num_hidden_layers_override is None:
@@ -402,8 +418,10 @@ class SiglipEncoder(nn.Module):
418
num_hidden_layers = num_hidden_layers_override
419
420
self.layers = nn.ModuleList([
405
- SiglipEncoderLayer(config, quant_config=quant_config)
406
- for _ in range(num_hidden_layers)
421
+ SiglipEncoderLayer(config,
422
+ quant_config=quant_config,
423
+ prefix=f"{prefix}.layers.{layer_idx}")
424
+ for layer_idx in range(num_hidden_layers)
425
])
426
427
def forward(
@@ -424,7 +442,8 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
442
self,
443
config: SiglipVisionConfig,
444
quant_config: Optional[QuantizationConfig] = None,
427
- ):
445
+ prefix: str = "",
446
+ ) -> None:
447
super().__init__()
448
449
self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
@@ -433,7 +452,9 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
452
config.hidden_size, config.num_attention_heads, batch_first=True)
453
self.layernorm = nn.LayerNorm(config.hidden_size,
454
eps=config.layer_norm_eps)
436
- self.mlp = SiglipMLP(config=config, quant_config=quant_config)
455
+ self.mlp = SiglipMLP(config=config,
456
+ quant_config=quant_config,
457
+ prefix=f"{prefix}.mlp")
458
459
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
460
batch_size = hidden_state.shape[0]
@@ -454,9 +475,13 @@ class SiglipVisionTransformer(nn.Module):
475
self,
476
config: SiglipVisionConfig,
477
quant_config: Optional[QuantizationConfig] = None,
478
+ *,
479
num_hidden_layers_override: Optional[int] = None,
458
- ):
480
+ require_post_norm: Optional[bool] = None,
481
+ prefix: str = "",
482
+ ) -> None:
483
super().__init__()
484
+
485
self.config = config
486
embed_dim = config.hidden_size
487
@@ -465,26 +490,34 @@ class SiglipVisionTransformer(nn.Module):
490
config,
491
quant_config=quant_config,
492
num_hidden_layers_override=num_hidden_layers_override,
493
+ prefix=f"{prefix}.encoder",
494
)
495
496
+ num_hidden_layers = config.num_hidden_layers
497
if len(self.encoder.layers) > config.num_hidden_layers:
498
raise ValueError(
472
- f"The original encoder only has {config.num_hidden_layers} "
499
+ f"The original encoder only has {num_hidden_layers} "
500
f"layers, but you requested {len(self.encoder.layers)} layers."
501
)
475
- elif len(self.encoder.layers) == config.num_hidden_layers:
502
+
503
+ # If possible, skip post_layernorm to conserve memory
504
+ if require_post_norm is None:
505
+ require_post_norm = len(self.encoder.layers) == num_hidden_layers
506
+
507
+ if require_post_norm:
508
self.post_layernorm = nn.LayerNorm(embed_dim,
509
eps=config.layer_norm_eps)
510
else:
479
- # post_layernorm is unused when we extract intermediate features
480
- # In this case, we can skip it to conserve memory
511
self.post_layernorm = None
512
513
self.use_head = (True if not hasattr(config, "vision_use_head") else
514
config.vision_use_head)
515
if self.use_head:
516
self.head = SiglipMultiheadAttentionPoolingHead(
487
- config=config, quant_config=quant_config)
517
+ config=config,
518
+ quant_config=quant_config,
519
+ prefix=f"{prefix}.head",
520
+ )
521
522
def forward(
523
self,
@@ -517,8 +550,11 @@ class SiglipVisionModel(nn.Module):
550
self,
551
config: SiglipVisionConfig,
552
quant_config: Optional[QuantizationConfig] = None,
553
+ *,
554
num_hidden_layers_override: Optional[int] = None,
521
- ):
555
+ require_post_norm: Optional[bool] = None,
556
+ prefix: str = "",
557
+ ) -> None:
558
super().__init__()
559
560
num_heads = config.num_attention_heads
@@ -529,6 +565,8 @@ class SiglipVisionModel(nn.Module):
565
config,
566
quant_config,
567
num_hidden_layers_override=num_hidden_layers_override,
568
+ require_post_norm=require_post_norm,
569
+ prefix=f"{prefix}.vision_model",
570
)
571
572
def get_input_embeddings(self) -> nn.Module: