@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c18e1a3418
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[VLM] Enable overriding whether post layernorm is used in vision encoder + fix quant args (#9217)

Co-authored-by: Isotr0py <2037008807@qq.com>

Cyrus Leung committed Oct 23, 2024 at 19:27 UTC c18e1a34189812af21aa504f9166de5ed4a86675
18 files changed +551 -253
vllm/model_executor/layers/quantization/awq.py
+16 -4
@@ -3,7 +3,8 @@ from typing import Any, Dict, List, Optional
3 import torch
4
5 from vllm import _custom_ops as ops
6 -from vllm.model_executor.layers.linear import LinearBase, LinearMethodBase
6 +from vllm.model_executor.layers.linear import (LinearBase, LinearMethodBase,
7 + UnquantizedLinearMethod)
8 from vllm.model_executor.layers.quantization.base_config import (
9 QuantizationConfig)
10 from vllm.model_executor.parameter import (GroupQuantScaleParameter,
@@ -21,10 +22,12 @@ class AWQConfig(QuantizationConfig):
22 weight_bits: int,
23 group_size: int,
24 zero_point: bool,
25 + modules_to_not_convert: Optional[List[str]] = None,
26 ) -> None:
27 self.weight_bits = weight_bits
28 self.group_size = group_size
29 self.zero_point = zero_point
30 + self.modules_to_not_convert = modules_to_not_convert or []
31
32 if self.weight_bits != 4:
33 raise ValueError(
@@ -35,7 +38,8 @@ class AWQConfig(QuantizationConfig):
38 def __repr__(self) -> str:
39 return (f"AWQConfig(weight_bits={self.weight_bits}, "
40 f"group_size={self.group_size}, "
38 - f"zero_point={self.zero_point})")
41 + f"zero_point={self.zero_point}, "
42 + f"modules_to_not_convert={self.modules_to_not_convert})")
43
44 def get_name(self) -> str:
45 return "awq"
@@ -61,11 +65,15 @@ class AWQConfig(QuantizationConfig):
65 weight_bits = cls.get_from_keys(config, ["w_bit", "bits"])
66 group_size = cls.get_from_keys(config, ["q_group_size", "group_size"])
67 zero_point = cls.get_from_keys(config, ["zero_point"])
64 - return cls(weight_bits, group_size, zero_point)
68 + modules_to_not_convert = cls.get_from_keys_or(
69 + config, ["modules_to_not_convert"], None)
70 + return cls(weight_bits, group_size, zero_point, modules_to_not_convert)
71
72 def get_quant_method(self, layer: torch.nn.Module,
67 - prefix: str) -> Optional["AWQLinearMethod"]:
73 + prefix: str) -> Optional["LinearMethodBase"]:
74 if isinstance(layer, LinearBase):
75 + if is_layer_skipped_awq(prefix, self.modules_to_not_convert):
76 + return UnquantizedLinearMethod()
77 return AWQLinearMethod(self)
78 return None
79
@@ -73,6 +81,10 @@ class AWQConfig(QuantizationConfig):
81 return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
82
83
84 +def is_layer_skipped_awq(prefix: str, modules_to_not_convert: List[str]):
85 + return any(module_name in prefix for module_name in modules_to_not_convert)
86 +
87 +
88 class AWQLinearMethod(LinearMethodBase):
89 """Linear method for AWQ.
90
vllm/model_executor/models/blip.py
+59 -28
@@ -122,7 +122,7 @@ def input_processor_for_blip(
122 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
123 class BlipVisionEmbeddings(nn.Module):
124
125 - def __init__(self, config: BlipVisionConfig):
125 + def __init__(self, config: Union[BlipVisionConfig, Blip2VisionConfig]):
126 super().__init__()
127
128 self.config = config
@@ -167,9 +167,10 @@ class BlipParallelAttention(nn.Module):
167
168 def __init__(
169 self,
170 - config: BlipVisionConfig,
170 + config: Union[BlipVisionConfig, Blip2VisionConfig],
171 quant_config: Optional[QuantizationConfig] = None,
172 - ):
172 + prefix: str = "",
173 + ) -> None:
174 super().__init__()
175 self.config = config
176 self.embed_dim = config.hidden_size
@@ -189,11 +190,13 @@ class BlipParallelAttention(nn.Module):
190 self.num_heads,
191 bias=config.qkv_bias,
192 quant_config=quant_config,
193 + prefix=f"{prefix}.qkv",
194 )
195 self.projection = RowParallelLinear(
196 self.embed_dim,
197 self.embed_dim,
198 quant_config=quant_config,
199 + prefix=f"{prefix}.projection",
200 )
201
202 self.tp_size = get_tensor_model_parallel_world_size()
@@ -235,9 +238,12 @@ class BlipParallelAttention(nn.Module):
238
239 class BlipMLP(nn.Module):
240
238 - def __init__(self,
239 - config: BlipVisionConfig,
240 - quant_config: Optional[QuantizationConfig] = None):
241 + def __init__(
242 + self,
243 + config: BlipVisionConfig,
244 + quant_config: Optional[QuantizationConfig] = None,
245 + prefix: str = "",
246 + ) -> None:
247 super().__init__()
248
249 self.config = config
@@ -246,11 +252,13 @@ class BlipMLP(nn.Module):
252 self.fc1 = ColumnParallelLinear(config.hidden_size,
253 config.intermediate_size,
254 bias=True,
249 - quant_config=quant_config)
255 + quant_config=quant_config,
256 + prefix=f"{prefix}.fc1")
257 self.fc2 = RowParallelLinear(config.intermediate_size,
258 config.hidden_size,
259 bias=True,
253 - quant_config=quant_config)
260 + quant_config=quant_config,
261 + prefix=f"{prefix}.fc2")
262
263 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
264 hidden_states, _ = self.fc1(hidden_states)
@@ -262,24 +270,32 @@ class BlipMLP(nn.Module):
270
271 class BlipEncoderLayer(nn.Module):
272
265 - def __init__(self,
266 - config: BlipVisionConfig,
267 - quant_config: Optional[QuantizationConfig] = None):
273 + def __init__(
274 + self,
275 + config: BlipVisionConfig,
276 + quant_config: Optional[QuantizationConfig] = None,
277 + prefix: str = "",
278 + ) -> None:
279 super().__init__()
280
281 # fallback to sdpa attention if tp unavailable
282 num_heads = config.num_attention_heads
283 tp_size = get_tensor_model_parallel_world_size()
284 if USE_XFORMERS_OPS and num_heads % tp_size == 0:
274 - self.self_attn = BlipParallelAttention(config,
275 - quant_config=quant_config)
285 + self.self_attn = BlipParallelAttention(
286 + config,
287 + quant_config=quant_config,
288 + prefix=f"{prefix}.self_attn",
289 + )
290 else:
291 # Blip doesn't have SDPA attention implemented in transformers
292 # use eager attention instead for cpu backend
293 self.self_attn = BlipAttention(config)
294 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
295 eps=config.layer_norm_eps)
282 - self.mlp = BlipMLP(config, quant_config=quant_config)
296 + self.mlp = BlipMLP(config,
297 + quant_config=quant_config,
298 + prefix=f"{prefix}.mlp")
299 self.layer_norm2 = nn.LayerNorm(config.hidden_size,
300 eps=config.layer_norm_eps)
301
@@ -307,10 +323,13 @@ class BlipEncoder(nn.Module):
323 config: BlipConfig
324 """
325
310 - def __init__(self,
311 - config: BlipVisionConfig,
312 - quant_config: Optional[QuantizationConfig] = None,
313 - num_hidden_layers_override: Optional[int] = None):
326 + def __init__(
327 + self,
328 + config: BlipVisionConfig,
329 + quant_config: Optional[QuantizationConfig] = None,
330 + num_hidden_layers_override: Optional[int] = None,
331 + prefix: str = "",
332 + ) -> None:
333 super().__init__()
334
335 self.config = config
@@ -321,8 +340,10 @@ class BlipEncoder(nn.Module):
340 num_hidden_layers = num_hidden_layers_override
341
342 self.layers = nn.ModuleList([
324 - BlipEncoderLayer(config=config, quant_config=quant_config)
325 - for _ in range(num_hidden_layers)
343 + BlipEncoderLayer(config=config,
344 + quant_config=quant_config,
345 + prefix=f"{prefix}.layers.{layer_idx}")
346 + for layer_idx in range(num_hidden_layers)
347 ])
348
349 def forward(self, inputs_embeds: torch.Tensor):
@@ -337,10 +358,15 @@ class BlipVisionModel(nn.Module):
358 config_class = BlipVisionConfig
359 main_input_name = "pixel_values"
360
340 - def __init__(self,
341 - config: BlipVisionConfig,
342 - quant_config: Optional[QuantizationConfig] = None,
343 - num_hidden_layers_override: Optional[int] = None):
361 + def __init__(
362 + self,
363 + config: BlipVisionConfig,
364 + quant_config: Optional[QuantizationConfig] = None,
365 + *,
366 + num_hidden_layers_override: Optional[int] = None,
367 + require_post_norm: Optional[bool] = None,
368 + prefix: str = "",
369 + ) -> None:
370 super().__init__()
371
372 tp_size = get_tensor_model_parallel_world_size()
@@ -354,19 +380,24 @@ class BlipVisionModel(nn.Module):
380 config=config,
381 quant_config=quant_config,
382 num_hidden_layers_override=num_hidden_layers_override,
383 + prefix=f"{prefix}.encoder",
384 )
385
386 + num_hidden_layers = config.num_hidden_layers
387 if len(self.encoder.layers) > config.num_hidden_layers:
388 raise ValueError(
361 - f"The original encoder only has {config.num_hidden_layers} "
389 + f"The original encoder only has {num_hidden_layers} "
390 f"layers, but you requested {len(self.encoder.layers)} layers."
391 )
364 - elif len(self.encoder.layers) == config.num_hidden_layers:
392 +
393 + # If possible, skip post_layernorm to conserve memory
394 + if require_post_norm is None:
395 + require_post_norm = len(self.encoder.layers) == num_hidden_layers
396 +
397 + if require_post_norm:
398 self.post_layernorm = nn.LayerNorm(config.hidden_size,
399 eps=config.layer_norm_eps)
400 else:
368 - # post_layernorm is unused when we extract intermediate features
369 - # In this case, we can skip it to conserve memory
401 self.post_layernorm = None
402
403 def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/blip2.py
+1 -1
@@ -490,7 +490,7 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
490 self.multimodal_config = multimodal_config
491
492 # TODO: Optionally initializes this for supporting embeddings.
493 - self.vision_model = BlipVisionModel(config.vision_config)
493 + self.vision_model = BlipVisionModel(config.vision_config, quant_config)
494
495 self.query_tokens = nn.Parameter(
496 torch.zeros(1, config.num_query_tokens,
vllm/model_executor/models/clip.py
+73 -31
@@ -192,6 +192,7 @@ class CLIPParallelAttention(nn.Module):
192 self,
193 config: CLIPVisionConfig,
194 quant_config: Optional[QuantizationConfig] = None,
195 + prefix: str = "",
196 ):
197 super().__init__()
198 self.config = config
@@ -211,12 +212,14 @@ class CLIPParallelAttention(nn.Module):
212 head_size=self.head_dim,
213 total_num_heads=self.num_heads,
214 quant_config=quant_config,
215 + prefix=f"{prefix}.qkv_proj",
216 )
217
218 self.out_proj = RowParallelLinear(
219 input_size=self.embed_dim,
220 output_size=self.embed_dim,
221 quant_config=quant_config,
222 + prefix=f"{prefix}.out_proj",
223 )
224
225 self.tp_size = get_tensor_model_parallel_world_size()
@@ -259,20 +262,25 @@ class CLIPParallelAttention(nn.Module):
262
263 class CLIPMLP(nn.Module):
264
262 - def __init__(self,
263 - config: CLIPVisionConfig,
264 - quant_config: Optional[QuantizationConfig] = None):
265 + def __init__(
266 + self,
267 + config: CLIPVisionConfig,
268 + quant_config: Optional[QuantizationConfig] = None,
269 + prefix: str = "",
270 + ) -> None:
271 super().__init__()
272 self.config = config
273 self.activation_fn = get_act_fn(config.hidden_act)
274 self.fc1 = ColumnParallelLinear(config.hidden_size,
275 config.intermediate_size,
276 bias=True,
271 - quant_config=quant_config)
277 + quant_config=quant_config,
278 + prefix=f"{prefix}.fc1")
279 self.fc2 = RowParallelLinear(config.intermediate_size,
280 config.hidden_size,
281 bias=True,
275 - quant_config=quant_config)
282 + quant_config=quant_config,
283 + prefix=f"{prefix}.fc2")
284
285 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
286 hidden_states, _ = self.fc1(hidden_states)
@@ -284,21 +292,29 @@ class CLIPMLP(nn.Module):
292
293 class CLIPEncoderLayer(nn.Module):
294
287 - def __init__(self,
288 - config: CLIPVisionConfig,
289 - quant_config: Optional[QuantizationConfig] = None):
295 + def __init__(
296 + self,
297 + config: CLIPVisionConfig,
298 + quant_config: Optional[QuantizationConfig] = None,
299 + prefix: str = "",
300 + ) -> None:
301 super().__init__()
302
303 num_heads = config.num_attention_heads
304 tp_size = get_tensor_model_parallel_world_size()
305 if USE_XFORMERS_OPS and num_heads % tp_size == 0:
295 - self.self_attn = CLIPParallelAttention(config,
296 - quant_config=quant_config)
306 + self.self_attn = CLIPParallelAttention(
307 + config,
308 + quant_config=quant_config,
309 + prefix=f"{prefix}.self_attn",
310 + )
311 else:
312 self.self_attn = CLIPSdpaAttention(config)
313 self.layer_norm1 = nn.LayerNorm(config.hidden_size,
314 eps=config.layer_norm_eps)
301 - self.mlp = CLIPMLP(config, quant_config=quant_config)
315 + self.mlp = CLIPMLP(config,
316 + quant_config=quant_config,
317 + prefix=f"{prefix}.mlp")
318 self.layer_norm2 = nn.LayerNorm(config.hidden_size,
319 eps=config.layer_norm_eps)
320
@@ -327,11 +343,15 @@ class CLIPEncoder(nn.Module):
343 config: CLIPConfig
344 """
345
330 - def __init__(self,
331 - config: CLIPVisionConfig,
332 - quant_config: Optional[QuantizationConfig] = None,
333 - num_hidden_layers_override: Optional[int] = None):
346 + def __init__(
347 + self,
348 + config: CLIPVisionConfig,
349 + quant_config: Optional[QuantizationConfig] = None,
350 + num_hidden_layers_override: Optional[int] = None,
351 + prefix: str = "",
352 + ) -> None:
353 super().__init__()
354 +
355 self.config = config
356
357 if num_hidden_layers_override is None:
@@ -339,8 +359,10 @@ class CLIPEncoder(nn.Module):
359 else:
360 num_hidden_layers = num_hidden_layers_override
361 self.layers = nn.ModuleList([
342 - CLIPEncoderLayer(config=config, quant_config=quant_config)
343 - for _ in range(num_hidden_layers)
362 + CLIPEncoderLayer(config=config,
363 + quant_config=quant_config,
364 + prefix=f"{prefix}.layers.{layer_idx}")
365 + for layer_idx in range(num_hidden_layers)
366 ])
367
368 def forward(self, inputs_embeds: torch.Tensor):
@@ -354,11 +376,17 @@ class CLIPEncoder(nn.Module):
376
377 class CLIPVisionTransformer(nn.Module):
378
357 - def __init__(self,
358 - config: CLIPVisionConfig,
359 - quant_config: Optional[QuantizationConfig] = None,
360 - num_hidden_layers_override: Optional[int] = None):
379 + def __init__(
380 + self,
381 + config: CLIPVisionConfig,
382 + quant_config: Optional[QuantizationConfig] = None,
383 + *,
384 + num_hidden_layers_override: Optional[int] = None,
385 + require_post_norm: Optional[bool] = None,
386 + prefix: str = "",
387 + ) -> None:
388 super().__init__()
389 +
390 self.config = config
391 embed_dim = config.hidden_size
392
@@ -370,19 +398,25 @@ class CLIPVisionTransformer(nn.Module):
398 self.encoder = CLIPEncoder(
399 config=config,
400 quant_config=quant_config,
373 - num_hidden_layers_override=num_hidden_layers_override)
401 + num_hidden_layers_override=num_hidden_layers_override,
402 + prefix=f"{prefix}.encoder",
403 + )
404
405 + num_hidden_layers = config.num_hidden_layers
406 if len(self.encoder.layers) > config.num_hidden_layers:
407 raise ValueError(
377 - f"The original encoder only has {config.num_hidden_layers} "
408 + f"The original encoder only has {num_hidden_layers} "
409 f"layers, but you requested {len(self.encoder.layers)} layers."
410 )
380 - elif len(self.encoder.layers) == config.num_hidden_layers:
411 +
412 + # If possible, skip post_layernorm to conserve memory
413 + if require_post_norm is None:
414 + require_post_norm = len(self.encoder.layers) == num_hidden_layers
415 +
416 + if require_post_norm:
417 self.post_layernorm = nn.LayerNorm(embed_dim,
418 eps=config.layer_norm_eps)
419 else:
384 - # post_layernorm is unused when we extract intermediate features
385 - # In this case, we can skip it to conserve memory
420 self.post_layernorm = None
421
422 def forward(
@@ -405,10 +439,15 @@ class CLIPVisionModel(nn.Module):
439 config_class = CLIPVisionConfig
440 main_input_name = "pixel_values"
441
408 - def __init__(self,
409 - config: CLIPVisionConfig,
410 - quant_config: Optional[QuantizationConfig] = None,
411 - num_hidden_layers_override: Optional[int] = None):
442 + def __init__(
443 + self,
444 + config: CLIPVisionConfig,
445 + quant_config: Optional[QuantizationConfig] = None,
446 + *,
447 + num_hidden_layers_override: Optional[int] = None,
448 + require_post_norm: Optional[bool] = None,
449 + prefix: str = "",
450 + ) -> None:
451 super().__init__()
452
453 tp_size = get_tensor_model_parallel_world_size()
@@ -418,7 +457,10 @@ class CLIPVisionModel(nn.Module):
457 self.vision_model = CLIPVisionTransformer(
458 config=config,
459 quant_config=quant_config,
421 - num_hidden_layers_override=num_hidden_layers_override)
460 + num_hidden_layers_override=num_hidden_layers_override,
461 + require_post_norm=require_post_norm,
462 + prefix=f"{prefix}.vision_model",
463 + )
464
465 def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
466 return self.vision_model(pixel_values)
vllm/model_executor/models/idefics2_vision_model.py
+41 -10
@@ -113,7 +113,8 @@ class Idefics2VisionAttention(nn.Module):
113 self,
114 config: Idefics2Config,
115 quant_config: Optional[QuantizationConfig] = None,
116 - ):
116 + prefix: str = "",
117 + ) -> None:
118 super().__init__()
119 self.config = config
120 self.embed_dim = config.hidden_size
@@ -130,12 +131,14 @@ class Idefics2VisionAttention(nn.Module):
131 self.head_dim,
132 self.num_heads,
133 quant_config=quant_config,
134 + prefix=f"{prefix}.qkv_proj",
135 )
136 self.out_proj = RowParallelLinear(
137 self.embed_dim,
138 self.embed_dim,
139 bias=True,
140 quant_config=quant_config,
141 + prefix=f"{prefix}.out_proj",
142 )
143 self.tp_size = get_tensor_model_parallel_world_size()
144 self.num_heads_per_partition = divide(self.num_heads, self.tp_size)
@@ -178,7 +181,8 @@ class Idefics2VisionMLP(nn.Module):
181 self,
182 config: Idefics2Config,
183 quant_config: Optional[QuantizationConfig] = None,
181 - ):
184 + prefix: str = "",
185 + ) -> None:
186 super().__init__()
187 self.config = config
188 self.activation_fn = get_act_fn(config.hidden_act)
@@ -187,12 +191,14 @@ class Idefics2VisionMLP(nn.Module):
191 config.intermediate_size,
192 bias=True,
193 quant_config=quant_config,
194 + prefix=f"{prefix}.fc1",
195 )
196 self.fc2 = RowParallelLinear(
197 config.intermediate_size,
198 config.hidden_size,
199 bias=True,
200 quant_config=quant_config,
201 + prefix=f"{prefix}.fc2",
202 )
203
204 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -204,13 +210,22 @@ class Idefics2VisionMLP(nn.Module):
210
211 class Idefics2EncoderLayer(nn.Module):
212
207 - def __init__(self, config: Idefics2Config):
213 + def __init__(
214 + self,
215 + config: Idefics2Config,
216 + quant_config: Optional[QuantizationConfig] = None,
217 + prefix: str = "",
218 + ) -> None:
219 super().__init__()
220 self.embed_dim = config.hidden_size
210 - self.self_attn = Idefics2VisionAttention(config)
221 + self.self_attn = Idefics2VisionAttention(config,
222 + quant_config=quant_config,
223 + prefix=f"{prefix}.self_attn")
224 self.layer_norm1 = nn.LayerNorm(self.embed_dim,
225 eps=config.layer_norm_eps)
213 - self.mlp = Idefics2VisionMLP(config)
226 + self.mlp = Idefics2VisionMLP(config,
227 + quant_config=quant_config,
228 + prefix=f"{prefix}.mlp")
229 self.layer_norm2 = nn.LayerNorm(self.embed_dim,
230 eps=config.layer_norm_eps)
231
@@ -245,12 +260,20 @@ class Idefics2Encoder(nn.Module):
260 config: Idefics2Config
261 """
262
248 - def __init__(self, config: Idefics2Config):
263 + def __init__(
264 + self,
265 + config: Idefics2Config,
266 + quant_config: Optional[QuantizationConfig] = None,
267 + prefix: str = "",
268 + ) -> None:
269 super().__init__()
270 +
271 self.config = config
272 self.layers = nn.ModuleList([
252 - Idefics2EncoderLayer(config)
253 - for _ in range(config.num_hidden_layers)
273 + Idefics2EncoderLayer(config,
274 + quant_config=quant_config,
275 + prefix=f"{prefix}.layers.{layer_idx}")
276 + for layer_idx in range(config.num_hidden_layers)
277 ])
278
279 def forward(
@@ -275,12 +298,20 @@ class Idefics2Encoder(nn.Module):
298
299 class Idefics2VisionTransformer(nn.Module):
300
278 - def __init__(self, config: Idefics2VisionConfig):
301 + def __init__(
302 + self,
303 + config: Idefics2VisionConfig,
304 + quant_config: Optional[QuantizationConfig] = None,
305 + prefix: str = "",
306 + ) -> None:
307 super().__init__()
308 +
309 embed_dim = config.hidden_size
310 self.config = config
311 self.embeddings = Idefics2VisionEmbeddings(config)
283 - self.encoder = Idefics2Encoder(config)
312 + self.encoder = Idefics2Encoder(config,
313 + quant_config=quant_config,
314 + prefix=f"{prefix}.encoder")
315 self.post_layernorm = nn.LayerNorm(embed_dim,
316 eps=config.layer_norm_eps)
317
vllm/model_executor/models/intern_vit.py
+30 -11
@@ -137,6 +137,7 @@ class InternParallelAttention(nn.Module):
137 quant_config: Optional[QuantizationConfig] = None,
138 *,
139 num_dummy_heads: int = 0,
140 + prefix: str = "",
141 ) -> None:
142 super().__init__()
143
@@ -165,6 +166,7 @@ class InternParallelAttention(nn.Module):
166 num_dummy_heads + self.num_heads,
167 bias=config.qkv_bias,
168 quant_config=quant_config,
169 + prefix=f"{prefix}.qkv",
170 )
171
172 self.qk_normalization = config.qk_normalization
@@ -181,6 +183,7 @@ class InternParallelAttention(nn.Module):
183 self.dummy_dim,
184 self.embed_dim,
185 quant_config=quant_config,
186 + prefix=f"{prefix}.proj",
187 )
188
189 def _apply_qk_norm(self, q: torch.Tensor, k: torch.Tensor):
@@ -284,20 +287,26 @@ class InternSdpaAttention(nn.Module):
287
288 class InternMLP(nn.Module):
289
287 - def __init__(self,
288 - config: PretrainedConfig,
289 - quant_config: Optional[QuantizationConfig] = None):
290 + def __init__(
291 + self,
292 + config: PretrainedConfig,
293 + quant_config: Optional[QuantizationConfig] = None,
294 + prefix: str = "",
295 + ) -> None:
296 super().__init__()
297 +
298 self.config = config
299 self.activation_fn = get_act_fn(config.hidden_act)
300 self.fc1 = ColumnParallelLinear(config.hidden_size,
301 config.intermediate_size,
302 bias=True,
296 - quant_config=quant_config)
303 + quant_config=quant_config,
304 + prefix=f"{prefix}.fc1")
305 self.fc2 = RowParallelLinear(config.intermediate_size,
306 config.hidden_size,
307 bias=True,
300 - quant_config=quant_config)
308 + quant_config=quant_config,
309 + prefix=f"{prefix}.fc2")
310
311 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
312 hidden_states, _ = self.fc1(hidden_states)
@@ -315,6 +324,7 @@ class InternVisionEncoderLayer(nn.Module):
324 quant_config: Optional[QuantizationConfig] = None,
325 *,
326 num_dummy_heads: int = 0,
327 + prefix: str = "",
328 ) -> None:
329 super().__init__()
330
@@ -324,9 +334,12 @@ class InternVisionEncoderLayer(nn.Module):
334
335 self.attn = self._init_attn(config,
336 quant_config,
327 - num_dummy_heads=num_dummy_heads)
337 + num_dummy_heads=num_dummy_heads,
338 + prefix=f"{prefix}.attn")
339
329 - self.mlp = InternMLP(config, quant_config=quant_config)
340 + self.mlp = InternMLP(config,
341 + quant_config=quant_config,
342 + prefix=f"{prefix}.mlp")
343 self.norm1 = NORM2FN[self.norm_type](self.embed_dim,
344 eps=config.layer_norm_eps)
345 self.norm2 = NORM2FN[self.norm_type](self.embed_dim,
@@ -343,6 +356,7 @@ class InternVisionEncoderLayer(nn.Module):
356 quant_config: Optional[QuantizationConfig],
357 *,
358 num_dummy_heads: int,
359 + prefix: str = "",
360 ):
361 # fallback to sdpa attention if tp unavailable
362 tp_size = get_tensor_model_parallel_world_size()
@@ -351,7 +365,8 @@ class InternVisionEncoderLayer(nn.Module):
365 if USE_XFORMERS_OPS and (num_heads + num_dummy_heads) % tp_size == 0:
366 return InternParallelAttention(config,
367 quant_config=quant_config,
354 - num_dummy_heads=num_dummy_heads)
368 + num_dummy_heads=num_dummy_heads,
369 + prefix=prefix)
370
371 return InternSdpaAttention(config, num_dummy_heads=num_dummy_heads)
372
@@ -377,6 +392,7 @@ class InternVisionEncoder(nn.Module):
392 *,
393 num_hidden_layers_override: Optional[int] = None,
394 num_dummy_heads: int = 0,
395 + prefix: str = "",
396 ):
397 super().__init__()
398
@@ -390,8 +406,9 @@ class InternVisionEncoder(nn.Module):
406 self.layers = nn.ModuleList([
407 InternVisionEncoderLayer(config,
408 quant_config,
393 - num_dummy_heads=num_dummy_heads)
394 - for _ in range(num_hidden_layers)
409 + num_dummy_heads=num_dummy_heads,
410 + prefix=f"{prefix}.layers.{layer_idx}")
411 + for layer_idx in range(num_hidden_layers)
412 ])
413
414 def forward(self, inputs_embeds: torch.Tensor):
@@ -412,7 +429,8 @@ class InternVisionModel(nn.Module):
429 *,
430 num_hidden_layers_override: Optional[int] = None,
431 num_dummy_heads: int = 0,
415 - ):
432 + prefix: str = "",
433 + ) -> None:
434 super().__init__()
435
436 self.config = config
@@ -423,6 +441,7 @@ class InternVisionModel(nn.Module):
441 quant_config=quant_config,
442 num_hidden_layers_override=num_hidden_layers_override,
443 num_dummy_heads=num_dummy_heads,
444 + prefix=f"{prefix}.encoder",
445 )
446
447 def get_input_embeddings(self):
vllm/model_executor/models/internvl.py
+35 -6
@@ -19,7 +19,8 @@ from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, MultiModalConfig
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
21 token_inputs)
22 -from vllm.model_executor.layers.quantization import QuantizationConfig
22 +from vllm.model_executor.layers.quantization import (AWQConfig,
23 + QuantizationConfig)
24 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
25 from vllm.model_executor.models.intern_vit import (InternVisionModel,
26 InternVisionPatchModel)
@@ -418,11 +419,11 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
419
420 self.config = config
421 self.multimodal_config = multimodal_config
422 + self._patch_quant_config(config, quant_config)
423
424 image_size = config.force_image_size or config.vision_config.image_size
425 patch_size = config.vision_config.patch_size
426 self.patch_size = patch_size
425 - self.select_layer = config.select_layer
427 self.num_image_token = int(
428 (image_size // patch_size)**2 * (config.downsample_ratio**2))
429 self.downsample_ratio = config.downsample_ratio
@@ -430,7 +431,12 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
431
432 self.llm_arch_name = config.text_config.architectures[0]
433 self.is_mono = self.llm_arch_name == 'InternLM2VEForCausalLM'
433 - self.vision_model = self._init_vision_model(config, self.is_mono)
434 + self.vision_model = self._init_vision_model(
435 + config,
436 + quant_config=quant_config,
437 + is_mono=self.is_mono,
438 + prefix="vision_model",
439 + )
440
441 self.language_model = init_vllm_registered_model(
442 config.text_config, cache_config, quant_config)
@@ -441,6 +447,18 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
447 self.make_empty_intermediate_tensors = (
448 self.language_model.make_empty_intermediate_tensors)
449
450 + def _patch_quant_config(self, config: PretrainedConfig,
451 + quant_config: QuantizationConfig):
452 + # the awq models from OpenGVLab missing `modules_to_not_convert`
453 + # patch the quant_config to add `modules_to_not_convert` back
454 + if isinstance(quant_config, AWQConfig):
455 + text_config = config.text_config
456 + llm_quant_config = getattr(text_config, "quantization_config",
457 + None)
458 + if (not quant_config.modules_to_not_convert) and \
459 + (llm_quant_config is not None):
460 + quant_config.modules_to_not_convert.append("vision_model")
461 +
462 @cached_property
463 def sampler(self):
464 if hasattr(self.language_model, "sampler"):
@@ -448,17 +466,28 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):
466
467 return Sampler()
468
451 - def _init_vision_model(self, config: PretrainedConfig, is_mono: bool):
469 + def _init_vision_model(
470 + self,
471 + config: PretrainedConfig,
472 + quant_config: Optional[QuantizationConfig],
473 + *,
474 + is_mono: bool,
475 + prefix: str,
476 + ):
477 if not is_mono:
453 - vision_feature_layer = self.select_layer
478 + vision_feature_layer = config.select_layer
479 if vision_feature_layer < 0:
480 num_hidden_layers = config.vision_config.num_hidden_layers \
481 + vision_feature_layer + 1
482 else:
483 num_hidden_layers = vision_feature_layer + 1
484 +
485 return InternVisionModel(
486 config.vision_config,
461 - num_hidden_layers_override=num_hidden_layers)
487 + quant_config=quant_config,
488 + num_hidden_layers_override=num_hidden_layers,
489 + prefix=prefix,
490 + )
491 else:
492 return InternVisionPatchModel(config.vision_config)
493
vllm/model_executor/models/llava.py
+25 -7
@@ -1,12 +1,12 @@
1 from functools import cached_property
2 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
3 - TypedDict, Union)
2 +from typing import (Iterable, List, Literal, Mapping, Optional, Protocol,
3 + Tuple, TypedDict, Union)
4
5 import torch
6 import torch.nn as nn
7 from PIL import Image
8 from transformers import (CLIPVisionConfig, LlavaConfig, PixtralVisionConfig,
9 - SiglipVisionConfig)
9 + PretrainedConfig, SiglipVisionConfig)
10
11 from vllm.attention import AttentionMetadata
12 from vllm.config import CacheConfig, MultiModalConfig
@@ -200,7 +200,17 @@ def input_processor_for_llava(ctx: InputContext, inputs: DecoderOnlyInputs):
200 raise NotImplementedError(msg)
201
202
203 -def _init_vision_tower(hf_config: LlavaConfig):
203 +class LlavaLikeConfig(Protocol):
204 + vision_config: PretrainedConfig
205 + vision_feature_layer: int
206 +
207 +
208 +def init_vision_tower_for_llava(
209 + hf_config: LlavaLikeConfig,
210 + quant_config: Optional[QuantizationConfig],
211 + *,
212 + require_post_norm: Optional[bool] = None,
213 +):
214 vision_config = hf_config.vision_config
215
216 # Initialize the vision tower only up to the required feature layer
@@ -214,16 +224,24 @@ def _init_vision_tower(hf_config: LlavaConfig):
224 if isinstance(vision_config, CLIPVisionConfig):
225 return CLIPVisionModel(
226 vision_config,
227 + quant_config,
228 num_hidden_layers_override=num_hidden_layers,
229 + require_post_norm=require_post_norm,
230 )
231 elif isinstance(vision_config, SiglipVisionConfig):
232 return SiglipVisionModel(
233 vision_config,
234 + quant_config,
235 num_hidden_layers_override=num_hidden_layers,
236 + require_post_norm=require_post_norm,
237 )
238 elif isinstance(vision_config, PixtralVisionConfig):
225 - # TODO: allow layer override?
226 - return PixtralHFVisionModel(vision_config)
239 + return PixtralHFVisionModel(
240 + vision_config,
241 + quant_config,
242 + num_hidden_layers_override=num_hidden_layers,
243 + require_post_norm=require_post_norm,
244 + )
245
246 msg = f"Unsupported vision config: {type(vision_config)}"
247 raise NotImplementedError(msg)
@@ -255,7 +273,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
273 config.projector_hidden_act = "gelu"
274
275 # TODO: Optionally initializes this for supporting embeddings.
258 - self.vision_tower = _init_vision_tower(config)
276 + self.vision_tower = init_vision_tower_for_llava(config, quant_config)
277 self.multi_modal_projector = LlavaMultiModalProjector(
278 vision_hidden_size=config.vision_config.hidden_size,
279 text_hidden_size=config.text_config.hidden_size,
vllm/model_executor/models/llava_next.py
+2 -28
@@ -26,7 +26,7 @@ from .clip import (CLIPVisionModel, dummy_image_for_clip,
26 dummy_seq_data_for_clip, get_clip_image_feature_size,
27 get_clip_patch_grid_length, input_processor_for_clip)
28 from .interfaces import SupportsMultiModal, SupportsPP
29 -from .llava import LlavaMultiModalProjector
29 +from .llava import LlavaMultiModalProjector, init_vision_tower_for_llava
30 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip, get_siglip_image_feature_size,
32 get_siglip_patch_grid_length, input_processor_for_siglip)
@@ -259,32 +259,6 @@ def input_processor_for_llava_next(ctx: InputContext,
259 raise NotImplementedError(msg)
260
261
262 -def _init_vision_tower(hf_config: LlavaNextConfig):
263 - vision_config = hf_config.vision_config
264 -
265 - # Initialize the vision tower only up to the required feature layer
266 - vision_feature_layer = hf_config.vision_feature_layer
267 - if vision_feature_layer < 0:
268 - num_hidden_layers = hf_config.vision_config.num_hidden_layers \
269 - + vision_feature_layer + 1
270 - else:
271 - num_hidden_layers = vision_feature_layer + 1
272 -
273 - if isinstance(vision_config, CLIPVisionConfig):
274 - return CLIPVisionModel(
275 - vision_config,
276 - num_hidden_layers_override=num_hidden_layers,
277 - )
278 - elif isinstance(vision_config, SiglipVisionConfig):
279 - return SiglipVisionModel(
280 - vision_config,
281 - num_hidden_layers_override=num_hidden_layers,
282 - )
283 -
284 - msg = f"Unsupported vision config: {type(vision_config)}"
285 - raise NotImplementedError(msg)
286 -
287 -
262 @MULTIMODAL_REGISTRY.register_image_input_mapper()
263 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_llava_next_image_tokens)
264 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
@@ -303,7 +277,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
277 self.multimodal_config = multimodal_config
278
279 # TODO: Optionally initializes this for supporting embeddings.
306 - self.vision_tower = _init_vision_tower(config)
280 + self.vision_tower = init_vision_tower_for_llava(config, quant_config)
281 self.image_newline = nn.Parameter(
282 torch.empty(config.text_config.hidden_size))
283 self.multi_modal_projector = LlavaMultiModalProjector(
vllm/model_executor/models/llava_next_video.py
+2 -27
@@ -26,6 +26,7 @@ from vllm.utils import is_list_of
26
27 from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
28 from .interfaces import SupportsMultiModal, SupportsPP
29 +from .llava import init_vision_tower_for_llava
30 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 dummy_seq_data_for_siglip)
32 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
@@ -179,32 +180,6 @@ def input_processor_for_llava_next_video(ctx: InputContext,
180 raise NotImplementedError(msg)
181
182
182 -def _init_vision_tower(hf_config: LlavaNextVideoConfig):
183 - vision_config = hf_config.vision_config
184 -
185 - # Initialize the vision tower only up to the required feature layer
186 - vision_feature_layer = hf_config.vision_feature_layer
187 - if vision_feature_layer < 0:
188 - num_hidden_layers = hf_config.vision_config.num_hidden_layers \
189 - + vision_feature_layer + 1
190 - else:
191 - num_hidden_layers = vision_feature_layer + 1
192 -
193 - if isinstance(vision_config, CLIPVisionConfig):
194 - return CLIPVisionModel(
195 - vision_config,
196 - num_hidden_layers_override=num_hidden_layers,
197 - )
198 - elif isinstance(vision_config, SiglipVisionConfig):
199 - return SiglipVisionModel(
200 - vision_config,
201 - num_hidden_layers_override=num_hidden_layers,
202 - )
203 -
204 - msg = f"Unsupported vision config: {type(vision_config)}"
205 - raise NotImplementedError(msg)
206 -
207 -
183 # adopted from transformers modeling_llava_next_video.py
184 class LlavaNextVideoPooler(nn.Module):
185
@@ -281,7 +256,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
256 self.multimodal_config = multimodal_config
257
258 # Initialize the vision tower only up to the required feature layer
284 - self.vision_tower = _init_vision_tower(config)
259 + self.vision_tower = init_vision_tower_for_llava(config, quant_config)
260 self.vision_resampler = LlavaNextVideoPooler(config)
261 self.multi_modal_projector = LlavaNextMultiModalProjector(
262 vision_hidden_size=config.vision_config.hidden_size,
vllm/model_executor/models/llava_onevision.py
+2 -27
@@ -31,6 +31,7 @@ from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
31 dummy_video_for_clip, get_clip_image_feature_size,
32 get_clip_patch_grid_length, input_processor_for_clip)
33 from .interfaces import SupportsMultiModal, SupportsPP
34 +from .llava import init_vision_tower_for_llava
35 from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36 dummy_video_for_siglip, get_siglip_image_feature_size,
37 get_siglip_patch_grid_length, input_processor_for_siglip)
@@ -357,32 +358,6 @@ def input_processor_for_llava_onevision(ctx: InputContext,
358 raise NotImplementedError(msg)
359
360
360 -def _init_vision_tower(hf_config: LlavaOnevisionConfig):
361 - vision_config = hf_config.vision_config
362 -
363 - # Initialize the vision tower only up to the required feature layer
364 - vision_feature_layer = hf_config.vision_feature_layer
365 - if vision_feature_layer < 0:
366 - num_hidden_layers = hf_config.vision_config.num_hidden_layers \
367 - + vision_feature_layer + 1
368 - else:
369 - num_hidden_layers = vision_feature_layer + 1
370 -
371 - if isinstance(vision_config, CLIPVisionConfig):
372 - return CLIPVisionModel(
373 - vision_config,
374 - num_hidden_layers_override=num_hidden_layers,
375 - )
376 - elif isinstance(vision_config, SiglipVisionConfig):
377 - return SiglipVisionModel(
378 - vision_config,
379 - num_hidden_layers_override=num_hidden_layers,
380 - )
381 -
382 - msg = f"Unsupported vision config: {type(vision_config)}"
383 - raise NotImplementedError(msg)
384 -
385 -
361 class LlavaOnevisionMultiModalProjector(nn.Module):
362
363 def __init__(self, config: LlavaOnevisionConfig):
@@ -425,7 +400,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
400 self.multimodal_config = multimodal_config
401
402 # Initialize the vision tower only up to the required feature layer
428 - self.vision_tower = _init_vision_tower(config)
403 + self.vision_tower = init_vision_tower_for_llava(config, quant_config)
404 self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
405 self.language_model = init_vllm_registered_model(
406 config.text_config, cache_config, quant_config)
vllm/model_executor/models/minicpmv.py
+25 -8
@@ -395,7 +395,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
395
396 self.version = get_version_by_config(self.config)
397 self.llm = self.init_llm(config, cache_config, quant_config)
398 - self.vpm = self.init_vision_module()
398 + self.vpm = self.init_vision_module(config, quant_config)
399 param_dtype = torch.get_default_dtype()
400 self.vpm.to(dtype=param_dtype)
401 self.vision_dim = (self.vpm.embed_dim if self.version == (2, 0) else
@@ -647,7 +647,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
647 ) -> nn.Module:
648 raise NotImplementedError
649
650 - def init_vision_module(self) -> nn.Module:
650 + def init_vision_module(
651 + self,
652 + config: PretrainedConfig,
653 + quant_config: Optional[QuantizationConfig],
654 + ) -> nn.Module:
655 raise NotImplementedError
656
657 def init_resampler(self, embed_dim: int, vision_dim: int) -> nn.Module:
@@ -693,7 +697,11 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
697 quant_config=quant_config),
698 name="model")
699
696 - def init_vision_module(self) -> nn.Module:
700 + def init_vision_module(
701 + self,
702 + config: PretrainedConfig,
703 + quant_config: Optional[QuantizationConfig],
704 + ) -> nn.Module:
705 # TODO :refactor this vision model
706 try:
707 import timm
@@ -817,8 +825,13 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
825 quant_config=quant_config),
826 name="model")
827
820 - def init_vision_module(self) -> nn.Module:
821 - model = Idefics2VisionTransformer(self.config.vision_config)
828 + def init_vision_module(
829 + self,
830 + config: PretrainedConfig,
831 + quant_config: Optional[QuantizationConfig],
832 + ) -> nn.Module:
833 + model = Idefics2VisionTransformer(config.vision_config,
834 + quant_config=quant_config)
835 if self.config.drop_vision_last_layer:
836 model.encoder.layers = model.encoder.layers[:-1]
837 return model
@@ -929,9 +942,13 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
942 quant_config=quant_config),
943 name="model")
944
932 - def init_vision_module(self) -> nn.Module:
933 -
934 - model = Idefics2VisionTransformer(self.config.vision_config)
945 + def init_vision_module(
946 + self,
947 + config: PretrainedConfig,
948 + quant_config: Optional[QuantizationConfig],
949 + ) -> nn.Module:
950 + model = Idefics2VisionTransformer(config.vision_config,
951 + quant_config=quant_config)
952 if self.config.drop_vision_last_layer:
953 model.encoder.layers = model.encoder.layers[:-1]
954 return model
vllm/model_executor/models/mllama.py
+89 -31
@@ -379,9 +379,13 @@ class MllamaVisionSdpaAttention(nn.Module):
379
380 class MllamaVisionEncoderLayer(nn.Module):
381
382 - def __init__(self,
383 - config: config_mllama.MllamaVisionConfig,
384 - is_gated: bool = False):
382 + def __init__(
383 + self,
384 + config: config_mllama.MllamaVisionConfig,
385 + quant_config: Optional[QuantizationConfig],
386 + prefix: str = "",
387 + is_gated: bool = False,
388 + ) -> None:
389 super().__init__()
390
391 self.hidden_size = config.hidden_size
@@ -390,7 +394,9 @@ class MllamaVisionEncoderLayer(nn.Module):
394 self.intermediate_size = config.intermediate_size
395
396 self.self_attn = MllamaVisionSdpaAttention(config)
393 - self.mlp = CLIPMLP(config)
397 + self.mlp = CLIPMLP(config,
398 + quant_config=quant_config,
399 + prefix=f"{prefix}.mlp")
400
401 self.input_layernorm = nn.LayerNorm(self.hidden_size,
402 eps=config.norm_eps)
@@ -427,16 +433,23 @@ class MllamaVisionEncoderLayer(nn.Module):
433
434 class MllamaVisionEncoder(nn.Module):
435
430 - def __init__(self,
431 - config: config_mllama.MllamaVisionConfig,
432 - num_layers=32,
433 - is_gated=False,
434 - output_hidden_states=None):
436 + def __init__(
437 + self,
438 + config: config_mllama.MllamaVisionConfig,
439 + quant_config: Optional[QuantizationConfig],
440 + num_layers: int = 32,
441 + is_gated: bool = False,
442 + output_hidden_states=None,
443 + prefix: str = "",
444 + ) -> None:
445 super().__init__()
446 self.config = config
447 self.layers = nn.ModuleList([
438 - MllamaVisionEncoderLayer(config, is_gated)
439 - for _ in range(num_layers)
448 + MllamaVisionEncoderLayer(config,
449 + quant_config=quant_config,
450 + is_gated=is_gated,
451 + prefix=f"{prefix}.layers.{layer_idx}")
452 + for layer_idx in range(num_layers)
453 ])
454 self.output_hidden_states = output_hidden_states or []
455
@@ -463,8 +476,14 @@ class MllamaVisionEncoder(nn.Module):
476
477 class MllamaVisionModel(nn.Module):
478
466 - def __init__(self, config: config_mllama.MllamaVisionConfig):
479 + def __init__(
480 + self,
481 + config: config_mllama.MllamaVisionConfig,
482 + quant_config: Optional[QuantizationConfig],
483 + prefix: str = "",
484 + ) -> None:
485 super().__init__()
486 +
487 self.image_size = config.image_size
488 self.patch_size = config.patch_size
489 self.max_num_tiles = config.max_num_tiles
@@ -500,12 +519,19 @@ class MllamaVisionModel(nn.Module):
519 # encoders
520 self.transformer = MllamaVisionEncoder(
521 config,
522 + quant_config,
523 config.num_hidden_layers,
524 is_gated=False,
505 - output_hidden_states=config.intermediate_layers_indices)
506 - self.global_transformer = MllamaVisionEncoder(config,
507 - config.num_global_layers,
508 - is_gated=True)
525 + output_hidden_states=config.intermediate_layers_indices,
526 + prefix=f"{prefix}.transformer",
527 + )
528 + self.global_transformer = MllamaVisionEncoder(
529 + config,
530 + quant_config,
531 + config.num_global_layers,
532 + is_gated=True,
533 + prefix=f"{prefix}.global_transformer",
534 + )
535
536 def apply_class_embedding(self,
537 hidden_state: torch.Tensor) -> torch.Tensor:
@@ -648,6 +674,7 @@ class MllamaTextCrossAttention(nn.Module):
674 config: Optional[config_mllama.MllamaTextConfig] = None,
675 layer_idx: Optional[int] = None,
676 quant_config: Optional[QuantizationConfig] = None,
677 + prefix: str = "",
678 ):
679 super().__init__()
680 self.config = config
@@ -673,6 +700,7 @@ class MllamaTextCrossAttention(nn.Module):
700 self.num_key_value_heads,
701 bias=False,
702 quant_config=quant_config,
703 + prefix=f"{prefix}.qkv_proj",
704 )
705 self.o_proj = RowParallelLinear(
706 self.num_heads * self.head_dim,
@@ -680,6 +708,7 @@ class MllamaTextCrossAttention(nn.Module):
708 bias=False,
709 input_is_parallel=True,
710 quant_config=quant_config,
711 + prefix=f"{prefix}.o_proj",
712 )
713 # vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
714 # use huggingface's instead
@@ -692,6 +721,7 @@ class MllamaTextCrossAttention(nn.Module):
721 self.head_dim,
722 self.scaling,
723 self.num_local_key_value_heads,
724 + prefix=f"{prefix}.attn",
725 )
726
727 def forward(
@@ -791,15 +821,21 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
821 """Cross-attention transformer block with tanh-gated attention
822 and feedforward."""
823
794 - def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int,
795 - quant_config: Optional[QuantizationConfig]) \
796 - -> None:
824 + def __init__(
825 + self,
826 + config: config_mllama.MllamaTextConfig,
827 + layer_idx: int,
828 + quant_config: Optional[QuantizationConfig],
829 + prefix: str = "",
830 + ) -> None:
831 super().__init__()
832 +
833 self.layer_idx = layer_idx
834 self.cross_attn = MllamaTextCrossAttention(
835 config=config,
836 layer_idx=layer_idx,
837 quant_config=quant_config,
838 + prefix=f"{prefix}.cross_attn",
839 )
840
841 self.input_layernorm = RMSNorm(config.hidden_size,
@@ -811,6 +847,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
847 intermediate_size=config.intermediate_size,
848 hidden_act=config.hidden_act,
849 quant_config=quant_config,
850 + prefix=f"{prefix}.mlp",
851 )
852 self.post_attention_layernorm = RMSNorm(config.hidden_size,
853 eps=config.rms_norm_eps)
@@ -854,10 +891,15 @@ class MllamaTextModel(nn.Module):
891 config_class = config_mllama.MllamaTextConfig
892 base_model_prefix = "model"
893
857 - def __init__(self, config: config_mllama.MllamaTextConfig,
858 - cache_config: Optional[CacheConfig],
859 - quant_config: Optional[QuantizationConfig]):
894 + def __init__(
895 + self,
896 + config: config_mllama.MllamaTextConfig,
897 + cache_config: Optional[CacheConfig],
898 + quant_config: Optional[QuantizationConfig],
899 + prefix: str = "",
900 + ) -> None:
901 super().__init__()
902 +
903 self.padding_idx = config.pad_token_id
904 self.vocab_size = config.vocab_size
905 self.embed_tokens = VocabParallelEmbedding(config.vocab_size + 8,
@@ -869,13 +911,20 @@ class MllamaTextModel(nn.Module):
911 if layer_idx in self.cross_attention_layers:
912 layers.append(
913 MllamaCrossAttentionDecoderLayer(
872 - config, layer_idx, quant_config=quant_config))
914 + config,
915 + layer_idx,
916 + quant_config=quant_config,
917 + prefix=f"{prefix}.layers.{layer_idx}",
918 + ))
919 else:
920 # TODO: force LlamaDecoderLayer to config.attention_bias=False
921 layers.append(
876 - LlamaDecoderLayer(config,
877 - cache_config=cache_config,
878 - quant_config=quant_config))
922 + LlamaDecoderLayer(
923 + config,
924 + cache_config=cache_config,
925 + quant_config=quant_config,
926 + prefix=f"{prefix}.layers.{layer_idx}",
927 + ))
928
929 self.layers = nn.ModuleList(layers)
930 self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -932,12 +981,19 @@ class MllamaForCausalLM(nn.Module):
981 "MllamaCrossAttentionDecoderLayer", "MllamaSelfAttentionDecoderLayer"
982 ]
983
935 - def __init__(self, config: config_mllama.MllamaTextConfig,
936 - cache_config: Optional[CacheConfig],
937 - quant_config: Optional[QuantizationConfig]):
984 + def __init__(
985 + self,
986 + config: config_mllama.MllamaTextConfig,
987 + cache_config: Optional[CacheConfig],
988 + quant_config: Optional[QuantizationConfig],
989 + prefix: str = "",
990 + ) -> None:
991 super().__init__()
992 self.vocab_size = config.vocab_size
940 - self.model = MllamaTextModel(config, cache_config, quant_config)
993 + self.model = MllamaTextModel(config,
994 + cache_config,
995 + quant_config,
996 + prefix=f"{prefix}.model")
997 self.lm_head = ParallelLMHead(
998 config.vocab_size,
999 config.hidden_size,
@@ -994,11 +1050,13 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1050 config.pad_token_id if config.pad_token_id is not None else -1
1051 self.image_size = config.vision_config.image_size
1052
997 - self.vision_model = MllamaVisionModel(config.vision_config)
1053 + self.vision_model = MllamaVisionModel(config.vision_config,
1054 + quant_config)
1055 self.language_model = MllamaForCausalLM(
1056 config.text_config,
1057 cache_config=cache_config,
1058 quant_config=quant_config,
1059 + prefix="language_model",
1060 )
1061 self.multi_modal_projector = nn.Linear(
1062 config.vision_config.vision_output_dim,
vllm/model_executor/models/nvlm_d.py
+5
@@ -4,10 +4,13 @@
4 # Copyright (c) 2024 NVIDIA
5 # Licensed under Apache 2.0 License [see LICENSE for details]
6 # --------------------------------------------------------
7 +from typing import Optional
8 +
9 import torch.nn as nn
10 from transformers import PretrainedConfig
11
12 from vllm.inputs import INPUT_REGISTRY
13 +from vllm.model_executor.layers.quantization import QuantizationConfig
14 from vllm.multimodal import MULTIMODAL_REGISTRY
15
16 from .intern_vit import InternVisionModel
@@ -56,9 +59,11 @@ class NVLM_D_Model(InternVLChatModel):
59 )
60
61 def _init_vision_model(self, config: PretrainedConfig,
62 + quant_config: Optional[QuantizationConfig],
63 num_hidden_layers: int):
64 # We added additional dummy heads to the original num of heads to make
65 # the number of heads divisible by 8.
66 return InternVisionModel(config.vision_config,
67 + quant_config=quant_config,
68 num_hidden_layers_override=num_hidden_layers,
69 num_dummy_heads=7)
vllm/model_executor/models/paligemma.py
+2 -1
@@ -142,7 +142,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
142 self.config = config
143 self.multimodal_config = multimodal_config
144
145 - self.vision_tower = SiglipVisionModel(config.vision_config)
145 + self.vision_tower = SiglipVisionModel(config.vision_config,
146 + quant_config)
147 self.multi_modal_projector = PaliGemmaMultiModalProjector(
148 vision_hidden_size=config.vision_config.hidden_size,
149 projection_dim=config.vision_config.projection_dim)
vllm/model_executor/models/phi3v.py
+10 -5
@@ -70,7 +70,8 @@ CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
70 projection_dim=768)
71
72
73 -def _init_img_processor(hf_config: PretrainedConfig):
73 +def _init_img_processor(hf_config: PretrainedConfig,
74 + quant_config: Optional[QuantizationConfig]):
75 clip_config = CLIP_VIT_LARGE_PATCH14_336_CONFIG
76 layer_idx = hf_config.img_processor.get('layer_idx', -2)
77
@@ -82,7 +83,10 @@ def _init_img_processor(hf_config: PretrainedConfig):
83 num_hidden_layers = layer_idx + 1
84
85 img_processor = CLIPVisionModel(
85 - clip_config, num_hidden_layers_override=num_hidden_layers)
86 + clip_config,
87 + quant_config,
88 + num_hidden_layers_override=num_hidden_layers,
89 + )
90
91 return img_processor
92
@@ -148,14 +152,15 @@ class Phi3ImageEmbeddingBase(nn.Module):
152 class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
153 """Phi3 Image embedding with HD transform."""
154
151 - def __init__(self, config: PretrainedConfig) -> None:
155 + def __init__(self, config: PretrainedConfig,
156 + quant_config: Optional[QuantizationConfig]) -> None:
157 super().__init__()
158
159 # n_embed or hidden_size
160 hidden_size = config.n_embd if hasattr(
161 config, 'n_embd') else config.hidden_size
162
158 - self.img_processor = _init_img_processor(config)
163 + self.img_processor = _init_img_processor(config, quant_config)
164
165 image_dim_out = config.img_processor['image_dim_out']
166 self.num_img_tokens = config.img_processor['num_img_tokens']
@@ -535,7 +540,7 @@ class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
540 )
541
542 # TODO: Optionally initializes this for supporting input embeddings.
538 - self.vision_embed_tokens = Phi3HDImageEmbedding(config)
543 + self.vision_embed_tokens = Phi3HDImageEmbedding(config, quant_config)
544
545 self.language_model = LlamaForCausalLM(config, cache_config,
546 quant_config)
vllm/model_executor/models/pixtral.py
+79 -11
@@ -767,9 +767,17 @@ def input_processor_for_pixtral_hf(
767
768 class PixtralHFMLP(nn.Module):
769
770 - def __init__(self, config: PixtralVisionConfig):
770 + def __init__(
771 + self,
772 + config: PixtralVisionConfig,
773 + quant_config: Optional[QuantizationConfig] = None,
774 + *,
775 + prefix: str = "",
776 + ) -> None:
777 super().__init__()
778 +
779 assert config.intermediate_size is not None
780 + # TODO: Use quant_config and prefix after optimizing this
781 self.gate_proj = nn.Linear(config.hidden_size,
782 config.intermediate_size,
783 bias=False)
@@ -787,8 +795,15 @@ class PixtralHFMLP(nn.Module):
795
796 class PixtralHFAttention(nn.Module):
797
790 - def __init__(self, config: PixtralVisionConfig):
798 + def __init__(
799 + self,
800 + config: PixtralVisionConfig,
801 + quant_config: Optional[QuantizationConfig] = None,
802 + *,
803 + prefix: str = "",
804 + ) -> None:
805 super().__init__()
806 +
807 self.config = config
808 assert not config.hidden_size % config.num_attention_heads
809 self.n_heads = config.num_attention_heads
@@ -796,6 +811,7 @@ class PixtralHFAttention(nn.Module):
811
812 self.scale = self.head_dim**-0.5
813
814 + # TODO: Use quant_config and prefix after optimizing this
815 self.q_proj = nn.Linear(config.hidden_size,
816 config.hidden_size,
817 bias=False)
@@ -840,11 +856,22 @@ class PixtralHFAttention(nn.Module):
856
857 class PixtralHFTransformerBlock(nn.Module):
858
843 - def __init__(self, config: PixtralVisionConfig):
859 + def __init__(
860 + self,
861 + config: PixtralVisionConfig,
862 + quant_config: Optional[QuantizationConfig] = None,
863 + *,
864 + prefix: str = "",
865 + ) -> None:
866 super().__init__()
867 +
868 self.attention_norm = RMSNorm(config.hidden_size, eps=1e-5)
846 - self.attention = PixtralHFAttention(config)
847 - self.feed_forward = PixtralHFMLP(config)
869 + self.attention = PixtralHFAttention(config,
870 + quant_config=quant_config,
871 + prefix=f"{prefix}.attention")
872 + self.feed_forward = PixtralHFMLP(config,
873 + quant_config=quant_config,
874 + prefix=f"{prefix}.feed_forward")
875 self.ffn_norm = RMSNorm(config.hidden_size, eps=1e-5)
876
877 def forward(
@@ -864,11 +891,27 @@ class PixtralHFTransformerBlock(nn.Module):
891
892 class PixtralHFTransformer(nn.Module):
893
867 - def __init__(self, config: PixtralVisionConfig):
894 + def __init__(
895 + self,
896 + config: PixtralVisionConfig,
897 + quant_config: Optional[QuantizationConfig] = None,
898 + *,
899 + num_hidden_layers_override: Optional[int] = None,
900 + prefix: str = "",
901 + ) -> None:
902 super().__init__()
869 - self.layers = torch.nn.ModuleList()
870 - for _ in range(config.num_hidden_layers):
871 - self.layers.append(PixtralHFTransformerBlock(config))
903 +
904 + if num_hidden_layers_override is None:
905 + num_hidden_layers = config.num_hidden_layers
906 + else:
907 + num_hidden_layers = num_hidden_layers_override
908 +
909 + self.layers = nn.ModuleList([
910 + PixtralHFTransformerBlock(config=config,
911 + quant_config=quant_config,
912 + prefix=f"{prefix}.layers.{layer_idx}")
913 + for layer_idx in range(num_hidden_layers)
914 + ])
915
916 def forward(
917 self,
@@ -883,7 +926,15 @@ class PixtralHFTransformer(nn.Module):
926
927 class PixtralHFVisionModel(nn.Module):
928
886 - def __init__(self, config: PixtralVisionConfig):
929 + def __init__(
930 + self,
931 + config: PixtralVisionConfig,
932 + quant_config: Optional[QuantizationConfig] = None,
933 + *,
934 + num_hidden_layers_override: Optional[int] = None,
935 + require_post_norm: Optional[bool] = None,
936 + prefix: str = "",
937 + ) -> None:
938 super().__init__()
939
940 self.config = config
@@ -895,7 +946,24 @@ class PixtralHFVisionModel(nn.Module):
946 bias=False,
947 )
948 self.ln_pre = RMSNorm(config.hidden_size, eps=1e-5)
898 - self.transformer = PixtralHFTransformer(config)
949 + self.transformer = PixtralHFTransformer(
950 + config,
951 + quant_config,
952 + num_hidden_layers_override=num_hidden_layers_override,
953 + prefix=f"{prefix}.transformer",
954 + )
955 +
956 + num_hidden_layers = config.num_hidden_layers
957 + if len(self.transformer.layers) > config.num_hidden_layers:
958 + raise ValueError(
959 + f"The original encoder only has {num_hidden_layers} "
960 + f"layers, but you requested {len(self.transformer.layers)} "
961 + "layers.")
962 +
963 + if require_post_norm is True:
964 + msg = "PixtralHFVisionModel does not have post-layernorm"
965 + raise ValueError(msg)
966 +
967 self.dtype = next(self.parameters()).dtype
968 self.device = next(self.parameters()).device
969 self.patch_positional_embedding = PixtralRotaryEmbedding(
vllm/model_executor/models/siglip.py
+55 -17
@@ -248,8 +248,10 @@ class SiglipParallelAttention(nn.Module):
248 self,
249 config: SiglipVisionConfig,
250 quant_config: Optional[QuantizationConfig] = None,
251 - ):
251 + prefix: str = "",
252 + ) -> None:
253 super().__init__()
254 +
255 self.config = config
256 self.embed_dim = config.hidden_size
257 self.num_heads = config.num_attention_heads
@@ -266,12 +268,14 @@ class SiglipParallelAttention(nn.Module):
268 head_size=self.head_dim,
269 total_num_heads=self.num_heads,
270 quant_config=quant_config,
271 + prefix=f"{prefix}.qkv_proj",
272 )
273
274 self.out_proj = RowParallelLinear(
275 input_size=self.embed_dim,
276 output_size=self.embed_dim,
277 quant_config=quant_config,
278 + prefix=f"{prefix}.out_proj",
279 )
280
281 self.tp_size = get_tensor_model_parallel_world_size()
@@ -314,8 +318,10 @@ class SiglipMLP(nn.Module):
318 self,
319 config: SiglipVisionConfig,
320 quant_config: Optional[QuantizationConfig] = None,
317 - ):
321 + prefix: str = "",
322 + ) -> None:
323 super().__init__()
324 +
325 self.config = config
326 self.activation_fn = get_act_fn(config.hidden_act)
327
@@ -326,11 +332,13 @@ class SiglipMLP(nn.Module):
332 config.hidden_size,
333 config.intermediate_size,
334 quant_config=quant_config if quantizable else None,
335 + prefix=f"{prefix}.fc1",
336 )
337 self.fc2 = RowParallelLinear(
338 config.intermediate_size,
339 config.hidden_size,
340 quant_config=quant_config if quantizable else None,
341 + prefix=f"{prefix}.fc2",
342 )
343
344 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
@@ -346,15 +354,20 @@ class SiglipEncoderLayer(nn.Module):
354 self,
355 config: SiglipVisionConfig,
356 quant_config: Optional[QuantizationConfig] = None,
349 - ):
357 + prefix: str = "",
358 + ) -> None:
359 super().__init__()
360 +
361 self.embed_dim = config.hidden_size
362
363 num_heads = config.num_attention_heads
364 tp_size = get_tensor_model_parallel_world_size()
365 if USE_XFORMERS_OPS and num_heads % tp_size == 0:
356 - self.self_attn = SiglipParallelAttention(config,
357 - quant_config=quant_config)
366 + self.self_attn = SiglipParallelAttention(
367 + config,
368 + quant_config=quant_config,
369 + prefix=f"{prefix}.self_attn",
370 + )
371 else:
372 self.self_attn = SiglipSdpaAttention(config)
373
@@ -363,6 +376,7 @@ class SiglipEncoderLayer(nn.Module):
376 self.mlp = SiglipMLP(
377 config,
378 quant_config=quant_config,
379 + prefix=f"{prefix}.mlp",
380 )
381 self.layer_norm2 = nn.LayerNorm(self.embed_dim,
382 eps=config.layer_norm_eps)
@@ -392,8 +406,10 @@ class SiglipEncoder(nn.Module):
406 config: SiglipVisionConfig,
407 quant_config: Optional[QuantizationConfig] = None,
408 num_hidden_layers_override: Optional[int] = None,
395 - ):
409 + prefix: str = "",
410 + ) -> None:
411 super().__init__()
412 +
413 self.config = config
414
415 if num_hidden_layers_override is None:
@@ -402,8 +418,10 @@ class SiglipEncoder(nn.Module):
418 num_hidden_layers = num_hidden_layers_override
419
420 self.layers = nn.ModuleList([
405 - SiglipEncoderLayer(config, quant_config=quant_config)
406 - for _ in range(num_hidden_layers)
421 + SiglipEncoderLayer(config,
422 + quant_config=quant_config,
423 + prefix=f"{prefix}.layers.{layer_idx}")
424 + for layer_idx in range(num_hidden_layers)
425 ])
426
427 def forward(
@@ -424,7 +442,8 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
442 self,
443 config: SiglipVisionConfig,
444 quant_config: Optional[QuantizationConfig] = None,
427 - ):
445 + prefix: str = "",
446 + ) -> None:
447 super().__init__()
448
449 self.probe = nn.Parameter(torch.randn(1, 1, config.hidden_size))
@@ -433,7 +452,9 @@ class SiglipMultiheadAttentionPoolingHead(nn.Module):
452 config.hidden_size, config.num_attention_heads, batch_first=True)
453 self.layernorm = nn.LayerNorm(config.hidden_size,
454 eps=config.layer_norm_eps)
436 - self.mlp = SiglipMLP(config=config, quant_config=quant_config)
455 + self.mlp = SiglipMLP(config=config,
456 + quant_config=quant_config,
457 + prefix=f"{prefix}.mlp")
458
459 def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
460 batch_size = hidden_state.shape[0]
@@ -454,9 +475,13 @@ class SiglipVisionTransformer(nn.Module):
475 self,
476 config: SiglipVisionConfig,
477 quant_config: Optional[QuantizationConfig] = None,
478 + *,
479 num_hidden_layers_override: Optional[int] = None,
458 - ):
480 + require_post_norm: Optional[bool] = None,
481 + prefix: str = "",
482 + ) -> None:
483 super().__init__()
484 +
485 self.config = config
486 embed_dim = config.hidden_size
487
@@ -465,26 +490,34 @@ class SiglipVisionTransformer(nn.Module):
490 config,
491 quant_config=quant_config,
492 num_hidden_layers_override=num_hidden_layers_override,
493 + prefix=f"{prefix}.encoder",
494 )
495
496 + num_hidden_layers = config.num_hidden_layers
497 if len(self.encoder.layers) > config.num_hidden_layers:
498 raise ValueError(
472 - f"The original encoder only has {config.num_hidden_layers} "
499 + f"The original encoder only has {num_hidden_layers} "
500 f"layers, but you requested {len(self.encoder.layers)} layers."
501 )
475 - elif len(self.encoder.layers) == config.num_hidden_layers:
502 +
503 + # If possible, skip post_layernorm to conserve memory
504 + if require_post_norm is None:
505 + require_post_norm = len(self.encoder.layers) == num_hidden_layers
506 +
507 + if require_post_norm:
508 self.post_layernorm = nn.LayerNorm(embed_dim,
509 eps=config.layer_norm_eps)
510 else:
479 - # post_layernorm is unused when we extract intermediate features
480 - # In this case, we can skip it to conserve memory
511 self.post_layernorm = None
512
513 self.use_head = (True if not hasattr(config, "vision_use_head") else
514 config.vision_use_head)
515 if self.use_head:
516 self.head = SiglipMultiheadAttentionPoolingHead(
487 - config=config, quant_config=quant_config)
517 + config=config,
518 + quant_config=quant_config,
519 + prefix=f"{prefix}.head",
520 + )
521
522 def forward(
523 self,
@@ -517,8 +550,11 @@ class SiglipVisionModel(nn.Module):
550 self,
551 config: SiglipVisionConfig,
552 quant_config: Optional[QuantizationConfig] = None,
553 + *,
554 num_hidden_layers_override: Optional[int] = None,
521 - ):
555 + require_post_norm: Optional[bool] = None,
556 + prefix: str = "",
557 + ) -> None:
558 super().__init__()
559
560 num_heads = config.num_attention_heads
@@ -529,6 +565,8 @@ class SiglipVisionModel(nn.Module):
565 config,
566 quant_config,
567 num_hidden_layers_override=num_hidden_layers_override,
568 + require_post_norm=require_post_norm,
569 + prefix=f"{prefix}.vision_model",
570 )
571
572 def get_input_embeddings(self) -> nn.Module: