@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d087bf863e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Support quantization of Qwen2VisionTransformer (#9817)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Oct 31, 2024 at 01:41 UTC d087bf863e0d228c8b5aaae6535de15c5817eb7b
1 file changed +35 -23
vllm/model_executor/models/qwen2_vl.py
+35 -23
@@ -126,15 +126,18 @@ class Qwen2VisionMLP(nn.Module):
126 hidden_features: int = None,
127 act_layer: Type[nn.Module] = QuickGELU,
128 quant_config: Optional[QuantizationConfig] = None,
129 + prefix: str = "",
130 ):
131 super().__init__()
132 self.fc1 = ColumnParallelLinear(in_features,
133 hidden_features,
133 - quant_config=quant_config)
134 + quant_config=quant_config,
135 + prefix=f"{prefix}.fc1")
136 self.act = act_layer()
137 self.fc2 = RowParallelLinear(hidden_features,
138 in_features,
137 - quant_config=quant_config)
139 + quant_config=quant_config,
140 + prefix=f"{prefix}.fc2")
141
142 def forward(self, x: torch.Tensor) -> torch.Tensor:
143 x_parallel, _ = self.fc1(x)
@@ -196,6 +199,7 @@ class Qwen2VisionAttention(nn.Module):
199 num_heads: Optional[int] = None,
200 projection_size: Optional[int] = None,
201 quant_config: Optional[QuantizationConfig] = None,
202 + prefix: str = "",
203 ) -> None:
204 super().__init__()
205 # Per attention head and per partition values.
@@ -207,10 +211,12 @@ class Qwen2VisionAttention(nn.Module):
211
212 self.qkv = ColumnParallelLinear(input_size=embed_dim,
213 output_size=3 * projection_size,
210 - quant_config=quant_config)
214 + quant_config=quant_config,
215 + prefix=f"{prefix}.qkv")
216 self.proj = RowParallelLinear(input_size=projection_size,
217 output_size=embed_dim,
213 - quant_config=quant_config)
218 + quant_config=quant_config,
219 + prefix=f"{prefix}.proj")
220
221 # Detect attention implementation.
222 self.attn_backend: _Backend = get_vit_attn_backend()
@@ -310,6 +316,7 @@ class Qwen2VisionBlock(nn.Module):
316 act_layer: Type[nn.Module] = QuickGELU,
317 norm_layer: Type[nn.Module] = None,
318 quant_config: Optional[QuantizationConfig] = None,
319 + prefix: str = "",
320 ) -> None:
321 super().__init__()
322 if norm_layer is None:
@@ -321,11 +328,13 @@ class Qwen2VisionBlock(nn.Module):
328 self.attn = Qwen2VisionAttention(embed_dim=dim,
329 num_heads=num_heads,
330 projection_size=dim,
324 - quant_config=quant_config)
331 + quant_config=quant_config,
332 + prefix=f"{prefix}.attn")
333 self.mlp = Qwen2VisionMLP(dim,
334 mlp_hidden_dim,
335 act_layer=act_layer,
328 - quant_config=quant_config)
336 + quant_config=quant_config,
337 + prefix=f"{prefix}.mlp")
338
339 def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
340 rotary_pos_emb: torch.Tensor) -> torch.Tensor:
@@ -374,6 +383,7 @@ class Qwen2VisionPatchMerger(nn.Module):
383 norm_layer: Type[nn.Module] = None,
384 spatial_merge_size: int = 2,
385 quant_config: Optional[QuantizationConfig] = None,
386 + prefix: str = "",
387 ) -> None:
388 super().__init__()
389 self.hidden_size = context_dim * (spatial_merge_size**2)
@@ -384,12 +394,14 @@ class Qwen2VisionPatchMerger(nn.Module):
394 ColumnParallelLinear(self.hidden_size,
395 self.hidden_size,
396 bias=True,
387 - quant_config=quant_config),
397 + quant_config=quant_config,
398 + prefix=f"{prefix}.mlp.0"),
399 nn.GELU(),
400 RowParallelLinear(self.hidden_size,
401 d_model,
402 bias=True,
392 - quant_config=quant_config),
403 + quant_config=quant_config,
404 + prefix=f"{prefix}.mlp.2"),
405 ])
406
407 def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -440,6 +452,7 @@ class Qwen2VisionTransformer(nn.Module):
452 vision_config: Qwen2VLVisionConfig,
453 norm_eps: float = 1e-6,
454 quant_config: Optional[QuantizationConfig] = None,
455 + prefix: str = "",
456 ) -> None:
457 super().__init__()
458
@@ -467,28 +480,29 @@ class Qwen2VisionTransformer(nn.Module):
480 self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2)
481
482 self.blocks = nn.ModuleList([
470 - Qwen2VisionBlock(
471 - dim=embed_dim,
472 - num_heads=num_heads,
473 - mlp_ratio=mlp_ratio,
474 - norm_layer=norm_layer,
475 - quant_config=quant_config,
476 - ) for _ in range(depth)
483 + Qwen2VisionBlock(dim=embed_dim,
484 + num_heads=num_heads,
485 + mlp_ratio=mlp_ratio,
486 + norm_layer=norm_layer,
487 + quant_config=quant_config,
488 + prefix=f"{prefix}.blocks.{layer_idx}")
489 + for layer_idx in range(depth)
490 ])
491 self.merger = Qwen2VisionPatchMerger(
492 d_model=hidden_size,
493 context_dim=embed_dim,
494 norm_layer=norm_layer,
495 quant_config=quant_config,
496 + prefix=f"{prefix}.merger",
497 )
498
499 @property
500 def dtype(self) -> torch.dtype:
487 - return self.blocks[0].mlp.fc2.weight.dtype
501 + return self.patch_embed.proj.weight.dtype
502
503 @property
504 def device(self) -> torch.device:
491 - return self.blocks[0].mlp.fc2.weight.device
505 + return self.patch_embed.proj.weight.device
506
507 def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
508 pos_ids = []
@@ -932,10 +946,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
946 self.visual = Qwen2VisionTransformer(
947 config.vision_config,
948 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
935 -
936 - # NOTE: Qwen2-VL vision encoder does not support any
937 - # quantization method now.
938 - quant_config=None,
949 + quant_config=quant_config,
950 + prefix="visual",
951 )
952
953 self.model = Qwen2Model(config,
@@ -1175,7 +1187,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1187 weight_loader(param, loaded_weight, shard_id)
1188 break
1189 else:
1178 - if "visual" in name and "qkv.weight" in name:
1190 + if "visual" in name and name.endswith("qkv.weight"):
1191 visual_num_heads = self.config.vision_config.num_heads
1192 visual_embed_dim = self.config.vision_config.embed_dim
1193 head_size = visual_embed_dim // visual_num_heads
@@ -1184,7 +1196,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1196 visual_embed_dim)
1197 loaded_weight = loaded_weight.transpose(0, 1)
1198 loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1187 - elif "visual" in name and "qkv.bias" in name:
1199 + elif "visual" in name and name.endswith("qkv.bias"):
1200 visual_num_heads = self.config.vision_config.num_heads
1201 visual_embed_dim = self.config.vision_config.embed_dim
1202 head_size = visual_embed_dim // visual_num_heads