A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Support quantization of Qwen2VisionTransformer (#9817)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Oct 31, 2024 at 01:41 UTC
d087bf863e0d228c8b5aaae6535de15c5817eb7b
1 file changed
+35
-23
vllm/model_executor/models/qwen2_vl.py
+35
-23
@@ -126,15 +126,18 @@ class Qwen2VisionMLP(nn.Module):
126
hidden_features: int = None,
127
act_layer: Type[nn.Module] = QuickGELU,
128
quant_config: Optional[QuantizationConfig] = None,
129
+ prefix: str = "",
130
):
131
super().__init__()
132
self.fc1 = ColumnParallelLinear(in_features,
133
hidden_features,
133
- quant_config=quant_config)
134
+ quant_config=quant_config,
135
+ prefix=f"{prefix}.fc1")
136
self.act = act_layer()
137
self.fc2 = RowParallelLinear(hidden_features,
138
in_features,
137
- quant_config=quant_config)
139
+ quant_config=quant_config,
140
+ prefix=f"{prefix}.fc2")
141
142
def forward(self, x: torch.Tensor) -> torch.Tensor:
143
x_parallel, _ = self.fc1(x)
@@ -196,6 +199,7 @@ class Qwen2VisionAttention(nn.Module):
199
num_heads: Optional[int] = None,
200
projection_size: Optional[int] = None,
201
quant_config: Optional[QuantizationConfig] = None,
202
+ prefix: str = "",
203
) -> None:
204
super().__init__()
205
# Per attention head and per partition values.
@@ -207,10 +211,12 @@ class Qwen2VisionAttention(nn.Module):
211
212
self.qkv = ColumnParallelLinear(input_size=embed_dim,
213
output_size=3 * projection_size,
210
- quant_config=quant_config)
214
+ quant_config=quant_config,
215
+ prefix=f"{prefix}.qkv")
216
self.proj = RowParallelLinear(input_size=projection_size,
217
output_size=embed_dim,
213
- quant_config=quant_config)
218
+ quant_config=quant_config,
219
+ prefix=f"{prefix}.proj")
220
221
# Detect attention implementation.
222
self.attn_backend: _Backend = get_vit_attn_backend()
@@ -310,6 +316,7 @@ class Qwen2VisionBlock(nn.Module):
316
act_layer: Type[nn.Module] = QuickGELU,
317
norm_layer: Type[nn.Module] = None,
318
quant_config: Optional[QuantizationConfig] = None,
319
+ prefix: str = "",
320
) -> None:
321
super().__init__()
322
if norm_layer is None:
@@ -321,11 +328,13 @@ class Qwen2VisionBlock(nn.Module):
328
self.attn = Qwen2VisionAttention(embed_dim=dim,
329
num_heads=num_heads,
330
projection_size=dim,
324
- quant_config=quant_config)
331
+ quant_config=quant_config,
332
+ prefix=f"{prefix}.attn")
333
self.mlp = Qwen2VisionMLP(dim,
334
mlp_hidden_dim,
335
act_layer=act_layer,
328
- quant_config=quant_config)
336
+ quant_config=quant_config,
337
+ prefix=f"{prefix}.mlp")
338
339
def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
340
rotary_pos_emb: torch.Tensor) -> torch.Tensor:
@@ -374,6 +383,7 @@ class Qwen2VisionPatchMerger(nn.Module):
383
norm_layer: Type[nn.Module] = None,
384
spatial_merge_size: int = 2,
385
quant_config: Optional[QuantizationConfig] = None,
386
+ prefix: str = "",
387
) -> None:
388
super().__init__()
389
self.hidden_size = context_dim * (spatial_merge_size**2)
@@ -384,12 +394,14 @@ class Qwen2VisionPatchMerger(nn.Module):
394
ColumnParallelLinear(self.hidden_size,
395
self.hidden_size,
396
bias=True,
387
- quant_config=quant_config),
397
+ quant_config=quant_config,
398
+ prefix=f"{prefix}.mlp.0"),
399
nn.GELU(),
400
RowParallelLinear(self.hidden_size,
401
d_model,
402
bias=True,
392
- quant_config=quant_config),
403
+ quant_config=quant_config,
404
+ prefix=f"{prefix}.mlp.2"),
405
])
406
407
def forward(self, x: torch.Tensor) -> torch.Tensor:
@@ -440,6 +452,7 @@ class Qwen2VisionTransformer(nn.Module):
452
vision_config: Qwen2VLVisionConfig,
453
norm_eps: float = 1e-6,
454
quant_config: Optional[QuantizationConfig] = None,
455
+ prefix: str = "",
456
) -> None:
457
super().__init__()
458
@@ -467,28 +480,29 @@ class Qwen2VisionTransformer(nn.Module):
480
self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2)
481
482
self.blocks = nn.ModuleList([
470
- Qwen2VisionBlock(
471
- dim=embed_dim,
472
- num_heads=num_heads,
473
- mlp_ratio=mlp_ratio,
474
- norm_layer=norm_layer,
475
- quant_config=quant_config,
476
- ) for _ in range(depth)
483
+ Qwen2VisionBlock(dim=embed_dim,
484
+ num_heads=num_heads,
485
+ mlp_ratio=mlp_ratio,
486
+ norm_layer=norm_layer,
487
+ quant_config=quant_config,
488
+ prefix=f"{prefix}.blocks.{layer_idx}")
489
+ for layer_idx in range(depth)
490
])
491
self.merger = Qwen2VisionPatchMerger(
492
d_model=hidden_size,
493
context_dim=embed_dim,
494
norm_layer=norm_layer,
495
quant_config=quant_config,
496
+ prefix=f"{prefix}.merger",
497
)
498
499
@property
500
def dtype(self) -> torch.dtype:
487
- return self.blocks[0].mlp.fc2.weight.dtype
501
+ return self.patch_embed.proj.weight.dtype
502
503
@property
504
def device(self) -> torch.device:
491
- return self.blocks[0].mlp.fc2.weight.device
505
+ return self.patch_embed.proj.weight.device
506
507
def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
508
pos_ids = []
@@ -932,10 +946,8 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
946
self.visual = Qwen2VisionTransformer(
947
config.vision_config,
948
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
935
-
936
- # NOTE: Qwen2-VL vision encoder does not support any
937
- # quantization method now.
938
- quant_config=None,
949
+ quant_config=quant_config,
950
+ prefix="visual",
951
)
952
953
self.model = Qwen2Model(config,
@@ -1175,7 +1187,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1187
weight_loader(param, loaded_weight, shard_id)
1188
break
1189
else:
1178
- if "visual" in name and "qkv.weight" in name:
1190
+ if "visual" in name and name.endswith("qkv.weight"):
1191
visual_num_heads = self.config.vision_config.num_heads
1192
visual_embed_dim = self.config.vision_config.embed_dim
1193
head_size = visual_embed_dim // visual_num_heads
@@ -1184,7 +1196,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1196
visual_embed_dim)
1197
loaded_weight = loaded_weight.transpose(0, 1)
1198
loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1187
- elif "visual" in name and "qkv.bias" in name:
1199
+ elif "visual" in name and name.endswith("qkv.bias"):
1200
visual_num_heads = self.config.vision_config.num_heads
1201
visual_embed_dim = self.config.vision_config.embed_dim
1202
head_size = visual_embed_dim // visual_num_heads