A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Fix Skywork R1V mlp (#26673)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Oct 13, 2025 at 13:42 UTC
98f30b8cba5bb7e16421ab4c75fc200357744310
1 file changed
+18
-3
vllm/model_executor/models/skyworkr1v.py
+18
-3
@@ -691,7 +691,9 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
691
prefix=maybe_prefix(prefix, "language_model"),
692
)
693
694
- self.mlp1 = self._init_mlp1(config)
694
+ self.mlp1 = self._init_mlp1(
695
+ config, quant_config, prefix=maybe_prefix(prefix, "mlp1")
696
+ )
697
698
self.img_context_token_id = None
699
self.visual_token_mask = None
@@ -738,7 +740,12 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
740
else:
741
return InternVisionPatchModel(config.vision_config)
742
741
- def _init_mlp1(self, config: PretrainedConfig) -> nn.Module:
743
+ def _init_mlp1(
744
+ self,
745
+ config: PretrainedConfig,
746
+ quant_config: QuantizationConfig,
747
+ prefix: str = "",
748
+ ) -> nn.Module:
749
vit_hidden_size = config.vision_config.hidden_size
750
llm_hidden_size = config.text_config.hidden_size
751
@@ -748,9 +755,17 @@ class SkyworkR1VChatModel(nn.Module, SupportsMultiModal, SupportsPP):
755
vit_hidden_size * int(1 / self.downsample_ratio) ** 2,
756
llm_hidden_size,
757
return_bias=False,
758
+ quant_config=quant_config,
759
+ prefix=f"{prefix}.1",
760
),
761
nn.GELU(),
753
- ReplicatedLinear(llm_hidden_size, llm_hidden_size, return_bias=False),
762
+ ReplicatedLinear(
763
+ llm_hidden_size,
764
+ llm_hidden_size,
765
+ return_bias=False,
766
+ quant_config=quant_config,
767
+ prefix=f"{prefix}.3",
768
+ ),
769
)
770
771
def pixel_shuffle(self, x, scale_factor=0.5):