@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5d98d56089
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

Support Pixtral-Large HF by using llava multimodal_projector_bias config (#12710)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Feb 3, 2025 at 22:55 UTC 5d98d56089426555d303d82bdf29b9ffc825ec20
4 files changed +13 -9
vllm/model_executor/models/llava.py
+4 -2
@@ -75,19 +75,20 @@ class LlavaMultiModalProjector(nn.Module):
75 vision_hidden_size: int,
76 text_hidden_size: int,
77 projector_hidden_act: str,
78 + multimodal_projector_bias: bool,
79 quant_config: Optional[QuantizationConfig] = None,
80 prefix: str = ""):
81 super().__init__()
82
83 self.linear_1 = ColumnParallelLinear(vision_hidden_size,
84 text_hidden_size,
84 - bias=True,
85 + bias=multimodal_projector_bias,
86 quant_config=quant_config,
87 prefix=f"{prefix}.linear_1")
88 self.act = get_act_fn(projector_hidden_act)
89 self.linear_2 = RowParallelLinear(text_hidden_size,
90 text_hidden_size,
90 - bias=True,
91 + bias=multimodal_projector_bias,
92 quant_config=quant_config,
93 prefix=f"{prefix}.linear_2")
94
@@ -503,6 +504,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
504 vision_hidden_size=config.vision_config.hidden_size,
505 text_hidden_size=config.text_config.hidden_size,
506 projector_hidden_act=config.projector_hidden_act,
507 + multimodal_projector_bias=config.multimodal_projector_bias,
508 quant_config=quant_config,
509 prefix=maybe_prefix(prefix, "multi_modal_projector"))
510
vllm/model_executor/models/llava_next.py
+2 -1
@@ -231,7 +231,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
231 self.multi_modal_projector = LlavaMultiModalProjector(
232 vision_hidden_size=vision_hidden_size,
233 text_hidden_size=config.text_config.hidden_size,
234 - projector_hidden_act=config.projector_hidden_act)
234 + projector_hidden_act=config.projector_hidden_act,
235 + multimodal_projector_bias=config.multimodal_projector_bias)
236
237 self.language_model = init_vllm_registered_model(
238 vllm_config=vllm_config,
vllm/model_executor/models/llava_next_video.py
+5 -4
@@ -253,16 +253,16 @@ class LlavaNextVideoPooler(nn.Module):
253 class LlavaNextMultiModalProjector(nn.Module):
254
255 def __init__(self, vision_hidden_size: int, text_hidden_size: int,
256 - projector_hidden_act: str):
256 + projector_hidden_act: str, multimodal_projector_bias: bool):
257 super().__init__()
258
259 self.linear_1 = nn.Linear(vision_hidden_size,
260 text_hidden_size,
261 - bias=True)
261 + bias=multimodal_projector_bias)
262 self.act = get_act_fn(projector_hidden_act)
263 self.linear_2 = nn.Linear(text_hidden_size,
264 text_hidden_size,
265 - bias=True)
265 + bias=multimodal_projector_bias)
266
267 def forward(self, image_features: torch.Tensor) -> torch.Tensor:
268 hidden_states = self.linear_1(image_features)
@@ -298,7 +298,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
298 self.multi_modal_projector = LlavaNextMultiModalProjector(
299 vision_hidden_size=config.vision_config.hidden_size,
300 text_hidden_size=config.text_config.hidden_size,
301 - projector_hidden_act=config.projector_hidden_act)
301 + projector_hidden_act=config.projector_hidden_act,
302 + multimodal_projector_bias=config.multimodal_projector_bias)
303 self.language_model = init_vllm_registered_model(
304 vllm_config=vllm_config,
305 hf_config=config.text_config,
vllm/model_executor/models/llava_onevision.py
+2 -2
@@ -372,11 +372,11 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
372
373 self.linear_1 = nn.Linear(config.vision_config.hidden_size,
374 config.text_config.hidden_size,
375 - bias=True)
375 + bias=config.multimodal_projector_bias)
376 self.act = get_act_fn(config.projector_hidden_act)
377 self.linear_2 = nn.Linear(config.text_config.hidden_size,
378 config.text_config.hidden_size,
379 - bias=True)
379 + bias=config.multimodal_projector_bias)
380
381 def forward(self, image_features: torch.Tensor) -> torch.Tensor:
382 hidden_states = self.linear_1(image_features)