A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Support Pixtral-Large HF by using llava multimodal_projector_bias config (#12710)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Feb 3, 2025 at 22:55 UTC
5d98d56089426555d303d82bdf29b9ffc825ec20
4 files changed
+13
-9
vllm/model_executor/models/llava.py
+4
-2
@@ -75,19 +75,20 @@ class LlavaMultiModalProjector(nn.Module):
75
vision_hidden_size: int,
76
text_hidden_size: int,
77
projector_hidden_act: str,
78
+ multimodal_projector_bias: bool,
79
quant_config: Optional[QuantizationConfig] = None,
80
prefix: str = ""):
81
super().__init__()
82
83
self.linear_1 = ColumnParallelLinear(vision_hidden_size,
84
text_hidden_size,
84
- bias=True,
85
+ bias=multimodal_projector_bias,
86
quant_config=quant_config,
87
prefix=f"{prefix}.linear_1")
88
self.act = get_act_fn(projector_hidden_act)
89
self.linear_2 = RowParallelLinear(text_hidden_size,
90
text_hidden_size,
90
- bias=True,
91
+ bias=multimodal_projector_bias,
92
quant_config=quant_config,
93
prefix=f"{prefix}.linear_2")
94
@@ -503,6 +504,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
504
vision_hidden_size=config.vision_config.hidden_size,
505
text_hidden_size=config.text_config.hidden_size,
506
projector_hidden_act=config.projector_hidden_act,
507
+ multimodal_projector_bias=config.multimodal_projector_bias,
508
quant_config=quant_config,
509
prefix=maybe_prefix(prefix, "multi_modal_projector"))
510
vllm/model_executor/models/llava_next.py
+2
-1
@@ -231,7 +231,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
231
self.multi_modal_projector = LlavaMultiModalProjector(
232
vision_hidden_size=vision_hidden_size,
233
text_hidden_size=config.text_config.hidden_size,
234
- projector_hidden_act=config.projector_hidden_act)
234
+ projector_hidden_act=config.projector_hidden_act,
235
+ multimodal_projector_bias=config.multimodal_projector_bias)
236
237
self.language_model = init_vllm_registered_model(
238
vllm_config=vllm_config,
vllm/model_executor/models/llava_next_video.py
+5
-4
@@ -253,16 +253,16 @@ class LlavaNextVideoPooler(nn.Module):
253
class LlavaNextMultiModalProjector(nn.Module):
254
255
def __init__(self, vision_hidden_size: int, text_hidden_size: int,
256
- projector_hidden_act: str):
256
+ projector_hidden_act: str, multimodal_projector_bias: bool):
257
super().__init__()
258
259
self.linear_1 = nn.Linear(vision_hidden_size,
260
text_hidden_size,
261
- bias=True)
261
+ bias=multimodal_projector_bias)
262
self.act = get_act_fn(projector_hidden_act)
263
self.linear_2 = nn.Linear(text_hidden_size,
264
text_hidden_size,
265
- bias=True)
265
+ bias=multimodal_projector_bias)
266
267
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
268
hidden_states = self.linear_1(image_features)
@@ -298,7 +298,8 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
298
self.multi_modal_projector = LlavaNextMultiModalProjector(
299
vision_hidden_size=config.vision_config.hidden_size,
300
text_hidden_size=config.text_config.hidden_size,
301
- projector_hidden_act=config.projector_hidden_act)
301
+ projector_hidden_act=config.projector_hidden_act,
302
+ multimodal_projector_bias=config.multimodal_projector_bias)
303
self.language_model = init_vllm_registered_model(
304
vllm_config=vllm_config,
305
hf_config=config.text_config,
vllm/model_executor/models/llava_onevision.py
+2
-2
@@ -372,11 +372,11 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
372
373
self.linear_1 = nn.Linear(config.vision_config.hidden_size,
374
config.text_config.hidden_size,
375
- bias=True)
375
+ bias=config.multimodal_projector_bias)
376
self.act = get_act_fn(config.projector_hidden_act)
377
self.linear_2 = nn.Linear(config.text_config.hidden_size,
378
config.text_config.hidden_size,
379
- bias=True)
379
+ bias=config.multimodal_projector_bias)
380
381
def forward(self, image_features: torch.Tensor) -> torch.Tensor:
382
hidden_states = self.linear_1(image_features)