@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c66c7f86ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix PaliGemma MMP (#6930)

Roger Wang committed Jul 30, 2024 at 02:20 UTC c66c7f86aca956014d9ec6cc7a3e6001037e4655
1 file changed +2 -5
vllm/model_executor/models/paligemma.py
+2 -5
@@ -9,7 +9,6 @@ from vllm.attention import AttentionMetadata
9 from vllm.config import CacheConfig, MultiModalConfig
10 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
11 from vllm.logger import init_logger
12 -from vllm.model_executor.layers.linear import ColumnParallelLinear
12 from vllm.model_executor.layers.logits_processor import LogitsProcessor
13 from vllm.model_executor.layers.quantization.base_config import (
14 QuantizationConfig)
@@ -133,12 +132,10 @@ class PaliGemmaMultiModalProjector(nn.Module):
132 def __init__(self, vision_hidden_size: int, projection_dim: int):
133 super().__init__()
134
136 - self.linear = ColumnParallelLinear(vision_hidden_size,
137 - projection_dim,
138 - bias=True)
135 + self.linear = nn.Linear(vision_hidden_size, projection_dim, bias=True)
136
137 def forward(self, image_features: torch.Tensor) -> torch.Tensor:
141 - hidden_states, _ = self.linear(image_features)
138 + hidden_states = self.linear(image_features)
139 return hidden_states
140
141