@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f2bd246c17
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[VLM] Fix paligemma, fuyu and persimmon with transformers 4.45 : use config.text_config.vocab_size (#8707)

Jani Monoses committed Sep 23, 2024 at 17:43 UTC f2bd246c17ba67d7749a2560a30711f74cd19177
3 files changed +9 -8
vllm/model_executor/models/fuyu.py
+1 -1
@@ -229,7 +229,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
229 self.multimodal_config = multimodal_config
230
231 self.padding_idx = config.pad_token_id
232 - self.vocab_size = config.vocab_size
232 + self.vocab_size = config.text_config.vocab_size
233 self.image_token_id = _IMAGE_TOKEN_ID
234 self.image_feature_size = config.patch_size**2 * config.num_channels
235
vllm/model_executor/models/paligemma.py
+2 -1
@@ -152,7 +152,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
152 self.unpadded_vocab_size = config.text_config.vocab_size
153 logit_scale = getattr(config, "logit_scale", 1.0)
154 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
155 - config.vocab_size, logit_scale)
155 + config.text_config.vocab_size,
156 + logit_scale)
157 self.sampler = Sampler()
158
159 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/persimmon.py
+6 -6
@@ -213,10 +213,10 @@ class PersimmonModel(nn.Module):
213 cache_config: Optional[CacheConfig] = None,
214 quant_config: Optional[QuantizationConfig] = None):
215 super().__init__()
216 - self.vocab_size = config.vocab_size
216 + self.vocab_size = config.text_config.vocab_size
217
218 - self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219 - config.hidden_size)
218 + self.embed_tokens = VocabParallelEmbedding(
219 + config.text_config.vocab_size, config.hidden_size)
220 self.layers = nn.ModuleList([
221 PersimmonDecoderLayer(config,
222 cache_config=cache_config,
@@ -257,14 +257,14 @@ class PersimmonForCausalLM(nn.Module):
257 quant_config: Optional[QuantizationConfig] = None):
258 super().__init__()
259 self.config = config
260 - self.vocab_size = config.vocab_size
260 + self.vocab_size = config.text_config.vocab_size
261 self.model = PersimmonModel(config,
262 cache_config=cache_config,
263 quant_config=quant_config)
264 - self.lm_head = ParallelLMHead(config.vocab_size,
264 + self.lm_head = ParallelLMHead(config.text_config.vocab_size,
265 config.hidden_size,
266 bias=False)
267 - self.logits_processor = LogitsProcessor(config.vocab_size)
267 + self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
268 self.sampler = Sampler()
269
270 def forward(