A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[VLM] Fix paligemma, fuyu and persimmon with transformers 4.45 : use config.text_config.vocab_size (#8707)
Jani Monoses committed
Sep 23, 2024 at 17:43 UTC
f2bd246c17ba67d7749a2560a30711f74cd19177
3 files changed
+9
-8
vllm/model_executor/models/fuyu.py
+1
-1
@@ -229,7 +229,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal):
229
self.multimodal_config = multimodal_config
230
231
self.padding_idx = config.pad_token_id
232
- self.vocab_size = config.vocab_size
232
+ self.vocab_size = config.text_config.vocab_size
233
self.image_token_id = _IMAGE_TOKEN_ID
234
self.image_feature_size = config.patch_size**2 * config.num_channels
235
vllm/model_executor/models/paligemma.py
+2
-1
@@ -152,7 +152,8 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal):
152
self.unpadded_vocab_size = config.text_config.vocab_size
153
logit_scale = getattr(config, "logit_scale", 1.0)
154
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
155
- config.vocab_size, logit_scale)
155
+ config.text_config.vocab_size,
156
+ logit_scale)
157
self.sampler = Sampler()
158
159
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/persimmon.py
+6
-6
@@ -213,10 +213,10 @@ class PersimmonModel(nn.Module):
213
cache_config: Optional[CacheConfig] = None,
214
quant_config: Optional[QuantizationConfig] = None):
215
super().__init__()
216
- self.vocab_size = config.vocab_size
216
+ self.vocab_size = config.text_config.vocab_size
217
218
- self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
219
- config.hidden_size)
218
+ self.embed_tokens = VocabParallelEmbedding(
219
+ config.text_config.vocab_size, config.hidden_size)
220
self.layers = nn.ModuleList([
221
PersimmonDecoderLayer(config,
222
cache_config=cache_config,
@@ -257,14 +257,14 @@ class PersimmonForCausalLM(nn.Module):
257
quant_config: Optional[QuantizationConfig] = None):
258
super().__init__()
259
self.config = config
260
- self.vocab_size = config.vocab_size
260
+ self.vocab_size = config.text_config.vocab_size
261
self.model = PersimmonModel(config,
262
cache_config=cache_config,
263
quant_config=quant_config)
264
- self.lm_head = ParallelLMHead(config.vocab_size,
264
+ self.lm_head = ParallelLMHead(config.text_config.vocab_size,
265
config.hidden_size,
266
bias=False)
267
- self.logits_processor = LogitsProcessor(config.vocab_size)
267
+ self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
268
self.sampler = Sampler()
269
270
def forward(