@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a838ba7254
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc]Fix Idefics3Model argument (#10255)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 12, 2024 at 21:07 UTC a838ba7254c98a7adc60a0976bdf277fb20b4221
1 file changed +4 -4
vllm/model_executor/models/idefics3.py
+4 -4
@@ -420,18 +420,18 @@ class Idefics3Model(nn.Module):
420 super().__init__()
421
422 config = vllm_config.model_config.hf_config
423 - cache_config = vllm_config.cache_config
423 quant_config = vllm_config.quant_config
424
425 self.config = config
426 self.padding_idx = self.config.text_config.pad_token_id
427 self.vocab_size = self.config.text_config.vocab_size
429 -
428 self.vision_model = Idefics3VisionTransformer(config.vision_config,
429 quant_config)
430 self.connector = Idefics3Connector(config)
433 - self.text_model = LlamaModel(config.text_config, cache_config,
434 - quant_config)
431 + self.text_model = LlamaModel(
432 + vllm_config=vllm_config.with_hf_config(config.text_config),
433 + prefix=maybe_prefix(prefix, "text_model"),
434 + )
435
436 self.image_seq_len = int(
437 ((config.vision_config.image_size //