@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89c1c6a196
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix `vocab_size` field access in `llava_next.py` (#6624)

Jae-Won Chung committed Jul 22, 2024 at 01:02 UTC 89c1c6a196e80536bb392c422e7f34439b1f3104
2 files changed +4 -2
vllm/model_executor/models/llava.py
+2 -1
@@ -155,7 +155,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
155 quant_config=quant_config)
156 logit_scale = getattr(config, "logit_scale", 1.0)
157 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
158 - config.vocab_size, logit_scale)
158 + config.text_config.vocab_size,
159 + logit_scale)
160 self.sampler = Sampler()
161
162 def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/llava_next.py
+2 -1
@@ -249,7 +249,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
249 quant_config=quant_config)
250 logit_scale = getattr(config, "logit_scale", 1.0)
251 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
252 - config.vocab_size, logit_scale)
252 + config.text_config.vocab_size,
253 + logit_scale)
254 self.sampler = Sampler()
255
256 self.image_newline = nn.Parameter(