A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix `vocab_size` field access in `llava_next.py` (#6624)
Jae-Won Chung committed
Jul 22, 2024 at 01:02 UTC
89c1c6a196e80536bb392c422e7f34439b1f3104
2 files changed
+4
-2
vllm/model_executor/models/llava.py
+2
-1
@@ -155,7 +155,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
155
quant_config=quant_config)
156
logit_scale = getattr(config, "logit_scale", 1.0)
157
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
158
- config.vocab_size, logit_scale)
158
+ config.text_config.vocab_size,
159
+ logit_scale)
160
self.sampler = Sampler()
161
162
def _validate_pixel_values(self, data: torch.Tensor) -> torch.Tensor:
vllm/model_executor/models/llava_next.py
+2
-1
@@ -249,7 +249,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
249
quant_config=quant_config)
250
logit_scale = getattr(config, "logit_scale", 1.0)
251
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
252
- config.vocab_size, logit_scale)
252
+ config.text_config.vocab_size,
253
+ logit_scale)
254
self.sampler = Sampler()
255
256
self.image_newline = nn.Parameter(