A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Align LoRA code between Mistral and Mixtral (fixes #2875) (#2880)
* Fix AttributeError: MixtralModel object has no attribute org_vocab_size. * Make LoRA logic for Mistral and Mixtral the same --------- Co-authored-by: Pernekhan Utemuratov <pernekhan@deepinfra.com>
Philipp Moritz committed
Feb 14, 2024 at 16:00 UTC
31348dff03d638eb66abda9bec94b8992de9c7a1
1 file changed
+10
-4
vllm/model_executor/models/mixtral.py
+10
-4
@@ -285,15 +285,19 @@ class MixtralModel(nn.Module):
285
self,
286
config: MixtralConfig,
287
linear_method: Optional[LinearMethodBase] = None,
288
+ lora_config: Optional[LoRAConfig] = None,
289
) -> None:
290
super().__init__()
291
self.padding_idx = config.pad_token_id
291
- self.vocab_size = config.vocab_size
292
+ lora_vocab = (lora_config.lora_extra_vocab_size *
293
+ (lora_config.max_loras or 1)) if lora_config else 0
294
+ self.vocab_size = config.vocab_size + lora_vocab
295
+ self.org_vocab_size = config.vocab_size
296
297
self.embed_tokens = VocabParallelEmbedding(
294
- config.vocab_size,
298
+ self.vocab_size,
299
config.hidden_size,
296
- org_num_embeddings=self.org_vocab_size,
300
+ org_num_embeddings=config.vocab_size,
301
)
302
self.layers = nn.ModuleList([
303
MixtralDecoderLayer(config, linear_method=linear_method)
@@ -350,7 +354,9 @@ class MixtralForCausalLM(nn.Module):
354
super().__init__()
355
self.config = config
356
self.linear_method = linear_method
353
- self.model = MixtralModel(config, linear_method)
357
+ self.model = MixtralModel(config,
358
+ linear_method,
359
+ lora_config=lora_config)
360
self.unpadded_vocab_size = config.vocab_size
361
if lora_config:
362
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size