@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 31348dff03
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Align LoRA code between Mistral and Mixtral (fixes #2875) (#2880)

* Fix AttributeError: MixtralModel object has no attribute org_vocab_size. * Make LoRA logic for Mistral and Mixtral the same --------- Co-authored-by: Pernekhan Utemuratov <pernekhan@deepinfra.com>

Philipp Moritz committed Feb 14, 2024 at 16:00 UTC 31348dff03d638eb66abda9bec94b8992de9c7a1
1 file changed +10 -4
vllm/model_executor/models/mixtral.py
+10 -4
@@ -285,15 +285,19 @@ class MixtralModel(nn.Module):
285 self,
286 config: MixtralConfig,
287 linear_method: Optional[LinearMethodBase] = None,
288 + lora_config: Optional[LoRAConfig] = None,
289 ) -> None:
290 super().__init__()
291 self.padding_idx = config.pad_token_id
291 - self.vocab_size = config.vocab_size
292 + lora_vocab = (lora_config.lora_extra_vocab_size *
293 + (lora_config.max_loras or 1)) if lora_config else 0
294 + self.vocab_size = config.vocab_size + lora_vocab
295 + self.org_vocab_size = config.vocab_size
296
297 self.embed_tokens = VocabParallelEmbedding(
294 - config.vocab_size,
298 + self.vocab_size,
299 config.hidden_size,
296 - org_num_embeddings=self.org_vocab_size,
300 + org_num_embeddings=config.vocab_size,
301 )
302 self.layers = nn.ModuleList([
303 MixtralDecoderLayer(config, linear_method=linear_method)
@@ -350,7 +354,9 @@ class MixtralForCausalLM(nn.Module):
354 super().__init__()
355 self.config = config
356 self.linear_method = linear_method
353 - self.model = MixtralModel(config, linear_method)
357 + self.model = MixtralModel(config,
358 + linear_method,
359 + lora_config=lora_config)
360 self.unpadded_vocab_size = config.vocab_size
361 if lora_config:
362 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size