@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 250e26a63e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix]Fix MiniCPM's LoRA bug (#9286)

Jee Jee Li committed Oct 13, 2024 at 00:36 UTC 250e26a63e241076d8182155b9c7ea4f9f157ea3
3 files changed +40 -17
vllm/lora/models.py
+5 -1
@@ -337,7 +337,11 @@ class LoRAModelManager(AdapterModelManager):
337 self.packed_modules_mapping = copy.deepcopy(
338 self.model.packed_modules_mapping)
339 # Used to indicate whether the model is a multimodal model
340 - self.supports_mm: bool = supports_multimodal(self.model)
340 + self.supports_mm: bool = (
341 + supports_multimodal(self.model)
342 + # In case the model only supports LoRA for
343 + # text modules (e.g. ChatGLM)
344 + and hasattr(self.model, "get_mm_mapping"))
345 self.packed_modules: Dict[str, List[str]] = {}
346 self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
347 # Dict instead of a Set for compatibility with LRUCache.
vllm/model_executor/models/minicpm.py
+13 -16
@@ -474,17 +474,18 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
474 unpadded_vocab_size = config.vocab_size
475 if lora_config:
476 unpadded_vocab_size += lora_config.lora_extra_vocab_size
477 - if not self.config.tie_word_embeddings:
478 - self.lm_head = ParallelLMHead(
479 - unpadded_vocab_size,
480 - config.hidden_size,
481 - org_num_embeddings=config.vocab_size,
482 - padding_size=DEFAULT_VOCAB_PADDING_SIZE
483 - # We need bigger padding if using lora for kernel
484 - # compatibility
485 - if not lora_config else lora_config.lora_vocab_padding_size,
486 - quant_config=quant_config,
487 - )
477 + self.lm_head = ParallelLMHead(
478 + unpadded_vocab_size,
479 + config.hidden_size,
480 + org_num_embeddings=config.vocab_size,
481 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
482 + # We need bigger padding if using lora for kernel
483 + # compatibility
484 + if not lora_config else lora_config.lora_vocab_padding_size,
485 + quant_config=quant_config,
486 + )
487 + if config.tie_word_embeddings:
488 + self.lm_head = self.lm_head.tie_weights(self.model.embed_tokens)
489 self.scale_width = self.config.hidden_size / self.config.dim_model_base
490
491 self.logits_processor = LogitsProcessor(unpadded_vocab_size,
@@ -517,11 +518,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
518 sampling_metadata: SamplingMetadata,
519 ) -> Optional[torch.Tensor]:
520 hidden_states = hidden_states / self.scale_width
520 - if self.config.tie_word_embeddings:
521 - lm_head = self.model.embed_tokens
522 - else:
523 - lm_head = self.lm_head
524 - logits = self.logits_processor(lm_head, hidden_states,
521 + logits = self.logits_processor(self.lm_head, hidden_states,
522 sampling_metadata)
523 return logits
524
vllm/model_executor/models/minicpm3.py
+22
@@ -216,6 +216,28 @@ class MiniCPM3Model(MiniCPMModel):
216
217
218 class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
219 + packed_modules_mapping = {
220 + "gate_up_proj": [
221 + "gate_proj",
222 + "up_proj",
223 + ],
224 + }
225 +
226 + # LoRA specific attributes
227 + supported_lora_modules = [
228 + "kv_a_proj_with_mqa",
229 + "q_a_proj",
230 + "q_b_proj",
231 + "kv_b_proj",
232 + "o_proj",
233 + "gate_up_proj",
234 + "down_proj",
235 + "embed_tokens",
236 + "lm_head",
237 + ]
238 +
239 + # `embedding_modules` and `embedding_padding_modules`
240 + # are inherited from MiniCPMForCausalLM
241
242 def _init_model(self):
243 self.model = MiniCPM3Model(config=self.config,