A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Bugfix][Model] Fix LoRA for Mistral-Small-3.1-24B-Instruct-2503 (#21183)
Signed-off-by: Varun Sundar Rabindranath <varun@neuralmagic.com> Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
Varun Sundar Rabindranath committed
Jul 19, 2025 at 09:45 UTC
9ffe905a4154d3ac373b5254fab72c995562137f
2 files changed
+27
-8
vllm/lora/models.py
+17
-2
@@ -498,6 +498,14 @@ class LoRAModelManager(AdapterModelManager):
498
self._active_adapters.clear()
499
500
def _create_lora_modules(self):
501
+
502
+ def _parent_module(module_name: str) -> str:
503
+ # module name is a dot separated name.
504
+ # for example:
505
+ # - given an input 'x.y.z' return 'x.y'
506
+ # - given an input 'x' return ''
507
+ return module_name.rpartition('.')[0]
508
+
509
for module_name, module in self.model.named_modules(
510
remove_duplicate=False):
511
if isinstance(module, PPMissingLayer):
@@ -529,10 +537,17 @@ class LoRAModelManager(AdapterModelManager):
537
new_module.scaling_factor_to_offset
538
# (yard1): TODO make this more robust
539
if "lm_head" in module_name:
540
+ logits_processor_module_name = 'logits_processor'
541
+ parent_module = _parent_module(module_name)
542
+ if parent_module:
543
+ logits_processor_module_name = (
544
+ f"{parent_module}.{logits_processor_module_name}")
545
+
546
logits_processor_module = self.model.get_submodule(
533
- "logits_processor")
547
+ logits_processor_module_name)
548
+
549
new_module = replace_submodule(
535
- self.model, "logits_processor",
550
+ self.model, logits_processor_module_name,
551
from_layer_logits_processor(logits_processor_module,
552
module, self.lora_slots,
553
self.lora_config,
vllm/lora/utils.py
+10
-6
@@ -188,16 +188,20 @@ def get_supported_lora_modules(model: nn.Module) -> list[str]:
188
"""
189
In vLLM, all linear layers support LoRA.
190
"""
191
+
192
supported_lora_modules: set[str] = set()
192
- # step1: traverse the model to get all the linear subfixes.
193
for name, module in model.named_modules():
194
+ # get the embedding modules if the module's embedding_modules
195
+ # is not empty.
196
+ embedding_modules = getattr(module, "embedding_modules", None)
197
+ if embedding_modules is not None:
198
+ for name in embedding_modules:
199
+ supported_lora_modules.add(name)
200
+
201
+ # get all the linear subfixes.
202
if isinstance(module, (LinearBase, )):
203
supported_lora_modules.add(name.split(".")[-1])
196
- # step 2: get the embedding modules if the model's mbedding_modules
197
- # is not empty.
198
- if model.embedding_modules:
199
- for name in model.embedding_modules:
200
- supported_lora_modules.add(name)
204
+
205
return list(supported_lora_modules)
206
207