A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
[Bugfix] Fix MoE LoRA bin/pt loading (#31161)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 23, 2025 at 19:09 UTC
27c6c2f98c68971fcf4e763938efd2350f517dc5
2 files changed
+1
-60
vllm/lora/lora_model.py
+1
-26
@@ -12,7 +12,6 @@ from vllm.lora.peft_helper import PEFTHelper
12
from vllm.lora.utils import (
13
get_lora_id,
14
is_base_embeddding_weights,
15
- is_regex_target_modules,
15
parse_fine_tuned_lora_name,
16
)
17
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -201,37 +200,13 @@ class LoRAModel:
200
for module in f.keys(): # noqa
201
tensors[module] = f.get_tensor(module)
202
elif os.path.isfile(lora_bin_file_path) or os.path.isfile(lora_pt_file_path):
204
- # When a bin/pt file is provided, we rely on config to find
205
- # unexpected modules.
206
- unexpected_modules = []
207
- target_modules = peft_helper.target_modules
208
- if not isinstance(target_modules, list):
209
- target_modules = [target_modules]
210
- for module in target_modules:
211
- # Compatible with more modules,
212
- # such as:layers.11.self_attn.k_proj
213
- part_name = module.split(".")[-1]
214
- if part_name not in expected_lora_modules:
215
- unexpected_modules.append(module)
216
- # loaded lora's target modules must be a subset of
217
- # expected_lora_modules. It is not reliable. See
218
- # https://github.com/vllm-project/vllm/pull/5909. But there's no
219
- # other better mechanism.
220
- if unexpected_modules and not is_regex_target_modules(
221
- peft_helper.target_modules, expected_lora_modules
222
- ):
223
- raise ValueError(
224
- f"While loading {lora_dir}, expected"
225
- f" target modules in {expected_lora_modules}"
226
- f" but received {unexpected_modules}."
227
- f" Please verify that the loaded LoRA module is correct"
228
- )
203
lora_file_path = (
204
lora_bin_file_path
205
if os.path.isfile(lora_bin_file_path)
206
else lora_pt_file_path
207
)
208
tensors = torch.load(lora_file_path, map_location=device, weights_only=True)
209
+ check_unexpected_modules(tensors)
210
else:
211
raise ValueError(f"{lora_dir} doesn't contain tensors")
212
vllm/lora/utils.py
-34
@@ -5,7 +5,6 @@ import os
5
from typing import TYPE_CHECKING, Optional
6
7
import huggingface_hub
8
-import regex as re
8
from huggingface_hub.utils import (
9
EntryNotFoundError,
10
HfHubHTTPError,
@@ -186,39 +185,6 @@ def is_base_embeddding_weights(name: str) -> bool:
185
return name.endswith(embedding_suffixes)
186
187
189
-def is_regex_target_modules(
190
- load_modules: str | list[str], expected_lora_modules: set[str]
191
-) -> bool:
192
- """
193
- PEFT supports passing `target_modules` in the form of regular expressions,
194
- such as `model.*(q_proj|k_proj|v_proj)$`. This function is mainly used to
195
- determine whether the suffix in the regular expression is present in the
196
- `expected_lora_modules`.
197
- """
198
-
199
- def is_valid_regex(pattern):
200
- try:
201
- re.compile(pattern)
202
- return True
203
- except re.error:
204
- return False
205
-
206
- def is_subset(sub_list, full_set):
207
- return set(sub_list).issubset(full_set)
208
-
209
- # Similar to PEFT's processing logic, regex-related operations are only
210
- # executed when the load_modules is a `str`.
211
- if not isinstance(load_modules, str):
212
- return False
213
-
214
- if is_valid_regex(load_modules):
215
- match = re.search(r"\((.*?)\)\$?$", load_modules)
216
- if match:
217
- suffix = match.group(1).split("|")
218
- return is_subset(suffix, expected_lora_modules)
219
- return False
220
-
221
-
188
def get_supported_lora_modules(model: nn.Module) -> list[str]:
189
"""
190
In vLLM, all linear layers support LoRA.