A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 01:00 UTC
[Misc] Qwen MoE model supports LoRA (#20932)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Jul 18, 2025 at 02:32 UTC
a3a6c695f43d6c95e8574dc0c47adc9bb46ce1ce
4 files changed
+20
-8
docs/models/supported_models.md
+2
-2
@@ -380,9 +380,9 @@ Specified using `--task generate`.
380
| `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | | | |
381
| `QWenLMHeadModel` | Qwen | `Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
382
| `Qwen2ForCausalLM` | QwQ, Qwen2 | `Qwen/QwQ-32B-Preview`, `Qwen/Qwen2-7B-Instruct`, `Qwen/Qwen2-7B`, etc. | ✅︎ | ✅︎ | ✅︎ |
383
-| `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | | ✅︎ | ✅︎ |
383
+| `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
384
| `Qwen3ForCausalLM` | Qwen3 | `Qwen/Qwen3-8B`, etc. | ✅︎ | ✅︎ | ✅︎ |
385
-| `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | | ✅︎ | ✅︎ |
385
+| `Qwen3MoeForCausalLM` | Qwen3MoE | `Qwen/Qwen3-30B-A3B`, etc. | ✅︎ | ✅︎ | ✅︎ |
386
| `StableLmForCausalLM` | StableLM | `stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc. | | | ✅︎ |
387
| `Starcoder2ForCausalLM` | Starcoder2 | `bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc. | | ✅︎ | ✅︎ |
388
| `SolarForCausalLM` | Solar Pro | `upstage/solar-pro-preview-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
vllm/lora/models.py
+13
@@ -29,6 +29,7 @@ from vllm.lora.utils import (from_layer, from_layer_logits_processor,
29
get_supported_lora_modules,
30
is_regex_target_modules,
31
parse_fine_tuned_lora_name, replace_submodule)
32
+from vllm.model_executor.layers.fused_moe import FusedMoE
33
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
34
from vllm.model_executor.models import SupportsLoRA, supports_multimodal
35
from vllm.model_executor.models.interfaces import is_pooling_model
@@ -60,6 +61,17 @@ def get_lora_id():
61
return _GLOBAL_LORA_ID
62
63
64
+def is_moe_model(model: nn.Module) -> bool:
65
+ """Checks if the model contains FusedMoE layers and warns the user."""
66
+ if any(isinstance(module, FusedMoE) for module in model.modules()):
67
+ logger.warning_once(
68
+ "For MoE models, vLLM currently does not support fused MoE LoRA "
69
+ "inference. Please ensure that the loaded LoRA model does not "
70
+ "contain expert weights.")
71
+ return True
72
+ return False
73
+
74
+
75
class LoRAModel(AdapterModel):
76
"""A LoRA fine-tuned model."""
77
@@ -375,6 +387,7 @@ class LoRAModelManager(AdapterModelManager):
387
# text modules (e.g. ChatGLM)
388
and hasattr(self.model, "get_mm_mapping"))
389
self.is_pooling_model = is_pooling_model(self.model)
390
+ self.is_moe_model = is_moe_model(self.model)
391
self.packed_modules: dict[str, list[str]] = {}
392
self.modules: dict[str, BaseLayerWithLoRA] = {}
393
# Dict instead of a set for compatibility with LRUCache.
vllm/model_executor/models/qwen2_moe.py
+3
-4
@@ -53,7 +53,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
53
from vllm.model_executor.sampling_metadata import SamplingMetadata
54
from vllm.sequence import IntermediateTensors
55
56
-from .interfaces import SupportsPP
56
+from .interfaces import SupportsLoRA, SupportsPP
57
from .utils import (AutoWeightsLoader, extract_layer_index,
58
is_pp_missing_parameter,
59
make_empty_intermediate_tensors_factory, make_layers,
@@ -448,8 +448,7 @@ class Qwen2MoeModel(nn.Module):
448
if weight_name not in name:
449
continue
450
name = name.replace(weight_name, param_name)
451
- if "layers.13.mlp.experts.w2_weight" in name:
452
- pass
451
+
452
# Skip layers on other devices.
453
if is_pp_missing_parameter(name, self):
454
continue
@@ -494,7 +493,7 @@ class Qwen2MoeModel(nn.Module):
493
return loaded_params
494
495
497
-class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
496
+class Qwen2MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
497
498
fall_back_to_pt_during_load = False
499
packed_modules_mapping = {
vllm/model_executor/models/qwen3_moe.py
+2
-2
@@ -50,7 +50,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
50
from vllm.model_executor.sampling_metadata import SamplingMetadata
51
from vllm.sequence import IntermediateTensors
52
53
-from .interfaces import SupportsPP
53
+from .interfaces import SupportsLoRA, SupportsPP
54
from .utils import (AutoWeightsLoader, extract_layer_index,
55
is_pp_missing_parameter,
56
make_empty_intermediate_tensors_factory, make_layers,
@@ -482,7 +482,7 @@ class Qwen3MoeModel(nn.Module):
482
return loaded_params
483
484
485
-class Qwen3MoeForCausalLM(nn.Module, SupportsPP):
485
+class Qwen3MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
486
packed_modules_mapping = {
487
"qkv_proj": [
488
"q_proj",