A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
Support LoRA and GPTQModel for PLaMo 2/3 (#31322)
Signed-off-by: Shinichi Hemmi <50256998+Alnusjaponica@users.noreply.github.com>
Shinichi Hemmi committed
Dec 26, 2025 at 12:41 UTC
9ee05cbe7f29d72580f613553dcd2af8e770ccca
3 files changed
+31
-17
docs/models/supported_models.md
+2
-2
@@ -438,8 +438,8 @@ th {
438
| `Phi3ForCausalLM` | Phi-4, Phi-3 | `microsoft/Phi-4-mini-instruct`, `microsoft/Phi-4`, `microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, `microsoft/Phi-3-medium-128k-instruct`, etc. | ✅︎ | ✅︎ |
439
| `PhiMoEForCausalLM` | Phi-3.5-MoE | `microsoft/Phi-3.5-MoE-instruct`, etc. | ✅︎ | ✅︎ |
440
| `PersimmonForCausalLM` | Persimmon | `adept/persimmon-8b-base`, `adept/persimmon-8b-chat`, etc. | | ✅︎ |
441
-| `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | | ✅︎ |
442
-| `Plamo3ForCausalLM` | PLaMo3 | `pfnet/plamo-3-nict-2b-base`, `pfnet/plamo-3-nict-8b-base`, etc. | | ✅︎ |
441
+| `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | ✅ | ✅︎ |
442
+| `Plamo3ForCausalLM` | PLaMo3 | `pfnet/plamo-3-nict-2b-base`, `pfnet/plamo-3-nict-8b-base`, etc. | ✅ | ✅︎ |
443
| `QWenLMHeadModel` | Qwen | `Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc. | ✅︎ | ✅︎ |
444
| `Qwen2ForCausalLM` | QwQ, Qwen2 | `Qwen/QwQ-32B-Preview`, `Qwen/Qwen2-7B-Instruct`, `Qwen/Qwen2-7B`, etc. | ✅︎ | ✅︎ |
445
| `Qwen2MoeForCausalLM` | Qwen2MoE | `Qwen/Qwen1.5-MoE-A2.7B`, `Qwen/Qwen1.5-MoE-A2.7B-Chat`, etc. | ✅︎ | ✅︎ |
vllm/model_executor/models/plamo2.py
+25
-8
@@ -50,8 +50,14 @@ from vllm.model_executor.model_loader.weight_utils import (
50
default_weight_loader,
51
sharded_weight_loader,
52
)
53
-from vllm.model_executor.models.interfaces import HasInnerState, IsHybrid, SupportsPP
53
+from vllm.model_executor.models.interfaces import (
54
+ HasInnerState,
55
+ IsHybrid,
56
+ SupportsLoRA,
57
+ SupportsPP,
58
+)
59
from vllm.model_executor.models.utils import (
60
+ AutoWeightsLoader,
61
is_pp_missing_parameter,
62
make_empty_intermediate_tensors_factory,
63
make_layers,
@@ -105,6 +111,7 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
111
self.cache_config = vllm_config.cache_config
112
self.model_config = vllm_config.model_config
113
self.quant_config = vllm_config.quant_config
114
+ self.is_lora_enabled = bool(vllm_config.lora_config)
115
self.hidden_size = self.config.hidden_size
116
self.ssm_state_size = self.config.mamba_d_state
117
self.conv_kernel_size = self.config.mamba_d_conv
@@ -202,7 +209,11 @@ class Plamo2MambaMixer(MambaBase, CustomOp):
209
self.prefix = prefix
210
211
def _project_ssm_parameters(self, hidden_states):
205
- ssm_parameters = self.bcdt_proj(hidden_states)
212
+ if self.is_lora_enabled:
213
+ # Lora kernel requires contiguous tensor.
214
+ ssm_parameters = self.bcdt_proj(hidden_states.contiguous())
215
+ else:
216
+ ssm_parameters = self.bcdt_proj(hidden_states)
217
B, C, time_step = torch.split(
218
ssm_parameters,
219
[self.ssm_state_size, self.ssm_state_size, self.time_step_rank],
@@ -780,13 +791,13 @@ class Plamo2Model(torch.nn.Module):
791
return hidden_states
792
793
783
-class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
794
+class Plamo2ForCausalLM(
795
+ torch.nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHybrid
796
+):
797
packed_modules_mapping = {
785
- "qkv_proj": [
786
- "q_proj",
787
- "k_proj",
788
- "v_proj",
789
- ],
798
+ "qkv_proj": ["qkv_proj"],
799
+ "gate_up_proj": ["gate_up_proj"],
800
+ "in_proj": ["in_proj"],
801
}
802
803
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
@@ -892,6 +903,12 @@ class Plamo2ForCausalLM(torch.nn.Module, HasInnerState, SupportsPP, IsHybrid):
903
if name == "lm_head.weight" and self.config.tie_word_embeddings:
904
assert "lm_head.weight" not in params_dict
905
continue
906
+ # Same workaround as AutoWeightsLoader for GPTQModel
907
+ if any(
908
+ substr in name
909
+ for substr in AutoWeightsLoader.ROTARY_EMBEDS_UNUSED_WEIGHTS
910
+ ):
911
+ continue
912
913
# Update the weight names to be compatible with the vllm version
914
# of the model.
vllm/model_executor/models/plamo3.py
+4
-7
@@ -35,7 +35,7 @@ from vllm.model_executor.model_loader.weight_utils import (
35
composed_weight_loader,
36
default_weight_loader,
37
)
38
-from vllm.model_executor.models.interfaces import SupportsPP
38
+from vllm.model_executor.models.interfaces import SupportsLoRA, SupportsPP
39
from vllm.model_executor.models.utils import (
40
AutoWeightsLoader,
41
extract_layer_index,
@@ -369,13 +369,10 @@ class Plamo3Model(nn.Module):
369
return hidden_states
370
371
372
-class Plamo3ForCausalLM(nn.Module, SupportsPP):
372
+class Plamo3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
373
packed_modules_mapping = {
374
- "qkv_proj": [
375
- "q_proj",
376
- "k_proj",
377
- "v_proj",
378
- ],
374
+ "qkv_proj": ["qkv_proj"],
375
+ "gate_up_proj": ["gate_up_proj"],
376
}
377
378
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: