A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Add LoRA support for DeepSeek models (V2, V3, R1-0528) (#23971)
Signed-off-by: sadeghja1070 <sadegh.ja1070@gmail.com> Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
sadegh.shokatian committed
Aug 30, 2025 at 06:40 UTC
379ea2823a751ef43de7b3e7bcb3262b1e14c510
3 files changed
+12
-7
docs/models/supported_models.md
+3
-3
@@ -335,9 +335,9 @@ th {
335
| `CohereForCausalLM`, `Cohere2ForCausalLM` | Command-R, Command-A | `CohereLabs/c4ai-command-r-v01`, `CohereLabs/c4ai-command-r7b-12-2024`, `CohereLabs/c4ai-command-a-03-2025`, `CohereLabs/command-a-reasoning-08-2025`, etc. | ✅︎ | ✅︎ | ✅︎ |
336
| `DbrxForCausalLM` | DBRX | `databricks/dbrx-base`, `databricks/dbrx-instruct`, etc. | | ✅︎ | ✅︎ |
337
| `DeciLMForCausalLM` | DeciLM | `nvidia/Llama-3_3-Nemotron-Super-49B-v1`, etc. | ✅︎ | ✅︎ | ✅︎ |
338
-| `DeepseekForCausalLM` | DeepSeek | `deepseek-ai/deepseek-llm-67b-base`, `deepseek-ai/deepseek-llm-7b-chat`, etc. | | ✅︎ | ✅︎ |
339
-| `DeepseekV2ForCausalLM` | DeepSeek-V2 | `deepseek-ai/DeepSeek-V2`, `deepseek-ai/DeepSeek-V2-Chat`, etc. | | ✅︎ | ✅︎ |
340
-| `DeepseekV3ForCausalLM` | DeepSeek-V3 | `deepseek-ai/DeepSeek-V3-Base`, `deepseek-ai/DeepSeek-V3`, etc. | | ✅︎ | ✅︎ |
338
+| `DeepseekForCausalLM` | DeepSeek | `deepseek-ai/deepseek-llm-67b-base`, `deepseek-ai/deepseek-llm-7b-chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
339
+| `DeepseekV2ForCausalLM` | DeepSeek-V2 | `deepseek-ai/DeepSeek-V2`, `deepseek-ai/DeepSeek-V2-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
340
+| `DeepseekV3ForCausalLM` | DeepSeek-V3 | `deepseek-ai/DeepSeek-V3`, `deepseek-ai/DeepSeek-R1`, `deepseek-ai/DeepSeek-V3.1`, etc. | ✅︎ | ✅︎ | ✅︎ |
341
| `Dots1ForCausalLM` | dots.llm1 | `rednote-hilab/dots.llm1.base`, `rednote-hilab/dots.llm1.inst`, etc. | | ✅︎ | ✅︎ |
342
| `Ernie4_5ForCausalLM` | Ernie4.5 | `baidu/ERNIE-4.5-0.3B-PT`, etc. | ✅︎ | ✅︎ | ✅︎ |
343
| `Ernie4_5_MoeForCausalLM` | Ernie4.5MoE | `baidu/ERNIE-4.5-21B-A3B-PT`, `baidu/ERNIE-4.5-300B-A47B-PT`, etc. |✅︎| ✅︎ | ✅︎ |
vllm/model_executor/models/deepseek.py
+6
-2
@@ -52,7 +52,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
52
from vllm.model_executor.sampling_metadata import SamplingMetadata
53
from vllm.sequence import IntermediateTensors
54
55
-from .interfaces import SupportsPP
55
+from .interfaces import SupportsLoRA, SupportsPP
56
from .utils import (AutoWeightsLoader, extract_layer_index,
57
is_pp_missing_parameter,
58
make_empty_intermediate_tensors_factory, make_layers,
@@ -439,7 +439,11 @@ class DeepseekModel(nn.Module):
439
return loaded_params
440
441
442
-class DeepseekForCausalLM(nn.Module, SupportsPP):
442
+class DeepseekForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
443
+ packed_modules_mapping = {
444
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
445
+ "gate_up_proj": ["gate_proj", "up_proj"],
446
+ }
447
448
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
449
super().__init__()
vllm/model_executor/models/deepseek_v2.py
+3
-2
@@ -56,7 +56,7 @@ from vllm.model_executor.model_loader.weight_utils import (
56
from vllm.model_executor.sampling_metadata import SamplingMetadata
57
from vllm.sequence import IntermediateTensors
58
59
-from .interfaces import MixtureOfExperts, SupportsPP
59
+from .interfaces import MixtureOfExperts, SupportsLoRA, SupportsPP
60
from .utils import (PPMissingLayer, is_pp_missing_parameter,
61
make_empty_intermediate_tensors_factory, make_layers,
62
maybe_prefix)
@@ -727,7 +727,8 @@ class DeepseekV2Model(nn.Module):
727
return hidden_states
728
729
730
-class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts):
730
+class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts,
731
+ SupportsLoRA):
732
packed_modules_mapping = {
733
"gate_up_proj": ["gate_proj", "up_proj"],
734
}