[LoRA] Add LoRA support for DeepSeek-V4 Flash Vision (#55897)
Signed-off-by: linitra24 <renshuang.zhou@daocloud.io>
linitra24 committed
Sep 13, 2026 at 22:10 UTC
dd4c8410707bdb833b155ff3258ce5271c0f476e
3 files changed
+42
-3
docs/models/supported_models.md
+1
-1
@@ -547,7 +547,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
547
| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
548
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
549
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
550
-| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | | ✅︎ |
550
+| `DeepseekV4ForConditionalGeneration` | DeepSeek-V4 (vision) | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-V4-Flash-Vision-Exp` | ✅︎ | ✅︎ |
551
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
552
| `Ernie4_5_VLMoeForConditionalGeneration` | Ernie4.5-VL | T + I<sup>+</sup>/ V<sup>+</sup> | `baidu/ERNIE-4.5-VL-28B-A3B-PT`, `baidu/ERNIE-4.5-VL-424B-A47B-PT` | | ✅︎ |
553
| `Exaone4_5_ForConditionalGeneration` | EXAONE-4.5 | T + I<sup>E+</sup> | `LGAI-EXAONE/EXAONE-4.5-33B`, etc. | ✅︎ | ✅︎ |
vllm/lora/ops/triton_ops/lora_shrink_op.py
+1
-1
@@ -179,7 +179,7 @@ def _lora_shrink(
179
assert weight.dtype in [torch.float16, torch.bfloat16]
180
181
assert inputs.size(1) == lora_a_weights[0].size(-1)
182
- assert inputs.is_contiguous()
182
+ inputs = inputs.contiguous()
183
assert output_tensor.is_contiguous()
184
185
# metadata sanity check
vllm/models/deepseek_v4/common/vl_model.py
+40
-1
@@ -25,9 +25,11 @@ from torch import nn
25
from vllm.model_executor.models.interfaces import (
26
MultiModalEmbeddings,
27
SupportsEagle3,
28
+ SupportsLoRA,
29
SupportsMultiModal,
30
SupportsPP,
31
)
32
+from vllm.model_executor.models.module_mapping import MultiModelKeys
33
from vllm.model_executor.models.utils import (
34
AutoWeightsLoader,
35
WeightsMapper,
@@ -35,6 +37,7 @@ from vllm.model_executor.models.utils import (
37
maybe_prefix,
38
)
39
from vllm.multimodal import MULTIMODAL_REGISTRY
40
+from vllm.multimodal.inputs import MultiModalKwargsItem
41
42
from .mm_preprocess import (
43
IMAGE_PLACEHOLDER,
@@ -81,7 +84,7 @@ def _make_deepseek_v4_vl_weights_mapper(
84
dummy_inputs=DeepseekV4VLDummyInputsBuilder,
85
)
86
class DeepseekV4ForConditionalGeneration(
84
- nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3
87
+ nn.Module, SupportsMultiModal, SupportsPP, SupportsEagle3, SupportsLoRA
88
):
89
"""Multimodal entry point for DeepSeek-V4 checkpoints with a vision tower.
90
@@ -89,9 +92,19 @@ class DeepseekV4ForConditionalGeneration(
92
delegates through ``language_model`` via the protocol defaults.
93
"""
94
95
+ packed_modules_mapping = {
96
+ "gate_up_proj": ["w1", "w3"],
97
+ "fused_wqa_wkv": ["wq_a", "wkv"],
98
+ "fused_wkv_wgate": ["wkv", "wgate"],
99
+ # for visual encoder
100
+ "wqkv": ["wqkv"],
101
+ "w1": ["w1"],
102
+ }
103
+
104
# The MoE router needs raw token ids to detect image sentinel tokens
105
# (borrowed reserved ids, see common/mm_preprocess.py) and apply bias_vl.
106
requires_raw_input_tokens = True
107
+ supports_tower_connector_lora = True
108
109
@classmethod
110
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
@@ -329,3 +342,29 @@ class DeepseekV4ForConditionalGeneration(
342
return
343
self.language_model.process_weights_after_loading()
344
self._weights_finalized = True
345
+
346
+ def get_mm_mapping(self) -> MultiModelKeys:
347
+ """Get the module prefixes in the multimodal model."""
348
+ return MultiModelKeys.from_string_field(
349
+ language_model="language_model",
350
+ connector="aligner",
351
+ tower_model="vision.",
352
+ )
353
+
354
+ def get_mm_lora_token_counts(
355
+ self,
356
+ *,
357
+ modality: str,
358
+ mm_kwargs: MultiModalKwargsItem | None,
359
+ num_mm_embeds: int,
360
+ ) -> tuple[int, int | None]:
361
+ if modality != "image":
362
+ raise ValueError(f"Unsupported modality: {modality!r}")
363
+
364
+ patches = mm_kwargs.get("patches") if mm_kwargs else None
365
+ if patches is not None and isinstance(patches.data, torch.Tensor):
366
+ tower_tokens = patches.data.shape[0]
367
+ else:
368
+ tower_tokens = num_mm_embeds * self.config.vision_downsample_ratio**2
369
+
370
+ return tower_tokens, num_mm_embeds