[LoRA][Nemotron] Add LoRA support for Nemotron VL models (for the language model only) (#56231)
Signed-off-by: Daniel Serebrenik <daserebrenik@nvidia.com>
danisereb committed
Sep 16, 2026 at 07:20 UTC
a31ec3a68bbbedd4b5d59490c762bcf32abf1a17
2 files changed
+34
-2
docs/models/supported_models.md
+7
@@ -602,6 +602,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
602
| `MossTranscribeDiarizeForConditionalGeneration` | MOSS-Transcribe-Diarize | T + A | `OpenMOSS-Team/MOSS-Transcribe-Diarize` | | ✅︎ |
603
| `Moondream3ForCausalLM` | Moondream3 | T + I | `moondream/moondream3-preview` | | ✅︎ |
604
| `MuseGlimmerForCausalLM`, `MuseGlimmerForConditionalGeneration` | Muse Glimmer | T + I<sup>+</sup> + V<sup>+</sup> | `meta-models/Muse-Glimmer-30B` | ✅︎ | ✅︎ |
605
+| `NemotronH_Nano_Omni_Reasoning_V3`, `NemotronH_Nano_VL_V2` | Nemotron Nano 3 Omni, Nemotron Nano V2 VL | T + I<sup>E+</sup> + V<sup>+</sup> + A<sup>*</sup> | `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16` | ✅︎ | |
606
| `NVLM_D_Model` | NVLM-D 1.0 | T + I<sup>+</sup> | `nvidia/NVLM-D-72B`, etc. | | ✅︎ |
607
| `OpenCUAForConditionalGeneration` | OpenCUA-7B | T + I<sup>E+</sup> | `xlangai/OpenCUA-7B` | ✅︎ | ✅︎ |
608
| `OpenPanguVLForConditionalGeneration` | openpangu-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `FreedomIntelligence/openPangu-VL-7B` | ✅︎ | ✅︎ |
@@ -701,6 +702,12 @@ Some models are supported only via the [Transformers modeling backend](#transfor
702
`meta-models/Muse-Glimmer-30B-assistant` checkpoint, which vLLM serves through
703
its [DFlash](../features/speculative_decoding/README.md) path.
704
705
+!!! note
706
+ Both Nemotron Nano architecture names map to the same vLLM implementation.
707
+ Audio inputs are only supported by the Omni variants; `NemotronH_Nano_VL_V2`
708
+ checkpoints such as `nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16` accept text,
709
+ image and video only.
710
+
711
!!! note
712
The official `openbmb/MiniCPM-V-2` doesn't work yet, so we need to use a fork (`HwwwH/MiniCPM-V-2`) for now.
713
For more details, please see: <https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630>
vllm/model_executor/models/nano_nemotron_vl.py
+27
-2
@@ -23,6 +23,7 @@ from vllm.config import VllmConfig
23
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
24
from vllm.inputs import MultiModalDataDict, MultiModalInput
25
from vllm.logger import init_logger
26
+from vllm.lora.layers.base import BaseLayerWithLoRA
27
from vllm.model_executor.layers.activation import ReLUSquaredActivation
28
from vllm.model_executor.layers.layernorm import RMSNorm
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -30,6 +31,7 @@ from vllm.model_executor.models.interfaces import (
31
HasInnerState,
32
IsHybrid,
33
MultiModalEmbeddings,
34
+ SupportsLoRA,
35
SupportsMultiModal,
36
SupportsMultiModalPruning,
37
)
@@ -897,11 +899,22 @@ class NanoNemotronVLDummyInputsBuilder(
899
dummy_inputs=NanoNemotronVLDummyInputsBuilder,
900
)
901
class NemotronH_Nano_VL_V2(
900
- nn.Module, HasInnerState, IsHybrid, SupportsMultiModal, SupportsMultiModalPruning
902
+ nn.Module,
903
+ HasInnerState,
904
+ IsHybrid,
905
+ SupportsMultiModal,
906
+ SupportsMultiModalPruning,
907
+ SupportsLoRA,
908
):
909
requires_sequential_video_encoding = True
910
"""Temporarily needed for dynamic res video w/ conv3d, doesn't support bs>1 yet"""
911
912
+ # LoRA covers the language model only
913
+ is_non_gated_moe = NemotronHForCausalLM.is_non_gated_moe
914
+ packed_modules_mapping = NemotronHForCausalLM.packed_modules_mapping
915
+ embedding_modules = NemotronHForCausalLM.embedding_modules
916
+ lora_skip_prefixes = NemotronHForCausalLM.lora_skip_prefixes
917
+
918
hf_to_vllm_mapper = WeightsMapper(
919
orig_to_new_prefix={
920
"language_model.backbone": "language_model.model",
@@ -1331,7 +1344,19 @@ class NemotronH_Nano_VL_V2(
1344
1345
# Create final video embeddings, merging text embeddings for indicator
1346
# tokens with video embeddings
1334
- text_embeddings = self.get_language_model().embed_input_ids(repl_token_ids)
1347
+
1348
+ # LoRA support -
1349
+ # These replacement tokens are produced inside the encoder, so they are
1350
+ # absent from the request-token batch that the LoRA adapter index
1351
+ # mapping is built from, and there can be more
1352
+ # of them than max_num_batched_tokens.
1353
+ # Embed them with the base weights -
1354
+ # the LoRA delta is undefined for tokens with no mapping entry.
1355
+ embed_tokens = self.get_language_model().model.embed_tokens
1356
+ if isinstance(embed_tokens, BaseLayerWithLoRA):
1357
+ embed_tokens = embed_tokens.base_layer
1358
+ text_embeddings = embed_tokens(repl_token_ids)
1359
+
1360
final_video_embeddings = _merge_multimodal_embeddings(
1361
inputs_embeds=text_embeddings,
1362
multimodal_embeddings=video_embeddings,