A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 02:15 UTC
Update registry for Nemotron-v3 VL Nano/Super (#39747)
Signed-off-by: Collin McCarthy <cmccarthy@nvidia.com>
Collin McCarthy committed
Apr 15, 2026 at 16:09 UTC
27c0ca50a0853e6087cd0cf8fe5caccc50f471f6
3 files changed
+50
tests/models/registry.py
+44
@@ -1038,6 +1038,50 @@ _MULTIMODAL_EXAMPLE_MODELS = {
1038
},
1039
trust_remote_code=True,
1040
),
1041
+ # NemotronH_Nano_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2
1042
+ # Use the same registry test as NemotronH_Nano_VL_V2 above
1043
+ "NemotronH_Nano_Omni_Reasoning_V3": _HfExamplesInfo(
1044
+ "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
1045
+ max_model_len=4096,
1046
+ use_original_num_layers=True,
1047
+ hf_overrides={
1048
+ "vision_config": PretrainedConfig(
1049
+ args={
1050
+ "min_num_patches": 1,
1051
+ "max_num_patches": 12,
1052
+ "model": "vit_huge_patch16_224",
1053
+ },
1054
+ video_temporal_patch_size=2,
1055
+ ),
1056
+ "text_config": {
1057
+ "num_hidden_layers": 2,
1058
+ "hybrid_override_pattern": "M*",
1059
+ },
1060
+ },
1061
+ trust_remote_code=True,
1062
+ ),
1063
+ # NemotronH_Super_Omni_Reasoning_V3 is an alias for NemotronH_Nano_VL_V2 as well
1064
+ # Use the same registry test as NemotronH_Nano_VL_V2 above
1065
+ "NemotronH_Super_Omni_Reasoning_V3": _HfExamplesInfo(
1066
+ "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",
1067
+ max_model_len=4096,
1068
+ use_original_num_layers=True,
1069
+ hf_overrides={
1070
+ "vision_config": PretrainedConfig(
1071
+ args={
1072
+ "min_num_patches": 1,
1073
+ "max_num_patches": 12,
1074
+ "model": "vit_huge_patch16_224",
1075
+ },
1076
+ video_temporal_patch_size=2,
1077
+ ),
1078
+ "text_config": {
1079
+ "num_hidden_layers": 2,
1080
+ "hybrid_override_pattern": "M*",
1081
+ },
1082
+ },
1083
+ trust_remote_code=True,
1084
+ ),
1085
"OpenCUAForConditionalGeneration": _HfExamplesInfo(
1086
"xlangai/OpenCUA-7B", trust_remote_code=True
1087
),
vllm/config/speculative.py
+4
@@ -300,6 +300,10 @@ class SpeculativeConfig:
300
{"n_predict": n_predict, "architectures": ["ErnieMTPModel"]}
301
)
302
303
+ if hf_config.architectures[0] == "NemotronH_Super_Omni_Reasoning_V3":
304
+ # Promote VLM's text_config so MTP detection below fires correctly
305
+ hf_config = hf_config.text_config
306
+
307
if (
308
hf_config.model_type in {"nemotron_h", "nemotron_h_puzzle"}
309
and hasattr(hf_config, "num_nextn_predict_layers")
vllm/model_executor/models/registry.py
+2
@@ -473,6 +473,8 @@ _MULTIMODAL_MODELS = {
473
"MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
474
"Molmo2ForConditionalGeneration": ("molmo2", "Molmo2ForConditionalGeneration"),
475
"NemotronH_Nano_VL_V2": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
476
+ "NemotronH_Nano_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
477
+ "NemotronH_Super_Omni_Reasoning_V3": ("nano_nemotron_vl", "NemotronH_Nano_VL_V2"),
478
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
479
"OpenCUAForConditionalGeneration": ("opencua", "OpenCUAForConditionalGeneration"),
480
"OpenPanguVLForConditionalGeneration": (