@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fa9e68022d

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 14:45 UTC

Fix Nano Nemotron VL regressions (#38655)

Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com>

Netanel Haber committed Apr 3, 2026 at 10:22 UTC fa9e68022d29c5396dfbb96d13587b6bc1bdb933

7 files changed +84 -52

tests/models/registry.py

+21 -1

@@ -7,6 +7,7 @@ from typing import Any, Literal

8 import pytest

9 from packaging.version import Version

10 +from transformers import PretrainedConfig

11 from transformers import __version__ as TRANSFORMERS_VERSION

13 from vllm.config.model import ModelDType, TokenizerMode

@@ -1004,7 +1005,26 @@ _MULTIMODAL_EXAMPLE_MODELS = {

1005 trust_remote_code=True,

1006 ),

1007 "NemotronH_Nano_VL_V2": _HfExamplesInfo(

1007 - "nano_vl_dummy", is_available_online=False, trust_remote_code=True

1008 + "nvidia/NVIDIA-Nemotron-Nano-12B-v2-VL-BF16",

1009 + max_model_len=4096,

1010 + # NemotronH layers are constructed via `hybrid_override_pattern`:

1011 + use_original_num_layers=True,

1012 + hf_overrides={

1013 + "vision_config": PretrainedConfig(

1014 + args={

1015 + "min_num_patches": 1, # Trigger image dynamic res

1016 + "max_num_patches": 12,

1017 + "model": "vit_huge_patch16_224",

1018 + },

1019 + # Trigger conv3d:

1020 + video_temporal_patch_size=2,

1021 + ),

1022 + "text_config": {

1023 + "num_hidden_layers": 2,

1024 + "hybrid_override_pattern": "M*",

1025 + },

1026 + },

1027 + trust_remote_code=True,

1028 ),

1029 "OpenCUAForConditionalGeneration": _HfExamplesInfo(

1030 "xlangai/OpenCUA-7B", trust_remote_code=True

tests/models/utils.py

+8 -1

@@ -447,9 +447,16 @@ def dummy_hf_overrides(

447 Dummy HF overrides function used to create dummy model

448 with only minimum nums of layer.

449 """

450 - hf_config.update(exist_overrides or {})

450 + # Copy because this helper is called more than once

451 + # while loading config, and we `.pop()`

452 + exist_overrides = (exist_overrides or {}).copy()

453 + text_config_override = exist_overrides.pop("text_config", None)

454 + hf_config.update(exist_overrides)

455

456 text_config = hf_config.get_text_config()

457 + if text_config_override is not None:

458 + # multimodal test models may override *some* text-model fields

459 + text_config.update(text_config_override)

460

461 # Ensure at least 2 expert per group

462 # Since `grouped_topk` assumes top-2

vllm/model_executor/models/nano_nemotron_vl.py

+29 -23

@@ -7,7 +7,6 @@

7 # LICENSE is in root directory.

8 # --------------------------------------------------------

10 -import copy

10 import math

11 import warnings

12 from collections.abc import Iterable, Mapping, Sequence

@@ -17,7 +16,7 @@ from typing import Annotated, Literal, TypeAlias

17 import torch

18 import torch.nn as nn

20 -from transformers import BatchFeature

19 +from transformers import BatchFeature, PretrainedConfig

21 from vllm.config import VllmConfig

22 from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions

@@ -210,11 +209,15 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):

209

210 @cached_property

211 def is_dynamic_tiler(self) -> bool:

213 - return self.get_hf_processor().dynamic_tiler is not None

212 + return BaseNanoNemotronVLProcessor.use_dynamic_resolution(self.get_hf_config())

213

215 - @cached_property

214 + @property

215 def supports_video(self):

217 - return self.get_hf_processor().supports_video

216 + return True

217 +

218 + @property

219 + def supports_audio(self) -> bool:

220 + return self.sound_config is not None

221

222 def get_video_token(self) -> str | None:

223 return IMG_CONTEXT

@@ -223,8 +226,8 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):

226 return self.ctx.get_mm_config().video_pruning_rate

227

228 @property

226 - def audio_extractor(self) -> ParakeetExtractor | None:

227 - return self.get_hf_processor().audio_extractor

229 + def sound_config(self) -> PretrainedConfig | None:

230 + return getattr(self.get_hf_config(), "sound_config", None)

231

232 def get_default_tok_params(self) -> TokenizeParams:

233 return super().get_default_tok_params().with_kwargs(add_special_tokens=False)

@@ -232,14 +235,14 @@ class NanoNemotronVLProcessingInfo(BaseProcessingInfo):

235 def get_supported_mm_limits(self) -> Mapping[str, int | None]:

236 image_limit = {"image": None}

237 video_limit = {"video": None} if self.supports_video else {}

235 - audio_limit = {"audio": None} if self.audio_extractor is not None else {}

238 + audio_limit = {"audio": None} if self.supports_audio else {}

239 return {**image_limit, **video_limit, **audio_limit}

240

241 def get_data_parser(self):

242 target_sr = None

243 target_channels = None

241 - if extractor := self.audio_extractor:

242 - target_sr = extractor.sampling_rate

244 + if self.sound_config:

245 + target_sr = self.sound_config.sampling_rate

246 target_channels = 1

247

248 return MultiModalDataParser(

@@ -371,7 +374,7 @@ class NanoNemotronVLMultiModalProcessor(

374 fields = self._get_image_fields_config(hf_inputs)

375 if self.info.supports_video:

376 fields |= self._get_video_fields_config(hf_inputs)

374 - if self.info.audio_extractor:

377 + if self.info.supports_audio:

378 fields |= self._get_audio_fields_config(hf_inputs)

379

380 return fields

@@ -399,9 +402,8 @@ class NanoNemotronVLMultiModalProcessor(

402

403 if isinstance(images, ImageEmbeddingItems):

404 feature_size = images.get_feature_size(item_idx)

402 - elif tiler := hf_processor.dynamic_tiler:

403 - image = images.get(item_idx)

404 - feature_size = tiler.get_cached_feature_size(image)

405 + elif self.info.is_dynamic_tiler:

406 + feature_size = out_mm_data["num_tokens_per_image"][item_idx]

407 else:

408 image_size = images.get_image_size(item_idx)

409 max_num_tiles = hf_processor.max_num_tiles

@@ -536,7 +538,7 @@ class NanoNemotronVLMultiModalProcessor(

538 prompt_repls.append(

539 self._get_prompt_repl_video(mm_items, hf_processor, out_mm_data)

540 )

539 - if self.info.audio_extractor:

541 + if self.info.supports_audio:

542 prompt_repls.append(

543 self._get_prompt_repl_audio(mm_items, hf_processor, out_mm_data)

544 )

@@ -772,12 +774,14 @@ class NanoNemotronVLDummyInputsBuilder(

774 else:

775 dummy_video = {}

776

775 - if extractor := self.info.audio_extractor:

777 + if sound_config := self.info.sound_config:

778 num_audios = mm_counts.get("audio", 0)

779 audio_overrides = mm_options.get("audio") if mm_options else None

780 tokens_per_audio = max(1, seq_len // max(num_audios, 1))

779 - max_audio_num_samples = MAX_AUDIO_LEN_S * extractor.sampling_rate

780 - calculated_max_audio_num_samples = extractor.audio_length(tokens_per_audio)

781 + max_audio_num_samples = MAX_AUDIO_LEN_S * sound_config.sampling_rate

782 + calculated_max_audio_num_samples = ParakeetExtractor.audio_length(

783 + sound_config, tokens_per_audio

784 + )

785 audio_len = min(max_audio_num_samples, calculated_max_audio_num_samples)

786 dummy_audio = {

787 "audio": self._get_dummy_audios(

@@ -1029,9 +1033,13 @@ class NemotronH_Nano_VL_V2(

1033 data=image_embeds,

1034 )

1035

1036 + pixel_values_flat = kwargs.pop("pixel_values_flat", None)

1037 + if pixel_values_flat is None:

1038 + return None

1039 +

1040 if self.dynamic_resolution:

1041 pixel_values_flat = DynamicResolutionImageTiler.stack(

1034 - kwargs.pop("pixel_values_flat"), self.patch_size

1042 + pixel_values_flat, self.patch_size

1043 )

1044 return NanoNemotronVLImagePixelInputsDynamic(

1045 pixel_values_flat=pixel_values_flat, **kwargs

@@ -1497,15 +1505,13 @@ class NemotronH_Nano_VL_V2(

1505 @classmethod

1506 def get_mamba_state_shape_from_config(cls, vllm_config: "VllmConfig"):

1507 text_config = vllm_config.model_config.hf_config.text_config

1500 - temp_vllm_config = copy.deepcopy(vllm_config)

1501 - temp_vllm_config.model_config.hf_config = text_config

1508 + temp_vllm_config = vllm_config.with_hf_config(text_config)

1509 return NemotronHForCausalLM.get_mamba_state_shape_from_config(temp_vllm_config)

1510

1511 @classmethod

1512 def get_mamba_state_dtype_from_config(cls, vllm_config: "VllmConfig"):

1513 text_config = vllm_config.model_config.hf_config.text_config

1507 - temp_vllm_config = copy.deepcopy(vllm_config)

1508 - temp_vllm_config.model_config.hf_config = text_config

1514 + temp_vllm_config = vllm_config.with_hf_config(text_config)

1515 return NemotronHForCausalLM.get_mamba_state_dtype_from_config(temp_vllm_config)

1516

1517 @classmethod

vllm/model_executor/models/parakeet.py

+4 -2

@@ -159,5 +159,7 @@ class ParakeetExtractor(ParakeetFeatureExtractor):

159 outputs["audio_num_clips"] = audio_num_clips

160 return outputs

161

162 - def audio_length(self, audio_tokens: int) -> int:

163 - return int(audio_tokens * self.config.subsampling_factor * self.hop_length)

162 + @staticmethod

163 + def audio_length(raw_config: PretrainedConfig, audio_tokens: int) -> int:

164 + config = ExtractorConfig.from_hf_config(raw_config)

165 + return int(audio_tokens * config.subsampling_factor * config.hop_length)

vllm/model_executor/models/radio.py

+1 -10

@@ -176,7 +176,6 @@ class ViTPatchGenerator(nn.Module):

176 temporal_patch_size=temporal_patch_size,

177 **factory,

178 )

179 - self._video_embedder_loaded = False

179

180 if abs_pos:

181 scale = embed_dim**-0.5

@@ -225,12 +224,7 @@ class ViTPatchGenerator(nn.Module):

224 Returns:

225 Embedded patches with temporal compression applied.

226 """

228 - if not self._video_embedder_loaded:

229 - raise ValueError(

230 - "Temporal compression (video_temporal_patch_size > 1) requires "

231 - "video_embedder weights, but they were never loaded. "

232 - "Ensure the checkpoint was trained with temporal compression."

233 - )

227 + assert self.temporal_patch_size > 1

228 T = self.temporal_patch_size

229 input_size = x.shape[2:]

230

@@ -794,9 +788,6 @@ class RadioModel(nn.Module):

788 weight_loader(param, weight)

789 loaded_params.add(vllm_key)

790

797 - if "model.patch_generator.video_embedder.weight" in loaded_params:

798 - self.model.patch_generator._video_embedder_loaded = True

799 -

791 return loaded_params

792

793 def _extract_final(

vllm/transformers_utils/configs/parakeet.py

@@ -44,15 +44,19 @@ class ExtractorConfig:

44 subsampling_factor: int

45 subsampling_conv_kernel_size: int

46 subsampling_conv_stride: int

47 + hop_length: int = 160

48 + """Default `160`: Matches HF default"""

49 clip_duration_s: int = 30

50 clip_min_duration_s: float = 0.1

52 @staticmethod

53 def from_hf_config(config: PretrainedConfig) -> "ExtractorConfig":

54 assert isinstance(config, PretrainedConfig)

55 + hop_length = int(getattr(config, "hop_length", ExtractorConfig.hop_length))

56 return ExtractorConfig(

57 feature_size=config.num_mel_bins,

58 sampling_rate=config.sampling_rate,

59 + hop_length=hop_length,

60 subsampling_factor=config.subsampling_factor,

61 subsampling_conv_kernel_size=config.subsampling_conv_kernel_size,

62 subsampling_conv_stride=config.subsampling_conv_stride,

vllm/transformers_utils/processors/nano_nemotron_vl.py

+17 -15

@@ -356,15 +356,6 @@ class DynamicResolutionImageTiler:

356 feature_sizes.append(param.num_embeddings)

357 return images, feature_sizes

358

359 - feature_size_cache: dict[Image.Image, int] = {}

360 -

361 - @classmethod

362 - def get_cached_feature_size(cls, image: Image.Image) -> int:

363 - feature_size = cls.feature_size_cache[id(image)]

364 - # hard assert that we only use the feature size once

365 - del cls.feature_size_cache[id(image)]

366 - return feature_size

367 -

359 @dataclass

360 class DynamicResolutionParams:

361 media: Image.Image

@@ -519,7 +510,6 @@ class DynamicResolutionImageTiler:

510 param, token_count = self.process_media(media, tokens_for_media)

511 params.append(param)

512 token_counts.append(token_count)

522 - self.feature_size_cache[id(param.media)] = param.num_embeddings

513

514 # Step 2: Check if total tokens is within budget

515 total_tokens = sum(token_counts)

@@ -857,13 +847,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

847

848 @property

849 def supports_video(self) -> bool:

860 - return self.video_token_id is not None

850 + return True

851

852 @property

863 - def video_token_id(self) -> int | None:

864 - if self.video_token is None:

865 - return None

866 - return self.tokenizer.get_vocab().get(self.video_token, None)

853 + def video_token_id(self) -> int:

854 + assert self.video_token is not None

855 + return self.tokenizer.get_vocab()[self.video_token]

856

857 @property

858 def image_token_id(self) -> int:

@@ -1055,6 +1044,13 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

1044 text_inputs = self.tokenizer(text, add_special_tokens=False)

1045

1046 combined_inputs = {**text_inputs, **video_inputs, **audio_inputs}

1047 + frames_indices = combined_inputs.get("frames_indices")

1048 + ragged_frames_indices = (

1049 + isinstance(frames_indices, list)

1050 + and len({len(frame_indices) for frame_indices in frames_indices}) > 1

1051 + )

1052 + if ragged_frames_indices:

1053 + combined_inputs.pop("frames_indices")

1054

1055 if self.dynamic_tiler is None:

1056 batch = BatchFeature(

@@ -1066,6 +1062,12 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):

1062 # allow images to be exempt from the BatchFeature validation:

1063 # We will .stack() them in _parse_and_validate_image_input

1064 batch.update(image_inputs)

1065 + if ragged_frames_indices:

1066 + assert isinstance(frames_indices, list)

1067 + batch["frames_indices"] = [

1068 + torch.as_tensor(frame_indices, dtype=torch.int64)

1069 + for frame_indices in frames_indices

1070 + ]

1071 return batch

1072

1073 def get_image_repl(