A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 09:45 UTC
nemotron-nano-vl: Allow `use_audio_in_video` to be passed at `vllm serve` time (#38538)
Signed-off-by: Andrii Skliar <askliar@nvidia.com> Co-authored-by: Andrii Skliar <askliar@nvidia.com>
Andrii Skliar committed
Apr 9, 2026 at 13:44 UTC
df2503e125f3c869b0f274e64530d09bf01ea30d
2 files changed
+74
-18
vllm/model_executor/models/nano_nemotron_vl.py
+72
-18
@@ -597,19 +597,26 @@ class NanoNemotronVLMultiModalProcessor(
597
def _extract_audio_from_videos(
598
self,
599
mm_items: MultiModalDataItems,
600
- ) -> tuple[MultiModalDataItems, list[AudioItem]]:
600
+ ) -> tuple[MultiModalDataItems, list[AudioItem], list[bool]]:
601
"""Extract audio tracks from video bytes in *mm_items*.
602
603
+ Videos whose bytes are missing or that contain no audio stream are
604
+ silently skipped. The returned *has_audio* mask is aligned with
605
+ the video list so callers know which ``<video>`` tokens need an
606
+ accompanying audio context.
607
+
608
Returns:
604
- The augmented *mm_items* (with audio added) and the list of
605
- extracted audio items.
609
+ A 3-tuple of (augmented mm_items, extracted audio items,
610
+ per-video boolean mask indicating which videos have audio).
611
"""
612
videos = mm_items.get_items("video", VideoProcessorItems)
613
assert isinstance(videos.metadata, list)
614
+
615
metadata_list = videos.metadata
616
617
audio_items: list[AudioItem] = []
612
- for metadata in metadata_list:
618
+ has_audio: list[bool] = []
619
+ for idx, metadata in enumerate(metadata_list):
620
video_bytes = metadata.get("original_video_bytes")
621
if video_bytes is None or len(video_bytes) == 0:
622
raise ValueError(
@@ -618,7 +625,16 @@ class NanoNemotronVLMultiModalProcessor(
625
"video must be loaded with keep_video_bytes=True (e.g. via "
626
"the chat API with a model that sets use_audio_in_video)."
627
)
621
- audio_items.append(load_audio_pyav(BytesIO(video_bytes)))
628
+ try:
629
+ audio_items.append(load_audio_pyav(BytesIO(video_bytes)))
630
+ has_audio.append(True)
631
+ except Exception:
632
+ logger.debug(
633
+ "Video %d: no audio stream found, skipping audio extraction.",
634
+ idx,
635
+ exc_info=True,
636
+ )
637
+ has_audio.append(False)
638
639
# Create a new VideoProcessorItems with metadata that does not contain
640
# the large video bytes, to avoid modifying the input `mm_items`.
@@ -628,45 +644,83 @@ class NanoNemotronVLMultiModalProcessor(
644
]
645
new_videos = VideoProcessorItems(data=videos.data, metadata=new_metadata_list)
646
631
- audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items})
647
+ audio_parsed = {}
648
+ if audio_items:
649
+ audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items})
650
651
# Create a new MultiModalDataItems with the new video and audio items.
652
new_mm_items_dict = {**mm_items, **audio_parsed, "video": new_videos}
653
mm_items = MultiModalDataItems(new_mm_items_dict)
654
637
- return mm_items, audio_items
655
+ return mm_items, audio_items, has_audio
656
657
def apply(
658
self,
659
inputs: ProcessorInputs,
660
timing_ctx: TimingContext,
661
) -> MultiModalInput:
644
- use_audio_in_video = bool(
645
- inputs.hf_processor_mm_kwargs.get("use_audio_in_video", False)
662
+ mm_config = self.info.ctx.model_config.get_multimodal_config()
663
+ merged_kwargs = mm_config.merge_mm_processor_kwargs(
664
+ inputs.hf_processor_mm_kwargs
665
)
666
+ use_audio_in_video = bool(merged_kwargs.get("use_audio_in_video", False))
667
+
668
inputs.hf_processor_mm_kwargs = {
669
k: v
670
for k, v in inputs.hf_processor_mm_kwargs.items()
671
if k != "use_audio_in_video"
672
}
673
653
- if not (
654
- use_audio_in_video
655
- and "video" in inputs.mm_data_items
656
- and "audio" not in inputs.mm_data_items
657
- ):
674
+ if not (use_audio_in_video and "video" in inputs.mm_data_items):
675
return super().apply(inputs, timing_ctx)
676
660
- mm_items, audio_items = self._extract_audio_from_videos(inputs.mm_data_items)
661
- inputs.mm_data_items = mm_items
677
+ mm_items = inputs.mm_data_items
678
+ if "audio" in mm_items:
679
+ # Audio was pre-populated by upstream (e.g., OpenAI chat endpoint).
680
+ # Reuse existing audio items; validate 1:1 correspondence.
681
+ videos = mm_items.get_items("video", VideoProcessorItems)
682
+ audios = mm_items.get_items("audio", AudioProcessorItems)
683
+ if len(audios) != len(videos):
684
+ raise ValueError(
685
+ "use_audio_in_video requires equal number of audio and "
686
+ f"video items, got num_audios={len(audios)}, "
687
+ f"num_videos={len(videos)}"
688
+ )
689
+ audio_items = audios.get_all()
690
+ has_audio = [True] * len(videos)
691
+ logger.info(
692
+ "Using %d pre-populated audio item(s) from upstream.",
693
+ len(audio_items),
694
+ )
695
+ else:
696
+ # Extract audio from video bytes (library usage path).
697
+ mm_items, audio_items, has_audio = self._extract_audio_from_videos(mm_items)
698
+ inputs.mm_data_items = mm_items
699
+ logger.info(
700
+ "Extracted audio from video bytes: %d audio(s), has_audio=%s.",
701
+ len(audio_items),
702
+ has_audio,
703
+ )
704
+
705
+ if not audio_items:
706
+ return super().apply(inputs, timing_ctx)
707
708
prompt = inputs.prompt
709
tokenizer = self.info.get_tokenizer()
710
if not isinstance(prompt, str):
711
prompt = tokenizer.decode(prompt, skip_special_tokens=False)
712
668
- for _ in audio_items:
669
- prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
713
+ # Inject AUDIO_CONTEXT only after <video> tokens whose video
714
+ # actually contained an audio stream (preserving video-audio pairing).
715
+ tag = "<video>"
716
+ head, *rest = prompt.split(tag)
717
+ rebuilt = [head]
718
+ for append_audio, part in zip(has_audio, rest, strict=True):
719
+ rebuilt.append(tag)
720
+ if append_audio:
721
+ rebuilt.append(AUDIO_CONTEXT)
722
+ rebuilt.append(part)
723
+ prompt = "".join(rebuilt)
724
725
inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
726
vllm/transformers_utils/processors/nano_nemotron_vl.py
+2
@@ -771,6 +771,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
771
max_num_tiles: int | None = None,
772
video_token: str | None = None,
773
video_pruning_rate: float | None = None,
774
+ use_audio_in_video: bool = False,
775
) -> None:
776
super().__init__(
777
config=config,
@@ -781,6 +782,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor):
782
# add extra video token for video processing
783
self.video_token = video_token
784
self.video_pruning_rate = video_pruning_rate
785
+ self.use_audio_in_video = use_audio_in_video
786
787
# Video params live exclusively in vision_config
788
vision_config = getattr(config, "vision_config", config)