A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 07:45 UTC
Support Audio Extraction from MP4 Video for Nemotron Nano VL (#35539)
Signed-off-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> Signed-off-by: Andrii Skliar <askliar@nvidia.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Signed-off-by: wangxiyuan <wangxiyuan1007@gmail.com> Signed-off-by: Andrii <askliar@nvidia.com> Co-authored-by: Netanel Haber <58652339+netanel-haber@users.noreply.github.com> Co-authored-by: Andrii Skliar <askliar@oci-nrt-cs-001-vscode-01.cm.cluster> Co-authored-by: Andrii <askliar@nvidia.com> Co-authored-by: root <root@pool0-03748.cm.cluster> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: root <root@pool0-02416.cm.cluster> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: wangxiyuan <wangxiyuan1007@gmail.com> Co-authored-by: root <root@pool0-04880.cm.cluster>
Andrii Skliar committed
Mar 4, 2026 at 08:20 UTC
5d199ac8f25a56495c24dcd8e6a63843002bba40
5 files changed
+225
-1
setup.py
+1
@@ -1056,6 +1056,7 @@ setup(
1056
"scipy",
1057
"soundfile",
1058
"mistral_common[audio]",
1059
+ "av",
1060
], # Required for audio processing
1061
"video": [], # Kept for backwards compatibility
1062
"flashinfer": [], # Kept for backwards compatibility
vllm/model_executor/models/config.py
+10
@@ -622,6 +622,15 @@ class NemotronHForCausalLMConfig(VerifyAndUpdateConfig):
622
cache_config.mamba_ssm_cache_dtype = mamba_ssm_cache_dtype
623
624
625
+class NemotronHNanoVLV2Config(VerifyAndUpdateConfig):
626
+ @staticmethod
627
+ def verify_and_update_model_config(model_config: "ModelConfig") -> None:
628
+ mm_config = model_config.multimodal_config
629
+ if mm_config is not None:
630
+ video_kwargs = mm_config.media_io_kwargs.setdefault("video", {})
631
+ video_kwargs.setdefault("video_backend", "nemotron_vl")
632
+
633
+
634
class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
635
@staticmethod
636
def verify_and_update_config(vllm_config: "VllmConfig") -> None:
@@ -661,6 +670,7 @@ MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
670
"GteNewModel": GteNewModelConfig,
671
"GteNewForSequenceClassification": GteNewModelConfig,
672
"Gemma3TextModel": Gemma3TextModelConfig,
673
+ "NemotronH_Nano_VL_V2": NemotronHNanoVLV2Config,
674
"LlamaBidirectionalForSequenceClassification": LlamaBidirectionalConfig,
675
"LlamaBidirectionalModel": LlamaBidirectionalConfig,
676
"LlamaNemotronVLModel": LlamaNemotronVLConfig,
vllm/model_executor/models/nano_nemotron_vl.py
+129
-1
@@ -59,9 +59,11 @@ from vllm.multimodal.inputs import (
59
AudioItem,
60
MultiModalDataDict,
61
MultiModalFieldConfig,
62
+ MultiModalInputs,
63
MultiModalKwargsItems,
64
VideoItem,
65
)
66
+from vllm.multimodal.media.audio import extract_audio_from_video_bytes
67
from vllm.multimodal.parse import (
68
AudioProcessorItems,
69
ImageEmbeddingItems,
@@ -69,8 +71,13 @@ from vllm.multimodal.parse import (
71
ImageSize,
72
MultiModalDataItems,
73
MultiModalDataParser,
74
+ VideoProcessorItems,
75
+)
76
+from vllm.multimodal.processing import (
77
+ BaseDummyInputsBuilder,
78
+ ProcessorInputs,
79
+ TimingContext,
80
)
73
-from vllm.multimodal.processing import BaseDummyInputsBuilder
81
from vllm.multimodal.processing.processor import (
82
BaseMultiModalProcessor,
83
BaseProcessingInfo,
@@ -1381,6 +1388,127 @@ class NanoNemotronVLMultiModalProcessor(
1388
):
1389
"""MultiModalProcessor extended for video support"""
1390
1391
+ def _extract_audio_from_videos(
1392
+ self,
1393
+ mm_items: MultiModalDataItems,
1394
+ ) -> tuple[MultiModalDataItems, list[AudioItem]]:
1395
+ """Extract audio tracks from video bytes in *mm_items*.
1396
+
1397
+ Returns:
1398
+ The augmented *mm_items* (with audio added) and the list of
1399
+ extracted audio items.
1400
+ """
1401
+ videos = mm_items.get_items("video", VideoProcessorItems)
1402
+ assert isinstance(videos.metadata, list)
1403
+ metadata_list = videos.metadata
1404
+
1405
+ audio_items: list[AudioItem] = []
1406
+ for metadata in metadata_list:
1407
+ video_bytes = metadata.get("original_video_bytes")
1408
+ if video_bytes is None or len(video_bytes) == 0:
1409
+ raise ValueError(
1410
+ "Cannot extract audio from video: original_video_bytes is "
1411
+ "missing or empty. When using use_audio_in_video=True, "
1412
+ "video must be loaded with keep_video_bytes=True (e.g. via "
1413
+ "the chat API with a model that sets use_audio_in_video)."
1414
+ )
1415
+ audio_items.append(extract_audio_from_video_bytes(video_bytes))
1416
+
1417
+ # Create a new VideoProcessorItems with metadata that does not contain
1418
+ # the large video bytes, to avoid modifying the input `mm_items`.
1419
+ new_metadata_list = [
1420
+ {k: v for k, v in meta.items() if k != "original_video_bytes"}
1421
+ for meta in metadata_list
1422
+ ]
1423
+ new_videos = VideoProcessorItems(data=videos.data, metadata=new_metadata_list)
1424
+
1425
+ audio_parsed = self.data_parser.parse_mm_data({"audio": audio_items})
1426
+
1427
+ # Create a new MultiModalDataItems with the new video and audio items.
1428
+ new_mm_items_dict = {**mm_items, **audio_parsed, "video": new_videos}
1429
+ mm_items = MultiModalDataItems(new_mm_items_dict)
1430
+
1431
+ return mm_items, audio_items
1432
+
1433
+ def apply(
1434
+ self,
1435
+ processor_inputs: ProcessorInputs,
1436
+ timing_ctx: TimingContext | None = None,
1437
+ ) -> MultiModalInputs:
1438
+ if (hf_processor_mm_kwargs := processor_inputs.hf_processor_mm_kwargs) is None:
1439
+ hf_processor_mm_kwargs = {}
1440
+
1441
+ use_audio_in_video = bool(
1442
+ hf_processor_mm_kwargs.get("use_audio_in_video", False)
1443
+ )
1444
+
1445
+ hf_processor_mm_kwargs = {
1446
+ k: v for k, v in hf_processor_mm_kwargs.items() if k != "use_audio_in_video"
1447
+ }
1448
+
1449
+ processor_inputs.hf_processor_mm_kwargs = hf_processor_mm_kwargs
1450
+
1451
+ if not (
1452
+ use_audio_in_video
1453
+ and "video" in processor_inputs.mm_data_items
1454
+ and "audio" not in processor_inputs.mm_data_items
1455
+ ):
1456
+ return super().apply(
1457
+ processor_inputs,
1458
+ timing_ctx,
1459
+ )
1460
+
1461
+ mm_items, audio_items = self._extract_audio_from_videos(
1462
+ processor_inputs.mm_data_items
1463
+ )
1464
+ processor_inputs.mm_data_items = mm_items
1465
+
1466
+ prompt = processor_inputs.prompt
1467
+ tokenizer = self.info.get_tokenizer()
1468
+ if not isinstance(prompt, str):
1469
+ prompt = tokenizer.decode(prompt, skip_special_tokens=False)
1470
+
1471
+ for _ in audio_items:
1472
+ prompt = prompt.replace("<video>", "<video>" + AUDIO_CONTEXT, 1)
1473
+
1474
+ processor_inputs.prompt = tokenizer.encode(prompt, add_special_tokens=False)
1475
+
1476
+ if processor_inputs.tokenization_kwargs is None:
1477
+ processor_inputs.tokenization_kwargs = {}
1478
+
1479
+ # Bypass the cached path: the HF processor must receive the
1480
+ # prompt (with injected <so_embedding>) and the audio data
1481
+ # together so it can perform audio-token replacement natively.
1482
+ (
1483
+ prompt_ids,
1484
+ mm_info,
1485
+ is_update_applied,
1486
+ ) = self._apply_hf_processor(
1487
+ processor_inputs,
1488
+ timing_ctx=timing_ctx,
1489
+ )
1490
+
1491
+ prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates(
1492
+ mm_items=mm_items,
1493
+ prompt_ids=prompt_ids,
1494
+ mm_kwargs=mm_info.kwargs,
1495
+ mm_prompt_updates=mm_info.prompt_updates,
1496
+ is_update_applied=is_update_applied,
1497
+ )
1498
+
1499
+ mm_placeholder_ranges = {
1500
+ modality: [item.to_range() for item in placeholders]
1501
+ for modality, placeholders in mm_placeholders.items()
1502
+ }
1503
+
1504
+ return MultiModalInputs(
1505
+ type="multimodal",
1506
+ prompt_token_ids=prompt_ids,
1507
+ mm_kwargs=mm_info.kwargs,
1508
+ mm_hashes=mm_info.hashes,
1509
+ mm_placeholders=mm_placeholder_ranges,
1510
+ )
1511
+
1512
def _get_mm_fields_config(
1513
self,
1514
hf_inputs: BatchFeature,
vllm/multimodal/media/audio.py
+58
@@ -4,6 +4,7 @@ import base64
4
from io import BytesIO
5
from pathlib import Path
6
7
+import numpy as np
8
import numpy.typing as npt
9
import pybase64
10
import torch
@@ -23,6 +24,63 @@ try:
24
except ImportError:
25
soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
26
27
+try:
28
+ import av
29
+except ImportError:
30
+ av = PlaceholderModule("av") # type: ignore[assignment]
31
+
32
+
33
+def extract_audio_from_video_bytes(
34
+ data: bytes,
35
+) -> tuple[npt.NDArray, float]:
36
+ """Extract the audio track from raw video bytes using PyAV.
37
+
38
+ PyAV wraps FFmpeg's C libraries in-process — no subprocess is
39
+ spawned, which is critical to avoid crashing CUDA-active vLLM
40
+ worker processes.
41
+
42
+ The returned waveform is at the native sample rate of the video's
43
+ audio stream. Resampling to a model-specific rate is left to the
44
+ downstream :class:`AudioResampler` in the parsing pipeline.
45
+
46
+ Args:
47
+ data: Raw video file bytes (e.g. from an mp4 file).
48
+
49
+ Returns:
50
+ A tuple of ``(waveform, sample_rate)`` suitable for use as an
51
+ :class:`AudioItem`.
52
+ """
53
+ if data is None or len(data) == 0:
54
+ raise ValueError(
55
+ "Cannot extract audio: video bytes are missing or empty. "
56
+ "Ensure video was loaded with keep_video_bytes=True for "
57
+ "audio-in-video extraction."
58
+ )
59
+ try:
60
+ with av.open(BytesIO(data)) as container:
61
+ if not container.streams.audio:
62
+ raise ValueError("No audio stream found in the video.")
63
+ stream = container.streams.audio[0]
64
+ native_sr = stream.rate
65
+
66
+ chunks: list[npt.NDArray] = []
67
+ for frame in container.decode(audio=0):
68
+ arr = frame.to_ndarray()
69
+ chunks.append(arr.mean(axis=0) if arr.ndim > 1 else arr)
70
+ except ValueError:
71
+ raise
72
+ except Exception as e:
73
+ raise ValueError(
74
+ "Invalid or corrupted video data when extracting audio. "
75
+ "Ensure the input is valid video bytes (e.g. a complete MP4)."
76
+ ) from e
77
+
78
+ if not chunks:
79
+ raise ValueError("No audio found in the video.")
80
+
81
+ audio = np.concatenate(chunks).astype(np.float32)
82
+ return audio, float(native_sr)
83
+
84
85
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
86
def __init__(self, **kwargs) -> None:
vllm/multimodal/video.py
+27
@@ -749,6 +749,33 @@ class Molmo2VideoBackend(VideoLoader):
749
return out
750
751
752
+@VIDEO_LOADER_REGISTRY.register("nemotron_vl")
753
+class NemotronVLVideoBackend(OpenCVVideoBackend):
754
+ @classmethod
755
+ def load_bytes(
756
+ cls,
757
+ data: bytes,
758
+ num_frames: int = -1,
759
+ fps: int = -1,
760
+ max_duration: int = 300,
761
+ frame_recovery: bool = False,
762
+ **kwargs,
763
+ ) -> tuple[npt.NDArray, dict[str, Any]]:
764
+ frames, metadata = OpenCVVideoBackend.load_bytes(
765
+ data,
766
+ num_frames=num_frames,
767
+ fps=fps,
768
+ max_duration=max_duration,
769
+ frame_recovery=frame_recovery,
770
+ **kwargs,
771
+ )
772
+
773
+ metadata = dict(metadata)
774
+ metadata["original_video_bytes"] = data
775
+
776
+ return frames, metadata
777
+
778
+
779
@VIDEO_LOADER_REGISTRY.register("openpangu")
780
class OpenCVDynamicOpenPanguVideoBackend(OpenCVVideoBackend):
781
@classmethod