A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 00:15 UTC
Support online use_audio_in_video (#36319)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Tianyu Guo committed
Mar 9, 2026 at 22:16 UTC
5578f2a4d33b3451203fa5d43e4e6847c00b55c6
10 files changed
+152
-10
vllm/entrypoints/chat_utils.py
+54
-8
@@ -564,7 +564,9 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
564
return self.model_cls.get_placeholder_str(modality, num_items)
565
566
@abstractmethod
567
- def create_parser(self) -> "BaseMultiModalContentParser":
567
+ def create_parser(
568
+ self, mm_processor_kwargs: dict[str, Any] | None = None
569
+ ) -> "BaseMultiModalContentParser":
570
raise NotImplementedError
571
572
@@ -690,8 +692,10 @@ class MultiModalItemTracker(BaseMultiModalItemTracker[tuple[object, str | None]]
692
dict(self._items_by_modality), self.mm_processor, self._modality_order
693
)
694
693
- def create_parser(self) -> "BaseMultiModalContentParser":
694
- return MultiModalContentParser(self)
695
+ def create_parser(
696
+ self, mm_processor_kwargs: dict[str, Any] | None = None
697
+ ) -> "BaseMultiModalContentParser":
698
+ return MultiModalContentParser(self, mm_processor_kwargs=mm_processor_kwargs)
699
700
701
class AsyncMultiModalItemTracker(
@@ -712,8 +716,12 @@ class AsyncMultiModalItemTracker(
716
resolved_items_by_modality, self.mm_processor, self._modality_order
717
)
718
715
- def create_parser(self) -> "BaseMultiModalContentParser":
716
- return AsyncMultiModalContentParser(self)
719
+ def create_parser(
720
+ self, mm_processor_kwargs: dict[str, Any] | None = None
721
+ ) -> "BaseMultiModalContentParser":
722
+ return AsyncMultiModalContentParser(
723
+ self, mm_processor_kwargs=mm_processor_kwargs
724
+ )
725
726
727
class BaseMultiModalContentParser(ABC):
@@ -778,7 +786,11 @@ class BaseMultiModalContentParser(ABC):
786
787
788
class MultiModalContentParser(BaseMultiModalContentParser):
781
- def __init__(self, tracker: MultiModalItemTracker) -> None:
789
+ def __init__(
790
+ self,
791
+ tracker: MultiModalItemTracker,
792
+ mm_processor_kwargs: dict[str, Any] | None = None,
793
+ ) -> None:
794
super().__init__()
795
796
self._tracker = tracker
@@ -790,6 +802,8 @@ class MultiModalContentParser(BaseMultiModalContentParser):
802
allowed_media_domains=tracker.allowed_media_domains,
803
)
804
805
+ self._mm_processor_kwargs = mm_processor_kwargs
806
+
807
@property
808
def model_config(self) -> ModelConfig:
809
return self._tracker.model_config
@@ -886,9 +900,23 @@ class MultiModalContentParser(BaseMultiModalContentParser):
900
placeholder = self._tracker.add("video", (video, uuid))
901
self._add_placeholder("video", placeholder)
902
903
+ # Extract audio from video if use_audio_in_video is True
904
+ if (
905
+ video_url
906
+ and self._mm_processor_kwargs
907
+ and self._mm_processor_kwargs.get("use_audio_in_video", False)
908
+ ):
909
+ audio = self._connector.fetch_audio(video_url) if video_url else None
910
+ audio_placeholder = self._tracker.add("audio", (audio, uuid))
911
+ self._add_placeholder("audio", audio_placeholder)
912
+
913
914
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
891
- def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
915
+ def __init__(
916
+ self,
917
+ tracker: AsyncMultiModalItemTracker,
918
+ mm_processor_kwargs: dict[str, Any] | None = None,
919
+ ) -> None:
920
super().__init__()
921
922
self._tracker = tracker
@@ -898,6 +926,7 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
926
allowed_local_media_path=tracker.allowed_local_media_path,
927
allowed_media_domains=tracker.allowed_media_domains,
928
)
929
+ self._mm_processor_kwargs: dict[str, Any] | None = mm_processor_kwargs
930
931
@property
932
def model_config(self) -> ModelConfig:
@@ -1033,6 +1062,16 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
1062
placeholder = self._tracker.add("video", coro)
1063
self._add_placeholder("video", placeholder)
1064
1065
+ # Extract audio from video if use_audio_in_video is True
1066
+ if (
1067
+ video_url
1068
+ and self._mm_processor_kwargs
1069
+ and self._mm_processor_kwargs.get("use_audio_in_video", False)
1070
+ ):
1071
+ audio_coro = self._audio_with_uuid_async(video_url, uuid)
1072
+ audio_placeholder = self._tracker.add("audio", audio_coro)
1073
+ self._add_placeholder("audio", audio_placeholder)
1074
+
1075
1076
@dataclass
1077
class ChatTemplateConfig:
@@ -1343,10 +1382,11 @@ def _parse_chat_message_content_parts(
1382
*,
1383
wrap_dicts: bool,
1384
interleave_strings: bool,
1385
+ mm_processor_kwargs: dict[str, Any] | None = None,
1386
) -> list[ConversationMessage]:
1387
content = list[_ContentPart]()
1388
1349
- mm_parser = mm_tracker.create_parser()
1389
+ mm_parser = mm_tracker.create_parser(mm_processor_kwargs=mm_processor_kwargs)
1390
1391
for part in parts:
1392
parse_res = _parse_chat_message_content_part(
@@ -1464,6 +1504,7 @@ def _parse_chat_message_content(
1504
mm_tracker: BaseMultiModalItemTracker,
1505
content_format: ChatTemplateContentFormat,
1506
interleave_strings: bool,
1507
+ mm_processor_kwargs: dict[str, Any] | None = None,
1508
) -> list[ConversationMessage]:
1509
role = message["role"]
1510
content = message.get("content")
@@ -1479,6 +1520,7 @@ def _parse_chat_message_content(
1520
mm_tracker,
1521
wrap_dicts=(content_format == "openai"),
1522
interleave_strings=interleave_strings,
1523
+ mm_processor_kwargs=mm_processor_kwargs,
1524
)
1525
1526
for result_msg in result:
@@ -1540,6 +1582,7 @@ def parse_chat_messages(
1582
model_config: ModelConfig,
1583
content_format: ChatTemplateContentFormat,
1584
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
1585
+ mm_processor_kwargs: dict[str, Any] | None = None,
1586
) -> tuple[
1587
list[ConversationMessage],
1588
MultiModalDataDict | None,
@@ -1558,6 +1601,7 @@ def parse_chat_messages(
1601
and model_config.multimodal_config is not None
1602
and model_config.multimodal_config.interleave_mm_strings
1603
),
1604
+ mm_processor_kwargs=mm_processor_kwargs,
1605
)
1606
1607
conversation.extend(sub_messages)
@@ -1574,6 +1618,7 @@ async def parse_chat_messages_async(
1618
model_config: ModelConfig,
1619
content_format: ChatTemplateContentFormat,
1620
media_io_kwargs: dict[str, dict[str, Any]] | None = None,
1621
+ mm_processor_kwargs: dict[str, Any] | None = None,
1622
) -> tuple[
1623
list[ConversationMessage],
1624
MultiModalDataDict | None,
@@ -1594,6 +1639,7 @@ async def parse_chat_messages_async(
1639
and model_config.multimodal_config is not None
1640
and model_config.multimodal_config.interleave_mm_strings
1641
),
1642
+ mm_processor_kwargs=mm_processor_kwargs,
1643
)
1644
1645
conversation.extend(sub_messages)
vllm/entrypoints/openai/engine/serving.py
+1
@@ -892,6 +892,7 @@ class OpenAIServing:
892
).with_defaults(
893
default_template_kwargs,
894
default_media_io_kwargs=(mm_config.media_io_kwargs if mm_config else None),
895
+ default_mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
896
)
897
898
(conversation,), (engine_prompt,) = await renderer.render_chat_async(
vllm/model_executor/models/qwen2_5_omni_thinker.py
+15
-1
@@ -78,7 +78,11 @@ from vllm.multimodal.parse import (
78
ModalityDataItems,
79
MultiModalDataItems,
80
)
81
-from vllm.multimodal.processing import BaseDummyInputsBuilder
81
+from vllm.multimodal.processing import (
82
+ BaseDummyInputsBuilder,
83
+ ProcessorInputs,
84
+ TimingContext,
85
+)
86
from vllm.multimodal.processing.processor import (
87
BaseMultiModalProcessor,
88
MultiModalPromptUpdates,
@@ -811,6 +815,16 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
815
),
816
]
817
818
+ def _cached_apply_hf_processor(
819
+ self,
820
+ inputs: ProcessorInputs,
821
+ timing_ctx: TimingContext,
822
+ ):
823
+ mm_processor_kwargs = inputs.hf_processor_mm_kwargs
824
+ if mm_processor_kwargs.get("use_audio_in_video", False):
825
+ return self._apply_hf_processor(inputs, timing_ctx)
826
+ return super()._cached_apply_hf_processor(inputs, timing_ctx)
827
+
828
def _apply_hf_processor_main(
829
self,
830
prompt: str | list[int],
vllm/multimodal/media/audio.py
+31
@@ -82,6 +82,35 @@ def extract_audio_from_video_bytes(
82
return audio, float(native_sr)
83
84
85
+def is_video(data: bytes) -> bool:
86
+ """Check if the fetched bytes are video"""
87
+ if len(data) < 12:
88
+ return False
89
+
90
+ box_type = data[4:8]
91
+ major_brand = data[8:12]
92
+
93
+ MP4_BRANDS = {
94
+ b"mp41",
95
+ b"mp42", # MP4
96
+ b"isom", # ISO Base Media
97
+ b"iso2",
98
+ b"iso4",
99
+ b"iso5",
100
+ b"iso6",
101
+ b"M4V ",
102
+ b"M4A ", # Apple
103
+ b"avc1", # H.264
104
+ b"dash", # DASH
105
+ b"mmp4",
106
+ b"MSNV",
107
+ }
108
+
109
+ is_avi = data[:4] == b"RIFF" and major_brand == b"AVI "
110
+ is_mp4 = box_type == b"ftyp" and major_brand in MP4_BRANDS
111
+ return is_mp4 or is_avi
112
+
113
+
114
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
115
"""Configuration values can be user-provided either by --media-io-kwargs or
116
by the runtime API field "media_io_kwargs". Ensure proper validation and
@@ -100,6 +129,8 @@ class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
129
self.kwargs = kwargs
130
131
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
132
+ if is_video(data):
133
+ return extract_audio_from_video_bytes(data)
134
return librosa.load(BytesIO(data), sr=None)
135
136
def load_base64(
vllm/renderers/deepseek_v32.py
+2
@@ -50,6 +50,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
50
self.model_config,
51
content_format="string",
52
media_io_kwargs=params.media_io_kwargs,
53
+ mm_processor_kwargs=params.mm_processor_kwargs,
54
)
55
56
prompt_raw = tokenizer.apply_chat_template(
@@ -77,6 +78,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]):
78
self.model_config,
79
content_format="string",
80
media_io_kwargs=params.media_io_kwargs,
81
+ mm_processor_kwargs=params.mm_processor_kwargs,
82
)
83
84
prompt_raw = tokenizer.apply_chat_template(
vllm/renderers/grok2.py
+2
@@ -50,6 +50,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
50
self.model_config,
51
content_format="string",
52
media_io_kwargs=params.media_io_kwargs,
53
+ mm_processor_kwargs=params.mm_processor_kwargs,
54
)
55
56
prompt_raw = tokenizer.apply_chat_template(
@@ -77,6 +78,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]):
78
self.model_config,
79
content_format="string",
80
media_io_kwargs=params.media_io_kwargs,
81
+ mm_processor_kwargs=params.mm_processor_kwargs,
82
)
83
84
prompt_raw = tokenizer.apply_chat_template(
vllm/renderers/hf.py
+2
@@ -636,6 +636,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
636
model_config=model_config,
637
),
638
media_io_kwargs=params.media_io_kwargs,
639
+ mm_processor_kwargs=params.mm_processor_kwargs,
640
)
641
642
prompt_raw = safe_apply_chat_template(
@@ -691,6 +692,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
692
model_config=model_config,
693
),
694
media_io_kwargs=params.media_io_kwargs,
695
+ mm_processor_kwargs=params.mm_processor_kwargs,
696
)
697
698
prompt_raw = safe_apply_chat_template(
vllm/renderers/mistral.py
+2
@@ -91,6 +91,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
91
self.model_config,
92
content_format="string",
93
media_io_kwargs=params.media_io_kwargs,
94
+ mm_processor_kwargs=params.mm_processor_kwargs,
95
)
96
97
prompt_raw = safe_apply_chat_template(
@@ -118,6 +119,7 @@ class MistralRenderer(BaseRenderer[MistralTokenizer]):
119
self.model_config,
120
content_format="string",
121
media_io_kwargs=params.media_io_kwargs,
122
+ mm_processor_kwargs=params.mm_processor_kwargs,
123
)
124
125
prompt_raw = await self._apply_chat_template_async(
vllm/renderers/params.py
+41
-1
@@ -40,6 +40,34 @@ def merge_kwargs(
40
return defaults | {k: v for k, v in overrides.items() if v not in unset_values}
41
42
43
+def recursively_merge_kwargs(
44
+ defaults: dict[str, Any] | None,
45
+ overrides: dict[str, Any] | None,
46
+ /,
47
+ *,
48
+ unset_values: tuple[object, ...] = (None, "auto"),
49
+) -> dict[str, Any]:
50
+ if defaults is None:
51
+ defaults = {}
52
+ if overrides is None:
53
+ overrides = {}
54
+
55
+ merged = dict(defaults)
56
+
57
+ for k, v in overrides.items():
58
+ if v in unset_values:
59
+ continue
60
+
61
+ if k in merged and isinstance(merged[k], dict) and isinstance(v, dict):
62
+ merged[k] = recursively_merge_kwargs(
63
+ merged[k], v, unset_values=unset_values
64
+ )
65
+ else:
66
+ merged[k] = v
67
+
68
+ return merged
69
+
70
+
71
@dataclass(frozen=True)
72
class ChatParams:
73
"""Configuration to control how to parse chat messages."""
@@ -56,12 +84,20 @@ class ChatParams:
84
media_io_kwargs: dict[str, dict[str, Any]] | None = None
85
"""Per-modality kwargs for media I/O (loading/decoding images, videos, etc.)."""
86
87
+ mm_processor_kwargs: dict[str, Any] | None = None
88
+ """The kwargs to pass to the multi-modal processor."""
89
+
90
def with_defaults(
91
self,
92
default_chat_template_kwargs: dict[str, Any] | None = None,
93
default_media_io_kwargs: dict[str, dict[str, Any]] | None = None,
94
+ default_mm_processor_kwargs: dict[str, Any] | None = None,
95
):
64
- if not default_chat_template_kwargs and not default_media_io_kwargs:
96
+ if (
97
+ not default_chat_template_kwargs
98
+ and not default_media_io_kwargs
99
+ and not default_mm_processor_kwargs
100
+ ):
101
return self
102
103
return ChatParams(
@@ -75,6 +111,10 @@ class ChatParams:
111
default_media_io_kwargs,
112
self.media_io_kwargs,
113
),
114
+ mm_processor_kwargs=recursively_merge_kwargs(
115
+ default_mm_processor_kwargs,
116
+ self.mm_processor_kwargs,
117
+ ),
118
)
119
120
def get_apply_chat_template_kwargs(self) -> dict[str, Any]:
vllm/renderers/terratorch.py
+2
@@ -44,6 +44,7 @@ class TerratorchRenderer(BaseRenderer):
44
model_config,
45
content_format="string",
46
media_io_kwargs=params.media_io_kwargs,
47
+ mm_processor_kwargs=params.mm_processor_kwargs,
48
)
49
50
prompt = parse_dec_only_prompt([1]) # Dummy token IDs
@@ -66,6 +67,7 @@ class TerratorchRenderer(BaseRenderer):
67
model_config,
68
content_format="string",
69
media_io_kwargs=params.media_io_kwargs,
70
+ mm_processor_kwargs=params.mm_processor_kwargs,
71
)
72
73
prompt = parse_dec_only_prompt([1]) # Dummy token IDs