A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Model][VLM] Add Qwen2.5-Omni model support (thinker only) (#15130)
Signed-off-by: fyabc <suyang.fy@alibaba-inc.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Xiong Wang <wangxiongts@163.com>
Yang Fan committed
Apr 19, 2025 at 14:14 UTC
2c1bd848a668787082c0a9364d96db13a9201baa
23 files changed
+1852
-82
docs/source/models/supported_models.md
+15
@@ -1040,6 +1040,13 @@ See [this page](#generative-models) for more information on how to use generativ
1040
* ✅︎
1041
* ✅︎
1042
* ✅︎
1043
+- * `Qwen2_5OmniThinkerForConditionalGeneration`
1044
+ * Qwen2.5-Omni
1045
+ * T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup>
1046
+ * `Qwen/Qwen2.5-Omni-7B`
1047
+ *
1048
+ * ✅︎
1049
+ * ✅︎\*
1050
- * `SkyworkR1VChatModel`
1051
* Skywork-R1V-38B
1052
* T + I
@@ -1109,6 +1116,14 @@ For more details, please see: <gh-pr:4087#issuecomment-2250397630>
1116
Our PaliGemma implementations have the same problem as Gemma 3 (see above) for both V0 and V1.
1117
:::
1118
1119
+:::{note}
1120
+To use Qwen2.5-Omni, you have to install a fork of Hugging Face Transformers library from source via
1121
+`pip install git+https://github.com/BakerBunker/transformers.git@qwen25omni`.
1122
+
1123
+Read audio from video pre-processing is currently supported on V0 (but not V1), because overlapping modalities is not yet supported in V1.
1124
+`--mm-processor-kwargs '{"use_audio_in_video": True}'`.
1125
+:::
1126
+
1127
### Pooling Models
1128
1129
See [this page](pooling-models) for more information on how to use pooling models.
examples/offline_inference/audio_language.py
+31
@@ -130,6 +130,36 @@ def run_qwen2_audio(question: str, audio_count: int) -> ModelRequestData:
130
)
131
132
133
+# Qwen2.5-Omni
134
+def run_qwen2_5_omni(question: str, audio_count: int):
135
+ model_name = "Qwen/Qwen2.5-Omni-7B"
136
+
137
+ engine_args = EngineArgs(
138
+ model=model_name,
139
+ max_model_len=4096,
140
+ max_num_seqs=5,
141
+ limit_mm_per_prompt={"audio": audio_count},
142
+ )
143
+
144
+ audio_in_prompt = "".join([
145
+ "<|audio_bos|><|AUDIO|><|audio_eos|>\n" for idx in range(audio_count)
146
+ ])
147
+
148
+ default_system = (
149
+ "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
150
+ "Group, capable of perceiving auditory and visual inputs, as well as "
151
+ "generating text and speech.")
152
+
153
+ prompt = (f"<|im_start|>system\n{default_system}<|im_end|>\n"
154
+ "<|im_start|>user\n"
155
+ f"{audio_in_prompt}{question}<|im_end|>\n"
156
+ "<|im_start|>assistant\n")
157
+ return ModelRequestData(
158
+ engine_args=engine_args,
159
+ prompt=prompt,
160
+ )
161
+
162
+
163
# Ultravox 0.5-1B
164
def run_ultravox(question: str, audio_count: int) -> ModelRequestData:
165
model_name = "fixie-ai/ultravox-v0_5-llama-3_2-1b"
@@ -182,6 +212,7 @@ model_example_map = {
212
"minicpmo": run_minicpmo,
213
"phi4_mm": run_phi4mm,
214
"qwen2_audio": run_qwen2_audio,
215
+ "qwen2_5_omni": run_qwen2_5_omni,
216
"ultravox": run_ultravox,
217
"whisper": run_whisper,
218
}
examples/offline_inference/qwen2_5_omni/README.md
new
+32
@@ -0,0 +1,32 @@
1
+# Qwen2.5-Omni Offline Inference Examples
2
+
3
+This folder provides several example scripts on how to inference Qwen2.5-Omni offline.
4
+
5
+## Thinker Only
6
+
7
+```bash
8
+# Audio + image + video
9
+python examples/offline_inference/qwen2_5_omni/only_thinker.py -q mixed_modalities
10
+
11
+# Read vision and audio inputs from a single video file
12
+# NOTE: V1 engine does not support interleaved modalities yet.
13
+VLLM_USE_V1=0 python examples/offline_inference/qwen2_5_omni/only_thinker.py -q use_audio_in_video
14
+
15
+# Multiple audios
16
+VLLM_USE_V1=0 python examples/offline_inference/qwen2_5_omni/only_thinker.py -q multi_audios
17
+```
18
+
19
+This script will run the thinker part of Qwen2.5-Omni, and generate text response.
20
+
21
+You can also test Qwen2.5-Omni on a single modality:
22
+
23
+```bash
24
+# Process audio inputs
25
+python examples/offline_inference/audio_language.py --model-type qwen2_5_omni
26
+
27
+# Process image inputs
28
+python examples/offline_inference/vision_language.py --modality image --model-type qwen2_5_omni
29
+
30
+# Process video inputs
31
+python examples/offline_inference/vision_language.py --modality video --model-type qwen2_5_omni
32
+```
examples/offline_inference/qwen2_5_omni/only_thinker.py
new
+160
@@ -0,0 +1,160 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""
3
+This example shows how to use vLLM for running offline inference
4
+with the correct prompt format on Qwen2.5-Omni (thinker only).
5
+"""
6
+
7
+from typing import NamedTuple
8
+
9
+import vllm.envs as envs
10
+from vllm import LLM, SamplingParams
11
+from vllm.assets.audio import AudioAsset
12
+from vllm.assets.image import ImageAsset
13
+from vllm.assets.video import VideoAsset
14
+from vllm.utils import FlexibleArgumentParser
15
+
16
+
17
+class QueryResult(NamedTuple):
18
+ inputs: dict
19
+ limit_mm_per_prompt: dict[str, int]
20
+
21
+
22
+# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
23
+# lower-end GPUs.
24
+# Unless specified, these settings have been tested to work on a single L4.
25
+
26
+default_system = (
27
+ "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
28
+ "Group, capable of perceiving auditory and visual inputs, as well as "
29
+ "generating text and speech.")
30
+
31
+
32
+def get_mixed_modalities_query() -> QueryResult:
33
+ question = ("What is recited in the audio? "
34
+ "What is the content of this image? Why is this video funny?")
35
+ prompt = (f"<|im_start|>system\n{default_system}<|im_end|>\n"
36
+ "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
37
+ "<|vision_bos|><|IMAGE|><|vision_eos|>"
38
+ "<|vision_bos|><|VIDEO|><|vision_eos|>"
39
+ f"{question}<|im_end|>\n"
40
+ f"<|im_start|>assistant\n")
41
+ return QueryResult(
42
+ inputs={
43
+ "prompt": prompt,
44
+ "multi_modal_data": {
45
+ "audio":
46
+ AudioAsset("mary_had_lamb").audio_and_sample_rate,
47
+ "image":
48
+ ImageAsset("cherry_blossom").pil_image.convert("RGB"),
49
+ "video":
50
+ VideoAsset(name="sample_demo_1.mp4",
51
+ num_frames=16).np_ndarrays,
52
+ },
53
+ },
54
+ limit_mm_per_prompt={
55
+ "audio": 1,
56
+ "image": 1,
57
+ "video": 1
58
+ },
59
+ )
60
+
61
+
62
+def get_use_audio_in_video_query() -> QueryResult:
63
+ question = ("Describe the content of the video, "
64
+ "then convert what the baby say into text.")
65
+ prompt = (f"<|im_start|>system\n{default_system}<|im_end|>\n"
66
+ "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
67
+ f"{question}<|im_end|>\n"
68
+ f"<|im_start|>assistant\n")
69
+ asset = VideoAsset(name="sample_demo_1.mp4", num_frames=16)
70
+ audio = asset.get_audio(sampling_rate=16000)
71
+ assert not envs.VLLM_USE_V1, ("V1 does not support use_audio_in_video. "
72
+ "Please launch this example with "
73
+ "`VLLM_USE_V1=0`.")
74
+ return QueryResult(
75
+ inputs={
76
+ "prompt": prompt,
77
+ "multi_modal_data": {
78
+ "video": asset.np_ndarrays,
79
+ "audio": audio,
80
+ },
81
+ "mm_processor_kwargs": {
82
+ "use_audio_in_video": True,
83
+ },
84
+ },
85
+ limit_mm_per_prompt={
86
+ "audio": 1,
87
+ "video": 1
88
+ },
89
+ )
90
+
91
+
92
+def get_multi_audios_query() -> QueryResult:
93
+ question = "Are these two audio clips the same?"
94
+ prompt = (f"<|im_start|>system\n{default_system}<|im_end|>\n"
95
+ "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
96
+ "<|audio_bos|><|AUDIO|><|audio_eos|>"
97
+ f"{question}<|im_end|>\n"
98
+ f"<|im_start|>assistant\n")
99
+ return QueryResult(
100
+ inputs={
101
+ "prompt": prompt,
102
+ "multi_modal_data": {
103
+ "audio": [
104
+ AudioAsset("winning_call").audio_and_sample_rate,
105
+ AudioAsset("mary_had_lamb").audio_and_sample_rate,
106
+ ],
107
+ },
108
+ },
109
+ limit_mm_per_prompt={
110
+ "audio": 2,
111
+ },
112
+ )
113
+
114
+
115
+query_map = {
116
+ "mixed_modalities": get_mixed_modalities_query,
117
+ "use_audio_in_video": get_use_audio_in_video_query,
118
+ "multi_audios": get_multi_audios_query,
119
+}
120
+
121
+
122
+def main(args):
123
+ model_name = "Qwen/Qwen2.5-Omni-7B"
124
+ query_result = query_map[args.query_type]()
125
+
126
+ llm = LLM(model=model_name,
127
+ max_model_len=5632,
128
+ max_num_seqs=5,
129
+ limit_mm_per_prompt=query_result.limit_mm_per_prompt,
130
+ seed=args.seed)
131
+
132
+ # We set temperature to 0.2 so that outputs can be different
133
+ # even when all prompts are identical when running batch inference.
134
+ sampling_params = SamplingParams(temperature=0.2, max_tokens=64)
135
+
136
+ outputs = llm.generate(query_result.inputs,
137
+ sampling_params=sampling_params)
138
+
139
+ for o in outputs:
140
+ generated_text = o.outputs[0].text
141
+ print(generated_text)
142
+
143
+
144
+if __name__ == "__main__":
145
+ parser = FlexibleArgumentParser(
146
+ description='Demo on using vLLM for offline inference with '
147
+ 'audio language models')
148
+ parser.add_argument('--query-type',
149
+ '-q',
150
+ type=str,
151
+ default="mixed_modalities",
152
+ choices=query_map.keys(),
153
+ help='Query type.')
154
+ parser.add_argument("--seed",
155
+ type=int,
156
+ default=None,
157
+ help="Set the seed when initializing `vllm.LLM`.")
158
+
159
+ args = parser.parse_args()
160
+ main(args)
examples/offline_inference/vision_language.py
+37
@@ -941,6 +941,42 @@ def run_qwen2_5_vl(questions: list[str], modality: str) -> ModelRequestData:
941
)
942
943
944
+# Qwen2.5-Omni
945
+def run_qwen2_5_omni(questions: list[str], modality: str):
946
+ model_name = "Qwen/Qwen2.5-Omni-7B"
947
+
948
+ engine_args = EngineArgs(
949
+ model=model_name,
950
+ max_model_len=4096,
951
+ max_num_seqs=5,
952
+ mm_processor_kwargs={
953
+ "min_pixels": 28 * 28,
954
+ "max_pixels": 1280 * 28 * 28,
955
+ "fps": [1],
956
+ },
957
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
958
+ )
959
+
960
+ if modality == "image":
961
+ placeholder = "<|IMAGE|>"
962
+ elif modality == "video":
963
+ placeholder = "<|VIDEO|>"
964
+
965
+ default_system = (
966
+ "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
967
+ "Group, capable of perceiving auditory and visual inputs, as well as "
968
+ "generating text and speech.")
969
+
970
+ prompts = [(f"<|im_start|>system\n{default_system}<|im_end|>\n"
971
+ f"<|im_start|>user\n<|vision_bos|>{placeholder}<|vision_eos|>"
972
+ f"{question}<|im_end|>\n"
973
+ "<|im_start|>assistant\n") for question in questions]
974
+ return ModelRequestData(
975
+ engine_args=engine_args,
976
+ prompts=prompts,
977
+ )
978
+
979
+
980
# SkyworkR1V
981
def run_skyworkr1v(questions: list[str], modality: str) -> ModelRequestData:
982
assert modality == "image"
@@ -1010,6 +1046,7 @@ model_example_map = {
1046
"qwen_vl": run_qwen_vl,
1047
"qwen2_vl": run_qwen2_vl,
1048
"qwen2_5_vl": run_qwen2_5_vl,
1049
+ "qwen2_5_omni": run_qwen2_5_omni,
1050
"skywork_chat": run_skyworkr1v,
1051
"smolvlm": run_smolvlm,
1052
}
tests/models/decoder_only/vision_language/test_models.py
+17
@@ -139,6 +139,23 @@ VLM_TEST_SETTINGS = {
139
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
140
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
141
),
142
+ "qwen2_5_omni": VLMTestInfo(
143
+ models=["Qwen/Qwen2.5-Omni-7B"],
144
+ test_type=(
145
+ VLMTestType.IMAGE,
146
+ VLMTestType.MULTI_IMAGE,
147
+ VLMTestType.VIDEO
148
+ ),
149
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
150
+ img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>", # noqa: E501
151
+ video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>", # noqa: E501
152
+ max_model_len=4096,
153
+ max_num_seqs=2,
154
+ auto_cls=AutoModelForVision2Seq,
155
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
156
+ image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
157
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
158
+ ),
159
#### Extended model tests
160
"aria": VLMTestInfo(
161
models=["rhymes-ai/Aria"],
tests/models/multimodal/processing/test_common.py
+1
@@ -280,6 +280,7 @@ def _test_processing_correctness_mistral(
280
"Qwen/Qwen2-VL-2B-Instruct",
281
"Qwen/Qwen2.5-VL-3B-Instruct",
282
"Qwen/Qwen2-Audio-7B-Instruct",
283
+ "Qwen/Qwen2.5-Omni-7B",
284
"Skywork/Skywork-R1V-38B",
285
"fixie-ai/ultravox-v0_5-llama-3_2-1b",
286
"openai/whisper-large-v3",
tests/models/registry.py
+2
@@ -362,6 +362,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
362
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
363
"Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
364
min_transformers_version="4.49"), # noqa: E501
365
+ "Qwen2_5OmniModel": _HfExamplesInfo("Qwen/Qwen2.5-Omni-7B", # noqa: E501
366
+ min_transformers_version="4.52"), # noqa: E501
367
"SkyworkR1VChatModel": _HfExamplesInfo("Skywork/Skywork-R1V-38B"),
368
"SmolVLMForConditionalGeneration": _HfExamplesInfo("HuggingFaceTB/SmolVLM2-2.2B-Instruct"), # noqa: E501
369
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_5-llama-3_2-1b", # noqa: E501
vllm/assets/video.py
+17
-1
@@ -2,7 +2,7 @@
2
3
from dataclasses import dataclass
4
from functools import lru_cache
5
-from typing import Literal
5
+from typing import Literal, Optional
6
7
import cv2
8
import numpy as np
@@ -10,8 +10,15 @@ import numpy.typing as npt
10
from huggingface_hub import hf_hub_download
11
from PIL import Image
12
13
+from vllm.utils import PlaceholderModule
14
+
15
from .base import get_cache_dir
16
17
+try:
18
+ import librosa
19
+except ImportError:
20
+ librosa = PlaceholderModule("librosa") # type: ignore[assignment]
21
+
22
23
@lru_cache
24
def download_video_asset(filename: str) -> str:
@@ -85,3 +92,12 @@ class VideoAsset:
92
video_path = download_video_asset(self.name)
93
ret = video_to_ndarrays(video_path, self.num_frames)
94
return ret
95
+
96
+ def get_audio(self, sampling_rate: Optional[float] = None) -> npt.NDArray:
97
+ """
98
+ Read audio data from the video asset, used in Qwen2.5-Omni examples.
99
+
100
+ See also: examples/offline_inference/qwen2_5_omni/only_thinker.py
101
+ """
102
+ video_path = download_video_asset(self.name)
103
+ return librosa.load(video_path, sr=sampling_rate)[0]
vllm/entrypoints/chat_utils.py
+5
-1
@@ -506,6 +506,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
506
return "<|image|>"
507
if model_type in ("qwen2_vl", "qwen2_5_vl"):
508
return "<|vision_start|><|image_pad|><|vision_end|>"
509
+ if model_type == "qwen2_5_omni":
510
+ return "<|vision_start|><|IMAGE|><|vision_end|>"
511
if model_type == "molmo":
512
return ""
513
if model_type == "aria":
@@ -521,7 +523,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
523
return "<|audio|>"
524
if model_type == "phi4mm":
525
return "<|endoftext11|>" # 200011 (see vocab.json in hf model)
524
- if model_type == "qwen2_audio":
526
+ if model_type in ("qwen2_audio", "qwen2_5_omni"):
527
return (f"Audio {current_count}: "
528
f"<|audio_bos|><|AUDIO|><|audio_eos|>")
529
if model_type == "minicpmo":
@@ -530,6 +532,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
532
elif modality == "video":
533
if model_type in ("qwen2_vl", "qwen2_5_vl"):
534
return "<|vision_start|><|video_pad|><|vision_end|>"
535
+ if model_type == "qwen2_5_omni":
536
+ return "<|vision_start|><|VIDEO|><|vision_end|>"
537
if model_type in ("minicpmo", "minicpmv"):
538
return "(<video>./</video>)"
539
if model_type.startswith("llava"):
vllm/envs.py
+1
-1
@@ -747,7 +747,7 @@ def compute_hash() -> str:
747
variables, ensure that it is included in the factors list if
748
it affects the computation graph. For example, different values
749
of VLLM_PP_LAYER_PARTITION will generate different computation
750
- graphs, so it is included in the factors list. The env vars that
750
+ graphs, so it is included in the factors list. The env vars that
751
affect the choice of different kernels or attention backends should
752
also be included in the factors list.
753
"""
vllm/model_executor/layers/rotary_embedding.py
+318
-8
@@ -988,8 +988,9 @@ class MRotaryEmbedding(RotaryEmbedding):
988
key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
989
return query, key
990
991
- @staticmethod
991
+ @classmethod
992
def get_input_positions(
993
+ cls,
994
input_tokens: List[int],
995
hf_config: PretrainedConfig,
996
image_grid_thw: Optional[Union[List[List[int]], torch.Tensor]],
@@ -997,6 +998,8 @@ class MRotaryEmbedding(RotaryEmbedding):
998
second_per_grid_ts: Optional[List[float]],
999
context_len: int = 0,
1000
seq_len: Optional[int] = None,
1001
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1002
+ use_audio_in_video: bool = False,
1003
) -> Tuple[List[List[int]], int]:
1004
"""Get mrope input positions and delta value."""
1005
@@ -1006,7 +1009,7 @@ class MRotaryEmbedding(RotaryEmbedding):
1009
second_per_grid_ts
1010
1011
llm_positions, mrope_position_delta = \
1009
- MRotaryEmbedding.get_input_positions_tensor(
1012
+ cls.get_input_positions_tensor(
1013
input_tokens=input_tokens,
1014
hf_config=hf_config,
1015
image_grid_thw=image_grid_thw,
@@ -1014,12 +1017,52 @@ class MRotaryEmbedding(RotaryEmbedding):
1017
second_per_grid_ts=second_per_grid_ts,
1018
context_len=context_len,
1019
seq_len=seq_len,
1020
+ audio_feature_lengths=audio_feature_lengths,
1021
+ use_audio_in_video=use_audio_in_video,
1022
)
1023
1024
return llm_positions.tolist(), mrope_position_delta
1025
1021
- @staticmethod
1026
+ @classmethod
1027
def get_input_positions_tensor(
1028
+ cls,
1029
+ input_tokens: List[int],
1030
+ hf_config: PretrainedConfig,
1031
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
1032
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
1033
+ second_per_grid_ts: List[float],
1034
+ context_len: int = 0,
1035
+ seq_len: Optional[int] = None,
1036
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1037
+ use_audio_in_video: bool = False,
1038
+ ) -> Tuple[torch.Tensor, int]:
1039
+ from vllm.transformers_utils.config import thinker_uses_mrope
1040
+ if thinker_uses_mrope(hf_config):
1041
+ return cls._omni_get_input_positions_tensor(
1042
+ input_tokens=input_tokens,
1043
+ hf_config=hf_config,
1044
+ image_grid_thw=image_grid_thw,
1045
+ video_grid_thw=video_grid_thw,
1046
+ second_per_grid_ts=second_per_grid_ts,
1047
+ context_len=context_len,
1048
+ seq_len=seq_len,
1049
+ audio_feature_lengths=audio_feature_lengths,
1050
+ use_audio_in_video=use_audio_in_video,
1051
+ )
1052
+ else:
1053
+ return cls._vl_get_input_positions_tensor(
1054
+ input_tokens=input_tokens,
1055
+ hf_config=hf_config,
1056
+ image_grid_thw=image_grid_thw,
1057
+ video_grid_thw=video_grid_thw,
1058
+ second_per_grid_ts=second_per_grid_ts,
1059
+ context_len=context_len,
1060
+ seq_len=seq_len,
1061
+ )
1062
+
1063
+ @classmethod
1064
+ def _vl_get_input_positions_tensor(
1065
+ cls,
1066
input_tokens: List[int],
1067
hf_config: PretrainedConfig,
1068
image_grid_thw: Union[List[List[int]], torch.Tensor],
@@ -1037,11 +1080,6 @@ class MRotaryEmbedding(RotaryEmbedding):
1080
tokens_per_second = getattr(hf_config.vision_config,
1081
"tokens_per_second", 1.0)
1082
1040
- if isinstance(image_grid_thw, torch.Tensor):
1041
- image_grid_thw = image_grid_thw.tolist()
1042
- if isinstance(video_grid_thw, torch.Tensor):
1043
- video_grid_thw = video_grid_thw.tolist()
1044
-
1083
input_tokens_tensor = torch.tensor(input_tokens)
1084
vision_start_indices = torch.argwhere(
1085
input_tokens_tensor == vision_start_token_id).squeeze(1)
@@ -1121,6 +1159,226 @@ class MRotaryEmbedding(RotaryEmbedding):
1159
1160
return llm_positions, mrope_position_delta
1161
1162
+ @classmethod
1163
+ def _omni_get_input_positions_tensor(
1164
+ cls,
1165
+ input_tokens: List[int],
1166
+ hf_config: PretrainedConfig,
1167
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
1168
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
1169
+ second_per_grid_ts: Optional[List[float]] = None,
1170
+ context_len: int = 0,
1171
+ seq_len: Optional[int] = None,
1172
+ audio_feature_lengths: Optional[torch.Tensor] = None,
1173
+ use_audio_in_video: bool = False,
1174
+ ) -> Tuple[torch.Tensor, int]:
1175
+ """Get mrope input positions and delta value (Qwen2.5-Omni version).
1176
+
1177
+ Differences from MRotaryEmbedding:
1178
+ 1. Add audio support (and related `audio_feature_lengths`).
1179
+ 2. Add `use_audio_in_video` option to read audio from video inputs.
1180
+ In this case, audio and vision position ids will be split into
1181
+ chunks and interleaved.
1182
+
1183
+ Example:
1184
+
1185
+ (V_i are vision position ids, A_i are audio position ids)
1186
+
1187
+ |V_1 ... V_n|A_1 ... A_n|V_n+1 ... V_2n|A_n+1 ... A_2n|...
1188
+ |vision chunk 1|audio chunk 1|vision chunk 2|audio chunk 2 |...
1189
+ """
1190
+
1191
+ # TODO(fyabc): refactor and share more code with
1192
+ # _vl_get_input_positions_tensor.
1193
+
1194
+ thinker_config = hf_config.thinker_config
1195
+ audio_token_id = thinker_config.audio_token_index
1196
+ image_token_id = thinker_config.image_token_index
1197
+ video_token_id = thinker_config.video_token_index
1198
+ audio_start_token_id = thinker_config.audio_start_token_id
1199
+ audio_end_token_id = thinker_config.audio_end_token_id
1200
+ vision_end_token_id = thinker_config.vision_end_token_id
1201
+ seconds_per_chunk = thinker_config.seconds_per_chunk
1202
+ spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1203
+ tokens_per_second = getattr(thinker_config.vision_config,
1204
+ "tokens_per_second", 25)
1205
+
1206
+ if isinstance(image_grid_thw, list):
1207
+ image_grid_thw = torch.tensor(image_grid_thw)
1208
+ if isinstance(video_grid_thw, list):
1209
+ video_grid_thw = torch.tensor(video_grid_thw)
1210
+
1211
+ src_item = input_tokens
1212
+ audio_seqlens = audio_feature_lengths
1213
+ if not second_per_grid_ts:
1214
+ second_per_grid_ts = [1] * video_grid_thw.shape[0]
1215
+ audio_idx = 0
1216
+ video_idx = 0
1217
+ image_idx = 0
1218
+ new_src_item: list[int] = []
1219
+ llm_pos_ids_list: list[torch.Tensor] = []
1220
+
1221
+ idx = 0
1222
+ while idx < len(src_item):
1223
+ new_src_item_len = len(new_src_item)
1224
+ start_idx = llm_pos_ids_list[-1].max() + 1 if len(
1225
+ llm_pos_ids_list) > 0 else 0
1226
+ if src_item[idx] not in [
1227
+ audio_token_id, video_token_id, image_token_id
1228
+ ]:
1229
+ if src_item[idx] == vision_end_token_id and use_audio_in_video:
1230
+ start_idx -= 1
1231
+ new_src_item.append(src_item[idx])
1232
+ llm_pos_ids = torch.tensor([start_idx],
1233
+ dtype=torch.long).expand(3, -1)
1234
+ llm_pos_ids_list.append(llm_pos_ids)
1235
+ elif src_item[idx] == audio_token_id:
1236
+ assert audio_seqlens is not None
1237
+ audio_seqlen = audio_seqlens[audio_idx]
1238
+ place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1)
1239
+ new_src_item.extend([audio_token_id] * place_num)
1240
+ llm_pos_ids = torch.arange(place_num).expand(3, -1) + start_idx
1241
+ llm_pos_ids_list.append(llm_pos_ids)
1242
+ audio_idx += 1
1243
+ elif src_item[idx] == image_token_id:
1244
+ grid_t = image_grid_thw[image_idx][0]
1245
+ grid_hs = image_grid_thw[:, 1]
1246
+ grid_ws = image_grid_thw[:, 2]
1247
+ t_index = (torch.arange(grid_t) * 1 * tokens_per_second).long()
1248
+ llm_pos_ids = cls._get_llm_pos_ids_for_vision(
1249
+ start_idx, image_idx, spatial_merge_size, t_index, grid_hs,
1250
+ grid_ws)
1251
+ llm_pos_ids_list.append(llm_pos_ids)
1252
+ vision_seqlen = image_grid_thw[image_idx].prod() // (
1253
+ spatial_merge_size**2)
1254
+ new_src_item.extend([image_token_id] * vision_seqlen)
1255
+ image_idx += 1
1256
+ elif src_item[idx] == video_token_id and not use_audio_in_video:
1257
+ grid_t = video_grid_thw[video_idx][0]
1258
+ grid_hs = video_grid_thw[:, 1]
1259
+ grid_ws = video_grid_thw[:, 2]
1260
+ t_index = (torch.arange(grid_t) *
1261
+ second_per_grid_ts[video_idx] *
1262
+ tokens_per_second).long()
1263
+ llm_pos_ids = cls._get_llm_pos_ids_for_vision(
1264
+ start_idx, video_idx, spatial_merge_size, t_index, grid_hs,
1265
+ grid_ws)
1266
+ llm_pos_ids_list.append(llm_pos_ids)
1267
+ vision_seqlen = video_grid_thw[video_idx].prod() // (
1268
+ spatial_merge_size**2)
1269
+ new_src_item.extend([video_token_id] * vision_seqlen)
1270
+ video_idx += 1
1271
+ else:
1272
+ # read audio from video
1273
+ assert audio_seqlens is not None
1274
+ audio_seqlen = audio_seqlens[audio_idx]
1275
+ vision_seqlen = video_grid_thw[video_idx].prod() // (
1276
+ spatial_merge_size**2)
1277
+ grid_t = video_grid_thw[video_idx][0]
1278
+ grid_h = video_grid_thw[video_idx][1]
1279
+ grid_w = video_grid_thw[video_idx][2]
1280
+ grid_hs = video_grid_thw[:, 1]
1281
+ grid_ws = video_grid_thw[:, 2]
1282
+ t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1283
+ t_index = (torch.arange(grid_t) *
1284
+ second_per_grid_ts[video_idx] *
1285
+ tokens_per_second).long()
1286
+ t_index_split_chunk = cls._split_list_into_ranges(
1287
+ t_index, t_ntoken_per_chunk)
1288
+ new_src_item.extend([audio_start_token_id])
1289
+ start_idx -= 1
1290
+ llm_pos_ids_list.extend([
1291
+ torch.tensor([start_idx], dtype=torch.long).expand(3, -1)
1292
+ ] * 1)
1293
+ place_num = (((audio_seqlen - 1) // 2 + 1 - 2) // 2 + 1) + 2
1294
+ pure_audio_len = place_num - 2
1295
+ added_audio_len = 0
1296
+ audio_llm_pos_ids_list: List[torch.Tensor] = []
1297
+ for t_chunk in t_index_split_chunk:
1298
+ vision_ntoken_per_chunk = len(
1299
+ t_chunk) * grid_h * grid_w // (spatial_merge_size**2)
1300
+ new_src_item.extend([video_token_id] *
1301
+ vision_ntoken_per_chunk)
1302
+ vision_llm_pos_ids_list = cls._get_llm_pos_ids_for_vision(
1303
+ start_idx + 1, video_idx, spatial_merge_size, t_chunk,
1304
+ grid_hs, grid_ws).split(1, dim=1)
1305
+ llm_pos_ids_list.extend(vision_llm_pos_ids_list)
1306
+ new_src_item.extend(
1307
+ min(t_ntoken_per_chunk, pure_audio_len -
1308
+ added_audio_len) * [audio_token_id])
1309
+ audio_start_idx = start_idx if len(
1310
+ audio_llm_pos_ids_list
1311
+ ) == 0 else audio_llm_pos_ids_list[-1][0].item()
1312
+ if min(t_ntoken_per_chunk,
1313
+ pure_audio_len - added_audio_len) > 0:
1314
+ audio_llm_pos_ids_list = (torch.arange(
1315
+ min(t_ntoken_per_chunk, pure_audio_len -
1316
+ added_audio_len)).expand(3, -1) +
1317
+ audio_start_idx + 1).split(
1318
+ 1, dim=1)
1319
+ else:
1320
+ audio_llm_pos_ids_list = []
1321
+ added_audio_len += min(t_ntoken_per_chunk,
1322
+ pure_audio_len - added_audio_len)
1323
+ llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1324
+ if added_audio_len < pure_audio_len:
1325
+ new_src_item.extend(
1326
+ (pure_audio_len - added_audio_len) * [audio_token_id])
1327
+ audio_llm_pos_ids_list = (
1328
+ torch.arange(pure_audio_len - added_audio_len).expand(
1329
+ 3, -1) + llm_pos_ids_list[-1].max() + 1).split(
1330
+ 1, dim=1)
1331
+ llm_pos_ids_list.extend(audio_llm_pos_ids_list)
1332
+ llm_pos_ids_list.extend([
1333
+ torch.tensor(
1334
+ [llm_pos_ids_list[-1].max() + 1] * 3).unsqueeze(1)
1335
+ ] * 1)
1336
+ new_src_item.extend([audio_end_token_id])
1337
+ audio_idx += 1
1338
+ video_idx += 1
1339
+ # move to the next token
1340
+ idx += len(new_src_item) - new_src_item_len
1341
+
1342
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1)
1343
+ mrope_position_delta = torch.cat(llm_pos_ids_list,
1344
+ dim=1).max() + 1 - len(src_item)
1345
+ llm_positions = llm_positions[:, context_len:seq_len]
1346
+
1347
+ return llm_positions, mrope_position_delta
1348
+
1349
+ @staticmethod
1350
+ def _get_llm_pos_ids_for_vision(
1351
+ start_idx: int,
1352
+ vision_idx: int,
1353
+ spatial_merge_size: int,
1354
+ t_index: List[int],
1355
+ grid_hs: torch.Tensor,
1356
+ grid_ws: torch.Tensor,
1357
+ ) -> torch.Tensor:
1358
+ llm_pos_ids_list = []
1359
+ llm_grid_h = grid_hs[vision_idx] // spatial_merge_size
1360
+ llm_grid_w = grid_ws[vision_idx] // spatial_merge_size
1361
+ h_index = (torch.arange(llm_grid_h).view(1, -1, 1).expand(
1362
+ len(t_index), -1, llm_grid_w).flatten())
1363
+ w_index = (torch.arange(llm_grid_w).view(1, 1, -1).expand(
1364
+ len(t_index), llm_grid_h, -1).flatten())
1365
+ t_index_tensor = torch.Tensor(t_index).to(llm_grid_h.device).view(
1366
+ -1, 1).expand(-1, llm_grid_h * llm_grid_w).long().flatten()
1367
+ _llm_pos_ids = torch.stack([t_index_tensor, h_index, w_index])
1368
+ llm_pos_ids_list.append(_llm_pos_ids + start_idx)
1369
+ llm_pos_ids = torch.cat(llm_pos_ids_list, dim=1)
1370
+ return llm_pos_ids
1371
+
1372
+ @staticmethod
1373
+ def _split_list_into_ranges(lst: torch.Tensor,
1374
+ interval: int) -> List[List[int]]:
1375
+ ranges: List[List[int]] = [[]
1376
+ for _ in range((max(lst) // interval) + 1)]
1377
+ for num in lst:
1378
+ index = num // interval
1379
+ ranges[index].append(num)
1380
+ return ranges
1381
+
1382
@staticmethod
1383
def get_next_input_positions(
1384
mrope_position_delta: int,
@@ -1144,6 +1402,58 @@ class MRotaryEmbedding(RotaryEmbedding):
1402
mrope_position_delta + seq_len,
1403
).expand(3, -1)
1404
1405
+ @classmethod
1406
+ def omni_get_updates_use_audio_in_video(
1407
+ cls,
1408
+ thinker_config: PretrainedConfig,
1409
+ audio_len: int,
1410
+ video_grid_thw: Union[List[int], torch.Tensor],
1411
+ video_second_per_grid_t: float,
1412
+ ) -> List[int]:
1413
+ """Get video prompt updates when `use_audio_in_video` is True.
1414
+
1415
+ In this case, audio and vision update ids will be split into
1416
+ chunks and interleaved (details in `_omni_get_input_positions_tensor`).
1417
+
1418
+ <|video_bos|><|VIDEO|><|video_eos|> =>
1419
+ <|video_bos|><|audio_bos|>(... chunks ...)<|audio_eos|><|video_eos|>
1420
+ """
1421
+
1422
+ audio_token_id = thinker_config.audio_token_index
1423
+ video_token_id = thinker_config.video_token_index
1424
+ audio_start_token_id = thinker_config.audio_start_token_id
1425
+ audio_end_token_id = thinker_config.audio_end_token_id
1426
+ seconds_per_chunk = thinker_config.seconds_per_chunk
1427
+ spatial_merge_size = thinker_config.vision_config.spatial_merge_size
1428
+ tokens_per_second = getattr(thinker_config.vision_config,
1429
+ "tokens_per_second", 25)
1430
+
1431
+ grid_t = video_grid_thw[0]
1432
+ grid_h = video_grid_thw[1]
1433
+ grid_w = video_grid_thw[2]
1434
+ t_ntoken_per_chunk = int(tokens_per_second * seconds_per_chunk)
1435
+ t_index = (torch.arange(grid_t) * video_second_per_grid_t *
1436
+ tokens_per_second).long()
1437
+ t_index_split_chunk = cls._split_list_into_ranges(
1438
+ t_index, t_ntoken_per_chunk)
1439
+
1440
+ updates = [audio_start_token_id]
1441
+ added_audio_len = 0
1442
+ for t_chunk in t_index_split_chunk:
1443
+ vision_ntoken_per_chunk = len(t_chunk) * grid_h * grid_w // (
1444
+ spatial_merge_size**2)
1445
+ updates.extend([video_token_id] * vision_ntoken_per_chunk)
1446
+
1447
+ audio_chunk_size = min(t_ntoken_per_chunk,
1448
+ audio_len - added_audio_len)
1449
+ updates.extend(audio_chunk_size * [audio_token_id])
1450
+ added_audio_len += audio_chunk_size
1451
+ if added_audio_len < audio_len:
1452
+ updates.extend((audio_len - added_audio_len) * [audio_token_id])
1453
+ updates.extend([audio_end_token_id])
1454
+
1455
+ return updates
1456
+
1457
1458
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
1459
vllm/model_executor/models/llava_onevision.py
+19
-19
@@ -583,21 +583,21 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
583
)
584
585
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
586
- modalities = {}
586
+ mm_input_by_modality = {}
587
588
# Preserve the order of modalities if there are multiple of them
589
# from the order of kwargs.
590
for input_key in kwargs:
591
- if input_key in ("pixel_values",
592
- "image_embeds") and "images" not in modalities:
593
- modalities["images"] = self._parse_and_validate_image_input(
594
- **kwargs)
595
- if input_key in ("pixel_values_videos",
596
- "video_embeds") and "videos" not in modalities:
597
- modalities["videos"] = self._parse_and_validate_video_input(
598
- **kwargs)
591
+ if input_key in ("pixel_values", "image_embeds"
592
+ ) and "image" not in mm_input_by_modality:
593
+ mm_input_by_modality[
594
+ "image"] = self._parse_and_validate_image_input(**kwargs)
595
+ if input_key in ("pixel_values_videos", "video_embeds"
596
+ ) and "video" not in mm_input_by_modality:
597
+ mm_input_by_modality[
598
+ "video"] = self._parse_and_validate_video_input(**kwargs)
599
600
- return modalities
600
+ return mm_input_by_modality
601
602
def _select_image_features(self, image_features: torch.Tensor, *,
603
strategy: str) -> torch.Tensor:
@@ -848,8 +848,9 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
848
849
def get_multimodal_embeddings(
850
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
851
- modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
852
- if not modalities:
851
+ mm_input_by_modality = self._parse_and_validate_multimodal_inputs(
852
+ **kwargs)
853
+ if not mm_input_by_modality:
854
return None
855
856
# The result multimodal_embeddings is tuple of tensors, with each
@@ -858,14 +859,13 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
859
860
# NOTE: It is important to iterate over the keys in this dictionary
861
# to preserve the order of the modalities.
861
- for modality in modalities:
862
- if modality == "images":
863
- image_input = modalities["images"]
864
- vision_embeddings = self._process_image_input(image_input)
862
+ for modality in mm_input_by_modality:
863
+ multimodal_input = mm_input_by_modality[modality]
864
+ if modality == "image":
865
+ vision_embeddings = self._process_image_input(multimodal_input)
866
multimodal_embeddings += tuple(vision_embeddings)
866
- if modality == "videos":
867
- video_input = modalities["videos"]
868
- video_embeddings = self._process_video_pixels(video_input)
867
+ if modality == "video":
868
+ video_embeddings = self._process_video_pixels(multimodal_input)
869
multimodal_embeddings += tuple(video_embeddings)
870
871
return multimodal_embeddings
vllm/model_executor/models/qwen2_5_omni_thinker.py
new
+977
@@ -0,0 +1,977 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# Copyright 2024 The Qwen team.
3
+# Copyright 2023 The vLLM team.
4
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5
+#
6
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7
+# and OPT implementations in this library. It has been modified from its
8
+# original forms to accommodate minor architectural differences compared
9
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10
+#
11
+# Licensed under the Apache License, Version 2.0 (the "License");
12
+# you may not use this file except in compliance with the License.
13
+# You may obtain a copy of the License at
14
+#
15
+# http://www.apache.org/licenses/LICENSE-2.0
16
+#
17
+# Unless required by applicable law or agreed to in writing, software
18
+# distributed under the License is distributed on an "AS IS" BASIS,
19
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+# See the License for the specific language governing permissions and
21
+# limitations under the License.
22
+"""Inference-only Qwen2.5-Omni model (thinker part)."""
23
+
24
+from copy import copy
25
+from functools import cached_property, partial
26
+from typing import (Any, Dict, Iterable, List, Mapping, Optional, Sequence,
27
+ Set, Tuple, Union)
28
+
29
+import torch
30
+import torch.nn as nn
31
+from transformers.feature_extraction_utils import BatchFeature
32
+from transformers.models.qwen2_5_omni.configuration_qwen2_5_omni import (
33
+ Qwen2_5OmniConfig, Qwen2_5OmniThinkerConfig)
34
+from transformers.models.qwen2_5_omni.modeling_qwen2_5_omni import (
35
+ Qwen2_5OmniAudioEncoder)
36
+from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
37
+ Qwen2_5OmniProcessor)
38
+from transformers.models.whisper import WhisperFeatureExtractor
39
+
40
+from vllm.config import VllmConfig
41
+from vllm.logger import init_logger
42
+from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
43
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
44
+from vllm.model_executor.models.qwen2_5_vl import (
45
+ Qwen2_5_VisionTransformer, Qwen2_5_VLImageEmbeddingInputs,
46
+ Qwen2_5_VLImageInputs, Qwen2_5_VLImagePixelInputs,
47
+ Qwen2_5_VLProcessingInfo, Qwen2_5_VLVideoEmbeddingInputs,
48
+ Qwen2_5_VLVideoInputs, Qwen2_5_VLVideoPixelInputs)
49
+from vllm.model_executor.models.qwen2_audio import (
50
+ Qwen2AudioInputs, Qwen2AudioProcessingInfo,
51
+ _get_feat_extract_output_lengths)
52
+from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
53
+from vllm.model_executor.sampling_metadata import SamplingMetadata
54
+from vllm.multimodal import MULTIMODAL_REGISTRY
55
+from vllm.multimodal.hasher import MultiModalHasher
56
+from vllm.multimodal.inputs import (ImageItem, ModalityData,
57
+ MultiModalDataDict, MultiModalFieldConfig,
58
+ MultiModalInputs, MultiModalKwargs,
59
+ NestedTensors)
60
+from vllm.multimodal.parse import (AudioProcessorItems, DictEmbeddingItems,
61
+ ModalityDataItems, MultiModalDataItems,
62
+ MultiModalDataParser)
63
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
64
+ PlaceholderFeaturesInfo,
65
+ PromptReplacement, PromptUpdate)
66
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
67
+from vllm.sequence import IntermediateTensors
68
+from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens
69
+
70
+from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
71
+from .utils import (AutoWeightsLoader, WeightsMapper,
72
+ init_vllm_registered_model, maybe_prefix,
73
+ merge_multimodal_embeddings)
74
+
75
+try:
76
+ import flash_attn
77
+except (ImportError, ModuleNotFoundError):
78
+ flash_attn = None
79
+
80
+logger = init_logger(__name__)
81
+
82
+
83
+def _qwen2_5_omni_thinker_field_config(hf_inputs: Mapping[str, torch.Tensor]):
84
+ audio_feature_lengths = hf_inputs.get("audio_feature_lengths",
85
+ torch.empty((0, )))
86
+
87
+ image_grid_thw = hf_inputs.get("image_grid_thw", torch.empty((0, 3)))
88
+ image_grid_sizes = image_grid_thw.prod(-1)
89
+
90
+ video_grid_thw = hf_inputs.get("video_grid_thw", torch.empty((0, 3)))
91
+ video_grid_sizes = video_grid_thw.prod(-1)
92
+
93
+ return dict(
94
+ input_audio_features=MultiModalFieldConfig.flat_from_sizes(
95
+ "audio", audio_feature_lengths, dim=1),
96
+ feature_attention_mask=MultiModalFieldConfig.batched("audio"),
97
+ audio_feature_lengths=MultiModalFieldConfig.batched("audio"),
98
+ pixel_values=MultiModalFieldConfig.flat_from_sizes(
99
+ "image", image_grid_sizes),
100
+ image_embeds=MultiModalFieldConfig.flat_from_sizes(
101
+ "image", image_grid_sizes),
102
+ image_grid_thw=MultiModalFieldConfig.batched("image"),
103
+ pixel_values_videos=MultiModalFieldConfig.flat_from_sizes(
104
+ "video", video_grid_sizes),
105
+ video_embeds=MultiModalFieldConfig.flat_from_sizes(
106
+ "video", video_grid_sizes),
107
+ video_grid_thw=MultiModalFieldConfig.batched("video"),
108
+ second_per_grid_ts=MultiModalFieldConfig.batched("video"),
109
+ )
110
+
111
+
112
+class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
113
+
114
+ def _parse_audio_data(
115
+ self,
116
+ data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
117
+ ) -> ModalityDataItems[Any, Any]:
118
+ if isinstance(data, dict):
119
+ return DictEmbeddingItems(
120
+ data,
121
+ modality="audio",
122
+ required_fields={
123
+ "input_audio_features", "audio_feature_lengths"
124
+ },
125
+ fields_factory=_qwen2_5_omni_thinker_field_config,
126
+ )
127
+
128
+ return super()._parse_audio_data(data)
129
+
130
+
131
+class Qwen2_5OmniThinkerProcessingInfo(Qwen2AudioProcessingInfo,
132
+ Qwen2_5_VLProcessingInfo):
133
+
134
+ def get_hf_config(self):
135
+ return self.ctx.get_hf_config(Qwen2_5OmniConfig).thinker_config
136
+
137
+ def get_hf_processor(
138
+ self,
139
+ *,
140
+ sampling_rate: Optional[int] = None,
141
+ min_pixels: Optional[int] = None,
142
+ max_pixels: Optional[int] = None,
143
+ size: Optional[dict[str, int]] = None,
144
+ fps: Optional[Union[float, List[float]]] = None,
145
+ **kwargs: object,
146
+ ) -> Qwen2_5OmniProcessor:
147
+ if fps is not None:
148
+ kwargs["fps"] = fps
149
+ processor = self.ctx.get_hf_processor(
150
+ Qwen2_5OmniProcessor,
151
+ image_processor=self.get_image_processor(min_pixels=min_pixels,
152
+ max_pixels=max_pixels,
153
+ size=size),
154
+ **kwargs,
155
+ )
156
+ if not hasattr(processor, "audio_token"):
157
+ processor.audio_token = "<|AUDIO|>"
158
+ if not hasattr(processor, "image_token"):
159
+ processor.image_token = "<|IMAGE|>"
160
+ if not hasattr(processor, "video_token"):
161
+ processor.video_token = "<|VIDEO|>"
162
+ return processor
163
+
164
+ def get_feature_extractor(
165
+ self,
166
+ *,
167
+ sampling_rate: Optional[int] = None,
168
+ **kwargs: object,
169
+ ):
170
+ hf_processor = self.get_hf_processor(sampling_rate=sampling_rate)
171
+ feature_extractor = hf_processor.feature_extractor # type: ignore
172
+ assert isinstance(feature_extractor, WhisperFeatureExtractor)
173
+ return feature_extractor
174
+
175
+ def get_max_audio_tokens(self) -> int:
176
+ hf_config = self.get_hf_config()
177
+ max_source_position = hf_config.audio_config.max_source_positions
178
+ output_lengths = (max_source_position - 2) // 2 + 1
179
+ return output_lengths
180
+
181
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
182
+ return {"audio": None, "image": None, "video": None}
183
+
184
+ def get_mm_max_tokens_per_item(
185
+ self,
186
+ seq_len: int,
187
+ mm_counts: Mapping[str, int],
188
+ ) -> Mapping[str, int]:
189
+ return {
190
+ "audio": self.get_max_audio_tokens(),
191
+ "image": self.get_max_image_tokens(),
192
+ "video": self.get_max_video_tokens(seq_len, mm_counts),
193
+ }
194
+
195
+
196
+class Qwen2_5OmniThinkerDummyInputsBuilder(
197
+ BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]):
198
+
199
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
200
+ num_audios = mm_counts.get("audio", 0)
201
+ num_images = mm_counts.get("image", 0)
202
+ num_videos = mm_counts.get("video", 0)
203
+
204
+ hf_processor = self.info.get_hf_processor()
205
+
206
+ audio_token: str = hf_processor.audio_token
207
+ image_token: str = hf_processor.image_token
208
+ video_token: str = hf_processor.video_token
209
+
210
+ return (audio_token * num_audios + image_token * num_images +
211
+ video_token * num_videos)
212
+
213
+ # TODO: @abstractmethod after transition
214
+ def get_dummy_mm_data(
215
+ self,
216
+ seq_len: int,
217
+ mm_counts: Mapping[str, int],
218
+ ) -> MultiModalDataDict:
219
+ num_audios = mm_counts.get("audio", 0)
220
+ num_images = mm_counts.get("image", 0)
221
+ num_videos = mm_counts.get("video", 0)
222
+
223
+ feature_extractor = self.info.get_feature_extractor()
224
+
225
+ target_audio_length = min(
226
+ feature_extractor.chunk_length,
227
+ 30,
228
+ ) * feature_extractor.sampling_rate
229
+ target_width, target_height = \
230
+ self.info.get_image_size_with_most_features()
231
+ target_num_frames = \
232
+ self.info.get_num_frames_with_most_features(seq_len, mm_counts)
233
+
234
+ mm_data = {
235
+ "audio":
236
+ self._get_dummy_audios(length=target_audio_length,
237
+ num_audios=num_audios),
238
+ "image":
239
+ self._get_dummy_images(width=target_width,
240
+ height=target_height,
241
+ num_images=num_images),
242
+ "video":
243
+ self._get_dummy_videos(width=target_width,
244
+ height=target_height,
245
+ num_frames=target_num_frames,
246
+ num_videos=num_videos),
247
+ }
248
+
249
+ return mm_data
250
+
251
+
252
+class Qwen2_5OmniThinkerMultiModalProcessor(
253
+ BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]):
254
+
255
+ def _get_data_parser(self) -> MultiModalDataParser:
256
+ feature_extractor = self.info.get_feature_extractor()
257
+ return Qwen2_5OmniThinkerMultiModalDataParser(
258
+ target_sr=feature_extractor.sampling_rate)
259
+
260
+ def _call_hf_processor(
261
+ self,
262
+ prompt: str,
263
+ mm_data: Mapping[str, object],
264
+ mm_kwargs: Mapping[str, object],
265
+ ) -> BatchFeature:
266
+ mm_data = dict(mm_data)
267
+ audios = mm_data.pop("audios", [])
268
+
269
+ # NOTE: WhisperFeatureExtractor cannot handle empty list of audios
270
+ if audios:
271
+ # NOTE: Qwen2.5-Omni processor accept "audio"
272
+ mm_data["audio"] = audios
273
+ mm_kwargs = dict(**mm_kwargs, )
274
+
275
+ hf_inputs = super()._call_hf_processor(
276
+ prompt=prompt,
277
+ mm_data=mm_data,
278
+ mm_kwargs=mm_kwargs,
279
+ )
280
+
281
+ input_features = hf_inputs.pop('input_features', None)
282
+ feature_attention_mask = hf_inputs.get('feature_attention_mask', None)
283
+ if ('input_audio_features' not in hf_inputs
284
+ and input_features is not None):
285
+ if feature_attention_mask is not None:
286
+ input_features = input_features.permute(
287
+ 0, 2, 1)[feature_attention_mask.bool()].permute(1, 0)
288
+ hf_inputs['input_audio_features'] = input_features
289
+ if ('audio_feature_lengths' not in hf_inputs
290
+ and feature_attention_mask is not None):
291
+ hf_inputs['audio_feature_lengths'] = feature_attention_mask.sum(-1)
292
+ return hf_inputs
293
+
294
+ def _get_mm_fields_config(
295
+ self,
296
+ hf_inputs: BatchFeature,
297
+ hf_processor_mm_kwargs: Mapping[str, object],
298
+ ) -> Mapping[str, MultiModalFieldConfig]:
299
+ return _qwen2_5_omni_thinker_field_config(hf_inputs)
300
+
301
+ def apply(
302
+ self,
303
+ prompt: Union[str, list[int]],
304
+ mm_data: MultiModalDataDict,
305
+ hf_processor_mm_kwargs: Mapping[str, object],
306
+ return_mm_hashes: bool = False,
307
+ ) -> MultiModalInputs:
308
+ """
309
+ Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
310
+ """
311
+ mm_items = self._to_mm_items(mm_data)
312
+
313
+ # Create MM hashes to be returned (only used in V1)
314
+ # TODO: Use these hash keys for caching operations in apply_hf_processor
315
+ # instead of rehashing.
316
+
317
+ if return_mm_hashes:
318
+ model_id = self.info.model_id
319
+ mm_hashes = {
320
+ modality: [
321
+ MultiModalHasher.hash_kwargs(model_id=model_id,
322
+ **{modality: item},
323
+ **hf_processor_mm_kwargs)
324
+ for item in items
325
+ ]
326
+ for modality, items in mm_items.items()
327
+ }
328
+ else:
329
+ mm_hashes = None
330
+
331
+ (
332
+ prompt_ids,
333
+ mm_kwargs,
334
+ is_update_applied,
335
+ ) = self._cached_apply_hf_processor(
336
+ prompt,
337
+ mm_items,
338
+ hf_processor_mm_kwargs,
339
+ )
340
+
341
+ unbound_prompt_updates = self._get_prompt_updates(
342
+ mm_items,
343
+ hf_processor_mm_kwargs,
344
+ mm_kwargs,
345
+ )
346
+ mm_prompt_updates = self._bind_and_group_updates(
347
+ unbound_prompt_updates)
348
+
349
+ mm_item_counts = mm_items.get_all_counts()
350
+ self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
351
+
352
+ use_audio_in_video = hf_processor_mm_kwargs.get(
353
+ "use_audio_in_video", False)
354
+
355
+ if is_update_applied:
356
+ mm_placeholders = self._find_mm_placeholders(
357
+ mm_prompt_updates,
358
+ prompt_ids,
359
+ mm_item_counts,
360
+ )
361
+ self._validate_mm_placeholders(
362
+ mm_placeholders,
363
+ mm_item_counts,
364
+ use_audio_in_video=use_audio_in_video)
365
+
366
+ tokenizer = self.info.get_tokenizer()
367
+ prompt = decode_tokens(tokenizer, prompt_ids)
368
+ else:
369
+ (
370
+ prompt_ids,
371
+ prompt,
372
+ mm_placeholders,
373
+ ) = self._apply_prompt_updates(
374
+ prompt_ids,
375
+ mm_prompt_updates,
376
+ mm_item_counts,
377
+ )
378
+ self._validate_mm_placeholders(
379
+ mm_placeholders,
380
+ mm_item_counts,
381
+ use_audio_in_video=use_audio_in_video)
382
+
383
+ tokenizer = self.info.get_tokenizer()
384
+ prompt = decode_tokens(tokenizer, prompt_ids)
385
+
386
+ mm_placeholder_ranges = {
387
+ modality: [item.to_range() for item in placeholders]
388
+ for modality, placeholders in mm_placeholders.items()
389
+ }
390
+
391
+ if use_audio_in_video:
392
+ mm_kwargs["use_audio_in_video"] = True
393
+
394
+ return MultiModalInputs(
395
+ type="multimodal",
396
+ prompt=prompt,
397
+ prompt_token_ids=prompt_ids,
398
+ mm_kwargs=mm_kwargs,
399
+ mm_hashes=mm_hashes,
400
+ mm_placeholders=mm_placeholder_ranges,
401
+ )
402
+
403
+ def _get_prompt_updates(
404
+ self,
405
+ mm_items: MultiModalDataItems,
406
+ hf_processor_mm_kwargs: Mapping[str, Any],
407
+ out_mm_kwargs: MultiModalKwargs,
408
+ ) -> Sequence[PromptUpdate]:
409
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
410
+ tokenizer = self.info.get_tokenizer()
411
+ image_processor = self.info.get_image_processor(
412
+ **hf_processor_mm_kwargs)
413
+ vocab = tokenizer.get_vocab()
414
+
415
+ audio_token = processor.audio_token
416
+ image_token = processor.image_token
417
+ video_token = processor.video_token
418
+ audio_token_id = vocab[audio_token]
419
+ image_token_id = vocab[image_token]
420
+ video_token_id = vocab[video_token]
421
+
422
+ audio_feature_lengths = out_mm_kwargs.get("audio_feature_lengths")
423
+ feature_attention_mask = out_mm_kwargs.get("feature_attention_mask")
424
+ if audio_feature_lengths is None and feature_attention_mask is None:
425
+ audio_output_lengths = []
426
+ elif audio_feature_lengths is not None:
427
+ _, audio_output_lens = _get_feat_extract_output_lengths(
428
+ audio_feature_lengths)
429
+ audio_output_lengths = audio_output_lens.tolist()
430
+ elif feature_attention_mask is not None:
431
+ assert isinstance(feature_attention_mask, torch.Tensor)
432
+ _, audio_output_lens = _get_feat_extract_output_lengths(
433
+ feature_attention_mask.sum(-1))
434
+ audio_output_lengths = audio_output_lens.tolist()
435
+
436
+ # number of audios read from video.
437
+ audio_in_video_item_idx = 0
438
+
439
+ def get_replacement_qwen2_audio(item_idx: int):
440
+ item_idx += audio_in_video_item_idx
441
+
442
+ num_features = audio_output_lengths[item_idx]
443
+ if num_features == 0:
444
+ audios = mm_items.get_items("audio", AudioProcessorItems)
445
+ audio = audios.get(item_idx)
446
+ raise ValueError(
447
+ f"The audio {audio} (len={len(audio)}) is too short "
448
+ "to be represented inside the model")
449
+
450
+ return [audio_token_id] * num_features
451
+
452
+ def get_replacement_qwen2_vision(item_idx: int, modality: str):
453
+ grid_thw = out_mm_kwargs[f"{modality}_grid_thw"][item_idx]
454
+ assert isinstance(grid_thw, torch.Tensor)
455
+ merge_length = image_processor.merge_size**2
456
+
457
+ token_id = image_token_id if modality == "image" else video_token_id
458
+ return [token_id] * (int(grid_thw.prod()) // merge_length)
459
+
460
+ use_audio_in_video = hf_processor_mm_kwargs.get(
461
+ "use_audio_in_video", False)
462
+ thinker_config = self.info.get_hf_config()
463
+
464
+ def get_replacement_qwen2_use_audio_in_video(item_idx: int):
465
+ nonlocal audio_in_video_item_idx
466
+
467
+ audio_num_features = audio_output_lengths[audio_in_video_item_idx +
468
+ item_idx]
469
+ video_grid_thw = out_mm_kwargs["video_grid_thw"][item_idx]
470
+
471
+ audio_in_video_item_idx += 1
472
+
473
+ second_per_grid_ts = hf_processor_mm_kwargs.get(
474
+ "second_per_grid_ts", None)
475
+ if second_per_grid_ts:
476
+ video_second_per_grid_t = second_per_grid_ts[item_idx]
477
+ else:
478
+ video_second_per_grid_t = 1.0
479
+
480
+ return MRotaryEmbedding.omni_get_updates_use_audio_in_video(
481
+ thinker_config=thinker_config,
482
+ audio_len=audio_num_features,
483
+ video_grid_thw=video_grid_thw,
484
+ video_second_per_grid_t=video_second_per_grid_t,
485
+ )
486
+
487
+ video_replacement_fn = (
488
+ get_replacement_qwen2_use_audio_in_video if use_audio_in_video else
489
+ partial(get_replacement_qwen2_vision, modality="video"))
490
+
491
+ return [
492
+ PromptReplacement(
493
+ modality="audio",
494
+ target=audio_token,
495
+ replacement=get_replacement_qwen2_audio,
496
+ ),
497
+ PromptReplacement(
498
+ modality="image",
499
+ target=image_token,
500
+ replacement=partial(get_replacement_qwen2_vision,
501
+ modality="image"),
502
+ ),
503
+ PromptReplacement(
504
+ modality="video",
505
+ target=video_token,
506
+ replacement=video_replacement_fn,
507
+ ),
508
+ ]
509
+
510
+ def _apply_hf_processor_main(
511
+ self,
512
+ prompt: Union[str, list[int]],
513
+ mm_items: MultiModalDataItems,
514
+ hf_processor_mm_kwargs: Mapping[str, object],
515
+ *,
516
+ enable_hf_prompt_update: bool,
517
+ ) -> tuple[list[int], MultiModalKwargs, bool]:
518
+ """
519
+ Qwen2.5-Omni reimplements this function to handle text only.
520
+ """
521
+ print(prompt)
522
+ print(hf_processor_mm_kwargs)
523
+ print(mm_items)
524
+ if isinstance(prompt, str):
525
+ if enable_hf_prompt_update:
526
+ return self._apply_hf_processor_text_mm(
527
+ prompt_text=prompt,
528
+ mm_items=mm_items,
529
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
530
+ )
531
+ tokenizer = self.info.get_tokenizer()
532
+ prompt_ids = encode_tokens(tokenizer, prompt)
533
+ else:
534
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt)
535
+
536
+ mm_kwargs = self._apply_hf_processor_mm_only(
537
+ mm_items=mm_items,
538
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
539
+ )
540
+
541
+ return prompt_ids, mm_kwargs, False
542
+
543
+ def _apply_hf_processor_mm_only(
544
+ self,
545
+ mm_items: MultiModalDataItems,
546
+ hf_processor_mm_kwargs: Mapping[str, object],
547
+ ) -> MultiModalKwargs:
548
+ """
549
+ Qwen2.5-Omni reimplements this function to handle `use_audio_in_video`.
550
+ """
551
+ mm_counts = mm_items.get_all_counts()
552
+
553
+ use_audio_in_video = hf_processor_mm_kwargs.get(
554
+ "use_audio_in_video", False)
555
+ if use_audio_in_video and "video" in mm_counts:
556
+ assert "audio" in mm_counts
557
+ mm_counts["audio"] -= mm_counts["video"]
558
+
559
+ _, mm_kwargs, _ = self._apply_hf_processor_text_mm(
560
+ prompt_text=self.dummy_inputs.get_dummy_text(mm_counts),
561
+ mm_items=mm_items,
562
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
563
+ )
564
+
565
+ return mm_kwargs
566
+
567
+ def _validate_mm_placeholders(
568
+ self,
569
+ mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]],
570
+ mm_item_counts: Mapping[str, int],
571
+ use_audio_in_video: bool = False,
572
+ ) -> None:
573
+ if use_audio_in_video:
574
+ mm_item_counts = copy(mm_item_counts)
575
+ if "video" in mm_item_counts:
576
+ assert "audio" in mm_item_counts
577
+ mm_item_counts["audio"] -= mm_item_counts["video"]
578
+ super()._validate_mm_placeholders(mm_placeholders, mm_item_counts)
579
+
580
+
581
+class Qwen2_5OmniConditionalGenerationMixin:
582
+
583
+ def _validate_and_reshape_mm_tensor(self,
584
+ mm_input: object,
585
+ name: str,
586
+ dim: int = 0) -> torch.Tensor:
587
+ if not isinstance(mm_input, (torch.Tensor, list)):
588
+ raise ValueError(f"Incorrect type of {name}. "
589
+ f"Got type: {type(mm_input)}")
590
+ if isinstance(mm_input, torch.Tensor):
591
+ return torch.concat(list(mm_input), dim=dim)
592
+ else:
593
+ return torch.concat(mm_input, dim=dim)
594
+
595
+ def _parse_and_validate_audio_input(
596
+ self, **kwargs: object) -> Optional[Qwen2AudioInputs]:
597
+ input_audio_features = kwargs.pop('input_audio_features', None)
598
+ audio_feature_lengths = kwargs.pop('audio_feature_lengths', None)
599
+ feature_attention_mask = kwargs.pop('feature_attention_mask', None)
600
+ if input_audio_features is None:
601
+ return None
602
+ input_audio_features = self._validate_and_reshape_mm_tensor(
603
+ input_audio_features, 'input_audio_features', dim=1)
604
+ if feature_attention_mask is not None:
605
+ feature_attention_mask = self._validate_and_reshape_mm_tensor(
606
+ feature_attention_mask, 'feature_attention_mask')
607
+ if not isinstance(input_audio_features, (torch.Tensor, list)):
608
+ raise ValueError("Incorrect type of audio input features. "
609
+ f"Got type: {type(input_audio_features)}")
610
+ return Qwen2AudioInputs(input_features=input_audio_features,
611
+ audio_feature_lengths=audio_feature_lengths,
612
+ feature_attention_mask=feature_attention_mask)
613
+
614
+ def _parse_and_validate_image_input(
615
+ self,
616
+ **kwargs: Dict[str, Any],
617
+ ) -> Optional[Qwen2_5_VLImageInputs]:
618
+ pixel_values = kwargs.pop("pixel_values", None)
619
+ image_embeds = kwargs.pop("image_embeds", None)
620
+ image_grid_thw = kwargs.pop("image_grid_thw", None)
621
+
622
+ if pixel_values is None and image_embeds is None:
623
+ return None
624
+
625
+ if pixel_values is not None:
626
+ pixel_values = self._validate_and_reshape_mm_tensor(
627
+ pixel_values, "image pixel values")
628
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
629
+ image_grid_thw, "image grid_thw")
630
+
631
+ if not isinstance(pixel_values, (torch.Tensor, list)):
632
+ raise ValueError("Incorrect type of image pixel values. "
633
+ f"Got type: {type(pixel_values)}")
634
+
635
+ return Qwen2_5_VLImagePixelInputs(type="pixel_values",
636
+ pixel_values=pixel_values,
637
+ image_grid_thw=image_grid_thw)
638
+
639
+ if image_embeds is not None:
640
+ image_embeds = self._validate_and_reshape_mm_tensor(
641
+ image_embeds, "image embeds")
642
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
643
+ image_grid_thw, "image grid_thw")
644
+
645
+ if not isinstance(image_embeds, torch.Tensor):
646
+ raise ValueError("Incorrect type of image embeddings. "
647
+ f"Got type: {type(image_embeds)}")
648
+ return Qwen2_5_VLImageEmbeddingInputs(
649
+ type="image_embeds",
650
+ image_embeds=image_embeds,
651
+ image_grid_thw=image_grid_thw)
652
+
653
+ def _parse_and_validate_video_input(
654
+ self,
655
+ **kwargs: Dict[str, Any],
656
+ ) -> Optional[Qwen2_5_VLVideoInputs]:
657
+ pixel_values_videos = kwargs.pop("pixel_values_videos", None)
658
+ video_embeds = kwargs.pop("video_embeds", None)
659
+ video_grid_thw = kwargs.pop("video_grid_thw", None)
660
+
661
+ if pixel_values_videos is None and video_embeds is None:
662
+ return None
663
+
664
+ if pixel_values_videos is not None:
665
+ pixel_values_videos = self._validate_and_reshape_mm_tensor(
666
+ pixel_values_videos, "video pixel values")
667
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
668
+ video_grid_thw, "video grid_thw")
669
+
670
+ return Qwen2_5_VLVideoPixelInputs(
671
+ type="pixel_values_videos",
672
+ pixel_values_videos=pixel_values_videos,
673
+ video_grid_thw=video_grid_thw,
674
+ )
675
+
676
+ if video_embeds is not None:
677
+ video_embeds = self._validate_and_reshape_mm_tensor(
678
+ video_embeds, "video embeds")
679
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
680
+ video_grid_thw, "video grid_thw")
681
+
682
+ if not isinstance(video_embeds, torch.Tensor):
683
+ raise ValueError("Incorrect type of video embeddings. "
684
+ f"Got type: {type(video_embeds)}")
685
+ return Qwen2_5_VLVideoEmbeddingInputs(
686
+ type="video_embeds",
687
+ video_embeds=video_embeds,
688
+ video_grid_thw=video_grid_thw)
689
+
690
+ def _process_audio_input(
691
+ self,
692
+ audio_input: Qwen2AudioInputs,
693
+ audio_hashes: List[str] = None,
694
+ cached_audio_features: torch.Tensor = None,
695
+ ) -> torch.Tensor:
696
+
697
+ input_features = audio_input["input_features"]
698
+ audio_feature_lengths = audio_input["audio_feature_lengths"]
699
+ if input_features.ndim == 3:
700
+ assert input_features.shape[0] == 1
701
+ input_features = input_features.squeeze(0)
702
+ if audio_feature_lengths.ndim == 2:
703
+ assert audio_feature_lengths.shape[
704
+ 0] == 1 or audio_feature_lengths.shape[1] == 1
705
+ if audio_feature_lengths.shape[0] == 1:
706
+ audio_feature_lengths = audio_feature_lengths.squeeze(0)
707
+ else:
708
+ audio_feature_lengths = audio_feature_lengths.squeeze(1)
709
+
710
+ audio_feat_lengths, audio_output_lengths = (
711
+ self.audio_tower._get_feat_extract_output_lengths(
712
+ audio_feature_lengths))
713
+
714
+ audio_outputs = self.audio_tower(
715
+ input_features.to(self.audio_tower.dtype),
716
+ feature_lens=audio_feature_lengths,
717
+ aftercnn_lens=audio_feat_lengths,
718
+ )
719
+ audio_features = audio_outputs.last_hidden_state
720
+ return audio_features.split(audio_output_lengths.tolist())
721
+
722
+ def _process_image_input(
723
+ self,
724
+ image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
725
+ if image_input["type"] == "image_embeds":
726
+ return image_input["image_embeds"].type(self.visual.dtype)
727
+
728
+ grid_thw = image_input["image_grid_thw"]
729
+ assert grid_thw.ndim == 2
730
+
731
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
732
+ image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
733
+ # Split concatenated embeddings for each image item.
734
+ merge_size = self.visual.spatial_merge_size
735
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
736
+
737
+ return image_embeds.split(sizes.tolist())
738
+
739
+ def _process_video_input(
740
+ self,
741
+ video_input: Qwen2_5_VLVideoInputs,
742
+ video_hashes: List[str] = None,
743
+ cached_video_embeds: torch.Tensor = None) -> torch.Tensor:
744
+ if video_input["type"] == "video_embeds":
745
+ return video_input["video_embeds"].type(self.visual.dtype)
746
+
747
+ grid_thw = video_input["video_grid_thw"]
748
+ assert grid_thw.ndim == 2
749
+
750
+ pixel_values_videos = video_input["pixel_values_videos"].type(
751
+ self.visual.dtype)
752
+ video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
753
+ # Split concatenated embeddings for each video item.
754
+ merge_size = self.visual.spatial_merge_size
755
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
756
+
757
+ return video_embeds.split(sizes.tolist())
758
+
759
+
760
+@MULTIMODAL_REGISTRY.register_processor(
761
+ Qwen2_5OmniThinkerMultiModalProcessor,
762
+ info=Qwen2_5OmniThinkerProcessingInfo,
763
+ dummy_inputs=Qwen2_5OmniThinkerDummyInputsBuilder,
764
+)
765
+class Qwen2_5OmniThinkerForConditionalGeneration(
766
+ nn.Module, SupportsMultiModal, SupportsPP,
767
+ Qwen2_5OmniConditionalGenerationMixin):
768
+ hf_to_vllm_mapper = WeightsMapper(
769
+ orig_to_new_prefix={
770
+ "thinker.lm_head.": "language_model.lm_head.",
771
+ "thinker.model.": "language_model.model.",
772
+ "thinker.": "",
773
+ })
774
+
775
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
776
+ super().__init__()
777
+ thinker_config: Qwen2_5OmniThinkerConfig = (
778
+ vllm_config.model_config.hf_config.thinker_config)
779
+ quant_config = vllm_config.quant_config
780
+ multimodal_config = vllm_config.model_config.multimodal_config
781
+ self.config = thinker_config
782
+ self.multimodal_config = multimodal_config
783
+
784
+ # force "use_flash_attention_2=True" to audio tower to align
785
+ # the results.
786
+ if flash_attn is not None:
787
+ audio_config = thinker_config.audio_config
788
+ audio_config._attn_implementation_autoset = True
789
+ audio_config._attn_implementation = "flash_attention_2"
790
+ else:
791
+ logger.warning(
792
+ "flash_attn is not available, the model may not yield the "
793
+ "exactly same result as the transformers implementation "
794
+ "in the audio tower part.")
795
+
796
+ self.audio_tower = Qwen2_5OmniAudioEncoder(thinker_config.audio_config)
797
+ self.visual = Qwen2_5_VisionTransformer(
798
+ vision_config=thinker_config.vision_config,
799
+ norm_eps=getattr(thinker_config.text_config, "rms_norm_eps", 1e-6),
800
+ quant_config=quant_config,
801
+ prefix=maybe_prefix(prefix, "visual"),
802
+ )
803
+ self.quant_config = quant_config
804
+ self.language_model = init_vllm_registered_model(
805
+ vllm_config=vllm_config,
806
+ prefix=maybe_prefix(prefix, "language_model"),
807
+ hf_config=thinker_config.text_config,
808
+ architectures=["Qwen2ForCausalLM"],
809
+ )
810
+
811
+ self.make_empty_intermediate_tensors = (
812
+ self.language_model.make_empty_intermediate_tensors)
813
+
814
+ @cached_property
815
+ def sampler(self):
816
+ if hasattr(self.language_model, "sampler"):
817
+ return self.language_model.sampler
818
+
819
+ return get_sampler()
820
+
821
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
822
+ mm_input_by_modality = {}
823
+
824
+ # Preserve the order of modalities if there are multiple of them
825
+ # from the order of kwargs.
826
+ for input_key in kwargs:
827
+ if input_key in ("pixel_values", "image_embeds"
828
+ ) and "image" not in mm_input_by_modality:
829
+ mm_input_by_modality[
830
+ "image"] = self._parse_and_validate_image_input(**kwargs)
831
+ if input_key in ("pixel_values_videos", "video_embeds"
832
+ ) and "video" not in mm_input_by_modality:
833
+ mm_input_by_modality[
834
+ "video"] = self._parse_and_validate_video_input(**kwargs)
835
+ if input_key in ("input_audio_features"
836
+ ) and "audio" not in mm_input_by_modality:
837
+ mm_input_by_modality[
838
+ "audio"] = self._parse_and_validate_audio_input(**kwargs)
839
+ return mm_input_by_modality
840
+
841
+ def get_multimodal_embeddings(
842
+ self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
843
+
844
+ mm_input_by_modality = self._parse_and_validate_multimodal_inputs(
845
+ **kwargs)
846
+ if not mm_input_by_modality:
847
+ return None
848
+
849
+ # The result multimodal_embeddings is tuple of tensors, with each
850
+ # tensor correspoending to a multimodal data item (image or video).
851
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
852
+
853
+ # NOTE: It is important to iterate over the keys in this dictionary
854
+ # to preserve the order of the modalities.
855
+ for modality in mm_input_by_modality:
856
+ multimodal_input = mm_input_by_modality[modality]
857
+ if modality == "image":
858
+ vision_embeddings = self._process_image_input(multimodal_input)
859
+ multimodal_embeddings += vision_embeddings
860
+ if modality == "video":
861
+ video_embeddings = self._process_video_input(multimodal_input)
862
+ multimodal_embeddings += video_embeddings
863
+ if modality == "audio":
864
+ audio_embeddings = self._process_audio_input(multimodal_input)
865
+ multimodal_embeddings += audio_embeddings
866
+ return multimodal_embeddings
867
+
868
+ def get_input_embeddings(
869
+ self,
870
+ input_ids: torch.Tensor,
871
+ multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
872
+ ) -> torch.Tensor:
873
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
874
+ if multimodal_embeddings is not None:
875
+
876
+ # TODO (ywang96): support overlapping modalitiy embeddings so that
877
+ # `use_audio_in_video` will work on V1.
878
+ inputs_embeds = merge_multimodal_embeddings(
879
+ input_ids, inputs_embeds, multimodal_embeddings, [
880
+ self.config.image_token_index,
881
+ self.config.video_token_index,
882
+ self.config.audio_token_index
883
+ ])
884
+ return inputs_embeds
885
+
886
+ def get_multimodal_embeddings_v0(
887
+ self, **kwargs: object) -> Optional[NestedTensors]:
888
+ audio_input = self._parse_and_validate_audio_input(**kwargs)
889
+ image_input = self._parse_and_validate_image_input(**kwargs)
890
+ video_input = self._parse_and_validate_video_input(**kwargs)
891
+
892
+ if audio_input is None and image_input is None and video_input is None:
893
+ return None
894
+
895
+ multimodal_embeddings: List[Tuple[NestedTensors, str]] = []
896
+
897
+ if audio_input is not None:
898
+ audio_embeds = self._process_audio_input(audio_input)
899
+ multimodal_embeddings.append((audio_embeds, "audio"))
900
+ if image_input is not None:
901
+ image_embeds = self._process_image_input(image_input)
902
+ multimodal_embeddings.append((image_embeds, "image"))
903
+ if video_input is not None:
904
+ video_embeds = self._process_video_input(video_input)
905
+ multimodal_embeddings.append((video_embeds, "video"))
906
+ return multimodal_embeddings
907
+
908
+ def get_input_embeddings_v0(
909
+ self,
910
+ input_ids: torch.Tensor,
911
+ multimodal_embeddings: Optional[NestedTensors] = None,
912
+ ) -> torch.Tensor:
913
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
914
+ if multimodal_embeddings is None:
915
+ return inputs_embeds
916
+
917
+ for embeddings, modality in multimodal_embeddings:
918
+ if modality == "audio":
919
+ placeholder_token_id = self.config.audio_token_index
920
+ if modality == "image":
921
+ placeholder_token_id = self.config.image_token_index
922
+ if modality == "video":
923
+ placeholder_token_id = self.config.video_token_index
924
+ inputs_embeds = merge_multimodal_embeddings(
925
+ input_ids, inputs_embeds, embeddings, placeholder_token_id)
926
+ return inputs_embeds
927
+
928
+ def forward(
929
+ self,
930
+ input_ids: torch.Tensor,
931
+ positions: torch.Tensor,
932
+ intermediate_tensors: Optional[IntermediateTensors] = None,
933
+ inputs_embeds: Optional[torch.Tensor] = None,
934
+ **kwargs: object,
935
+ ) -> Union[torch.Tensor, IntermediateTensors]:
936
+ if intermediate_tensors is not None:
937
+ inputs_embeds = None
938
+
939
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
940
+ # condition is for v0 compatibility.
941
+ elif inputs_embeds is None:
942
+ multimodal_embeddings = self.get_multimodal_embeddings_v0(**kwargs)
943
+ inputs_embeds = self.get_input_embeddings_v0(
944
+ input_ids, multimodal_embeddings)
945
+ input_ids = None
946
+
947
+ hidden_states = self.language_model.model(input_ids,
948
+ positions,
949
+ intermediate_tensors,
950
+ inputs_embeds=inputs_embeds)
951
+ return hidden_states
952
+
953
+ def compute_logits(
954
+ self,
955
+ hidden_states: torch.Tensor,
956
+ sampling_metadata: SamplingMetadata,
957
+ ) -> Optional[torch.Tensor]:
958
+ return self.language_model.compute_logits(hidden_states,
959
+ sampling_metadata)
960
+
961
+ def sample(
962
+ self,
963
+ logits: torch.Tensor,
964
+ sampling_metadata: SamplingMetadata,
965
+ ) -> Optional[SamplerOutput]:
966
+ return self.language_model.sample(logits, sampling_metadata)
967
+
968
+ def load_weights(self, weights: Iterable[Tuple[str,
969
+ torch.Tensor]]) -> Set[str]:
970
+ loader = AutoWeightsLoader(
971
+ self,
972
+ skip_prefixes=["talker.", "token2wav."],
973
+ )
974
+ loaded_weights = loader.load_weights(weights,
975
+ mapper=self.hf_to_vllm_mapper)
976
+
977
+ return loaded_weights
vllm/model_executor/models/qwen2_5_vl.py
+51
-28
@@ -38,13 +38,14 @@ from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
38
Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig)
39
40
from vllm.config import VllmConfig
41
-from vllm.distributed import parallel_state, tensor_model_parallel_all_gather
41
+from vllm.distributed import parallel_state
42
from vllm.distributed import utils as dist_utils
43
from vllm.logger import init_logger
44
from vllm.model_executor import SamplingMetadata
45
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
46
from vllm.model_executor.layers.layernorm import RMSNorm
47
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48
+ QKVParallelLinear,
49
RowParallelLinear)
50
from vllm.model_executor.layers.quantization import QuantizationConfig
51
from vllm.model_executor.layers.quantization.gptq import GPTQConfig
@@ -195,6 +196,23 @@ class Qwen2_5_VisionMLP(nn.Module):
196
return x_down
197
198
199
+def all_gather_interleave(local_tensor, hidden_size: int, tp_size: int):
200
+ """All-gather the input tensor interleavely across model parallel group."""
201
+ import torch.distributed as dist
202
+ gathered_tensors = [torch.zeros_like(local_tensor) for _ in range(tp_size)]
203
+ dist.all_gather(gathered_tensors, local_tensor)
204
+
205
+ gathered_tensors_split = [
206
+ torch.split(tensor, hidden_size // tp_size, -1)
207
+ for tensor in gathered_tensors
208
+ ]
209
+ ordered_tensors = [
210
+ tensor for pair in zip(*gathered_tensors_split) for tensor in pair
211
+ ]
212
+ result_tensor = torch.cat(ordered_tensors, dim=-1)
213
+ return result_tensor
214
+
215
+
216
class Qwen2_5_VisionAttention(nn.Module):
217
218
def __init__(
@@ -214,10 +232,14 @@ class Qwen2_5_VisionAttention(nn.Module):
232
self.num_attention_heads_per_partition = dist_utils.divide(
233
num_heads, self.tp_size)
234
217
- self.qkv = ColumnParallelLinear(input_size=embed_dim,
218
- output_size=3 * projection_size,
219
- quant_config=quant_config,
220
- prefix=f"{prefix}.qkv")
235
+ self.qkv = QKVParallelLinear(
236
+ hidden_size=embed_dim,
237
+ head_size=self.hidden_size_per_attention_head,
238
+ total_num_heads=num_heads,
239
+ total_num_kv_heads=num_heads,
240
+ bias=True,
241
+ quant_config=quant_config,
242
+ prefix=f"{prefix}.qkv")
243
self.proj = RowParallelLinear(input_size=projection_size,
244
output_size=embed_dim,
245
quant_config=quant_config,
@@ -236,7 +258,8 @@ class Qwen2_5_VisionAttention(nn.Module):
258
# [s, b, 3 * head * head_dim]
259
seq_len, bs, _ = qkv.shape
260
if self.tp_size > 1:
239
- qkv = tensor_model_parallel_all_gather(qkv)
261
+ qkv = all_gather_interleave(qkv, self.qkv.hidden_size,
262
+ self.tp_size)
263
264
# [s, b, 3 * head * head_dim] -> 3 * [s, b, head * head_dim]
265
q, k, v = qkv.chunk(3, dim=2)
@@ -694,9 +717,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
717
torch.Tensor]]) -> Set[str]:
718
stacked_params_mapping = [
719
# (param_name, shard_name, shard_id)
697
- ("qkv_proj", "q_proj", "q"),
698
- ("qkv_proj", "k_proj", "k"),
699
- ("qkv_proj", "v_proj", "v"),
720
+ ("attn.qkv.", "attn.q.", "q"),
721
+ ("attn.qkv.", "attn.k.", "k"),
722
+ ("attn.qkv.", "attn.v.", "v"),
723
]
724
params_dict = dict(self.named_parameters(remove_duplicate=False))
725
loaded_params: Set[str] = set()
@@ -952,20 +975,20 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
975
return video_embeds.split(sizes.tolist())
976
977
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
955
- modalities = {}
978
+ mm_input_by_modality = {}
979
980
# Preserve the order of modalities if there are multiple of them
981
# from the order of kwargs.
982
for input_key in kwargs:
960
- if input_key in ("pixel_values",
961
- "image_embeds") and "images" not in modalities:
962
- modalities["images"] = self._parse_and_validate_image_input(
963
- **kwargs)
964
- if input_key in ("pixel_values_videos",
965
- "video_embeds") and "videos" not in modalities:
966
- modalities["videos"] = self._parse_and_validate_video_input(
967
- **kwargs)
968
- return modalities
983
+ if input_key in ("pixel_values", "image_embeds"
984
+ ) and "image" not in mm_input_by_modality:
985
+ mm_input_by_modality[
986
+ "image"] = self._parse_and_validate_image_input(**kwargs)
987
+ if input_key in ("pixel_values_videos", "video_embeds"
988
+ ) and "video" not in mm_input_by_modality:
989
+ mm_input_by_modality[
990
+ "video"] = self._parse_and_validate_video_input(**kwargs)
991
+ return mm_input_by_modality
992
993
def get_language_model(self) -> torch.nn.Module:
994
return self.language_model
@@ -973,8 +996,9 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
996
def get_multimodal_embeddings(
997
self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
998
976
- modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
977
- if not modalities:
999
+ mm_input_by_modality = self._parse_and_validate_multimodal_inputs(
1000
+ **kwargs)
1001
+ if not mm_input_by_modality:
1002
return None
1003
1004
# The result multimodal_embeddings is tuple of tensors, with each
@@ -983,14 +1007,13 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1007
1008
# NOTE: It is important to iterate over the keys in this dictionary
1009
# to preserve the order of the modalities.
986
- for modality in modalities:
987
- if modality == "images":
988
- image_input = modalities["images"]
989
- vision_embeddings = self._process_image_input(image_input)
1010
+ for modality in mm_input_by_modality:
1011
+ multimodal_input = mm_input_by_modality[modality]
1012
+ if modality == "image":
1013
+ vision_embeddings = self._process_image_input(multimodal_input)
1014
multimodal_embeddings += vision_embeddings
991
- if modality == "videos":
992
- video_input = modalities["videos"]
993
- video_embeddings = self._process_video_input(video_input)
1015
+ if modality == "video":
1016
+ video_embeddings = self._process_video_input(multimodal_input)
1017
multimodal_embeddings += video_embeddings
1018
return multimodal_embeddings
1019
vllm/model_executor/models/registry.py
+1
@@ -200,6 +200,7 @@ _MULTIMODAL_MODELS = {
200
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
201
"Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
202
"Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
203
+ "Qwen2_5OmniModel": ("qwen2_5_omni_thinker", "Qwen2_5OmniThinkerForConditionalGeneration"), # noqa: E501
204
"UltravoxModel": ("ultravox", "UltravoxModel"),
205
"Phi4MMForCausalLM": ("phi4mm", "Phi4MMForCausalLM"),
206
# [Encoder-decoder]
vllm/model_executor/models/transformers.py
+1
-1
@@ -84,7 +84,7 @@ def replace_linear_class(
84
) -> Union[ColumnParallelLinear, RowParallelLinear]:
85
"""
86
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
87
-
87
+
88
Args:
89
linear (nn.Linear): `nn.Linear` to be replaced.
90
style (str): Tensor parallel style of the new linear, e.g. "colwise".
vllm/multimodal/inputs.py
+63
-15
@@ -320,7 +320,8 @@ class MultiModalFlatField(BaseMultiModalField):
320
:func:`MultiModalFieldConfig.flat`
321
:func:`MultiModalFieldConfig.flat_from_sizes`
322
"""
323
- slices: Sequence[slice]
323
+ slices: Union[Sequence[slice], Sequence[Sequence[slice]]]
324
+ dim: int = 0
325
326
def build_elems(
327
self,
@@ -329,7 +330,10 @@ class MultiModalFlatField(BaseMultiModalField):
330
data: NestedTensors,
331
) -> Sequence[MultiModalFieldElem]:
332
field_factory = self._field_factory(modality=modality, key=key)
332
- return [field_factory(data[s]) for s in self.slices]
333
+ if not is_list_of(self.slices, slice, check="all"):
334
+ assert isinstance(data, torch.Tensor), \
335
+ "torch.Tensor is required for multiple slices"
336
+ return [field_factory(data[cast(slice, s)]) for s in self.slices]
337
338
def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
339
if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
@@ -338,10 +342,16 @@ class MultiModalFlatField(BaseMultiModalField):
342
# - produce exactly same result as `torch.concat(batch)`
343
# - will achieve zero-copy if the tensor is contiguous
344
return batch[0].contiguous()
341
- first_shape = batch[0].shape
342
- if all(elem.shape[1:] == first_shape[1:] for elem in batch):
343
- return torch.concat(batch)
345
346
+ def _expect_same_shape(tensor: torch.Tensor):
347
+ return tensor.shape[:self.dim] + tensor.shape[self.dim + 1:]
348
+
349
+ first_shape = _expect_same_shape(batch[0])
350
+
351
+ if all(_expect_same_shape(elem) == first_shape for elem in batch):
352
+ return torch.concat(batch, dim=self.dim)
353
+
354
+ assert self.dim == 0, "dim == 0 is required for nested list"
355
return [e for elem in batch for e in elem]
356
357
@@ -398,7 +408,9 @@ class MultiModalFieldConfig:
408
)
409
410
@staticmethod
401
- def flat(modality: str, slices: Sequence[slice]):
411
+ def flat(modality: str,
412
+ slices: Union[Sequence[slice], Sequence[Sequence[slice]]],
413
+ dim: int = 0):
414
"""
415
Defines a field where an element in the batch is obtained by
416
slicing along the first dimension of the underlying data.
@@ -406,8 +418,10 @@ class MultiModalFieldConfig:
418
Args:
419
modality: The modality of the multi-modal item that uses this
420
keyword argument.
409
- slices: For each multi-modal item, a slice that is used to extract
410
- the data corresponding to it.
421
+ slices: For each multi-modal item, a slice (dim=0) or a tuple of
422
+ slices (dim>0) that is used to extract the data corresponding
423
+ to it.
424
+ dim: The dimension to extract data, default to 0.
425
426
Example:
427
@@ -423,14 +437,33 @@ class MultiModalFieldConfig:
437
Element 1: [AAA]
438
Element 2: [BBBB]
439
Element 3: [CC]
440
+
441
+ .. code-block::
442
+
443
+ Given:
444
+ slices: [
445
+ (slice(None), slice(0, 3)),
446
+ (slice(None), slice(3, 7)),
447
+ (slice(None), slice(7, 9))]
448
+ dim: 1
449
+
450
+ Input:
451
+ Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
452
+
453
+ Output:
454
+ Element 1: [[A],[A],[A]]
455
+ Element 2: [[B],[B],[B],[B]]
456
+ Element 3: [[C],[C]]
457
"""
458
return MultiModalFieldConfig(
428
- field=MultiModalFlatField(slices=slices),
459
+ field=MultiModalFlatField(slices=slices, dim=dim),
460
modality=modality,
461
)
462
463
@staticmethod
433
- def flat_from_sizes(modality: str, size_per_item: torch.Tensor):
464
+ def flat_from_sizes(modality: str,
465
+ size_per_item: torch.Tensor,
466
+ dim: int = 0):
467
"""
468
Defines a field where an element in the batch is obtained by
469
slicing along the first dimension of the underlying data.
@@ -440,6 +473,7 @@ class MultiModalFieldConfig:
473
keyword argument.
474
slices: For each multi-modal item, the size of the slice that
475
is used to extract the data corresponding to it.
476
+ dim: The dimension to slice, default to 0.
477
478
Example:
479
@@ -455,6 +489,21 @@ class MultiModalFieldConfig:
489
Element 1: [AAA]
490
Element 2: [BBBB]
491
Element 3: [CC]
492
+
493
+
494
+ .. code-block::
495
+
496
+ Given:
497
+ slices: [3, 4, 2]
498
+ dim: 1
499
+
500
+ Input:
501
+ Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
502
+
503
+ Output:
504
+ Element 1: [[A],[A],[A]]
505
+ Element 2: [[B],[B],[B],[B]]
506
+ Element 3: [[C],[C]]
507
508
See also:
509
:func:`MultiModalFieldConfig.flat`
@@ -465,12 +514,11 @@ class MultiModalFieldConfig:
514
f"but found shape: {size_per_item.shape}")
515
516
slice_idxs = [0, *accumulate(size_per_item)]
468
- slices = [
469
- slice(slice_idxs[i], slice_idxs[i + 1])
470
- for i in range(len(size_per_item))
471
- ]
517
+ slices = [(slice(None, None, None), ) * dim +
518
+ (slice(slice_idxs[i], slice_idxs[i + 1]), )
519
+ for i in range(len(size_per_item))]
520
473
- return MultiModalFieldConfig.flat(modality, slices)
521
+ return MultiModalFieldConfig.flat(modality, slices, dim=dim)
522
523
@staticmethod
524
def shared(modality: str, batch_size: int):
vllm/transformers_utils/config.py
+24
-2
@@ -222,8 +222,7 @@ def patch_rope_scaling_dict(rope_scaling: Dict[str, Any]) -> None:
222
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
223
224
225
-def uses_mrope(config: PretrainedConfig) -> bool:
226
- """Detect if the model with this config uses M-ROPE."""
225
+def _uses_mrope(config: PretrainedConfig) -> bool:
226
rope_scaling = getattr(config, "rope_scaling", None)
227
if rope_scaling is None:
228
return False
@@ -231,6 +230,24 @@ def uses_mrope(config: PretrainedConfig) -> bool:
230
return "mrope_section" in rope_scaling
231
232
233
+def uses_mrope(config: PretrainedConfig) -> bool:
234
+ """Detect if the model with this config uses M-ROPE."""
235
+ return _uses_mrope(config) or thinker_uses_mrope(config)
236
+
237
+
238
+def thinker_uses_mrope(config: PretrainedConfig) -> bool:
239
+ """Detect if the model contains a thinker config and it uses M-ROPE."""
240
+ thinker_config = getattr(config, "thinker_config", None)
241
+ if thinker_config is None:
242
+ return False
243
+
244
+ thinker_text_config = getattr(thinker_config, "text_config", None)
245
+ if thinker_text_config is None:
246
+ return False
247
+
248
+ return uses_mrope(thinker_text_config)
249
+
250
+
251
def is_encoder_decoder(config: PretrainedConfig) -> bool:
252
"""Detect if the model with this config is used as an encoder/decoder."""
253
text_config = getattr(config, "text_config", None)
@@ -740,6 +757,11 @@ def get_hf_text_config(config: PretrainedConfig):
757
# if transformers config doesn't align with this assumption.
758
assert hasattr(config.text_config, "num_attention_heads")
759
return config.text_config
760
+ elif hasattr(config, "thinker_config"):
761
+ # TODO(suyang.fy): Refactor code.
762
+ # For Qwen2.5-Omni, change hf_text_config to
763
+ # thinker_config.text_config.
764
+ return config.thinker_config.text_config
765
else:
766
return config
767
vllm/transformers_utils/processor.py
+49
@@ -111,6 +111,55 @@ def cached_processor_from_config(
111
)
112
113
114
+def get_feature_extractor(
115
+ processor_name: str,
116
+ *args: Any,
117
+ trust_remote_code: bool = False,
118
+ **kwargs: Any,
119
+):
120
+ """Load an audio feature extractor for the given model name
121
+ via HuggingFace."""
122
+ # don't put this import at the top level
123
+ # it will call torch.cuda.device_count()
124
+ from transformers import AutoFeatureExtractor
125
+ from transformers.feature_extraction_utils import FeatureExtractionMixin
126
+ try:
127
+ feature_extractor = AutoFeatureExtractor.from_pretrained(
128
+ processor_name,
129
+ *args,
130
+ trust_remote_code=trust_remote_code,
131
+ **kwargs)
132
+ except ValueError as e:
133
+ # If the error pertains to the processor class not existing or not
134
+ # currently being imported, suggest using the --trust-remote-code flag.
135
+ # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
136
+ if not trust_remote_code:
137
+ err_msg = (
138
+ "Failed to load the feature extractor. If the feature "
139
+ "extractor is a custom extractor not yet available in the "
140
+ "HuggingFace transformers library, consider setting "
141
+ "`trust_remote_code=True` in LLM or using the "
142
+ "`--trust-remote-code` flag in the CLI.")
143
+ raise RuntimeError(err_msg) from e
144
+ else:
145
+ raise e
146
+ return cast(FeatureExtractionMixin, feature_extractor)
147
+
148
+
149
+cached_get_feature_extractor = lru_cache(get_feature_extractor)
150
+
151
+
152
+def cached_feature_extractor_from_config(
153
+ model_config: "ModelConfig",
154
+ **kwargs: Any,
155
+):
156
+ return cached_get_feature_extractor(
157
+ model_config.model,
158
+ trust_remote_code=model_config.trust_remote_code,
159
+ **_merge_mm_kwargs(model_config, **kwargs),
160
+ )
161
+
162
+
163
def get_image_processor(
164
processor_name: str,
165
*args: Any,
vllm/v1/worker/gpu_model_runner.py
+9
@@ -355,6 +355,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
355
image_grid_thw = []
356
video_grid_thw = []
357
second_per_grid_ts = []
358
+ audio_feature_lengths = []
359
+ use_audio_in_video = False
360
for mm_input in self.requests[req_id].mm_inputs:
361
if mm_input.get("image_grid_thw") is not None:
362
image_grid_thw.extend(
@@ -365,6 +367,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
367
if mm_input.get("second_per_grid_ts") is not None:
368
second_per_grid_ts.extend(
369
mm_input["second_per_grid_ts"])
370
+ if mm_input.get("audio_feature_lengths") is not None:
371
+ audio_feature_lengths.extend(
372
+ mm_input["audio_feature_lengths"])
373
+ if mm_input.get("use_audio_in_video") is True:
374
+ use_audio_in_video = True
375
376
hf_config = self.model_config.hf_config
377
@@ -376,6 +383,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
383
image_grid_thw=image_grid_thw,
384
video_grid_thw=video_grid_thw,
385
second_per_grid_ts=second_per_grid_ts,
386
+ audio_feature_lengths=audio_feature_lengths,
387
+ use_audio_in_video=use_audio_in_video,
388
)
389
390
req_ids_to_add.append(req_id)
vllm/worker/cpu_model_runner.py
+11
-3
@@ -382,11 +382,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
382
383
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
384
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
385
- assert image_grid_thw is not None or video_grid_thw is not None, (
386
- "mrope embedding type requires multi-modal input mapper "
387
- "returns 'image_grid_thw' or 'video_grid_thw'.")
385
+ audio_feature_lengths = mm_kwargs.get("audio_feature_lengths",
386
+ None)
387
+ assert (
388
+ image_grid_thw is not None or video_grid_thw is not None
389
+ or audio_feature_lengths is not None), (
390
+ "mrope embedding type requires multi-modal input mapper "
391
+ "returns 'image_grid_thw' or 'video_grid_thw' or "
392
+ "'audio_feature_lengths'.")
393
394
second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
395
+ use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
396
hf_config = self.runner.model_config.hf_config
397
token_ids = seq_data.get_token_ids()
398
@@ -398,6 +404,8 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
404
video_grid_thw=video_grid_thw,
405
second_per_grid_ts=second_per_grid_ts,
406
context_len=computed_len,
407
+ audio_feature_lengths=audio_feature_lengths,
408
+ use_audio_in_video=use_audio_in_video,
409
)
410
seq_data.mrope_position_delta = mrope_position_delta
411
vllm/worker/model_runner.py
+11
-3
@@ -699,11 +699,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
699
if self.runner.model_config.uses_mrope:
700
image_grid_thw = mm_kwargs.get("image_grid_thw", None)
701
video_grid_thw = mm_kwargs.get("video_grid_thw", None)
702
- assert image_grid_thw is not None or video_grid_thw is not None, (
703
- "mrope embedding type requires multi-modal input mapper "
704
- "returns 'image_grid_thw' or 'video_grid_thw'.")
702
+ audio_feature_lengths = mm_kwargs.get("audio_feature_lengths",
703
+ None)
704
+ assert (
705
+ image_grid_thw is not None or video_grid_thw is not None
706
+ or audio_feature_lengths is not None), (
707
+ "mrope embedding type requires multi-modal input mapper "
708
+ "returns 'image_grid_thw' or 'video_grid_thw' or "
709
+ "'audio_feature_lengths'.")
710
711
second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
712
+ use_audio_in_video = mm_kwargs.get("use_audio_in_video", False)
713
hf_config = self.runner.model_config.hf_config
714
715
inter_data.mrope_input_positions = [None] * inter_data.n_seqs
@@ -721,6 +727,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
727
second_per_grid_ts=second_per_grid_ts,
728
context_len=inter_data.context_lens[seq_idx],
729
seq_len=inter_data.seq_lens[seq_idx],
730
+ audio_feature_lengths=audio_feature_lengths,
731
+ use_audio_in_video=use_audio_in_video,
732
)
733
734
seq_data.mrope_position_delta = mrope_position_delta