A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[Model] Refactor Phi-4-multimodal to use merged processor and support V1 (#15477)
Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Isotr0py committed
Apr 19, 2025 at 17:26 UTC
83f3c3bd91f7ef549cb4b355659e91388cb32656
15 files changed
+792
-1220
docs/source/models/supported_models.md
+1
-1
@@ -1004,7 +1004,7 @@ See [this page](#generative-models) for more information on how to use generativ
1004
* `microsoft/Phi-4-multimodal-instruct`, etc.
1005
* ✅︎
1006
*
1007
- *
1007
+ * ✅︎
1008
- * `PixtralForConditionalGeneration`
1009
* Pixtral
1010
* T + I<sup>+</sup>
examples/offline_inference/audio_language.py
+1
-1
@@ -89,7 +89,7 @@ def run_phi4mm(question: str, audio_count: int) -> ModelRequestData:
89
engine_args = EngineArgs(
90
model=model_path,
91
trust_remote_code=True,
92
- max_model_len=4096,
92
+ max_model_len=12800,
93
max_num_seqs=2,
94
enable_lora=True,
95
max_lora_rank=320,
examples/offline_inference/vision_language.py
+4
-1
@@ -814,10 +814,13 @@ def run_phi4mm(questions: list[str], modality: str) -> ModelRequestData:
814
engine_args = EngineArgs(
815
model=model_path,
816
trust_remote_code=True,
817
- max_model_len=4096,
817
+ max_model_len=5120,
818
max_num_seqs=2,
819
+ max_num_batched_tokens=12800,
820
enable_lora=True,
821
max_lora_rank=320,
822
+ # Note - mm_processor_kwargs can also be passed to generate/chat calls
823
+ mm_processor_kwargs={"dynamic_hd": 16},
824
limit_mm_per_prompt={"image": 1},
825
)
826
examples/offline_inference/vision_language_multi_image.py
+3
-1
@@ -503,11 +503,13 @@ def load_phi4mm(question: str, image_urls: list[str]) -> ModelRequestData:
503
engine_args = EngineArgs(
504
model=model_path,
505
trust_remote_code=True,
506
- max_model_len=10000,
506
+ max_model_len=4096,
507
max_num_seqs=2,
508
limit_mm_per_prompt={"image": len(image_urls)},
509
enable_lora=True,
510
max_lora_rank=320,
511
+ # Note - mm_processor_kwargs can also be passed to generate/chat calls
512
+ mm_processor_kwargs={"dynamic_hd": 4},
513
)
514
515
placeholders = "".join(f"<|image_{i}|>"
requirements/docs.txt
+1
@@ -18,6 +18,7 @@ transformers
18
mistral_common >= 1.5.4
19
aiohttp
20
starlette
21
+scipy
22
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
23
fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
24
partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
tests/models/decoder_only/audio_language/test_ultravox.py
+18
-9
@@ -1,14 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import json
4
-from typing import Optional
4
+from typing import Any, Optional
5
6
import numpy as np
7
import pytest
8
import pytest_asyncio
9
from transformers import AutoModel, AutoTokenizer
10
11
-from vllm.multimodal.audio import resample_audio
11
+from vllm.multimodal.audio import resample_audio_librosa
12
from vllm.sequence import SampleLogprobs
13
14
from ....conftest import HfRunner, VllmRunner
@@ -43,6 +43,18 @@ def audio(request):
43
return AudioAsset(request.param)
44
45
46
+def params_kwargs_to_cli_args(params_kwargs: dict[str, Any]) -> list[str]:
47
+ """Convert kwargs to CLI args."""
48
+ args = []
49
+ for key, value in params_kwargs.items():
50
+ if isinstance(value, bool):
51
+ if value:
52
+ args.append(f"--{key.replace('_','-')}")
53
+ else:
54
+ args.append(f"--{key.replace('_','-')}={value}")
55
+ return args
56
+
57
+
58
@pytest.fixture(params=[
59
pytest.param({}, marks=pytest.mark.cpu_model),
60
pytest.param(CHUNKED_PREFILL_KWARGS),
@@ -52,10 +64,7 @@ def server(request, audio_assets):
64
"--dtype", "bfloat16", "--max-model-len", "4096", "--enforce-eager",
65
"--limit-mm-per-prompt",
66
json.dumps({"audio": len(audio_assets)}), "--trust-remote-code"
55
- ] + [
56
- f"--{key.replace('_','-')}={value}"
57
- for key, value in request.param.items()
58
- ]
67
+ ] + params_kwargs_to_cli_args(request.param)
68
69
with RemoteOpenAIServer(MODEL_NAME,
70
args,
@@ -136,9 +145,9 @@ def run_test(
145
[hf_prompt],
146
max_tokens,
147
num_logprobs=num_logprobs,
139
- audios=[(resample_audio(audio[0],
140
- orig_sr=audio[1],
141
- target_sr=16000), 16000)])
148
+ audios=[(resample_audio_librosa(audio[0],
149
+ orig_sr=audio[1],
150
+ target_sr=16000), 16000)])
151
for _, hf_prompt, audio in prompts_and_audios
152
]
153
tests/models/decoder_only/vision_language/test_phi4mm.py
+3
-3
@@ -181,7 +181,7 @@ def run_test(
181
],
182
)
183
@pytest.mark.parametrize("dtype", [target_dtype])
184
-@pytest.mark.parametrize("max_model_len", [4096])
184
+@pytest.mark.parametrize("max_model_len", [12800])
185
@pytest.mark.parametrize("max_tokens", [128])
186
@pytest.mark.parametrize("num_logprobs", [10])
187
def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
@@ -225,7 +225,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
225
],
226
)
227
@pytest.mark.parametrize("dtype", [target_dtype])
228
-@pytest.mark.parametrize("max_model_len", [10000])
228
+@pytest.mark.parametrize("max_model_len", [25600])
229
@pytest.mark.parametrize("max_tokens", [128])
230
@pytest.mark.parametrize("num_logprobs", [10])
231
def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
@@ -258,7 +258,7 @@ def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
258
259
@pytest.mark.parametrize("model", models)
260
@pytest.mark.parametrize("dtype", [target_dtype])
261
-@pytest.mark.parametrize("max_model_len", [10000])
261
+@pytest.mark.parametrize("max_model_len", [12800])
262
@pytest.mark.parametrize("max_tokens", [128])
263
@pytest.mark.parametrize("num_logprobs", [10])
264
def test_vision_speech_models(hf_runner, vllm_runner, model, dtype: str,
tests/models/multimodal/processing/test_common.py
+1
@@ -274,6 +274,7 @@ def _test_processing_correctness_mistral(
274
"nvidia/NVLM-D-72B",
275
"google/paligemma-3b-mix-224",
276
"google/paligemma2-3b-ft-docci-448",
277
+ "microsoft/Phi-4-multimodal-instruct",
278
"mistralai/Pixtral-12B-2409",
279
"mistral-community/pixtral-12b",
280
"Qwen/Qwen-VL-Chat",
tests/models/multimodal/processing/test_phi4mm.py
new
+59
@@ -0,0 +1,59 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+"""Tests for phi4mm's multimodal preprocessing kwargs."""
3
+import pytest
4
+
5
+from vllm.multimodal import MULTIMODAL_REGISTRY
6
+
7
+from ....conftest import _ImageAssets
8
+from ...utils import build_model_context
9
+
10
+
11
+@pytest.mark.parametrize("model_id", ["microsoft/Phi-4-multimodal-instruct"])
12
+# yapf: disable
13
+@pytest.mark.parametrize(
14
+ ("mm_processor_kwargs", "expected_toks_per_img"),
15
+ [
16
+ ({"dynamic_hd": 4}, 1329),
17
+ ({"dynamic_hd": 16}, 4433),
18
+ # the default num_crops of phi-4-multimodal is 36
19
+ ({}, 9585),
20
+ ])
21
+# yapf: enable
22
+@pytest.mark.parametrize("num_imgs", [1, 2])
23
+@pytest.mark.parametrize("kwargs_on_init", [True, False])
24
+def test_processor_override(
25
+ image_assets: _ImageAssets,
26
+ model_id: str,
27
+ mm_processor_kwargs: dict[str, int],
28
+ expected_toks_per_img: int,
29
+ num_imgs: int,
30
+ kwargs_on_init: bool,
31
+):
32
+ """Ensure Phi4MMMultiModalProcessor handles dynamic_hd properly."""
33
+ # Avoid initializing CUDA early
34
+ from vllm.model_executor.models.phi4mm import _IMAGE_PLACEHOLDER_TOKEN_ID
35
+
36
+ ctx = build_model_context(
37
+ model_id,
38
+ mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
39
+ limit_mm_per_prompt={"image": num_imgs},
40
+ )
41
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
42
+ hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
43
+
44
+ # Build the image str / prompt based on the number of images we pass
45
+ img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
46
+ prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
47
+
48
+ image_size = ctx.get_hf_config(
49
+ ).embd_layer["image_embd_layer"]["crop_size"]
50
+ dummy_image_size = (image_size * 7, image_size * 7)
51
+ dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
52
+ mm_data = {"image": [dummy_image] * num_imgs}
53
+
54
+ processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
55
+
56
+ # Ensure we have the right number of placeholders per num_crops size
57
+ img_tok_count = processed_inputs["prompt_token_ids"].count(
58
+ _IMAGE_PLACEHOLDER_TOKEN_ID)
59
+ assert img_tok_count == expected_toks_per_img * num_imgs
vllm/entrypoints/chat_utils.py
+2
-5
@@ -482,11 +482,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
482
if modality in ("image", "image_embeds"):
483
if model_type == "chatglm":
484
return "<|begin_of_image|><|endoftext|><|end_of_image|>"
485
- if model_type == "phi3_v":
486
- # Workaround since this token is not defined in the tokenizer
485
+ if model_type in ("phi3_v", "phi4mm"):
486
return f"<|image_{current_count}|>"
488
- if model_type == "phi4mm":
489
- return "<|endoftext10|>" # 200010 (see vocab.json in hf model)
487
if model_type in ("minicpmo", "minicpmv"):
488
return "(<image>./</image>)"
489
if model_type in ("blip-2", "florence2", "fuyu", "paligemma",
@@ -522,7 +519,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
519
if model_type == "ultravox":
520
return "<|audio|>"
521
if model_type == "phi4mm":
525
- return "<|endoftext11|>" # 200011 (see vocab.json in hf model)
522
+ return f"<|audio_{current_count}|>"
523
if model_type in ("qwen2_audio", "qwen2_5_omni"):
524
return (f"Audio {current_count}: "
525
f"<|audio_bos|><|AUDIO|><|audio_eos|>")
vllm/model_executor/models/phi3v.py
+1
-1
@@ -327,7 +327,7 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
327
*,
328
image_width: int,
329
image_height: int,
330
- processor: Optional[ProcessorMixin],
330
+ processor: Optional[ProcessorMixin] = None,
331
) -> int:
332
if processor is None:
333
processor = self.get_hf_processor()
vllm/model_executor/models/phi4mm.py
+614
-1125
@@ -1,41 +1,42 @@
1
# SPDX-License-Identifier: Apache-2.0
2
import math
3
-import re
4
-from functools import lru_cache
5
-from typing import (Dict, Iterable, List, Literal, Mapping, Optional, Tuple,
6
- TypedDict, Union)
3
+from collections.abc import Iterable, Mapping, Sequence
4
+from typing import Any, Dict, List, Literal, Optional, Tuple, TypedDict, Union
5
6
import numpy as np
9
-import scipy.signal
7
import torch
8
import torch.nn as nn
12
-import torchvision.transforms as T
13
-from PIL import Image
14
-from transformers import PretrainedConfig, SiglipVisionConfig
15
-from transformers.utils import logging
9
+from transformers import (BatchFeature, PretrainedConfig, ProcessorMixin,
10
+ SequenceFeatureExtractor, SiglipVisionConfig)
11
12
from vllm.config import VllmConfig
13
from vllm.distributed import get_pp_group
19
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
20
- InputContext)
21
-from vllm.inputs.data import TokenInputs, token_inputs
14
from vllm.model_executor.layers.logits_processor import LogitsProcessor
15
from vllm.model_executor.layers.quantization import QuantizationConfig
24
-from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
16
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
17
from vllm.model_executor.layers.vocab_parallel_embedding import (
18
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead)
19
from vllm.model_executor.models.llama import LlamaModel
20
from vllm.model_executor.models.module_mapping import MultiModelKeys
21
from vllm.model_executor.sampling_metadata import SamplingMetadata
22
from vllm.multimodal import MULTIMODAL_REGISTRY
31
-from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
32
-from vllm.sequence import IntermediateTensors, SequenceData
33
-from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
23
+from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
24
+ MultiModalKwargs, NestedTensors)
25
+from vllm.multimodal.parse import (AudioProcessorItems, ImageEmbeddingItems,
26
+ ImageProcessorItems, ImageSize,
27
+ MultiModalDataItems, MultiModalDataParser)
28
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
+ BaseProcessingInfo, PromptReplacement,
30
+ PromptUpdate)
31
+from vllm.multimodal.profiling import BaseDummyInputsBuilder
32
+from vllm.sequence import IntermediateTensors
33
+from vllm.utils import is_list_of
34
35
from .idefics2_vision_model import Idefics2VisionTransformer
36
-from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsV0Only
36
+from .interfaces import MultiModalEmbeddings, SupportsLoRA, SupportsMultiModal
37
from .phi4mm_audio import AudioEmbedding
38
-from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
38
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn, maybe_prefix,
39
+ merge_multimodal_embeddings)
40
41
# <|endoftext10|> (see vocab.json in hf model)
42
_IMAGE_PLACEHOLDER_TOKEN_ID = 200010
@@ -43,115 +44,19 @@ _IMAGE_PLACEHOLDER_TOKEN_ID = 200010
44
_AUDIO_PLACEHOLDER_TOKEN_ID = 200011
45
46
_AUDIO_MAX_SOUNDFILE_SIZE = 241_000
46
-DUMMY_SAMPLING_FREQUENCY = 16_000 # kHz
47
-
48
-DYNAMIC_HD = 16
49
-AUDIO_TOKEN_PATTERN = r"<\|audio_(\d+)\|>"
50
-IMAGE_TOKEN_PATTERN = r"<\|image_(\d+)\|>"
47
48
SIGLIP_NAME = "siglip-so400m-patch14-448"
49
VISION_ENCODER_TO_PROCESSING_CONFIG = {
50
'siglip-so400m-patch14-448': {
55
- 'dynamic_hd': 16,
51
'vit_image_size': 448,
52
'vit_patch_size': 14,
53
'token_compression_factor': 2,
54
},
55
}
61
-logger = logging.get_logger(__name__)
62
-# This is a workaround to prevent text (user input) + audio + image
63
-# from being used in the same prompt.
64
-# It includes token ids for "/n" and tokens in added_tokens_decoder
65
-# from the tokenizer_confg.json file.
66
-NON_USER_INPUT_TOKENS = {
67
- 198, 200010, 200011, 199999, 200018, 200019, 200020, 200021, 200022,
68
- 200023, 200024, 200025, 200026, 200027, 200028
69
-}
56
57
72
-def get_max_dummy_image(ctx: InputContext):
73
- hf_config = ctx.get_hf_config()
74
- vision_encoder_name = hf_config.img_processor
75
- if vision_encoder_name is None:
76
- vision_encoder_name = SIGLIP_NAME
77
- prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[vision_encoder_name]
78
- dynamic_hd_size = prepro_config['dynamic_hd']
79
- vit_image_size = prepro_config['vit_image_size']
80
-
81
- max_side = vit_image_size * dynamic_hd_size
82
- dummy_image = dummy_image_for_phi4mm(vit_image_size, max_side)
83
- return dummy_image
84
-
85
-
86
-# image token length
87
-def get_max_phi4mm_image_tokens(ctx: InputContext):
88
- dummy_image = get_max_dummy_image(ctx)
89
-
90
- hf_config = ctx.get_hf_config()
91
- vision_encoder_name = hf_config.img_processor
92
- if vision_encoder_name is None:
93
- vision_encoder_name = SIGLIP_NAME
94
- prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[vision_encoder_name]
95
- dynamic_hd_size = prepro_config['dynamic_hd']
96
- vit_image_size = prepro_config['vit_image_size']
97
- vit_patch_size = prepro_config['vit_patch_size']
98
- token_compression_factor = prepro_config['token_compression_factor']
99
-
100
- image_num_tokens = _compute_num_image_tokens(dummy_image, dynamic_hd_size,
101
- vit_image_size,
102
- vit_patch_size,
103
- token_compression_factor)
104
- return image_num_tokens
105
-
106
-
107
-def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height,
108
- image_size):
109
- best_ratio_diff = float('inf')
110
- best_ratio = (1, 1)
111
- area = width * height
112
- for ratio in target_ratios:
113
- target_aspect_ratio = ratio[0] / ratio[1]
114
- ratio_diff = abs(aspect_ratio - target_aspect_ratio)
115
- if ratio_diff < best_ratio_diff:
116
- best_ratio_diff = ratio_diff
117
- best_ratio = ratio
118
- elif ratio_diff == best_ratio_diff:
119
- if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
120
- best_ratio = ratio
121
- return best_ratio
122
-
123
-
124
-def _find_target_aspect_ratio(image, image_size, max_num, min_num):
125
- orig_width, orig_height = image.size
126
-
127
- w_crop_num = math.ceil(orig_width / float(image_size))
128
- h_crop_num = math.ceil(orig_height / float(image_size))
129
- if w_crop_num * h_crop_num > max_num:
130
- aspect_ratio = orig_width / orig_height
131
-
132
- # calculate the existing image aspect ratio
133
- target_ratios = set((i, j) for i in range(1, max_num + 1)
134
- for j in range(1, max_num + 1)
135
- if i * j <= max_num and i * j >= min_num)
136
- target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
137
-
138
- # find the closest aspect ratio to the target
139
- target_aspect_ratio = find_closest_aspect_ratio(
140
- aspect_ratio, target_ratios, orig_width, orig_height, image_size)
141
-
142
- # calculate the target width and height
143
- target_width = image_size * target_aspect_ratio[0]
144
- target_height = image_size * target_aspect_ratio[1]
145
- logger.debug("target_aspect_ratio: %s", target_aspect_ratio)
146
- else:
147
- target_width = image_size * w_crop_num
148
- target_height = image_size * h_crop_num
149
- target_aspect_ratio = (w_crop_num, h_crop_num)
150
- return target_aspect_ratio, target_height, target_width
151
-
152
-
153
-def _get_padding_size(image, target_height, target_width):
154
- orig_width, orig_height = image.size
58
+def _get_padding_size(orig_width: int, orig_height: int, target_height: int,
59
+ target_width: int):
60
ratio_width = target_width / orig_width
61
ratio_height = target_height / orig_height
62
@@ -164,181 +69,6 @@ def _get_padding_size(image, target_height, target_width):
69
return padding_height, padding_width
70
71
167
-def dynamic_preprocess(image,
168
- min_num=1,
169
- max_num=12,
170
- image_size=384,
171
- mask_size=27):
172
- target_aspect_ratio, target_height, target_width =\
173
- _find_target_aspect_ratio(
174
- image, image_size, max_num, min_num)
175
- padding_height, padding_width = _get_padding_size(image, target_height,
176
- target_width)
177
-
178
- # Calculate the ratio
179
- orig_width, orig_height = image.size
180
- ratio_width = target_width / orig_width
181
- ratio_height = target_height / orig_height
182
- if ratio_width < ratio_height:
183
- new_size = (target_width, int(orig_height * ratio_width))
184
- else:
185
- new_size = (int(orig_width * ratio_height), target_height)
186
-
187
- attention_mask = torch.ones((int(mask_size * target_aspect_ratio[1]),
188
- int(mask_size * target_aspect_ratio[0])))
189
- if padding_width >= 14:
190
- attention_mask[:, -math.floor(padding_width / 14):] = 0
191
- if padding_height >= 14:
192
- attention_mask[-math.floor(padding_height / 14):, :] = 0
193
- assert attention_mask.sum(
194
- ) > 0, f'attention mask is empty {attention_mask}'
195
-
196
- if min(new_size[1], target_height) < 10 or min(new_size[0],
197
- target_width) < 10:
198
- raise ValueError(f'the aspect ratio is very extreme {new_size}')
199
-
200
- image = T.functional.resize(
201
- image,
202
- [new_size[1], new_size[0]],
203
- )
204
-
205
- resized_img = T.functional.pad(image,
206
- [0, 0, padding_width, padding_height],
207
- fill=[255, 255, 255])
208
-
209
- return resized_img, attention_mask
210
-
211
-
212
-def pad_to_max_num_crops(images, max_crops=5):
213
- """
214
- images: B x 3 x H x W, B<=max_crops
215
- """
216
- B, _, H, W = images.shape
217
- if max_crops > B:
218
- pad = torch.zeros(max_crops - B,
219
- 3,
220
- H,
221
- W,
222
- dtype=images.dtype,
223
- device=images.device)
224
- images = torch.cat([images, pad], dim=0)
225
- return images
226
-
227
-
228
-def pad_mask_to_max_num_crops(masks, max_crops=5):
229
- B, H, W = masks.shape
230
- if max_crops > B:
231
- pad = torch.ones(max_crops - B,
232
- H,
233
- W,
234
- dtype=masks.dtype,
235
- device=masks.device)
236
- masks = torch.cat([masks, pad], dim=0)
237
- return masks
238
-
239
-
240
-def preprocess(images, dynamic_hd_size, vit_resolution, vit_patch_size):
241
-
242
- # Basic settings.
243
- img_processor = T.Compose([
244
- T.ToTensor(),
245
- T.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
246
- ])
247
- # Dynamic HD
248
- base_resolution = vit_resolution
249
- images = [image.convert('RGB') for image in images]
250
- # cover 384 and 448 resolution
251
- mask_resolution = base_resolution // vit_patch_size
252
- elems, image_attention_masks = [], []
253
- for im in images:
254
- elem, attention_mask = dynamic_preprocess(im,
255
- max_num=dynamic_hd_size,
256
- image_size=base_resolution,
257
- mask_size=mask_resolution)
258
- elems.append(elem)
259
- image_attention_masks.append(attention_mask)
260
- hd_images = [img_processor(im) for im in elems]
261
- global_image = [
262
- torch.nn.functional.interpolate(
263
- im.unsqueeze(0).float(),
264
- size=(base_resolution, base_resolution),
265
- mode='bicubic',
266
- ).to(im.dtype) for im in hd_images
267
- ]
268
- shapes = [[im.size(1), im.size(2)] for im in hd_images]
269
- mask_shapes = [[mask.size(0), mask.size(1)]
270
- for mask in image_attention_masks]
271
- global_attention_mask = [
272
- torch.ones((1, mask_resolution, mask_resolution)) for _ in hd_images
273
- ]
274
- hd_images_reshape = [
275
- im.reshape(1, 3, h // base_resolution, base_resolution,
276
- w // base_resolution, base_resolution).permute(
277
- 0, 2, 4, 1, 3, 5).reshape(-1, 3, base_resolution,
278
- base_resolution).contiguous()
279
- for im, (h, w) in zip(hd_images, shapes)
280
- ]
281
- attention_masks_reshape = [
282
- mask.reshape(1, h // mask_resolution, mask_resolution,
283
- w // mask_resolution, mask_resolution).permute(
284
- 0, 1, 3, 2, 4).reshape(-1, mask_resolution,
285
- mask_resolution).contiguous()
286
- for mask, (h, w) in zip(image_attention_masks, mask_shapes)
287
- ]
288
- # NOTE token compression is hard coded here, and odd numbers seems to fail
289
- downsample_attention_masks = [
290
- mask[:, 0::2,
291
- 0::2].reshape(1, h // mask_resolution, w // mask_resolution,
292
- mask_resolution // 2 + mask_resolution % 2,
293
- mask_resolution // 2 + mask_resolution % 2).permute(
294
- 0, 1, 3, 2, 4)
295
- for mask, (h, w) in zip(attention_masks_reshape, mask_shapes)
296
- ]
297
- downsample_attention_masks = [
298
- mask.reshape(mask.size(1) * mask.size(2),
299
- mask.size(3) * mask.size(4))
300
- for mask in downsample_attention_masks
301
- ]
302
- # NOTE hard coded number of tokens
303
- num_img_tokens = [
304
- 256 + 1 + int(mask.sum().item()) + int(mask[:, 0].sum().item()) + 16
305
- for mask in downsample_attention_masks
306
- ]
307
-
308
- hd_images_reshape = [
309
- torch.cat([_global_image] + [_im], dim=0)
310
- for _global_image, _im in zip(global_image, hd_images_reshape)
311
- ]
312
- hd_masks_reshape = [
313
- torch.cat([_global_mask] + [_mask],
314
- dim=0) for _global_mask, _mask in zip(
315
- global_attention_mask, attention_masks_reshape)
316
- ]
317
- max_crops = max([img.size(0) for img in hd_images_reshape])
318
- image_transformed = [
319
- pad_to_max_num_crops(im, max_crops) for im in hd_images_reshape
320
- ]
321
- image_transformed = torch.stack(image_transformed, dim=0)
322
- mask_transformed = [
323
- pad_mask_to_max_num_crops(mask, max_crops) \
324
- for mask in hd_masks_reshape
325
- ]
326
- mask_transformed = torch.stack(mask_transformed, dim=0)
327
-
328
- returned_input_image_embeds = image_transformed
329
- returned_image_sizes = torch.tensor(shapes, dtype=torch.long)
330
- returned_image_attention_mask = mask_transformed
331
- returned_num_img_tokens = num_img_tokens
332
-
333
- data = {
334
- "pixel_values": returned_input_image_embeds,
335
- "image_sizes": returned_image_sizes,
336
- "image_attention_mask": returned_image_attention_mask,
337
- "num_img_tokens": returned_num_img_tokens,
338
- }
339
- return data
340
-
341
-
72
def get_navit_vision_model(layer_idx: int = -1, **kwargs):
73
vision_config = {
74
"hidden_size": 1152,
@@ -492,7 +222,7 @@ class Phi4MMImageEncoder(nn.Module):
222
223
def forward(self, pixel_values: torch.FloatTensor,
224
image_sizes: torch.Tensor,
495
- image_attention_mask: torch.Tensor) -> torch.FloatTensor:
225
+ image_attention_mask: torch.Tensor) -> list[torch.FloatTensor]:
226
"""
227
process image and return vision embeddings.
228
@@ -656,785 +386,528 @@ class Phi4MMImageEncoder(nn.Module):
386
for _output_img in output_imgs:
387
img_feature_proj = self.img_projection(
388
_output_img.to(target_device).to(target_dtype))
659
- img_set_tensor.append(img_feature_proj)
389
+ img_set_tensor.append(img_feature_proj.squeeze(0))
390
391
return img_set_tensor
392
393
664
-class Phi4MMAudioFeatureInputs(TypedDict):
665
- type: Literal["audio_features"]
666
- data: Tuple[NestedTensors]
667
- """Shape: `((batch_size, num_audios, 80, M), )"""
668
-
669
-
670
-class Phi4MMAudioEmbeddingInputs(TypedDict):
671
- type: Literal["audio_embeds"]
672
- data: NestedTensors
673
- """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
674
-
675
-
676
-Phi4MMAudioInputs = Union[Phi4MMAudioFeatureInputs, Phi4MMAudioEmbeddingInputs]
677
-
678
-
679
-def speechlib_mel(sample_rate, n_fft, n_mels, fmin=None, fmax=None):
680
- """Create a Mel filter-bank the same as SpeechLib FbankFC.
681
-
682
- Args:
683
- sample_rate (int): Sample rate in Hz. number > 0 [scalar]
684
- n_fft (int): FFT size. int > 0 [scalar]
685
- n_mel (int): Mel filter size. int > 0 [scalar]
686
- fmin (float): lowest frequency (in Hz). If None use 0.0.
687
- float >= 0 [scalar]
688
- fmax: highest frequency (in Hz). If None use sample_rate / 2.
689
- float >= 0 [scalar]
690
-
691
- Returns
692
- out (numpy.ndarray): Mel transform matrix
693
- [shape=(n_mels, 1 + n_fft/2)]
394
+class Phi4MMImagePixelInputs(TypedDict):
395
+ type: Literal["pixel_values"]
396
+ data: Union[torch.Tensor, List[torch.Tensor]]
397
"""
398
+ Shape:
399
+ `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`
400
696
- bank_width = int(n_fft // 2 + 1)
697
- if fmax is None:
698
- fmax = sample_rate / 2
699
- if fmin is None:
700
- fmin = 0
701
- assert fmin >= 0, "fmin cannot be negative"
702
- assert (fmin < fmax <=
703
- sample_rate / 2), "fmax must be between (fmin, samplerate / 2]"
704
-
705
- def mel(f):
706
- return 1127.0 * np.log(1.0 + f / 700.0)
707
-
708
- def bin2mel(fft_bin):
709
- return 1127.0 * np.log(1.0 + fft_bin * sample_rate / (n_fft * 700.0))
710
-
711
- def f2bin(f):
712
- return int((f * n_fft / sample_rate) + 0.5)
713
-
714
- # Spec 1: FFT bin range [f2bin(fmin) + 1, f2bin(fmax) - 1]
715
- klo = f2bin(fmin) + 1
716
- khi = f2bin(fmax)
717
-
718
- khi = max(khi, klo)
719
-
720
- # Spec 2: SpeechLib uses triangles in Mel space
721
- mlo = mel(fmin)
722
- mhi = mel(fmax)
723
- m_centers = np.linspace(mlo, mhi, n_mels + 2)
724
- ms = (mhi - mlo) / (n_mels + 1)
725
-
726
- matrix = np.zeros((n_mels, bank_width), dtype=np.float32)
727
- for m in range(0, n_mels):
728
- left = m_centers[m]
729
- center = m_centers[m + 1]
730
- right = m_centers[m + 2]
731
- for fft_bin in range(klo, khi):
732
- mbin = bin2mel(fft_bin)
733
- if left < mbin < right:
734
- matrix[m, fft_bin] = 1.0 - abs(center - mbin) / ms
735
-
736
- return matrix
737
-
738
-
739
-class LogFbankProcessor:
740
-
741
- def __init__(self):
742
-
743
- self._eightk_method = "fillzero"
744
- self._mel = speechlib_mel(16000, 512, 80, fmin=None, fmax=7690).T
745
-
746
- self._hamming400 = np.hamming(400) # for 16k audio
747
- self._hamming200 = np.hamming(200) # for 8k audio
401
+ Note that `num_patches` may be different per batch and image,
402
+ in which case the data is passed as a list instead of a batched tensor.
403
+ """
404
749
- def extract_spectrogram(self, wav, fs):
750
- """Extract spectrogram features from waveform.
751
- Args:
752
- wav (1D array): waveform of the input
753
- fs (int): sampling rate of the waveform, 16000 or 8000.
754
- If fs=8000, the waveform will be resampled to 16000Hz.
755
- Output:
756
- log_fbank (2D array): a TxD matrix of log Mel filterbank features.
757
- D=80, and T is the number of frames.
758
- """
759
- if wav.ndim > 1:
760
- wav = np.squeeze(wav)
405
+ image_sizes: torch.Tensor
406
+ """
407
+ Shape: `(batch_size * num_images, 2)`
408
762
- # by default, we extract the mean if stereo
763
- if len(wav.shape) == 2:
764
- wav = wav.mean(1)
409
+ This should be in `(height, width)` format.
410
+ """
411
766
- # Resample to 16000 or 8000 if needed
767
- if fs > 16000:
768
- wav = scipy.signal.resample_poly(wav, 1, fs // 16000)
769
- fs = 16000
770
- elif 8000 < fs < 16000:
771
- wav = scipy.signal.resample_poly(wav, 1, fs // 8000)
772
- fs = 8000
773
- elif fs < 8000:
774
- raise RuntimeError(f"Unsupported sample rate {fs}")
775
-
776
- if fs == 8000:
777
- if self._eightk_method == "resample":
778
- # Input audio is 8 kHz. Convert to 16 kHz before feature
779
- # extraction
780
- wav = scipy.signal.resample_poly(wav, 2, 1)
781
- fs = 16000
782
- # Do nothing here for fillzero method
783
- elif fs != 16000:
784
- # Input audio is not a supported sample rate.
785
- raise RuntimeError(
786
- f"Input data using an unsupported sample rate: {fs}")
787
-
788
- preemphasis = 0.97
789
-
790
- if fs == 8000:
791
- n_fft = 256
792
- win_length = 200
793
- hop_length = 80
794
- fft_window = self._hamming200
795
- elif fs == 16000:
796
- n_fft = 512
797
- win_length = 400
798
- hop_length = 160
799
- fft_window = self._hamming400
800
-
801
- # Spec 1: SpeechLib cut remaining sample insufficient for a hop
802
- n_batch = (wav.shape[0] - win_length) // hop_length + 1
803
- # Here we don't use stride_tricks since the input array may not satisfy
804
- # memory layout requirement and we need writeable output
805
- # Here we only use list of views before copy to destination
806
- # so it is more efficient than broadcasting
807
- y_frames = np.array(
808
- [
809
- wav[_stride:_stride + win_length]
810
- for _stride in range(0, hop_length * n_batch, hop_length)
811
- ],
812
- dtype=np.float32,
813
- )
412
+ num_img_tokens: list[int]
413
+ """Shape: `(batch_size * num_images)`"""
414
815
- # Spec 2: SpeechLib applies preemphasis within each batch
816
- y_frames_prev = np.roll(y_frames, 1, axis=1)
817
- y_frames_prev[:, 0] = y_frames_prev[:, 1]
818
- y_frames = (y_frames - preemphasis * y_frames_prev) * 32768
415
+ image_attention_mask: torch.Tensor
416
+ """Shape: `(batch_size * num_images, H_mask, W_mask)`"""
417
820
- S = np.fft.rfft(fft_window * y_frames, n=n_fft,
821
- axis=1).astype(np.complex64)
418
823
- if fs == 8000:
824
- # Need to pad the output to look like 16 kHz data but with zeros in
825
- # the 4 to 8 kHz bins.
826
- frames, bins = S.shape
827
- padarray = np.zeros((frames, bins))
828
- S = np.concatenate((S[:, 0:-1], padarray),
829
- axis=1) # Nyquist bin gets set to zero
419
+class Phi4MMImageEmbeddingInputs(TypedDict):
420
+ type: Literal["image_embeds"]
421
+ data: Union[torch.Tensor, List[torch.Tensor]]
422
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
423
831
- spec = np.abs(S).astype(np.float32)
832
- return spec
424
+ `hidden_size` must match the hidden size of language model backbone.
425
+ """
426
834
- def extract_features(self, wav, fs):
835
- """Extract log filterbank features from waveform.
836
- Args:
837
- wav (1D array): waveform of the input
838
- fs (int): sampling rate of the waveform, 16000 or 8000.
839
- If fs=8000, the waveform will be resampled to 16000Hz.
840
- Output:
841
- log_fbank (2D array): a TxD matrix of log Mel filterbank features.
842
- D=80, and T is the number of frames.
843
- """
844
- spec = self.extract_spectrogram(wav, fs)
845
- spec_power = spec**2
427
847
- fbank_power = np.clip(spec_power.dot(self._mel), 1.0, None)
848
- log_fbank = np.log(fbank_power).astype(np.float32)
428
+class Phi4MMAudioFeatureInputs(TypedDict):
429
+ type: Literal["audio_features"]
430
+ data: Union[torch.Tensor, List[torch.Tensor]]
431
+ """Shape: `(batch_size * num_audios, 80, M)"""
432
850
- return log_fbank
433
434
+class Phi4MMAudioEmbeddingInputs(TypedDict):
435
+ type: Literal["audio_embeds"]
436
+ data: NestedTensors
437
+ """Shape: `(batch_size, num_audios, audio_feature_size, hidden_size)"""
438
853
-@lru_cache
854
-def audio_feature_extractor() -> LogFbankProcessor:
855
- # Creates an instance of the audio processor, needed to extract the
856
- # the audio features from the sound file
857
- # LRU cache ensures that we only make one copy
858
- return LogFbankProcessor()
439
440
+Phi4MMImageInput = Union[Phi4MMImagePixelInputs, Phi4MMImageEmbeddingInputs]
441
+Phi4MMAudioInputs = Union[Phi4MMAudioFeatureInputs, Phi4MMAudioEmbeddingInputs]
442
861
-def _compute_num_image_tokens(image, dynamic_hd_size, vit_image_size,
862
- vit_patch_size, token_compression_factor):
863
- """
864
- compute the number of tokens an image is expected to take up considering
865
- the image encoder architecture and exclude output features containing
866
- only padding pixels
443
868
- for siglip, vit_image_size=448, vit_patch_size=14, so output will be
869
- 32x32 feature map
870
- NOTE right now, Phi4MM uses hard-coded token_compression_factor=2
871
- """
872
- assert vit_image_size % vit_patch_size == 0, \
873
- "vit_image_size must be divisible by vit_patch_size"
874
- assert vit_image_size // vit_patch_size % token_compression_factor == 0, \
875
- "vit_image_size // vit_patch_size must be divisible by "\
876
- "token_compression_factor"
877
-
878
- target_aspect_ratio, target_height, target_width = (
879
- _find_target_aspect_ratio(image,
880
- vit_image_size,
881
- dynamic_hd_size,
882
- min_num=1))
883
- assert target_aspect_ratio[
884
- 0] * vit_image_size == target_width, \
885
- f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}"
886
- assert target_aspect_ratio[
887
- 1] * vit_image_size == target_height, \
888
- f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}"
889
- assert (target_height % vit_image_size == 0
890
- and target_width % vit_image_size == 0)
891
-
892
- padding_height, padding_width = _get_padding_size(image, target_height,
893
- target_width)
894
- assert padding_width == 0 or padding_height == 0, \
895
- "padding_width or padding_height must be 0"
896
-
897
- target_feat_width = target_width // vit_patch_size
898
- target_feat_height = target_height // vit_patch_size
899
- if padding_width >= vit_patch_size:
900
- assert padding_height == 0, "padding_height not 0"
901
- non_pad_feat_width = target_feat_width - math.floor(
902
- padding_width / vit_patch_size)
903
- non_pad_feat_height = target_feat_height
904
- elif padding_height >= vit_patch_size:
905
- assert padding_width == 0, "padding_width not 0"
906
- non_pad_feat_height = target_feat_height - math.floor(
907
- padding_height / vit_patch_size)
908
- non_pad_feat_width = target_feat_width
909
- else:
910
- # small padding shorter than a vit patch
911
- non_pad_feat_width = target_feat_width
912
- non_pad_feat_height = target_feat_height
913
-
914
- feat_width = non_pad_feat_width // token_compression_factor
915
- feat_height = non_pad_feat_height // token_compression_factor
916
- # NOTE it's possible that the non-padding feature is not divisible
917
- if non_pad_feat_width % token_compression_factor != 0:
918
- feat_width += 1
919
- if non_pad_feat_height % token_compression_factor != 0:
920
- feat_height += 1
921
- num_hd_patch_tokens = feat_width * feat_height
922
- num_hd_newline_tokens = feat_height
923
- vit_feature_size = vit_image_size // vit_patch_size
924
- num_global_image_tokens = (vit_feature_size // token_compression_factor)**2
925
- num_sep_tokens = 1
926
- num_global_image_newline_tokens = \
927
- vit_feature_size // token_compression_factor
928
-
929
- return (num_global_image_tokens + num_sep_tokens + num_hd_patch_tokens +
930
- num_hd_newline_tokens + num_global_image_newline_tokens)
931
-
932
-
933
-def compute_logfbank_output_size(wav_length: int, fs: int) -> Tuple[int, int]:
444
+def cat_with_pad(tensors, dim, padding_value=0):
445
"""
935
- Compute the output size of the `extract_features` method.
936
-
937
- Args:
938
- wav_length (int): Length of the input waveform in samples.
939
- fs (int): Sampling rate of the waveform, either 16000 or 8000.
940
-
941
- Returns:
942
- tuple (int, int): Output size as (T, D), where:
943
- T: Number of time frames.
944
- D: Number of Mel filterbank bins (80).
446
+ cat along dim, while pad to max for all other dims
447
"""
448
+ ndim = tensors[0].dim()
449
+ assert all(
450
+ t.dim() == ndim for t in
451
+ tensors[1:]), "All tensors must have the same number of dimensions"
452
947
- # Resample to 16000 or 8000 if needed
948
- if fs > 16000:
949
- wav_length //= fs // 16000
950
- fs = 16000
951
- elif 8000 <= fs < 16000:
952
- # We'll resample to 16K from 8K
953
- wav_length *= 2
954
- fs = 16000
955
- elif fs < 8000:
956
- raise RuntimeError(f"Unsupported sample rate {fs}")
453
+ out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]
454
+ out_size[dim] = sum(t.shape[dim] for t in tensors)
455
+ output = tensors[0].new_full(out_size, padding_value)
456
958
- # Spectrogram parameters for 16 kHz
959
- win_length = 400 # Frame length in samples
960
- hop_length = 160 # Frame shift in samples
961
- mel_bins = 80 # Number of mel filterbank bins
457
+ index = 0
458
+ for t in tensors:
459
+ # Create a slice list where every dimension except dim is full slice
460
+ slices = [slice(0, t.shape[d]) for d in range(ndim)]
461
+ # Update only the concat dimension slice
462
+ slices[dim] = slice(index, index + t.shape[dim])
463
963
- # Calculate number of frames (T)
964
- T = (wav_length - win_length) // hop_length + 1
965
- if T < 1:
966
- raise ValueError("Waveform too short for given parameters.")
464
+ output[slices] = t
465
+ index += t.shape[dim]
466
968
- # Return time frames (T) and mel bins (D)
969
- return T, mel_bins
467
+ return output
468
469
972
-def _get_audio_embed_sizes(audios, ctx: InputContext):
973
- """
974
- Get the audio embedding sizes for each audio file.
470
+class Phi4MMProcessingInfo(BaseProcessingInfo):
471
976
- Args:
977
- audios (List[Tuple[np.ndarray, int]]): List of audio files as tuples of
978
- waveform and sample rate.
979
- ctx (InputContext): Input context.
472
+ def get_hf_processor(
473
+ self,
474
+ *,
475
+ dynamic_hd: Optional[int] = None,
476
+ **kwargs: object,
477
+ ) -> ProcessorMixin:
478
+ if dynamic_hd is not None:
479
+ kwargs["dynamic_hd"] = dynamic_hd
480
981
- Returns:
982
- List[int]: List of audio embedding sizes.
983
- """
984
- audio_embed_sizes = []
985
- for audio in audios:
986
- audio_data, sf = audio
987
- audio_frames, _ = compute_logfbank_output_size(len(audio_data), sf)
988
- audio_embed_size = _compute_audio_embed_size(ctx.get_hf_config(),
989
- audio_frames)
990
- audio_embed_sizes.append(audio_embed_size)
991
- return audio_embed_sizes
481
+ return self.ctx.get_hf_processor(**kwargs)
482
483
+ @property
484
+ def image_tokens(self) -> list[str]:
485
+ return [f"<|image_{i+1}|>" for i in range(100)]
486
994
-def _get_audio_id_to_input_ids(audios, ctx: InputContext, prompt_str=""):
995
- """
996
- The following will search for `<|audio_{idx}|>` tokens and
997
- return a mapping of audio placeholder tokens to audio placeholder token ids
998
- based on the size of the audio embeddings.
487
+ @property
488
+ def audio_tokens(self) -> list[str]:
489
+ return [f"<|audio_{i+1}|>" for i in range(100)]
490
1000
- Args:
1001
- audios (List[Tuple[np.ndarray, int]]): List of audio files as tuples of
1002
- waveform and sample rate.
1003
- ctx (InputContext): Input context.
1004
- prompt_str (str): The prompt string.
491
+ def get_dynamic_hd(
492
+ self,
493
+ processor: Optional[ProcessorMixin] = None,
494
+ ) -> int:
495
+ if processor is None:
496
+ processor = self.get_hf_processor()
497
+ image_processor = processor.image_processor
498
+ return image_processor.dynamic_hd
499
1006
- Returns:
1007
- Dict[str, List[int]]: Mapping of audio placeholder tokens to audio
1008
- placeholder token ids.
500
+ def get_feature_extractor(self) -> SequenceFeatureExtractor:
501
+ return self.get_hf_processor().audio_processor
502
1010
- """
1011
- if len(audios) == 0:
1012
- return {}
1013
-
1014
- audio_embed_sizes = _get_audio_embed_sizes(audios, ctx)
1015
- audio_ids = re.findall(AUDIO_TOKEN_PATTERN, prompt_str)
1016
- audio_ids = [int(audio_id) for audio_id in audio_ids]
1017
- assert len(audio_ids) == len(
1018
- audio_embed_sizes
1019
- ), "Number of audio tokens and audio features do not match"
1020
- assert tuple(audio_ids) == tuple(range(1,
1021
- len(audio_ids) +
1022
- 1)), "Audio ids are not in order!"
1023
- audio_id_to_input_ids = {
1024
- f"<|audio_{audio_id}|>":
1025
- [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
1026
- for audio_id, audio_embed_size in zip(audio_ids, audio_embed_sizes)
1027
- }
503
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
504
+ return {"audio": None, "image": None}
505
1029
- return audio_id_to_input_ids
1030
-
1031
-
1032
-def _count_image_tokens(images, ctx: InputContext):
1033
- hf_config = ctx.get_hf_config()
1034
- vision_encoder_name = hf_config.img_processor
1035
- if vision_encoder_name is None:
1036
- vision_encoder_name = SIGLIP_NAME
1037
- prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[vision_encoder_name]
1038
- dynamic_hd_size = prepro_config['dynamic_hd']
1039
- vit_image_size = prepro_config['vit_image_size']
1040
- vit_patch_size = prepro_config['vit_patch_size']
1041
- token_compression_factor = prepro_config['token_compression_factor']
1042
-
1043
- image_token_counts = [
1044
- _compute_num_image_tokens(image, dynamic_hd_size, vit_image_size,
1045
- vit_patch_size, token_compression_factor)
1046
- for image in images
1047
- ]
1048
- return image_token_counts
1049
-
1050
-
1051
-def _get_image_id_to_input_ids(images, prompt, ctx: InputContext):
1052
- if len(images) == 0:
1053
- return {}
1054
-
1055
- image_ids = re.findall(IMAGE_TOKEN_PATTERN, prompt)
1056
- image_ids = [int(image_id) for image_id in image_ids]
1057
- assert len(image_ids) == len(
1058
- set(image_ids)), "Duplicate image tokens in prompt"
1059
- assert len(images) == len(
1060
- image_ids), "Number of images and image tokens in prompt do not match"
1061
-
1062
- # NOTE the following assertion is not strictly necessary
1063
- assert tuple(image_ids) == tuple(range(1,
1064
- len(image_ids) +
1065
- 1)), "Image ids are not in order"
1066
-
1067
- image_token_counts = _count_image_tokens(images, ctx)
1068
- image_id_to_input_ids = {
1069
- f"<|image_{image_id}|>": [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_tokens
1070
- for image_id, num_tokens in zip(image_ids, image_token_counts)
1071
- }
1072
- return image_id_to_input_ids
506
+ def get_mm_max_tokens_per_item(
507
+ self,
508
+ seq_len: int,
509
+ mm_counts: Mapping[str, int],
510
+ ) -> Mapping[str, int]:
511
+ return {
512
+ "image": self.get_max_image_tokens(),
513
+ "audio": self.get_max_audio_tokens(),
514
+ }
515
516
+ def get_max_audio_tokens(self) -> int:
517
+ sr = self.get_feature_extractor().sampling_rate
518
+ num_frames = self.get_audio_num_frames(_AUDIO_MAX_SOUNDFILE_SIZE, sr)
519
+ return self._compute_audio_embed_size(num_frames)
520
1075
-def input_processor_for_phi4mm(ctx: InputContext,
1076
- inputs: DecoderOnlyInputs) -> TokenInputs:
1077
- """
1078
- Implements the input processor, which transforms the input prompt ids
1079
- to include the audio placeholder token. This will become the `input_ids`
1080
- in `forward` for the model.
521
+ def get_max_image_tokens(self) -> int:
522
+ target_width, target_height = self.get_image_size_with_most_features()
523
+ return self.get_num_image_tokens(image_width=target_width,
524
+ image_height=target_height)
525
1082
- Args:
1083
- ctx (InputContext): Input context.
1084
- inputs (DecoderOnlyInputs): The inputs (e.g. prompt, prompt_token_ids)
1085
- to process.
526
+ def _find_target_aspect_ratio(
527
+ self,
528
+ orig_width: int,
529
+ orig_height: int,
530
+ image_size: int,
531
+ max_num: int,
532
+ min_num: int,
533
+ ):
534
+ w_crop_num = math.ceil(orig_width / float(image_size))
535
+ h_crop_num = math.ceil(orig_height / float(image_size))
536
+ if w_crop_num * h_crop_num > max_num:
537
+ aspect_ratio = orig_width / orig_height
538
+
539
+ # calculate the existing image aspect ratio
540
+ target_ratios = set((i, j) for i in range(1, max_num + 1)
541
+ for j in range(1, max_num + 1)
542
+ if i * j <= max_num and i * j >= min_num)
543
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
544
+
545
+ # find the closest aspect ratio to the target
546
+ image_processor = self.get_hf_processor().image_processor
547
+ target_aspect_ratio = image_processor.find_closest_aspect_ratio(
548
+ aspect_ratio,
549
+ target_ratios,
550
+ orig_width,
551
+ orig_height,
552
+ image_size,
553
+ )
554
+
555
+ # calculate the target width and height
556
+ target_width = image_size * target_aspect_ratio[0]
557
+ target_height = image_size * target_aspect_ratio[1]
558
+ else:
559
+ target_width = image_size * w_crop_num
560
+ target_height = image_size * h_crop_num
561
+ target_aspect_ratio = (w_crop_num, h_crop_num)
562
+ return target_aspect_ratio, target_height, target_width
563
1087
- Returns:
1088
- TokenInputs: Processed inputs
1089
- """
1090
- multi_modal_data = inputs.get("multi_modal_data")
1091
- if (multi_modal_data is None or
1092
- ("audio" not in multi_modal_data and "image" not in multi_modal_data)):
1093
- # pure text input, so no need to do pre-processing
1094
- return inputs
1095
-
1096
- prompt_str = inputs.get("prompt")
1097
- prompt_token_ids = inputs.get("prompt_token_ids")
1098
- # for offline_inference, we will get str input and we parse MM special
1099
- # tokens from it
1100
- # (ignore prompt_token_ids)
1101
- # for OAI server, we will get prompt_token_ids, where MM special tokens
1102
- # are already parsed
1103
-
1104
- if 'audio' in multi_modal_data:
1105
- audios = multi_modal_data["audio"]
1106
-
1107
- if not isinstance(audios, list):
1108
- audios = [audios]
1109
- if prompt_str is not None:
1110
- audio_id_to_input_ids = _get_audio_id_to_input_ids(
1111
- audios, ctx, prompt_str=prompt_str)
1112
- audio_embed_sizes = []
1113
- elif prompt_token_ids is not None:
1114
- audio_id_to_input_ids = {}
1115
- audio_embed_sizes = _get_audio_embed_sizes(audios, ctx)
1116
- else:
1117
- audio_id_to_input_ids = {}
1118
- audio_embed_sizes = []
1119
-
1120
- if 'image' in multi_modal_data:
1121
- # PIL Image or list of PIL Images
1122
- images = multi_modal_data["image"]
1123
- if not isinstance(images, list):
1124
- images = [images]
1125
- if prompt_str is not None:
1126
- image_id_to_input_ids = _get_image_id_to_input_ids(
1127
- images, prompt_str, ctx)
1128
- image_token_counts = []
1129
- elif prompt_token_ids is not None:
1130
- image_id_to_input_ids = {}
1131
- image_token_counts = _count_image_tokens(images, ctx)
1132
- else:
1133
- image_id_to_input_ids = {}
1134
- image_token_counts = []
1135
-
1136
- # Handle the case where the prompt is a string and we need to manually
1137
- # tokenize it.
1138
- # In this case, the `audio_id_to_input_ids` dict will be mapping from
1139
- # an audio placeholder
1140
- # string (e.g. `<|audio_1|>`) to the audio placeholder tokens for the
1141
- # given audio length.
1142
- if prompt_str:
1143
- pattern = r"(<\|image_\d+\|>|<\|audio_\d+\|>)"
1144
- prompt_chunk_strings = re.split(pattern, prompt_str)
1145
- prompt_chunk_strings = [s for s in prompt_chunk_strings if s != ""]
1146
-
1147
- # Create the new input_ids with the placeholder image and audio
1148
- # tokens inserted
1149
- tokenizer = cached_tokenizer_from_config(ctx.model_config)
1150
- input_ids = []
1151
- has_imag, has_audio, has_user_text_input = False, False, False
1152
- for prompt_chunk_string in prompt_chunk_strings:
1153
- if re.match(IMAGE_TOKEN_PATTERN, prompt_chunk_string):
1154
- input_ids.extend(image_id_to_input_ids[prompt_chunk_string])
1155
- has_imag = True
1156
- elif re.match(AUDIO_TOKEN_PATTERN, prompt_chunk_string):
1157
- input_ids.extend(audio_id_to_input_ids[prompt_chunk_string])
1158
- has_audio = True
1159
- else:
1160
- curr_token_ids = tokenizer(prompt_chunk_string).input_ids
1161
- if not has_user_text_input:
1162
- for token_id in curr_token_ids:
1163
- if token_id not in NON_USER_INPUT_TOKENS:
1164
- has_user_text_input = True
1165
- break
1166
- input_ids.extend(curr_token_ids)
1167
- if has_audio and has_imag and has_user_text_input:
1168
- raise ValueError(
1169
- "Phi4MMForCausalLM does not support text + audio + image" +
1170
- " inputs in the same prompt")
1171
- # Handle the case where the prompt is already tokenized
1172
- else:
1173
- assert prompt_token_ids is not None, \
1174
- "If string prompt isn't provided, prompt_token_ids must be"
1175
-
1176
- i = 0
1177
- input_ids = prompt_token_ids
1178
- # only needed for later assertion
1179
- img_cnt, audio_cnt, user_text_input_cnt = 0, 0, 0
1180
- image_token_count_iter = iter(image_token_counts)
1181
- audio_embed_size_iter = iter(audio_embed_sizes)
1182
- while i < len(input_ids):
1183
- token_id = input_ids[i]
1184
- if token_id == _AUDIO_PLACEHOLDER_TOKEN_ID:
1185
- token_count = next(audio_embed_size_iter)
1186
- audio_cnt += 1
1187
- elif token_id == _IMAGE_PLACEHOLDER_TOKEN_ID:
1188
- token_count = next(image_token_count_iter)
1189
- img_cnt += 1
1190
- else:
1191
- user_text_input_cnt += 1 if token_id not in \
1192
- NON_USER_INPUT_TOKENS else 0
1193
- i += 1
1194
- continue
1195
- tokens = [token_id] * token_count
1196
- input_ids = input_ids[:i] + tokens + input_ids[i + 1:]
1197
- i += token_count
1198
-
1199
- if audio_cnt > 0 and img_cnt > 0 and user_text_input_cnt > 0:
1200
- raise ValueError(
1201
- "Phi4MMForCausalLM does not support text + audio + image" +
1202
- " inputs in the same prompt")
1203
- # If the below assertion fails, it might be that input pure-text
1204
- # messages contain image/audio special tokens literally
1205
- # (<|endoftext10|>, <|endoftext11|>).
1206
- assert (img_cnt == len(image_token_counts)), (
1207
- f"Number of image tokens in prompt_token_ids ({img_cnt}) "
1208
- f"does not match number of images ({len(image_token_counts)})")
1209
- assert (audio_cnt == len(audio_embed_sizes)), (
1210
- f"Number of audio tokens in prompt_token_ids ({audio_cnt}) "
1211
- f"does not match number of audios ({len(audio_embed_sizes)})")
1212
-
1213
- # NOTE: Create a defensive copy of the original inputs
1214
- return token_inputs(
1215
- prompt_token_ids=input_ids,
1216
- prompt=prompt_str,
1217
- multi_modal_data=multi_modal_data,
1218
- )
564
+ def _compute_num_image_tokens(
565
+ self,
566
+ orig_width: int,
567
+ orig_height: int,
568
+ dynamic_hd_size: int,
569
+ vit_image_size: int,
570
+ vit_patch_size: int,
571
+ token_compression_factor: int = 2,
572
+ ):
573
+ """
574
+ compute the number of tokens an image is expected to take up considering
575
+ the image encoder architecture and exclude output features containing
576
+ only padding pixels
577
578
+ for siglip, vit_image_size=448, vit_patch_size=14, so output will be
579
+ 32x32 feature map
580
+ NOTE right now, Phi4MM uses hard-coded token_compression_factor=2
581
+ """
582
+ assert vit_image_size % vit_patch_size == 0, (
583
+ "vit_image_size must be divisible by vit_patch_size")
584
+ assert (vit_image_size // vit_patch_size %
585
+ token_compression_factor == 0), (
586
+ "vit_image_size // vit_patch_size must be divisible by "
587
+ "token_compression_factor")
588
+
589
+ target_aspect_ratio, target_height, target_width = (
590
+ self._find_target_aspect_ratio(orig_width,
591
+ orig_height,
592
+ vit_image_size,
593
+ dynamic_hd_size,
594
+ min_num=1))
595
+ assert target_aspect_ratio[0] * vit_image_size == target_width, (
596
+ f"{target_aspect_ratio[0]} * {vit_image_size} != {target_width}")
597
+ assert target_aspect_ratio[1] * vit_image_size == target_height, (
598
+ f"{target_aspect_ratio[1]} * {vit_image_size} != {target_height}")
599
+ assert (target_height % vit_image_size == 0
600
+ and target_width % vit_image_size == 0)
601
+
602
+ padding_height, padding_width = _get_padding_size(
603
+ orig_width, orig_height, target_height, target_width)
604
+ assert padding_width == 0 or padding_height == 0, \
605
+ "padding_width or padding_height must be 0"
606
+
607
+ target_feat_width = target_width // vit_patch_size
608
+ target_feat_height = target_height // vit_patch_size
609
+ if padding_width >= vit_patch_size:
610
+ assert padding_height == 0, "padding_height not 0"
611
+ non_pad_feat_width = target_feat_width - math.floor(
612
+ padding_width / vit_patch_size)
613
+ non_pad_feat_height = target_feat_height
614
+ elif padding_height >= vit_patch_size:
615
+ assert padding_width == 0, "padding_width not 0"
616
+ non_pad_feat_height = target_feat_height - math.floor(
617
+ padding_height / vit_patch_size)
618
+ non_pad_feat_width = target_feat_width
619
+ else:
620
+ # small padding shorter than a vit patch
621
+ non_pad_feat_width = target_feat_width
622
+ non_pad_feat_height = target_feat_height
623
+
624
+ feat_width = non_pad_feat_width // token_compression_factor
625
+ feat_height = non_pad_feat_height // token_compression_factor
626
+ # NOTE it's possible that the non-padding feature is not divisible
627
+ if non_pad_feat_width % token_compression_factor != 0:
628
+ feat_width += 1
629
+ if non_pad_feat_height % token_compression_factor != 0:
630
+ feat_height += 1
631
+ num_hd_patch_tokens = feat_width * feat_height
632
+ num_hd_newline_tokens = feat_height
633
+ vit_feature_size = vit_image_size // vit_patch_size
634
+ num_global_image_tokens = (vit_feature_size //
635
+ token_compression_factor)**2
636
+ num_sep_tokens = 1
637
+ num_global_image_newline_tokens = \
638
+ vit_feature_size // token_compression_factor
639
+
640
+ return (num_global_image_tokens + num_sep_tokens +
641
+ num_hd_patch_tokens + num_hd_newline_tokens +
642
+ num_global_image_newline_tokens)
643
+
644
+ def get_num_image_tokens(
645
+ self,
646
+ *,
647
+ image_width: int,
648
+ image_height: int,
649
+ processor: Optional[ProcessorMixin] = None,
650
+ ) -> int:
651
+ hf_config = self.get_hf_config()
652
+ vision_encoder_name = hf_config.img_processor
653
+ if vision_encoder_name is None:
654
+ vision_encoder_name = SIGLIP_NAME
655
+ prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[
656
+ vision_encoder_name]
657
+ vit_image_size = prepro_config['vit_image_size']
658
+ vit_patch_size = prepro_config['vit_patch_size']
659
+ token_compression_factor = prepro_config['token_compression_factor']
660
+
661
+ dynamic_hd_size = self.get_dynamic_hd(processor=processor)
662
+
663
+ image_num_tokens = self._compute_num_image_tokens(
664
+ image_width,
665
+ image_height,
666
+ dynamic_hd_size=dynamic_hd_size,
667
+ vit_image_size=vit_image_size,
668
+ vit_patch_size=vit_patch_size,
669
+ token_compression_factor=token_compression_factor,
670
+ )
671
1221
-def _compute_audio_embed_size(hf_config, audio_frames):
1222
- """
1223
- Compute the audio embedding size based on the audio frames and
1224
- compression rate.
1225
- """
1226
- compression_rate = hf_config.embd_layer['audio_embd_layer'][
1227
- 'compression_rate']
1228
- # NOTE: this is a hard-coded value but might be configurable in the future
1229
- qformer_compression_rate = 1
1230
- integer = audio_frames // compression_rate
1231
- remainder = audio_frames % compression_rate
672
+ return image_num_tokens
673
1233
- result = integer if remainder == 0 else integer + 1
674
+ def get_image_size_with_most_features(
675
+ self,
676
+ processor: Optional[ProcessorMixin] = None,
677
+ ) -> ImageSize:
678
+ hf_config = self.get_hf_config()
679
+ vision_encoder_name = hf_config.img_processor
680
+ if vision_encoder_name is None:
681
+ vision_encoder_name = SIGLIP_NAME
682
+ prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[
683
+ vision_encoder_name]
684
+ vit_image_size = prepro_config['vit_image_size']
685
+
686
+ max_side = vit_image_size * self.get_dynamic_hd(processor=processor)
687
+ return ImageSize(height=max_side, width=vit_image_size)
688
+
689
+ def get_audio_num_frames(self, audio_len: int, sr: float) -> int:
690
+ """
691
+ Compute the output size of the `extract_features` method.
692
1235
- integer = result // qformer_compression_rate
1236
- remainder = result % qformer_compression_rate
1237
- result = integer if remainder == 0 else integer + 1 # qformer compression
693
+ Args:
694
+ audio_len (int): Length of the input waveform in samples.
695
+ sr (float): Sampling rate of the waveform, either 16000 or 8000.
696
1239
- return result
697
+ Returns:
698
+ tuple (int, int): Output size as (T, D), where:
699
+ T: Number of time frames.
700
+ D: Number of Mel filterbank bins (80).
701
+ """
702
703
+ # Resample to 16000 or 8000 if needed
704
+ if sr > 16000:
705
+ audio_len //= sr // 16000
706
+ elif 8000 <= sr < 16000:
707
+ # We'll resample to 16K from 8K
708
+ audio_len *= 2
709
+ elif sr < 8000:
710
+ raise RuntimeError(f"Unsupported sample rate {sr}")
711
+
712
+ # Spectrogram parameters for 16 kHz
713
+ win_length = 400 # Frame length in samples
714
+ hop_length = 160 # Frame shift in samples
715
+
716
+ # Calculate number of frames (T)
717
+ num_frames = (audio_len - win_length) // hop_length + 1
718
+ if num_frames < 1:
719
+ raise ValueError("Waveform too short for given parameters.")
720
+
721
+ # Return time frames (T)
722
+ return num_frames
723
+
724
+ def _compute_audio_embed_size(self, audio_frames: int) -> int:
725
+ """
726
+ Compute the audio embedding size based on the audio frames and
727
+ compression rate.
728
+ """
729
+ hf_config = self.get_hf_config()
730
+ compression_rate = hf_config.embd_layer['audio_embd_layer'][
731
+ 'compression_rate']
732
+ # NOTE: this is a hard-coded value but might be configurable
733
+ # in the future
734
+ qformer_compression_rate = 1
735
+ integer = audio_frames // compression_rate
736
+ remainder = audio_frames % compression_rate
737
1242
-def get_max_phi4mm_audio_tokens(ctx: InputContext) -> int:
1243
- return 10000
738
+ result = integer if remainder == 0 else integer + 1
739
740
+ integer = result // qformer_compression_rate
741
+ remainder = result % qformer_compression_rate
742
+ # qformer compression
743
+ result = integer if remainder == 0 else integer + 1
744
1246
-def dummy_audio_for_phi4mm(audio_count: int) -> dict:
1247
- """
1248
- Create dummy audio data for the Phi4MM model, which is used for profiling.
745
+ return result
746
1250
- Args:
1251
- audio_count (int): Number of audio samples.
747
1253
- Returns:
1254
- dict: Dummy audio data.
1255
- """
1256
- dummy_audio = np.full((_AUDIO_MAX_SOUNDFILE_SIZE, ), 0.0)
1257
- return [(dummy_audio, DUMMY_SAMPLING_FREQUENCY)] * audio_count
748
+class Phi4MMDummyInputsBuilder(BaseDummyInputsBuilder[Phi4MMProcessingInfo]):
749
750
+ def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
751
+ num_audios = mm_counts.get("audio", 0)
752
+ num_images = mm_counts.get("image", 0)
753
1260
-def dummy_image_for_phi4mm(width: int, height: int):
1261
- image = Image.new('RGB', (width, height), color='black')
1262
- return image
754
+ image_tokens: list[str] = self.info.image_tokens[:num_images]
755
+ audio_tokens: list[str] = self.info.audio_tokens[:num_audios]
756
757
+ return "".join(image_tokens + audio_tokens)
758
1265
-def dummy_data_for_phi4mm(ctx: InputContext, seq_len: int,
1266
- mm_counts: Mapping[str, int]) -> DummyData:
1267
- """
1268
- Create dummy sequence (input_ids) and audio data for the Phi4MM model,
1269
- which is used for profiling.
759
+ def get_dummy_mm_data(
760
+ self,
761
+ seq_len: int,
762
+ mm_counts: Mapping[str, int],
763
+ ) -> MultiModalDataDict:
764
+ num_audios = mm_counts.get("audio", 0)
765
+ num_images = mm_counts.get("image", 0)
766
1271
- In this case, the sequence data is a bunch of 0s with a number of audio
1272
- tokens that correspond to the audio embed size of the
1273
- _AUDIO_MAX_SOUNDFILE_SIZE.
767
+ target_width, target_height = \
768
+ self.info.get_image_size_with_most_features()
769
1275
- Args:
1276
- ctx (InputContext): Input context.
1277
- seq_len (int): Length of the sequence.
1278
- mm_counts (Mapping[str, int]): Multi-modal counts.
770
+ target_width, target_height = \
771
+ self.info.get_image_size_with_most_features()
772
1280
- Returns:
1281
- Tuple: Dummy sequence data and dummy audio data.
1282
- """
1283
- audio_count = mm_counts["audio"]
1284
- audio_frames, _ = compute_logfbank_output_size(_AUDIO_MAX_SOUNDFILE_SIZE,
1285
- DUMMY_SAMPLING_FREQUENCY)
1286
- audio_feature_size = _compute_audio_embed_size(ctx.get_hf_config(),
1287
- audio_frames)
1288
-
1289
- image_count = mm_counts["image"]
1290
- dummy_image = get_max_dummy_image(ctx)
1291
- max_image_tokens = get_max_phi4mm_image_tokens(ctx)
1292
- total_image_tokens = image_count * max_image_tokens
1293
-
1294
- if seq_len - audio_feature_size * audio_count - total_image_tokens < 0:
1295
- raise RuntimeError(
1296
- f"Phi4MM cannot process {audio_count} audios and {image_count}"
1297
- f"images in a prompt, please increase max_model_len to be at"
1298
- f" larger than "
1299
- f"{audio_feature_size * audio_count + total_image_tokens}"
1300
- " or reduce audio/image limit by --limit-mm-per-prompt.")
1301
-
1302
- if audio_feature_size * audio_count > total_image_tokens:
1303
- seq_data = SequenceData.from_prompt_token_counts(
1304
- (_AUDIO_PLACEHOLDER_TOKEN_ID, audio_feature_size * audio_count),
1305
- (0, seq_len - audio_feature_size * audio_count),
1306
- )
773
mm_data = {
1308
- "audio": dummy_audio_for_phi4mm(audio_count),
774
+ "image":
775
+ self._get_dummy_images(width=target_width,
776
+ height=target_height,
777
+ num_images=num_images),
778
+ "audio":
779
+ self._get_dummy_audios(length=_AUDIO_MAX_SOUNDFILE_SIZE,
780
+ num_audios=num_audios),
781
}
1310
- else:
1311
- seq_data = SequenceData.from_prompt_token_counts(
1312
- (_IMAGE_PLACEHOLDER_TOKEN_ID, total_image_tokens),
1313
- (0, seq_len - total_image_tokens),
1314
- )
1315
- mm_data = {
1316
- "image": [dummy_image] * image_count,
1317
- }
1318
- return DummyData(seq_data, mm_data)
782
783
+ return mm_data
784
1321
-def input_mapper_for_phi4mm_audio(ctx: InputContext,
1322
- data: object) -> MultiModalKwargs:
1323
- """
1324
- This function is used to create the MultiModalKwargs for the Phi4MM
1325
- (audio) model.
1326
- Specifically, for audio, we extract the audio features from the sound
1327
- file and create pairs of audio features and audio embed lengths (the
1328
- latter of which is used to repeat the audio placeholder token in the
1329
- input prompt IDs).
1330
- These pairs are used, downstream, in `_audio_features_to_embeddings`
1331
- (via `_process_audio_input`).
1332
-
1333
- Note that the incoming audio data (each entry in `data`) is a tuple of
1334
- the audio data and the sampling frequency (e.g. from soundfile.read).
1335
-
1336
- Args:
1337
- ctx (InputContext): Input context.
1338
- data (object): Audio data.
1339
-
1340
- Returns:
1341
- MultiModalKwargs: Multi-modal inputs.
1342
- """
1343
- if not isinstance(data, list):
1344
- data = [data]
1345
-
1346
- if len(data) == 0:
1347
- return MultiModalKwargs()
1348
-
1349
- audio_features = []
1350
- for audio_input in data:
1351
- if not isinstance(audio_input, tuple):
1352
- raise NotImplementedError(
1353
- f"Unsupported data type: {type(audio_input)}")
1354
-
1355
- audio, sf = audio_input
1356
- feature_extractor = audio_feature_extractor()
1357
- single_audio_features = feature_extractor.extract_features(audio, sf)
1358
- feat_stride = (1 if not hasattr(feature_extractor, "stride") else
1359
- feature_extractor.stride)
1360
- audio_frames = len(single_audio_features) * feat_stride
1361
- single_audio_embed_size = _compute_audio_embed_size(
1362
- ctx.get_hf_config(), audio_frames)
1363
- single_audio_feature_audio_len_pair = (
1364
- single_audio_features,
1365
- [single_audio_embed_size],
1366
- )
1367
- audio_features.append(single_audio_feature_audio_len_pair)
1368
- return MultiModalKwargs({"audio_features": audio_features})
1369
-
1370
-
1371
-def input_mapper_for_phi4mm_image(ctx: InputContext, data: object):
1372
- if not isinstance(data, list):
1373
- data = [data]
1374
- # data: list of PIL images
1375
- if len(data) == 0:
1376
- return MultiModalKwargs()
1377
- hf_config = ctx.get_hf_config()
1378
- vision_encoder_name = hf_config.img_processor
1379
- if vision_encoder_name is None:
1380
- vision_encoder_name = SIGLIP_NAME
1381
- prepro_config = VISION_ENCODER_TO_PROCESSING_CONFIG[vision_encoder_name]
1382
- dynamic_hd_size = prepro_config['dynamic_hd']
1383
- vit_image_size = prepro_config['vit_image_size']
1384
- vit_patch_size = prepro_config['vit_patch_size']
1385
-
1386
- image_input_dict = preprocess(data, dynamic_hd_size, vit_image_size,
1387
- vit_patch_size)
1388
- return MultiModalKwargs({
1389
- "pixel_values":
1390
- image_input_dict["pixel_values"],
1391
- "image_sizes":
1392
- image_input_dict["image_sizes"],
1393
- "image_attention_mask":
1394
- image_input_dict["image_attention_mask"],
1395
- "num_img_tokens":
1396
- image_input_dict["num_img_tokens"],
1397
- })
785
786
+class Phi4MMMultiModalProcessor(BaseMultiModalProcessor[Phi4MMProcessingInfo]):
787
1400
-def cat_with_pad(tensors, dim, padding_value=0):
1401
- """
1402
- cat along dim, while pad to max for all other dims
1403
- """
1404
- ndim = tensors[0].dim()
1405
- assert all(
1406
- t.dim() == ndim for t in
1407
- tensors[1:]), "All tensors must have the same number of dimensions"
788
+ def _get_data_parser(self) -> MultiModalDataParser:
789
+ feature_extractor = self.info.get_feature_extractor()
790
+ return MultiModalDataParser(target_sr=feature_extractor.sampling_rate,
791
+ audio_resample_method="scipy")
792
1409
- out_size = [max(t.shape[i] for t in tensors) for i in range(ndim)]
1410
- out_size[dim] = sum(t.shape[dim] for t in tensors)
1411
- output = tensors[0].new_full(out_size, padding_value)
793
+ def _call_hf_processor(
794
+ self,
795
+ prompt: str,
796
+ mm_data: Mapping[str, object],
797
+ mm_kwargs: Mapping[str, object],
798
+ ) -> BatchFeature:
799
+ if not mm_data:
800
+ prompt_ids = self.info.get_tokenizer().encode(prompt)
801
+ prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
802
+ return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
803
+
804
+ sr = self.info.get_feature_extractor().sampling_rate
805
+ if (audio_data := mm_data.get("audios", [])):
806
+ mm_data['audios'] = [(data, sr) for data in audio_data]
807
+
808
+ processed_outputs = super()._call_hf_processor(prompt, mm_data,
809
+ mm_kwargs)
810
+
811
+ num_img_tokens = [
812
+ self.info.get_num_image_tokens(image_width=img_size[0],
813
+ image_height=img_size[1])
814
+ for img_size in processed_outputs["image_sizes"]
815
+ ]
816
+ processed_outputs["num_img_tokens"] = num_img_tokens
817
1413
- index = 0
1414
- for t in tensors:
1415
- # Create a slice list where every dimension except dim is full slice
1416
- slices = [slice(0, t.shape[d]) for d in range(ndim)]
1417
- # Update only the concat dimension slice
1418
- slices[dim] = slice(index, index + t.shape[dim])
818
+ audio_features = processed_outputs['input_audio_embeds']
819
+ feature_sizes = [
820
+ self.info.get_audio_num_frames(len(audio), sr)
821
+ for audio in audio_data
822
+ ]
823
+ processed_outputs['input_audio_embeds'] = [
824
+ audio_features[idx, :size]
825
+ for idx, size in enumerate(feature_sizes)
826
+ ]
827
1420
- output[slices] = t
1421
- index += t.shape[dim]
828
+ return processed_outputs
829
1423
- return output
830
+ def _get_mm_fields_config(
831
+ self,
832
+ hf_inputs: BatchFeature,
833
+ hf_processor_mm_kwargs: Mapping[str, object],
834
+ ) -> Mapping[str, MultiModalFieldConfig]:
835
+ return dict(
836
+ input_image_embeds=MultiModalFieldConfig.batched("image"),
837
+ image_attention_mask=MultiModalFieldConfig.batched("image"),
838
+ image_sizes=MultiModalFieldConfig.batched("image"),
839
+ num_img_tokens=MultiModalFieldConfig.batched("image"),
840
+ input_audio_embeds=MultiModalFieldConfig.batched("audio"),
841
+ )
842
+
843
+ def _get_prompt_updates(
844
+ self,
845
+ mm_items: MultiModalDataItems,
846
+ hf_processor_mm_kwargs: Mapping[str, Any],
847
+ out_mm_kwargs: MultiModalKwargs,
848
+ ) -> Sequence[PromptUpdate]:
849
+ image_tokens: list[str] = self.info.image_tokens # type: ignore
850
+ audio_tokens: list[str] = self.info.audio_tokens # type: ignore
851
+ feature_extractor = self.info.get_feature_extractor()
852
+ hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
853
+
854
+ def get_image_replacement_phi4mm(item_idx: int):
855
+ images = mm_items.get_items(
856
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
857
+
858
+ if isinstance(images, ImageEmbeddingItems):
859
+ num_image_tokens = images.get_feature_size(item_idx)
860
+ else:
861
+ image_size = images.get_image_size(item_idx)
862
+ num_image_tokens = self.info.get_num_image_tokens(
863
+ image_width=image_size.width,
864
+ image_height=image_size.height,
865
+ processor=hf_processor,
866
+ )
867
+
868
+ image_tokens = [_IMAGE_PLACEHOLDER_TOKEN_ID] * num_image_tokens
869
+
870
+ return image_tokens
871
+
872
+ def get_audio_replacement_phi4mm(item_idx: int):
873
+ audios = mm_items.get_items("audio", AudioProcessorItems)
874
+ # TODO(Isotr0py): support embedding inputs
875
+ audio_len = audios.get_audio_length(item_idx)
876
+ audio_frames = self.info.get_audio_num_frames(
877
+ audio_len, feature_extractor.sampling_rate)
878
+ audio_embed_size = self.info._compute_audio_embed_size(
879
+ audio_frames)
880
+
881
+ audio_tokens = [_AUDIO_PLACEHOLDER_TOKEN_ID] * audio_embed_size
882
+
883
+ return audio_tokens
884
+
885
+ num_images = mm_items.get_count("image", strict=False)
886
+ num_audios = mm_items.get_count("audio", strict=False)
887
+
888
+ image_repl = [
889
+ PromptReplacement(
890
+ modality="image",
891
+ target=image_token,
892
+ replacement=get_image_replacement_phi4mm,
893
+ ) for image_token in image_tokens[:num_images]
894
+ ]
895
+ audio_repl = [
896
+ PromptReplacement(
897
+ modality="audio",
898
+ target=audio_token,
899
+ replacement=get_audio_replacement_phi4mm,
900
+ ) for audio_token in audio_tokens[:num_audios]
901
+ ]
902
+ return image_repl + audio_repl
903
904
1426
-@MULTIMODAL_REGISTRY.register_input_mapper("audio",
1427
- input_mapper_for_phi4mm_audio)
1428
-@MULTIMODAL_REGISTRY.register_input_mapper("image",
1429
- input_mapper_for_phi4mm_image)
1430
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
1431
- "audio", get_max_phi4mm_audio_tokens)
1432
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
1433
- "image", get_max_phi4mm_image_tokens)
1434
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi4mm)
1435
-@INPUT_REGISTRY.register_input_processor(input_processor_for_phi4mm)
1436
-class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1437
- SupportsV0Only):
905
+@MULTIMODAL_REGISTRY.register_processor(
906
+ Phi4MMMultiModalProcessor,
907
+ info=Phi4MMProcessingInfo,
908
+ dummy_inputs=Phi4MMDummyInputsBuilder,
909
+)
910
+class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal):
911
"""
912
Implements the Phi-4-multimodal-instruct model in vLLM.
913
"""
@@ -1518,48 +991,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
991
logit_scale = getattr(config, "logit_scale", 1.0)
992
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
993
config.vocab_size, logit_scale)
1521
- self.sampler = Sampler()
1522
-
1523
- def _audio_features_to_embeddings(
1524
- self,
1525
- input_ids: torch.Tensor,
1526
- input_features: List[torch.Tensor],
1527
- audio_input_sizes: torch.Tensor,
1528
- audio_projection_mode: str,
1529
- ) -> torch.Tensor:
1530
- """
1531
- Convert audio features to embeddings, which are used as input to the
1532
- model (via `inputs_embeds`).
1533
-
1534
- Args:
1535
- input_ids (torch.Tensor): Input IDs (the prompt in this case).
1536
- input_features (list[torch.Tensor]): Input features (the audio
1537
- embeddings).
1538
- audio_input_sizes (list[torch.Tensor]): Audio input sizes (the
1539
- audio embed lengths to use for padding the audio placeholder token
1540
- in the input prompt IDs).
1541
- """
1542
- # The audio projection can either be a single linear or Sequential,
1543
- # so handle both cases
1544
- if isinstance(self.embed_tokens_extend.audio_projection,
1545
- nn.Sequential):
1546
- target_dtype = self.embed_tokens_extend.audio_projection[
1547
- 0].bias.dtype
1548
- else:
1549
- target_dtype = self.embed_tokens_extend.audio_projection.bias.dtype
1550
-
1551
- audio_input = [
1552
- input.unsqueeze(0).to(target_dtype) for input in input_features
1553
- ]
1554
- kwargs = {
1555
- "wte": self.model.embed_tokens,
1556
- 'audio_projection_mode': audio_projection_mode
1557
- }
1558
- audio_embeddings = self.embed_tokens_extend(input_ids, audio_input,
1559
- audio_input_sizes,
1560
- **kwargs)
1561
- audio_embeddings = audio_embeddings.to(target_dtype)
1562
- return audio_embeddings
994
+ self.sampler = get_sampler()
995
996
def _parse_and_validate_audio_input(
997
self, **kwargs: object) -> Optional[Phi4MMAudioInputs]:
@@ -1574,7 +1006,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1006
Returns:
1007
Optional[Phi4MMAudioInputs]: Parsed and validated audio inputs.
1008
"""
1577
- audio_features = kwargs.pop("audio_features", None)
1009
+ audio_features = kwargs.pop("input_audio_embeds", None)
1010
audio_embeds = kwargs.pop("audio_embeds", None)
1011
1012
if audio_features is None and audio_embeds is None:
@@ -1586,7 +1018,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1018
f"Got type: {type(audio_features)}")
1019
1020
return Phi4MMAudioFeatureInputs(type="audio_features",
1589
- data=audio_features)
1021
+ data=flatten_bn(audio_features))
1022
1023
if audio_embeds is not None:
1024
if not isinstance(audio_embeds, (torch.Tensor, list)):
@@ -1598,8 +1030,7 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1030
1031
raise AssertionError("This line should be unreachable.")
1032
1601
- def _process_audio_input(self, input_ids: torch.Tensor,
1602
- audio_input: Phi4MMAudioInputs,
1033
+ def _process_audio_input(self, audio_input: Phi4MMAudioInputs,
1034
audio_projection_mode: str) -> NestedTensors:
1035
"""
1036
Create the audio embeddings from the audio input, where the audio input
@@ -1607,8 +1038,6 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1038
created by `input_mapper_for_phi4mm_audio`.
1039
1040
Args:
1610
- input_ids (torch.Tensor): Input IDs (the prompt in this case,
1611
- before the audio token replication).
1041
audio_input (Phi4MMAudioInputs): Audio input.
1042
1043
Returns:
@@ -1620,21 +1049,20 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1049
audio_features = audio_input["data"]
1050
# (e.g. multiple examples) and the second dim is the multi-audio dim
1051
# (e.g. multiple audios in the same example)
1623
- audio_feature = [i[0] for j in audio_features for i in j]
1624
- audio_feature_len = [i[1].item() for j in audio_features for i in j]
1625
- # Add the batch dim via `squeeze`
1052
1627
- return self._audio_features_to_embeddings(
1628
- input_ids.unsqueeze(0),
1629
- audio_feature,
1630
- audio_feature_len,
1631
- audio_projection_mode,
1632
- ).squeeze(0)
1053
+ dtype = next(self.embed_tokens_extend.parameters()).dtype
1054
+ audio_embeds = [
1055
+ self.embed_tokens_extend(
1056
+ features.to(dtype),
1057
+ audio_projection_mode=audio_projection_mode,
1058
+ ) for features in audio_features
1059
+ ]
1060
+ return audio_embeds
1061
1062
def _parse_and_validate_image_input(self,
1063
**kwargs: object) -> Optional[Dict]:
1636
- pixel_values: Optional[Dict] = kwargs.get("pixel_values")
1637
- if pixel_values is None:
1064
+ input_image_embeds: NestedTensors = kwargs.get("input_image_embeds")
1065
+ if input_image_embeds is None:
1066
return None
1067
1068
image_sizes = kwargs.get("image_sizes")
@@ -1643,23 +1071,24 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1071
assert image_sizes is not None and image_attention_mask is not None\
1072
and num_img_tokens is not None, "Missing image inputs"
1073
1646
- if isinstance(pixel_values, list):
1647
- assert pixel_values[0].dim() == 5, "Incorrect image inputs"
1074
+ if is_list_of(input_image_embeds, torch.Tensor):
1075
+ assert all(p.dim() == 5
1076
+ for p in input_image_embeds), "Incorrect image inputs"
1077
# list len is batch_size.
1078
# each tensor has dimension: num_img_per_example, num_hd_patches,
1079
# channels, height, width.
1080
# need to pad along num_hd_patches.
1081
# mask size num_img_per_prompt, num_hd_patches, feat_h, heat_w.
1653
- pixel_values = cat_with_pad(pixel_values, dim=0)
1654
- elif isinstance(pixel_values, torch.Tensor):
1082
+ input_image_embeds = cat_with_pad(input_image_embeds, dim=0)
1083
+ elif isinstance(input_image_embeds, torch.Tensor):
1084
# dimension: batch_size, num_img_per_example, num_hd_patches,
1085
# channels, height, width.
1086
# we flatten first 2 dims to make it a single large batch for
1087
# SigLIP Encoder.
1659
- assert pixel_values.dim() == 6, "Incorrect image inputs"
1660
- pixel_values = pixel_values.flatten(0, 1)
1088
+ assert input_image_embeds.dim() == 6, "Incorrect image inputs"
1089
+ input_image_embeds = input_image_embeds.flatten(0, 1)
1090
else:
1662
- raise ValueError("Incorrect pixel_values inputs")
1091
+ raise ValueError("Incorrect input_image_embeds inputs")
1092
1093
if isinstance(image_attention_mask, list):
1094
image_attention_mask = cat_with_pad(image_attention_mask, dim=0)
@@ -1685,80 +1114,140 @@ class Phi4MMForCausalLM(nn.Module, SupportsLoRA, SupportsMultiModal,
1114
else:
1115
raise ValueError("Incorrect image_attention_mask inputs")
1116
1688
- return {
1689
- 'pixel_values': pixel_values,
1690
- 'image_sizes': image_sizes,
1691
- 'image_attention_mask': image_attention_mask,
1692
- 'num_img_tokens': num_img_tokens,
1693
- }
1117
+ return Phi4MMImagePixelInputs(
1118
+ type="pixel_values",
1119
+ data=input_image_embeds,
1120
+ image_sizes=image_sizes,
1121
+ image_attention_mask=image_attention_mask,
1122
+ num_img_tokens=num_img_tokens,
1123
+ )
1124
+
1125
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
1126
+ modalities = {}
1127
+
1128
+ # Preserve the order of modalities if there are multiple of them
1129
+ # from the order of kwargs.
1130
+ for input_key in kwargs:
1131
+ if input_key in ("input_image_embeds",
1132
+ "image_embeds") and "images" not in modalities:
1133
+ modalities["images"] = self._parse_and_validate_image_input(
1134
+ **kwargs)
1135
+ if input_key in ("input_audio_embeds",
1136
+ "audio_embeds") and "audios" not in modalities:
1137
+ modalities["audios"] = self._parse_and_validate_audio_input(
1138
+ **kwargs)
1139
+
1140
+ return modalities
1141
+
1142
+ def _process_image_input(
1143
+ self, image_input: Phi4MMImagePixelInputs) -> list[torch.Tensor]:
1144
+ if image_input["type"] == "image_embeds":
1145
+ image_embeds = image_input["image_embeds"].type(self.visual.dtype)
1146
+ else:
1147
+ dtype = next(self.vision_encoder.parameters()).dtype
1148
+ pixel_values = image_input['data'].to(dtype)
1149
+ image_sizes = image_input['image_sizes']
1150
+ image_attention_mask = image_input['image_attention_mask']
1151
+ image_embeds = self.vision_encoder(pixel_values, image_sizes,
1152
+ image_attention_mask)
1153
+ return image_embeds
1154
+
1155
+ def get_multimodal_embeddings(
1156
+ self, **kwargs: object) -> Optional[MultiModalEmbeddings]:
1157
+
1158
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
1159
+ if not modalities:
1160
+ return None
1161
1695
- def merge_image_features_to_inputs_embeds(
1162
+ # The result multimodal_embeddings is tuple of tensors, with each
1163
+ # tensor correspoending to a multimodal data item (image or video).
1164
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
1165
+
1166
+ # NOTE: It is important to iterate over the keys in this dictionary
1167
+ # to preserve the order of the modalities.
1168
+ audio_projection_mode = 'speech'
1169
+ for modality in modalities:
1170
+ # make sure process images first
1171
+ if modality == "images":
1172
+ audio_projection_mode = "vision"
1173
+ image_input = modalities["images"]
1174
+ vision_embeddings = self._process_image_input(image_input)
1175
+ multimodal_embeddings += tuple(vision_embeddings)
1176
+ if modality == "audios":
1177
+ audio_input = modalities["audios"]
1178
+ audio_embeddings = self._process_audio_input(
1179
+ audio_input, audio_projection_mode=audio_projection_mode)
1180
+ multimodal_embeddings += tuple(audio_embeddings)
1181
+
1182
+ return multimodal_embeddings
1183
+
1184
+ def get_input_embeddings(
1185
self,
1186
input_ids: torch.Tensor,
1698
- inputs_embeds: torch.Tensor,
1699
- image_set_tensors: List[torch.Tensor],
1700
- ):
1701
- position_tuple = (input_ids == _IMAGE_PLACEHOLDER_TOKEN_ID).nonzero(
1702
- as_tuple=True)
1703
-
1704
- assert all([t.shape[0] == 1 for t in image_set_tensors
1705
- ]), 'img_set_tensor should have shape (1, N_tokens, C)'
1706
- # Shape: (merged_N_tokens, C)
1707
- image_set_tensor = torch.cat(image_set_tensors, dim=1).squeeze(0)
1708
- image_set_tensor = image_set_tensor.to(inputs_embeds.dtype).to(
1709
- inputs_embeds.device)
1710
- merged_embeds = inputs_embeds.index_put(
1711
- indices=position_tuple,
1712
- values=image_set_tensor,
1713
- accumulate=False,
1714
- )
1715
- return merged_embeds
1187
+ multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
1188
+ ) -> torch.Tensor:
1189
+ inputs_embeds = self.model.embed_tokens(input_ids)
1190
+ if multimodal_embeddings is not None:
1191
+ inputs_embeds = merge_multimodal_embeddings(
1192
+ input_ids, inputs_embeds, multimodal_embeddings,
1193
+ [_IMAGE_PLACEHOLDER_TOKEN_ID, _AUDIO_PLACEHOLDER_TOKEN_ID])
1194
+ return inputs_embeds
1195
+
1196
+ def get_input_embeddings_v0(
1197
+ self,
1198
+ input_ids: torch.Tensor,
1199
+ image_input: Optional[Phi4MMImagePixelInputs] = None,
1200
+ audio_input: Optional[Phi4MMAudioFeatureInputs] = None,
1201
+ ) -> torch.Tensor:
1202
+ audio_projection_mode = 'speech'
1203
+ inputs_embeds = self.get_input_embeddings(input_ids)
1204
+ if image_input is not None:
1205
+ image_embeds = self._process_image_input(image_input)
1206
+ inputs_embeds = merge_multimodal_embeddings(
1207
+ input_ids,
1208
+ inputs_embeds,
1209
+ image_embeds,
1210
+ placeholder_token_id=_IMAGE_PLACEHOLDER_TOKEN_ID,
1211
+ )
1212
+ audio_projection_mode = 'vision'
1213
+
1214
+ if audio_input is not None:
1215
+ audio_embeds = self._process_audio_input(
1216
+ audio_input, audio_projection_mode=audio_projection_mode)
1217
+ inputs_embeds = merge_multimodal_embeddings(
1218
+ input_ids,
1219
+ inputs_embeds,
1220
+ audio_embeds,
1221
+ placeholder_token_id=_AUDIO_PLACEHOLDER_TOKEN_ID,
1222
+ )
1223
+ return inputs_embeds
1224
1225
def forward(
1226
self,
1227
input_ids: torch.Tensor,
1228
positions: torch.Tensor,
1229
intermediate_tensors: Optional[IntermediateTensors] = None,
1230
+ inputs_embeds: Optional[torch.Tensor] = None,
1231
**kwargs: object,
1232
) -> torch.Tensor:
1233
if intermediate_tensors is not None:
1725
- input_ids = None
1234
inputs_embeds = None
1727
- else:
1728
- # Each entry in this is a pair of audio_features and audio_embed
1729
- # lengths
1235
+
1236
+ # NOTE: In v1, inputs_embeds is always generated at model runner from
1237
+ # `get_multimodal_embeddings` and `get_input_embeddings`, this
1238
+ # condition is only for v0 compatibility.
1239
+ elif inputs_embeds is None:
1240
+ image_input = self._parse_and_validate_image_input(**kwargs)
1241
audio_input = self._parse_and_validate_audio_input(**kwargs)
1731
- image_inputs = self._parse_and_validate_image_input(**kwargs)
1732
-
1733
- has_audio = audio_input is not None
1734
- has_image = image_inputs is not None
1735
-
1736
- if has_audio:
1737
- audio_projection_mode = 'vision' if has_image else 'speech'
1738
- inputs_embeds = self._process_audio_input(
1739
- input_ids, audio_input, audio_projection_mode)
1740
-
1741
- if has_image:
1742
- dtype = self.vision_encoder.img_processor.embeddings.\
1743
- patch_embedding.weight.dtype
1744
- pixel_values = image_inputs['pixel_values'].to(dtype)
1745
- image_sizes = image_inputs['image_sizes']
1746
- image_attention_mask = image_inputs['image_attention_mask']
1747
- image_set_tensors = self.vision_encoder(
1748
- pixel_values, image_sizes, image_attention_mask)
1749
- if not has_audio:
1750
- inputs_embeds = self.model.embed_tokens(input_ids)
1751
-
1752
- inputs_embeds = self.merge_image_features_to_inputs_embeds(
1753
- input_ids, inputs_embeds, image_set_tensors)
1754
-
1755
- if has_image or has_audio:
1756
- # multi-modal input, we have set inputs_embeds properly in
1757
- # previous steps
1758
- input_ids = None
1759
- else:
1760
- # text-only, we keep using original input_ids
1242
+
1243
+ if image_input is None and audio_input is None:
1244
inputs_embeds = None
1245
+ else:
1246
+ inputs_embeds = self.get_input_embeddings_v0(
1247
+ input_ids,
1248
+ image_input=image_input,
1249
+ audio_input=audio_input)
1250
+ input_ids = None
1251
1252
hidden_states = self.model(
1253
input_ids,
vllm/model_executor/models/phi4mm_audio.py
+18
-57
@@ -1159,8 +1159,11 @@ class AudioEmbedding(nn.Module):
1159
input_embeds: torch.FloatTensor,
1160
audio_attention_mask: torch.Tensor = None,
1161
audio_projection_mode: str = "speech",
1162
- ):
1163
-
1162
+ ) -> torch.FloatTensor:
1163
+ """
1164
+ arguments:
1165
+ input_embeds: audio features (B, T, D) B: num audios in a sequence
1166
+ """
1167
if self.freeze_audio_processor:
1168
with torch.no_grad():
1169
audio_features, masks = self.encoder(input_embeds,
@@ -1210,62 +1213,20 @@ class AudioEmbedding(nn.Module):
1213
1214
def forward(
1215
self,
1213
- input_ids: torch.LongTensor,
1214
- input_embeds: torch.FloatTensor,
1215
- audio_embed_sizes,
1216
- **kwargs,
1216
+ audio_features: torch.FloatTensor,
1217
+ audio_attention_mask: torch.Tensor = None,
1218
+ audio_projection_mode: str = "speech",
1219
) -> torch.FloatTensor:
1220
"""
1221
arguments:
1220
- input_ids: input text ids (B, U)
1221
- input_embeds: audio features (B, T, D) B: num audios in a sequence
1222
+ audio_features: audio features (T, D)
1223
+
1224
+ returns:
1225
+ audio_embeds: audio embeddings (num_audio_tokens, hidden_dim)
1226
"""
1223
- assert input_embeds is not None and len(input_embeds) == len(
1224
- audio_embed_sizes)
1225
-
1226
- input_shape = input_ids.size()
1227
- input_ids = input_ids.view(-1, input_shape[-1])
1228
-
1229
- with torch.no_grad():
1230
- positions = (input_ids == _AUDIO_PLACEHOLDER_TOKEN_ID).nonzero(
1231
- as_tuple=False)
1232
-
1233
- if not isinstance(input_embeds, list):
1234
- input_embeds = [input_embeds]
1235
-
1236
- audio_projection_mode = kwargs.get("audio_projection_mode", "speech")
1237
- audio_set_tensor = [
1238
- self.get_audio_features(
1239
- input_embed, audio_projection_mode=audio_projection_mode)
1240
- for input_embed in input_embeds
1241
- ]
1242
-
1243
- with torch.no_grad():
1244
- input_ids.clamp_min_(0).clamp_max_(self.vocab_size)
1245
-
1246
- if "wte" in kwargs:
1247
- # we use the token embedding layer from the huggingface model, this
1248
- # is REQUIRED to make sure we are using the loaded weights.
1249
- hidden_states = kwargs["wte"](input_ids)
1250
- else:
1251
- # otherwise, we use token embedding in pretrained mixformer from
1252
- # phi team
1253
- hidden_states = self.wte(input_ids)
1254
-
1255
- if len(positions.tolist()) > 0:
1256
- assert sum(audio_embed_sizes) == len(
1257
- positions
1258
- ), "please ensure the encoder outputs have the same length as"\
1259
- " defined in input_ids!"
1260
- idx = 0
1261
- for i in range(len(audio_embed_sizes)):
1262
- cnt = audio_embed_sizes[i]
1263
- assert audio_set_tensor[i].shape[0] == 1
1264
- hidden_states[
1265
- positions[idx, 0],
1266
- positions[idx, 1]:positions[idx, 1] + cnt,
1267
- ] = (audio_set_tensor[i][0, :audio_embed_sizes[i], :].to(
1268
- hidden_states.dtype).to(hidden_states.device))
1269
- idx += cnt
1270
-
1271
- return hidden_states
1227
+ audio_embeds = self.get_audio_features(
1228
+ audio_features.unsqueeze(0),
1229
+ audio_attention_mask=audio_attention_mask,
1230
+ audio_projection_mode=audio_projection_mode,
1231
+ )
1232
+ return audio_embeds.squeeze(0)
vllm/multimodal/audio.py
+51
-2
@@ -1,8 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
-
2
import base64
3
from io import BytesIO
4
from pathlib import Path
5
+from typing import Literal, Optional
6
7
import numpy as np
8
import numpy.typing as npt
@@ -43,7 +43,7 @@ class AudioPlugin(MultiModalPlugin):
43
"There is no default maximum multimodal tokens")
44
45
46
-def resample_audio(
46
+def resample_audio_librosa(
47
audio: npt.NDArray[np.floating],
48
*,
49
orig_sr: float,
@@ -52,6 +52,55 @@ def resample_audio(
52
return librosa.resample(audio, orig_sr=orig_sr, target_sr=target_sr)
53
54
55
+def resample_audio_scipy(
56
+ audio: npt.NDArray[np.floating],
57
+ *,
58
+ orig_sr: float,
59
+ target_sr: float,
60
+):
61
+ # lazy import scipy.signal, otherwise it will crash doc build.
62
+ import scipy.signal
63
+
64
+ if orig_sr > target_sr:
65
+ return scipy.signal.resample_poly(audio, 1, orig_sr // target_sr)
66
+ elif orig_sr < target_sr:
67
+ return scipy.signal.resample_poly(audio, target_sr // orig_sr, 1)
68
+ return audio
69
+
70
+
71
+class AudioResampler:
72
+ """Resample audio data to a target sample rate."""
73
+
74
+ def __init__(
75
+ self,
76
+ target_sr: Optional[float] = None,
77
+ method: Literal["librosa", "scipy"] = "librosa",
78
+ ):
79
+ self.target_sr = target_sr
80
+ self.method = method
81
+
82
+ def resample(
83
+ self,
84
+ audio: npt.NDArray[np.floating],
85
+ *,
86
+ orig_sr: float,
87
+ ) -> npt.NDArray[np.floating]:
88
+ if self.target_sr is None:
89
+ raise RuntimeError("Audio resampling is not supported when "
90
+ "`target_sr` is not provided")
91
+ if self.method == "librosa":
92
+ return resample_audio_librosa(audio,
93
+ orig_sr=orig_sr,
94
+ target_sr=self.target_sr)
95
+ elif self.method == "scipy":
96
+ return resample_audio_scipy(audio,
97
+ orig_sr=orig_sr,
98
+ target_sr=self.target_sr)
99
+ else:
100
+ raise ValueError(f"Invalid resampling method: {self.method}. "
101
+ "Supported methods are 'librosa' and 'scipy'.")
102
+
103
+
104
class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
105
106
def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
vllm/multimodal/parse.py
+15
-14
@@ -3,8 +3,8 @@
3
from abc import ABC, abstractmethod
4
from collections import UserDict
5
from collections.abc import Callable, Iterator, Mapping, Sequence
6
-from typing import (TYPE_CHECKING, Any, Generic, NamedTuple, Optional, TypeVar,
7
- Union)
6
+from typing import (TYPE_CHECKING, Any, Generic, Literal, NamedTuple, Optional,
7
+ TypeVar, Union)
8
9
import numpy as np
10
import torch
@@ -14,7 +14,7 @@ from typing_extensions import TypeAlias, TypeGuard, assert_never
14
15
from vllm.utils import is_list_of
16
17
-from .audio import resample_audio
17
+from .audio import AudioResampler
18
from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,
19
ImageItem, ModalityData, MultiModalDataDict,
20
MultiModalFieldConfig, MultiModalKwargs, VideoItem)
@@ -308,10 +308,18 @@ class MultiModalDataParser:
308
items to the model's expected sampling rate.
309
"""
310
311
- def __init__(self, *, target_sr: Optional[float] = None) -> None:
311
+ def __init__(
312
+ self,
313
+ *,
314
+ target_sr: Optional[float] = None,
315
+ audio_resample_method: Literal["librosa", "scipy"] = "librosa",
316
+ ) -> None:
317
super().__init__()
318
314
- self.target_sr = target_sr
319
+ self.audio_resampler = AudioResampler(
320
+ target_sr=target_sr,
321
+ method=audio_resample_method,
322
+ )
323
324
def _is_embeddings(
325
self, data: object
@@ -374,15 +382,8 @@ class MultiModalDataParser:
382
if orig_sr is None:
383
new_audio = audio
384
else:
377
- target_sr = self.target_sr
378
- if target_sr is None:
379
- raise RuntimeError(
380
- "Audio resampling is not supported when "
381
- "`target_sr` is not provided")
382
-
383
- new_audio = resample_audio(audio,
384
- orig_sr=orig_sr,
385
- target_sr=target_sr)
385
+ new_audio = self.audio_resampler.resample(audio,
386
+ orig_sr=orig_sr)
387
388
new_audios.append(new_audio)
389