A high-throughput and memory-efficient inference and serving engine for LLMs
[ROCm][CI] Gate incompatible HF references on Transformers v5 (#41532)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Andreas Karatzas committed
Jun 16, 2026 at 07:34 UTC
7e179e4bc0c125d8f06a1779a3712c3173ad13fb
10 files changed
+217
-145
setup.py
+1
@@ -1240,6 +1240,7 @@ setup(
1240
"av",
1241
"scipy",
1242
"soundfile",
1243
+ "soxr",
1244
"mistral_common[audio]",
1245
], # Required for audio processing
1246
"video": [], # Kept for backwards compatibility
tests/models/language/generation/test_common.py
+6
-1
@@ -148,7 +148,11 @@ def test_models(
148
"def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - "
149
)
150
151
- with hf_runner(model) as hf_model:
151
+ with hf_runner(
152
+ model,
153
+ revision=model_info.revision,
154
+ trust_remote_code=model_info.trust_remote_code,
155
+ ) as hf_model:
156
hf_outputs = hf_model.generate_greedy_logprobs_limit(
157
example_prompts, max_tokens, num_logprobs
158
)
@@ -184,6 +188,7 @@ def test_models(
188
model,
189
tokenizer_name=model_info.tokenizer or model,
190
tokenizer_mode=model_info.tokenizer_mode,
191
+ revision=model_info.revision,
192
trust_remote_code=model_info.trust_remote_code,
193
# Remove the effects of batch variance on ROCm since batch invariance
194
# is not yet supported.
tests/models/multimodal/generation/test_musicflamingo.py
+19
-13
@@ -59,6 +59,12 @@ def get_fixture_path(filename):
59
)
60
61
62
+def load_expected_fixture(filename):
63
+ fixture_path = get_fixture_path(filename)
64
+ with open(fixture_path) as f:
65
+ return json.load(f)
66
+
67
+
68
def assert_output_matches(output, expected_text, expected_token_ids):
69
generated = output.outputs[0]
70
assert generated.text == expected_text
@@ -76,7 +82,7 @@ def llm():
82
model_info.check_transformers_version(on_fail="skip")
83
84
try:
79
- return LLM(
85
+ llm = LLM(
86
model=MODEL_NAME,
87
dtype="bfloat16",
88
enforce_eager=True,
@@ -86,14 +92,19 @@ def llm():
92
except Exception as e:
93
pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
94
95
+ # ROCm may compile decoder kernels on the first inference pass; warm up
96
+ # once so exact fixture assertions cover the steady-state path.
97
+ llm.chat(
98
+ messages=SINGLE_CONVERSATION,
99
+ sampling_params=SamplingParams(temperature=0.0, max_tokens=1),
100
+ use_tqdm=False,
101
+ )
102
90
-def test_single_generation(llm):
91
- fixture_path = get_fixture_path("expected_results_single.json")
92
- if not os.path.exists(fixture_path):
93
- pytest.skip(f"Fixture not found: {fixture_path}")
103
+ return llm
104
95
- with open(fixture_path) as f:
96
- expected = json.load(f)
105
+
106
+def test_single_generation(llm):
107
+ expected = load_expected_fixture("expected_results_single.json")
108
109
outputs = llm.chat(
110
messages=SINGLE_CONVERSATION,
@@ -108,12 +119,7 @@ def test_single_generation(llm):
119
120
121
def test_batched_generation(llm):
111
- fixture_path = get_fixture_path("expected_results_batched.json")
112
- if not os.path.exists(fixture_path):
113
- pytest.skip(f"Fixture not found: {fixture_path}")
114
-
115
- with open(fixture_path) as f:
116
- expected = json.load(f)
122
+ expected = load_expected_fixture("expected_results_batched.json")
123
124
outputs = llm.chat(
125
messages=BATCHED_CONVERSATIONS,
tests/models/multimodal/processing/test_audioflamingo3.py
+26
-15
@@ -42,15 +42,30 @@ class MockAudioFlamingo3Processor:
42
self.audio_token_id = 12345
43
self.max_audio_len = 60
44
self.feature_extractor = MockFeatureExtractor()
45
+ self.tokenizer = self._tokenize
46
46
- def __call__(self, text=None, audios=None, **kwargs):
47
- return {"input_ids": [1, 2, 3], "input_features": [np.zeros((3000, 80))]}
47
+ def __call__(self, text=None, audio=None, **kwargs):
48
+ return {
49
+ "input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
50
+ "input_features": torch.zeros((3, 80, 3000)),
51
+ "input_features_mask": torch.ones((3, 3000), dtype=torch.long),
52
+ }
53
+
54
+ def _tokenize(self, text, **kwargs):
55
+ return {"input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long)}
56
57
58
class MockFeatureExtractor:
59
def __init__(self):
60
self.sampling_rate = 16000
61
self.chunk_length = 30
62
+ self.hop_length = 160
63
+
64
+ def __call__(self, audios, **kwargs):
65
+ return {
66
+ "input_features": torch.zeros((len(audios), 80, 3000)),
67
+ "attention_mask": torch.ones((len(audios), 3000), dtype=torch.long),
68
+ }
69
70
71
@pytest.fixture
@@ -60,6 +75,9 @@ def mock_ctx():
75
ctx = MagicMock()
76
ctx.get_hf_config.return_value = config
77
ctx.get_hf_processor.return_value = MockAudioFlamingo3Processor()
78
+ ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
79
+ **data, **kwargs
80
+ )
81
ctx.model_config.hf_config = config
82
return ctx
83
@@ -89,21 +107,14 @@ def test_audio_chunk_counting(mock_ctx):
107
mm_data = {"audio": [audio_1, audio_2]}
108
prompt = "<|user|>Listen.<|end|>"
109
92
- from vllm.multimodal.processing import BaseMultiModalProcessor
93
-
94
- def mock_base_call(self, prompt, mm_data, mm_kwargs, tok_kwargs):
95
- return {"input_ids": [1, 2, 3], "input_features": torch.randn(1, 80, 3000)}
96
-
97
- with pytest.MonkeyPatch.context() as mp:
98
- mp.setattr(BaseMultiModalProcessor, "_call_hf_processor", mock_base_call)
99
-
100
- processed = processor._call_hf_processor(prompt, mm_data, {}, {})
110
+ processed = processor._call_hf_processor(prompt, mm_data, {}, {})
111
102
- chunk_counts = processed["chunk_counts"]
112
+ chunk_counts = processed["chunk_counts"]
113
104
- assert chunk_counts[0].item() == 1
105
- assert chunk_counts[1].item() == 2
106
- assert len(chunk_counts) == 2
114
+ assert chunk_counts[0].item() == 1
115
+ assert chunk_counts[1].item() == 2
116
+ assert len(chunk_counts) == 2
117
+ assert processed["feature_attention_mask"].shape == (3, 3000)
118
119
120
def test_dummy_data_generation(mock_ctx):
tests/models/multimodal/processing/test_musicflamingo.py
+13
-14
@@ -49,6 +49,13 @@ class MockMusicFlamingoProcessor:
49
self.max_audio_len = 1200
50
self.feature_extractor = MockFeatureExtractor()
51
52
+ def __call__(self, text=None, audio=None, **kwargs):
53
+ return {
54
+ "input_ids": torch.tensor([[1, 2, 3]], dtype=torch.long),
55
+ "input_features": torch.zeros((3, 80, 3000)),
56
+ "input_features_mask": torch.ones((3, 3000), dtype=torch.long),
57
+ }
58
+
59
60
class MockFeatureExtractor:
61
def __init__(self):
@@ -63,6 +70,9 @@ def mock_ctx():
70
ctx = MagicMock()
71
ctx.get_hf_config.return_value = config
72
ctx.get_hf_processor.return_value = MockMusicFlamingoProcessor()
73
+ ctx.call_hf_processor.side_effect = lambda processor, data, kwargs: processor(
74
+ **data, **kwargs
75
+ )
76
ctx.model_config.hf_config = config
77
return ctx
78
@@ -73,7 +83,7 @@ def check_transformers_version():
83
model_info.check_transformers_version(on_fail="skip")
84
85
76
-def test_musicflamingo_chunk_counting_uses_rote_timestamps(mock_ctx, monkeypatch):
86
+def test_musicflamingo_chunk_counting_without_rote_timestamps(mock_ctx):
87
from vllm.model_executor.models.musicflamingo import (
88
MusicFlamingoDummyInputsBuilder,
89
MusicFlamingoMultiModalProcessor,
@@ -92,24 +102,13 @@ def test_musicflamingo_chunk_counting_uses_rote_timestamps(mock_ctx, monkeypatch
102
mm_data = {"audio": [audio_1, audio_2]}
103
prompt = "<|user|>Listen.<|end|>"
104
95
- from vllm.multimodal.processing import BaseMultiModalProcessor
96
-
97
- def mock_base_call(self, prompt, mm_data, mm_kwargs, tok_kwargs):
98
- del self, prompt, mm_data, mm_kwargs, tok_kwargs
99
- return {
100
- "input_ids": [1, 2, 3],
101
- "input_features": torch.randn(3, 80, 3000),
102
- "rote_timestamps": torch.randn(3, 750),
103
- }
104
-
105
- monkeypatch.setattr(BaseMultiModalProcessor, "_call_hf_processor", mock_base_call)
106
-
105
processed = processor._call_hf_processor(prompt, mm_data, {}, {})
106
107
chunk_counts = processed["chunk_counts"]
108
109
assert chunk_counts.tolist() == [1, 2]
112
- assert "rote_timestamps" in processed
110
+ assert "rote_timestamps" not in processed
111
+ assert processed["feature_attention_mask"].shape == (3, 3000)
112
113
114
def test_musicflamingo_dummy_text_uses_plain_audio_tokens(mock_ctx):
tests/models/registry.py
+24
-10
@@ -145,22 +145,28 @@ class _HfExamplesInfo:
145
# Only check the base version for the min/max version, otherwise preview
146
# models cannot be run because `x.yy.0.dev0`<`x.yy.0`
147
if min_version and Version(cur_base_version) < Version(min_version):
148
- is_version_valid = not check_min_version
148
+ is_version_valid = False
149
+ should_check_version = check_min_version
150
msg += f">={min_version}` is required to run this model."
151
elif max_version and Version(cur_base_version) > Version(max_version):
151
- is_version_valid = not check_max_version
152
+ is_version_valid = False
153
+ should_check_version = check_max_version
154
msg += f"<={max_version}` is required to run this model."
155
else:
156
is_version_valid = True
157
+ should_check_version = False
158
156
- # check if Transformers version breaks the corresponding model runner,
157
- # skip test when model runner not compatible
158
- is_reason_valid = not (
159
- check_version_reason
160
- and self.transformers_version_reason
159
+ # Reasons explain a known incompatibility with a violated version
160
+ # bound. They should not skip models when the installed version is
161
+ # already within the requested range.
162
+ is_reason_applicable = (
163
+ not is_version_valid
164
+ and self.transformers_version_reason is not None
165
and check_version_reason in self.transformers_version_reason
166
)
163
- is_transformers_valid = is_version_valid and is_reason_valid
167
+ is_transformers_valid = is_version_valid or (
168
+ not should_check_version and not is_reason_applicable
169
+ )
170
if is_transformers_valid:
171
return None
172
elif self.transformers_version_reason:
@@ -405,7 +411,15 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
411
"openbmb/MiniCPM3-4B", trust_remote_code=True
412
),
413
"MiniCPM4ForCausalLM": _HfExamplesInfo(
408
- "openbmb/MiniCPM4.1-8B", trust_remote_code=True
414
+ "openbmb/MiniCPM4.1-8B",
415
+ min_transformers_version="4.56",
416
+ max_transformers_version="4.57",
417
+ transformers_version_reason={
418
+ "hf": "HF remote code imports removed `is_torch_fx_available`; "
419
+ "the upstream compatibility shim request was closed as not planned: "
420
+ "https://github.com/huggingface/transformers/issues/44561"
421
+ },
422
+ trust_remote_code=True,
423
),
424
"MiniMaxForCausalLM": _HfExamplesInfo("MiniMaxAI/MiniMax-Text-01-hf"),
425
"MiniMaxText01ForCausalLM": _HfExamplesInfo(
@@ -786,7 +800,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
800
),
801
"MusicFlamingoForConditionalGeneration": _HfExamplesInfo(
802
"nvidia/music-flamingo-2601-hf",
789
- min_transformers_version="5.3.0",
803
+ min_transformers_version="5.5.0",
804
transformers_version_reason={
805
"vllm": "Needs https://github.com/huggingface/transformers/pull/43538"
806
},
vllm/model_executor/models/audioflamingo3.py
+27
-29
@@ -202,7 +202,7 @@ class AudioFlamingo3ProcessingInfo(BaseProcessingInfo):
202
)
203
204
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
205
- return {"audio": None}
205
+ return {"audio": 1}
206
207
208
class AudioFlamingo3DummyInputsBuilder(
@@ -379,32 +379,37 @@ class AudioFlamingo3MultiModalProcessor(
379
mm_kwargs: Mapping[str, Any],
380
tok_kwargs: Mapping[str, object],
381
) -> BatchFeature:
382
- audios = mm_data.pop("audios", [])
383
- if audios:
384
- mm_data["audio"] = audios
382
+ processor_mm_data = dict(mm_data)
383
+ audios = processor_mm_data.pop("audios", None)
384
+ if audios is not None:
385
+ processor_mm_data["audio"] = audios
386
386
- if not mm_data.get("audio", []):
387
- prompt_ids = self.info.get_tokenizer().encode(prompt)
388
- prompt_ids = self._apply_hf_processor_tokens_only(prompt_ids)
389
- return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")
390
-
391
- processor = self.info.get_hf_processor(**mm_kwargs)
392
- feature_extractor = processor.feature_extractor
393
- mm_kwargs = dict(
394
- **mm_kwargs,
395
- sampling_rate=feature_extractor.sampling_rate,
387
+ outputs = super()._call_hf_processor(
388
+ prompt=prompt,
389
+ mm_data=processor_mm_data,
390
+ mm_kwargs=mm_kwargs,
391
+ tok_kwargs=tok_kwargs,
392
)
393
398
- audio_list = mm_data.get("audio")
399
- if not isinstance(audio_list, list):
400
- audio_list = [audio_list]
394
+ if "input_features_mask" in outputs:
395
+ outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
396
402
- chunk_counts = []
397
+ audio_data = processor_mm_data.get("audio")
398
+ if audio_data is None:
399
+ return outputs
400
+
401
+ audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
402
+ if len(audio_list) == 0:
403
+ return outputs
404
+
405
+ processor = self.info.get_hf_processor(**mm_kwargs)
406
+ feature_extractor = processor.feature_extractor
407
sampling_rate = feature_extractor.sampling_rate
408
chunk_length = feature_extractor.chunk_length
409
window_size = int(sampling_rate * chunk_length)
410
max_windows = int(processor.max_audio_len // chunk_length)
411
412
+ chunk_counts = []
413
for audio in audio_list:
414
# audio is numpy array or list
415
n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
@@ -414,18 +419,7 @@ class AudioFlamingo3MultiModalProcessor(
419
n_win = max_windows
420
chunk_counts.append(n_win)
421
417
- outputs = super()._call_hf_processor(
418
- prompt=prompt,
419
- mm_data=mm_data,
420
- mm_kwargs=mm_kwargs,
421
- tok_kwargs=tok_kwargs,
422
- )
423
-
424
- if "input_features_mask" in outputs:
425
- outputs["feature_attention_mask"] = outputs.pop("input_features_mask")
426
-
422
outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
428
-
423
return outputs
424
425
def _get_mm_fields_config(
@@ -611,6 +605,10 @@ class AudioFlamingo3ForConditionalGeneration(
605
input_features: torch.Tensor,
606
feature_attention_mask: torch.Tensor,
607
) -> torch.Tensor:
608
+ input_features = input_features.to(
609
+ dtype=self.audio_tower.conv1.weight.dtype,
610
+ device=self.audio_tower.conv1.weight.device,
611
+ )
612
audio_attention_mask = _build_audio_encoder_attention_mask(
613
feature_attention_mask,
614
dtype=self.audio_tower.conv1.weight.dtype,
vllm/model_executor/models/musicflamingo.py
+67
-60
@@ -120,9 +120,11 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
120
) -> tuple["torch.Tensor", float]:
121
del seq_len
122
base = config.rope_parameters["rope_theta"]
123
- dim = getattr(config, "head_dim", None) or (
123
+ partial_rotary_factor = config.rope_parameters.get("partial_rotary_factor", 1.0)
124
+ head_dim = getattr(config, "head_dim", None) or (
125
config.hidden_size // config.num_attention_heads
126
)
127
+ dim = int(head_dim * partial_rotary_factor)
128
attention_factor = 1.0
129
130
inv_freq = 1.0 / (
@@ -148,21 +150,40 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
150
position_angles = torch.repeat_interleave(position_angles, 2, dim=-1)
151
return position_angles.to(dtype=inv_freq.dtype)
152
153
+ def _restore_fp32_rope_buffers(self) -> None:
154
+ rope_init_fn: Callable = self.compute_default_rope_parameters
155
+ if self.rope_type != "default":
156
+ rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
157
+ inv_freq, self.attention_scaling = rope_init_fn(
158
+ self.config, self.inv_freq.device
159
+ )
160
+
161
+ self.inv_freq = inv_freq
162
+ self.original_inv_freq = inv_freq.clone()
163
+ self.position_angles = self._compute_position_angles(inv_freq)
164
+
165
+ def _apply(self, fn):
166
+ super()._apply(fn)
167
+ self._restore_fp32_rope_buffers()
168
+ return self
169
+
170
@torch.no_grad()
171
def forward(self, timestamps: Tensor, seq_len: int) -> tuple[Tensor, Tensor]:
153
- batch_positions = torch.arange(
154
- timestamps.shape[0],
172
+ window_starts = timestamps[:, 0].to(
173
device=self.inv_freq.device,
174
dtype=self.inv_freq.dtype,
175
)
158
- batch_positions = batch_positions / self.max_seq_len_cached
159
- batch_freqs = batch_positions.unsqueeze(-1) * self.inv_freq
160
- batch_freqs = torch.repeat_interleave(batch_freqs, 2, dim=-1)
176
+ window_duration = self.config.audio_frame_step * 4 * seq_len
177
+ window_positions = (
178
+ torch.round(window_starts / window_duration) / self.max_seq_len_cached
179
+ )
180
+ window_freqs = window_positions.unsqueeze(-1) * self.inv_freq
181
+ window_freqs = torch.repeat_interleave(window_freqs, 2, dim=-1)
182
162
- batch_freqs = batch_freqs[:, None, :]
183
+ window_freqs = window_freqs[:, None, :]
184
time_freqs = self.position_angles[:seq_len][None, :, :]
164
- batch_freqs, time_freqs = broadcast_tensors(batch_freqs, time_freqs)
165
- freqs = torch.cat((batch_freqs, time_freqs), dim=-1)
185
+ window_freqs, time_freqs = broadcast_tensors(window_freqs, time_freqs)
186
+ freqs = torch.cat((window_freqs, time_freqs), dim=-1)
187
angle = (-timestamps * 2 * pi).to(freqs)
188
freqs = freqs * angle.unsqueeze(-1)
189
return freqs.cos(), freqs.sin()
@@ -170,7 +191,7 @@ class MusicFlamingoRotaryEmbedding(nn.Module):
191
192
class MusicFlamingoFeatureInputs(AudioFlamingo3FeatureInputs):
193
rote_timestamps: Annotated[
173
- torch.Tensor,
194
+ torch.Tensor | None,
195
TensorShape(
196
"num_chunks",
197
"num_audio_time_steps",
@@ -205,6 +226,7 @@ class MusicFlamingoProcessingInfo(AudioFlamingo3ProcessingInfo):
226
feature_extractor = self.get_feature_extractor()
227
return MusicFlamingoMultiModalDataParser(
228
target_sr=feature_extractor.sampling_rate,
229
+ audio_resample_method="soxr",
230
expected_hidden_size=self._get_expected_hidden_size(),
231
)
232
@@ -268,49 +290,6 @@ class MusicFlamingoMultiModalDataParser(AudioFlamingo3MultiModalDataParser):
290
291
292
class MusicFlamingoMultiModalProcessor(AudioFlamingo3MultiModalProcessor):
271
- def _call_hf_processor(
272
- self,
273
- prompt: str,
274
- mm_data: dict[str, object],
275
- mm_kwargs: Mapping[str, Any],
276
- tok_kwargs: Mapping[str, object],
277
- ) -> BatchFeature:
278
- outputs = super()._call_hf_processor(
279
- prompt=prompt,
280
- mm_data=mm_data,
281
- mm_kwargs=mm_kwargs,
282
- tok_kwargs=tok_kwargs,
283
- )
284
-
285
- audio_data = mm_data.get("audio")
286
- if audio_data is None:
287
- return outputs
288
-
289
- audio_list = audio_data if isinstance(audio_data, list) else [audio_data]
290
- if len(audio_list) == 0:
291
- return outputs
292
-
293
- processor = self.info.get_hf_processor(**mm_kwargs)
294
- feature_extractor = processor.feature_extractor
295
- sampling_rate = feature_extractor.sampling_rate
296
- chunk_length = feature_extractor.chunk_length
297
- window_size = int(sampling_rate * chunk_length)
298
- max_windows = int(processor.max_audio_len // chunk_length)
299
-
300
- chunk_counts = []
301
- for audio in audio_list:
302
- n_samples = len(audio) if isinstance(audio, list) else audio.shape[0]
303
- n_win = max(1, (n_samples + window_size - 1) // window_size)
304
- chunk_counts.append(min(n_win, max_windows))
305
- outputs["chunk_counts"] = torch.tensor(chunk_counts, dtype=torch.long)
306
-
307
- if "rote_timestamps" not in outputs:
308
- raise KeyError(
309
- "MusicFlamingoProcessor output must include `rote_timestamps`."
310
- )
311
-
312
- return outputs
313
-
293
def _get_mm_fields_config(
294
self,
295
hf_inputs: BatchFeature,
@@ -405,6 +384,32 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
384
rote_timestamps=rote_timestamps,
385
)
386
387
+ def _build_audio_timestamps(
388
+ self,
389
+ chunk_counts: list[int],
390
+ seq_len: int,
391
+ device: torch.device,
392
+ ) -> torch.Tensor:
393
+ audio_embed_frame_step = self.config.audio_frame_step * 4
394
+ frame_offsets = (
395
+ torch.arange(seq_len, device=device, dtype=torch.float32)
396
+ * audio_embed_frame_step
397
+ )
398
+
399
+ if not chunk_counts:
400
+ return frame_offsets.new_empty((0, seq_len))
401
+
402
+ window_indices = torch.cat(
403
+ [
404
+ torch.arange(count, device=device, dtype=torch.float32)
405
+ for count in chunk_counts
406
+ ]
407
+ )
408
+ return (
409
+ window_indices.unsqueeze(1) * seq_len * audio_embed_frame_step
410
+ + frame_offsets
411
+ )
412
+
413
def _process_audio_input(
414
self, audio_input: MusicFlamingoInputs
415
) -> torch.Tensor | tuple[torch.Tensor, ...]:
@@ -412,13 +417,6 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
417
return super()._process_audio_input(audio_input)
418
419
rote_timestamps = audio_input["rote_timestamps"]
415
- if rote_timestamps is None:
416
- raise ValueError(
417
- "MusicFlamingo audio feature inputs must include `rote_timestamps`."
418
- )
419
- if isinstance(rote_timestamps, list):
420
- rote_timestamps = torch.cat(rote_timestamps, dim=0)
421
-
420
(
421
input_features,
422
feature_attention_mask,
@@ -428,6 +426,15 @@ class MusicFlamingoForConditionalGeneration(AudioFlamingo3ForConditionalGenerati
426
input_features,
427
feature_attention_mask,
428
)
429
+ if rote_timestamps is None:
430
+ rote_timestamps = self._build_audio_timestamps(
431
+ chunk_counts,
432
+ seq_len=hidden_states.shape[-2],
433
+ device=hidden_states.device,
434
+ )
435
+ elif isinstance(rote_timestamps, list):
436
+ rote_timestamps = torch.cat(rote_timestamps, dim=0)
437
+
438
cos, sin = self.pos_emb(
439
rote_timestamps.to(hidden_states.device),
440
seq_len=hidden_states.shape[-2],
vllm/multimodal/audio.py
+33
-2
@@ -21,6 +21,11 @@ try:
21
except ImportError:
22
scipy_signal = PlaceholderModule("scipy").placeholder_attr("signal") # type: ignore[assignment]
23
24
+try:
25
+ import soxr as soxr
26
+except ImportError:
27
+ soxr = PlaceholderModule("soxr") # type: ignore[assignment]
28
+
29
30
# ============================================================
31
# Aligned with `librosa.get_duration` function
@@ -245,13 +250,37 @@ def resample_audio_scipy(
250
)
251
252
253
+def resample_audio_soxr(
254
+ audio: npt.NDArray[np.floating],
255
+ *,
256
+ orig_sr: float,
257
+ target_sr: float,
258
+) -> npt.NDArray[np.floating]:
259
+ orig_sr_int = int(round(orig_sr))
260
+ target_sr_int = int(round(target_sr))
261
+
262
+ if orig_sr_int == target_sr_int:
263
+ return audio
264
+
265
+ if audio.ndim == 2:
266
+ return np.stack(
267
+ [
268
+ resample_audio_soxr(ch, orig_sr=orig_sr, target_sr=target_sr)
269
+ for ch in audio
270
+ ],
271
+ axis=0,
272
+ )
273
+
274
+ return soxr.resample(audio, orig_sr_int, target_sr_int)
275
+
276
+
277
class AudioResampler:
278
"""Resample audio data to a target sample rate."""
279
280
def __init__(
281
self,
282
target_sr: float | None = None,
254
- method: Literal["pyav", "scipy"] = "pyav",
283
+ method: Literal["pyav", "scipy", "soxr"] = "pyav",
284
):
285
self.target_sr = target_sr
286
self.method = method
@@ -279,10 +308,12 @@ class AudioResampler:
308
return resample_audio_scipy(
309
audio, orig_sr=orig_sr, target_sr=self.target_sr
310
)
311
+ elif self.method == "soxr":
312
+ return resample_audio_soxr(audio, orig_sr=orig_sr, target_sr=self.target_sr)
313
else:
314
raise ValueError(
315
f"Invalid resampling method: {self.method}. "
285
- "Supported methods are 'pyav' and 'scipy'."
316
+ "Supported methods are 'pyav', 'scipy', and 'soxr'."
317
)
318
319
vllm/multimodal/parse.py
+1
-1
@@ -508,7 +508,7 @@ class MultiModalDataParser:
508
*,
509
target_sr: float | None = None,
510
target_channels: int | None = None,
511
- audio_resample_method: Literal["pyav", "scipy"] = "pyav",
511
+ audio_resample_method: Literal["pyav", "scipy", "soxr"] = "pyav",
512
video_needs_metadata: bool = False,
513
expected_hidden_size: int | None = None,
514
) -> None: