A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 18:45 UTC
[Bugfix][Voxtral] Add fetch_audio to MistralCommonFeatureExtractor (transformers>=5.10 compat) (#44559)
Signed-off-by: Yadan Wei <weiyadan@amazon.com> Co-authored-by: Yadan Wei <weiyadan@amazon.com> Co-authored-by: Claude <noreply@anthropic.com>
Yadan-Wei committed
Jun 6, 2026 at 00:58 UTC
c9b4b184b408ca1ad779828cd1394c6c5217ded2
3 files changed
+177
tests/transformers_utils/processors/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
tests/transformers_utils/processors/test_voxtral.py
new
+127
@@ -0,0 +1,127 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Unit tests for ``MistralCommonFeatureExtractor.fetch_audio``.
4
+
5
+``transformers>=5.10`` adds a ``ProcessorMixin.prepare_inputs_layout`` helper
6
+that calls ``self.feature_extractor.fetch_audio(...)`` unconditionally. The
7
+duck-typed :class:`MistralCommonFeatureExtractor` previously did not implement
8
+that method, so loading any voxtral model under transformers 5.10.x raised
9
+``AttributeError: 'MistralCommonFeatureExtractor' object has no attribute
10
+'fetch_audio'``. These tests pin the new ``fetch_audio`` method to the same
11
+contract as ``transformers.SequenceFeatureExtractor.fetch_audio``.
12
+"""
13
+
14
+import numpy as np
15
+import pytest
16
+import torch
17
+
18
+from vllm.tokenizers.mistral import MistralTokenizer
19
+from vllm.transformers_utils.processors.voxtral import (
20
+ MistralCommonFeatureExtractor,
21
+)
22
+
23
+
24
+@pytest.fixture(scope="module")
25
+def feature_extractor() -> MistralCommonFeatureExtractor:
26
+ tokenizer = MistralTokenizer.from_pretrained("mistralai/Voxtral-Mini-3B-2507")
27
+ return MistralCommonFeatureExtractor(tokenizer.instruct.audio_encoder)
28
+
29
+
30
+@pytest.mark.parametrize(
31
+ "audio",
32
+ [
33
+ np.zeros(1024, dtype=np.float32),
34
+ torch.zeros(1024),
35
+ [0.0, 1.0, 2.0],
36
+ ],
37
+ ids=["numpy_array", "torch_tensor", "list_of_floats"],
38
+)
39
+def test_fetch_audio_passes_through(
40
+ feature_extractor: MistralCommonFeatureExtractor, audio
41
+):
42
+ result = feature_extractor.fetch_audio(audio)
43
+ assert result is audio
44
+
45
+
46
+def test_fetch_audio_recurses_over_list_of_arrays(
47
+ feature_extractor: MistralCommonFeatureExtractor,
48
+):
49
+ a = np.zeros(8, dtype=np.float32)
50
+ b = np.ones(8, dtype=np.float32)
51
+ result = feature_extractor.fetch_audio([a, b])
52
+ assert isinstance(result, list)
53
+ assert len(result) == 2
54
+ assert result[0] is a
55
+ assert result[1] is b
56
+
57
+
58
+def test_fetch_audio_uses_self_sampling_rate_when_none(
59
+ monkeypatch, feature_extractor: MistralCommonFeatureExtractor
60
+):
61
+ """If ``sampling_rate`` is None, ``self.sampling_rate`` must be used.
62
+
63
+ Verified indirectly via the recursion path: when we pass a list of arrays
64
+ without sampling_rate, recursive calls receive the resolved rate.
65
+ """
66
+ captured: list[int | None] = []
67
+ original = feature_extractor.fetch_audio
68
+
69
+ def spy(audio, sampling_rate=None):
70
+ captured.append(sampling_rate)
71
+ return original(audio, sampling_rate=sampling_rate)
72
+
73
+ monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
74
+ feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)])
75
+ # Top-level call has sampling_rate=None; inner recursive call sees the
76
+ # resolved rate from self.sampling_rate.
77
+ assert captured[0] is None
78
+ assert captured[1] == 16000
79
+
80
+
81
+def test_fetch_audio_explicit_sampling_rate_propagates(
82
+ monkeypatch, feature_extractor: MistralCommonFeatureExtractor
83
+):
84
+ captured: list[int | None] = []
85
+ original = feature_extractor.fetch_audio
86
+
87
+ def spy(audio, sampling_rate=None):
88
+ captured.append(sampling_rate)
89
+ return original(audio, sampling_rate=sampling_rate)
90
+
91
+ monkeypatch.setattr(feature_extractor, "fetch_audio", spy)
92
+ feature_extractor.fetch_audio([np.zeros(4, dtype=np.float32)], sampling_rate=8000)
93
+ assert captured[0] == 8000
94
+ assert captured[1] == 8000
95
+
96
+
97
+def test_fetch_audio_rejects_unsupported_type(
98
+ feature_extractor: MistralCommonFeatureExtractor,
99
+):
100
+ with pytest.raises(TypeError, match="only a numpy array"):
101
+ feature_extractor.fetch_audio(42) # type: ignore[arg-type]
102
+
103
+
104
+def test_fetch_audio_str_delegates_to_load_audio(
105
+ monkeypatch, feature_extractor: MistralCommonFeatureExtractor
106
+):
107
+ """A str input must round-trip through ``transformers.audio_utils.load_audio``.
108
+
109
+ We monkey-patch ``load_audio`` so the test stays offline (no real URL/path
110
+ fetched) and still asserts the delegation contract.
111
+ """
112
+ sentinel = np.array([0.5, -0.5], dtype=np.float32)
113
+ received: dict[str, object] = {}
114
+
115
+ def fake_load_audio(path, sampling_rate=None):
116
+ received["path"] = path
117
+ received["sampling_rate"] = sampling_rate
118
+ return sentinel
119
+
120
+ import transformers.audio_utils
121
+
122
+ monkeypatch.setattr(transformers.audio_utils, "load_audio", fake_load_audio)
123
+
124
+ result = feature_extractor.fetch_audio("/tmp/fake.wav")
125
+ assert result is sentinel
126
+ assert received["path"] == "/tmp/fake.wav"
127
+ assert received["sampling_rate"] == 16000
vllm/transformers_utils/processors/voxtral.py
+48
@@ -53,6 +53,54 @@ class MistralCommonFeatureExtractor:
53
def get_num_audio_tokens(self, audio_length: int) -> int:
54
return ceil(audio_length / (self.sampling_rate // self.frame_rate))
55
56
+ def fetch_audio(self, audio_url_or_urls, sampling_rate=None):
57
+ """HF-compatible duck-typed ``fetch_audio``.
58
+
59
+ Mirrors :meth:`transformers.SequenceFeatureExtractor.fetch_audio` so
60
+ :class:`transformers.ProcessorMixin.prepare_inputs_layout` (added in
61
+ transformers 5.10) works on this duck-typed feature extractor. Older
62
+ transformers versions never invoke this method, so the addition is a
63
+ no-op there.
64
+
65
+ Accepts the same shapes as ``SequenceFeatureExtractor.fetch_audio``:
66
+
67
+ * ``np.ndarray`` / ``torch.Tensor`` — returned as-is.
68
+ * ``list[float]`` — returned as-is (a single audio sample).
69
+ * ``str`` URL or path — delegated to
70
+ :func:`transformers.audio_utils.load_audio`.
71
+ * ``list`` of any of the above — recursed element-wise.
72
+
73
+ ``ProcessorMixin.prepare_inputs_layout`` always passes already-decoded
74
+ audio (numpy array or torch tensor), so the str / list-of-str branches
75
+ exist only to keep the contract identical to the upstream method.
76
+
77
+ The semantics of ``transformers.audio_utils.is_valid_audio`` differ
78
+ between transformers versions (5.9 only accepts ndarray/tensor; 5.10
79
+ also accepts ``list[float]``). We detect ``list[float]`` explicitly to
80
+ keep behavior identical across versions.
81
+ """
82
+ from transformers.audio_utils import is_valid_audio
83
+
84
+ sampling_rate = sampling_rate if sampling_rate else self.sampling_rate
85
+ if is_valid_audio(audio_url_or_urls):
86
+ return audio_url_or_urls
87
+ if isinstance(audio_url_or_urls, (list, tuple)):
88
+ if audio_url_or_urls and isinstance(audio_url_or_urls[0], float):
89
+ # A single audio represented as ``list[float]``.
90
+ return audio_url_or_urls
91
+ return [
92
+ self.fetch_audio(x, sampling_rate=sampling_rate)
93
+ for x in audio_url_or_urls
94
+ ]
95
+ if isinstance(audio_url_or_urls, str):
96
+ from transformers.audio_utils import load_audio
97
+
98
+ return load_audio(audio_url_or_urls, sampling_rate=sampling_rate)
99
+ raise TypeError(
100
+ "only a numpy array, torch tensor, str URL/path, or list of those "
101
+ f"is supported but got type={type(audio_url_or_urls)}"
102
+ )
103
+
104
105
class MistralCommonVoxtralProcessor(ProcessorMixin):
106
attributes = ["feature_extractor", "tokenizer"]