fix(processor): route MiMo-V2-Omni media fetch through MediaConnector (#43117)
Signed-off-by: Ievgen Bondarenko <ibondarenko@student.sierracollege.edu> Signed-off-by: Ievgen (Jack) Bondarenko <ibondarenko@student.sierracollege.edu> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Ievgen Bondarenko committed
Jul 11, 2026 at 08:52 UTC
54503ecec0f3ac31e5ecfc5f28652e4cc42307b5
1 file changed
+22
-76
vllm/transformers_utils/processors/mimo_v2_omni.py
+22
-76
@@ -7,33 +7,21 @@ Ported from SGLang's MiMoV2OmniProcessor / MiMoVLProcessor implementations.
7
"""
8
9
import contextlib
10
-import copy
11
-import io
10
import logging
11
import math
12
from collections import OrderedDict
13
from concurrent.futures import ThreadPoolExecutor, as_completed
14
from dataclasses import dataclass, field
17
-from io import BytesIO
15
from typing import Any, Literal
16
17
import numpy as np
18
import regex as re
22
-import requests
19
import torch
20
import torch.nn.functional as F
21
from PIL import Image
22
from transformers import BatchFeature, TensorType
23
from transformers.processing_utils import ProcessorMixin
24
29
-try:
30
- from torchcodec.decoders import AudioDecoder
31
-
32
- _HAS_TORCHCODEC = True
33
-except ImportError:
34
- AudioDecoder = None
35
- _HAS_TORCHCODEC = False
36
-
25
try:
26
import torchaudio
27
from torchaudio.transforms import MelSpectrogram as _MelSpectrogram
@@ -62,7 +50,7 @@ _mean_std_cache: dict[str, tuple[torch.Tensor, torch.Tensor]] = {}
50
51
@dataclass
52
class ImageInput:
65
- # PIL.Image | str (path/url/base64) | bytes | torch.Tensor (C,H,W)
53
+ # PIL.Image | torch.Tensor (C,H,W)
54
image: Any
55
max_pixels: int | None = None
56
min_pixels: int | None = None
@@ -87,7 +75,7 @@ class VideoInput:
75
76
@dataclass
77
class AudioInput:
90
- # str (path/url/base64) | bytes | tuple[waveform_1D, sr]
78
+ # tuple[waveform_1D, sr]
79
# | np.ndarray | torch.Tensor (T,n_vq)
80
audio: Any
81
@@ -168,14 +156,6 @@ def _smart_resize(
156
return int(h_bar), int(w_bar)
157
158
171
-def _to_rgb(img: Image.Image) -> Image.Image:
172
- if img.mode == "RGBA":
173
- bg = Image.new("RGB", img.size, (255, 255, 255))
174
- bg.paste(img, mask=img.split()[3])
175
- return bg
176
- return img.convert("RGB")
177
-
178
-
159
def _standardize(images: torch.Tensor) -> torch.Tensor:
160
key = str(images.device)
161
if key not in _mean_std_cache:
@@ -228,27 +208,6 @@ def _transform_single(
208
return _standardize(out).squeeze(0), w_bar, h_bar
209
210
231
-def _fetch_image(src: Any) -> Image.Image:
232
- if isinstance(src, Image.Image):
233
- return _to_rgb(src)
234
- if isinstance(src, bytes):
235
- return _to_rgb(copy.deepcopy(Image.open(BytesIO(src))))
236
- if isinstance(src, str):
237
- if src.startswith(("http://", "https://")):
238
- r = requests.get(src, timeout=30)
239
- r.raise_for_status()
240
- return _to_rgb(copy.deepcopy(Image.open(BytesIO(r.content))))
241
- if src.startswith("file://"):
242
- return _to_rgb(Image.open(src[7:]))
243
- if src.startswith("data:image"):
244
- import pybase64 as _b64
245
-
246
- _, b64 = src.split("base64,", 1)
247
- return _to_rgb(copy.deepcopy(Image.open(BytesIO(_b64.b64decode(b64)))))
248
- return _to_rgb(Image.open(src))
249
- raise ValueError(f"Unrecognized image source: {type(src)}")
250
-
251
-
211
# ---------------------------------------------------------------------------
212
# Core processor
213
# ---------------------------------------------------------------------------
@@ -259,6 +218,10 @@ class MiMoVLProcessor:
218
219
Handles image/video/audio preprocessing and token sequence construction.
220
Ported from SGLang's MiMoVLProcessor.
221
+
222
+ Media strings (URL / path / data:) are resolved upstream in vLLM's media
223
+ pipeline; this processor accepts only already-decoded inputs (``PIL.Image``
224
+ for images, ``(waveform, sr)`` tuples for audio).
225
"""
226
227
def __init__(
@@ -451,33 +414,14 @@ class MiMoVLProcessor:
414
return kw
415
416
def preprocess_audio(self, audio: Any) -> tuple[torch.Tensor, int]:
454
- """Decode audio bytes/path/tuple → (mel_spec (T, n_mels), token_len)."""
455
- if isinstance(audio, tuple):
456
- waveform, original_sr = audio
457
- else:
458
- if AudioDecoder is None:
459
- raise RuntimeError(
460
- "torchcodec is required for audio. "
461
- "Install with: pip install torchcodec"
462
- )
463
- if isinstance(audio, bytes):
464
- file_obj: Any = io.BytesIO(audio)
465
- elif isinstance(audio, str):
466
- if audio.startswith("data:"):
467
- import pybase64 as _b64
468
-
469
- file_obj = io.BytesIO(_b64.b64decode(audio.split(",")[1]))
470
- elif audio.startswith(("http://", "https://")):
471
- r = requests.get(audio, timeout=30)
472
- r.raise_for_status()
473
- file_obj = io.BytesIO(r.content)
474
- else:
475
- file_obj = audio
476
- else:
477
- raise ValueError(f"Unsupported audio source type: {type(audio)}")
478
- samples = AudioDecoder(file_obj).get_all_samples()
479
- waveform = samples.data
480
- original_sr = samples.sample_rate
417
+ """Convert a pre-loaded ``(waveform, sr)`` tuple into (mel_spec, token_len)."""
418
+ if not isinstance(audio, tuple):
419
+ raise ValueError(
420
+ f"Unsupported audio source type: {type(audio)}. Audio must be a "
421
+ "pre-decoded (waveform, sample_rate) tuple; URL/path/bytes "
422
+ "resolution is handled upstream in vLLM's media pipeline."
423
+ )
424
+ waveform, original_sr = audio
425
426
if original_sr != self.audio_sampling_rate:
427
if original_sr not in self._resamplers:
@@ -504,8 +448,6 @@ class MiMoVLProcessor:
448
def process_image(self, image: ImageInput) -> torch.Tensor:
449
kw = self._resolve_img_kw(image)
450
src = image.image
507
- if isinstance(src, (str, bytes)):
508
- src = _fetch_image(src)
451
tensor, _, _ = _transform_single(
452
src,
453
factor=self.patch_size * self.merge_size,
@@ -584,7 +526,7 @@ class MiMoVLProcessor:
526
src = audio.audio
527
if isinstance(src, np.ndarray):
528
src = (torch.from_numpy(src).float(), self.audio_sampling_rate)
587
- if isinstance(src, (str, bytes, tuple)):
529
+ if isinstance(src, tuple):
530
return self.preprocess_audio(src)
531
# Pre-tokenized tensor (T, n_vq)
532
assert isinstance(src, torch.Tensor) and src.ndim == 2
@@ -899,7 +841,7 @@ class MiMoOmniProcessor(ProcessorMixin):
841
"""HuggingFace-compatible ProcessorMixin wrapper for MiMo-Omni.
842
843
Accepts PIL images, pre-decoded video tuples (frames_TCHW, timestamps_T),
902
- and audio (file path / bytes / (waveform, sr) tuple / numpy array).
844
+ and audio (file path / (waveform, sr) tuple / numpy array).
845
"""
846
847
attributes = ["tokenizer"]
@@ -1011,8 +953,12 @@ class MiMoOmniProcessor(ProcessorMixin):
953
)
954
955
@classmethod
1014
- def from_hf_config(cls, tokenizer: Any, hf_config: Any) -> "MiMoOmniProcessor":
1015
- """Convenience factory: instantiate directly from an HF model config object."""
956
+ def from_hf_config(
957
+ cls,
958
+ tokenizer: Any,
959
+ hf_config: Any,
960
+ ) -> "MiMoOmniProcessor":
961
+ """Instantiate directly from an HF model config object."""
962
vc = hf_config.vision_config
963
if isinstance(vc, dict):
964
patch_size = vc.get("patch_size", 14)