@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28459785ff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 10, 02:45 UTC

[3/N] Group together media-related code (#32406)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 15, 2026 at 19:52 UTC 28459785ff905648f08a42049904d7d162646398
25 files changed +833 -725
tests/conftest.py
+1 -1
@@ -63,7 +63,7 @@ from vllm.distributed import (
63 )
64 from vllm.logger import init_logger
65 from vllm.logprobs import Logprob
66 -from vllm.multimodal.base import MediaWithBytes
66 +from vllm.multimodal.media import MediaWithBytes
67 from vllm.multimodal.utils import fetch_image
68 from vllm.outputs import RequestOutput
69 from vllm.sampling_params import BeamSearchParams
tests/entrypoints/openai/test_sparse_tensor_validation.py
+1 -2
@@ -14,8 +14,7 @@ import pytest
14 import torch
15
16 from vllm.entrypoints.renderer import CompletionRenderer
17 -from vllm.multimodal.audio import AudioEmbeddingMediaIO
18 -from vllm.multimodal.image import ImageEmbeddingMediaIO
17 +from vllm.multimodal.media import AudioEmbeddingMediaIO, ImageEmbeddingMediaIO
18
19
20 def _encode_tensor(tensor: torch.Tensor) -> bytes:
tests/entrypoints/openai/test_vision.py
+1 -1
@@ -8,7 +8,7 @@ import pytest
8 import pytest_asyncio
9 from transformers import AutoProcessor
10
11 -from vllm.multimodal.base import MediaWithBytes
11 +from vllm.multimodal.media import MediaWithBytes
12 from vllm.multimodal.utils import encode_image_url, fetch_image
13 from vllm.platforms import current_platform
14
tests/entrypoints/pooling/embed/test_online_vision.py
+1 -1
@@ -9,7 +9,7 @@ from transformers import AutoProcessor
9
10 from tests.utils import VLLM_PATH, RemoteOpenAIServer
11 from vllm.entrypoints.pooling.embed.protocol import EmbeddingResponse
12 -from vllm.multimodal.base import MediaWithBytes
12 +from vllm.multimodal.media import MediaWithBytes
13 from vllm.multimodal.utils import fetch_image
14
15 MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
tests/multimodal/media/__init__.py
tests/multimodal/media/test_audio.py new
+73
@@ -0,0 +1,73 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import base64
4 +from pathlib import Path
5 +from unittest.mock import patch
6 +
7 +import numpy as np
8 +import pytest
9 +
10 +from vllm.multimodal.media import AudioMediaIO
11 +
12 +pytestmark = pytest.mark.cpu_test
13 +
14 +ASSETS_DIR = Path(__file__).parent.parent / "assets"
15 +assert ASSETS_DIR.exists()
16 +
17 +
18 +@pytest.fixture
19 +def dummy_audio():
20 + return np.array([0.0, 0.1, 0.2, 0.3, 0.4], dtype=float)
21 +
22 +
23 +@pytest.fixture
24 +def dummy_audio_bytes():
25 + return b"FAKEAUDIOBYTES"
26 +
27 +
28 +def test_audio_media_io_load_bytes(dummy_audio_bytes):
29 + audio_io = AudioMediaIO()
30 + with patch("librosa.load") as mock_load:
31 + mock_load.return_value = (np.array([0.1, 0.2]), 16000)
32 + out = audio_io.load_bytes(dummy_audio_bytes)
33 + mock_load.assert_called_once()
34 + assert isinstance(out[0], np.ndarray)
35 + assert out[1] == 16000
36 +
37 +
38 +def test_audio_media_io_load_base64(dummy_audio_bytes):
39 + audio_io = AudioMediaIO()
40 + encoded = base64.b64encode(dummy_audio_bytes).decode("utf-8")
41 + with patch.object(AudioMediaIO, "load_bytes") as mock_load_bytes:
42 + mock_load_bytes.return_value = (np.array([0.1, 0.2]), 16000)
43 + out = audio_io.load_base64("audio/wav", encoded)
44 + mock_load_bytes.assert_called_once()
45 + assert isinstance(out[0], np.ndarray)
46 + assert out[1] == 16000
47 +
48 +
49 +def test_audio_media_io_load_file():
50 + audio_io = AudioMediaIO()
51 + path = Path("/fake/path.wav")
52 + with patch("librosa.load") as mock_load:
53 + mock_load.return_value = (np.array([0.1, 0.2]), 16000)
54 + out = audio_io.load_file(path)
55 + mock_load.assert_called_once_with(path, sr=None)
56 + assert isinstance(out[0], np.ndarray)
57 + assert out[1] == 16000
58 +
59 +
60 +def test_audio_media_io_encode_base64(dummy_audio):
61 + audio_io = AudioMediaIO()
62 + media = (dummy_audio, 16000)
63 + with patch("soundfile.write") as mock_write:
64 +
65 + def write_to_buffer(buffer, *_args, **_kwargs):
66 + buffer.write(b"dummy_wav_data")
67 +
68 + mock_write.side_effect = write_to_buffer
69 +
70 + out = audio_io.encode_base64(media)
71 + decoded = base64.b64decode(out)
72 + assert decoded == b"dummy_wav_data"
73 + mock_write.assert_called_once()
tests/multimodal/media/test_base.py new
+45
@@ -0,0 +1,45 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import pickle
4 +from pathlib import Path
5 +
6 +import pytest
7 +from PIL import Image
8 +
9 +from vllm.multimodal.media import MediaWithBytes
10 +
11 +pytestmark = pytest.mark.cpu_test
12 +
13 +ASSETS_DIR = Path(__file__).parent.parent / "assets"
14 +assert ASSETS_DIR.exists()
15 +
16 +
17 +def test_media_with_bytes_pickle_roundtrip():
18 + """Regression test for pickle/unpickle of MediaWithBytes.
19 +
20 + Verifies that MediaWithBytes can be pickled and unpickled without
21 + RecursionError. See: https://github.com/vllm-project/vllm/issues/30818
22 + """
23 + original_image = Image.open(ASSETS_DIR / "image1.png").convert("RGB")
24 + original_bytes = b"test_bytes_data"
25 +
26 + wrapper = MediaWithBytes(media=original_image, original_bytes=original_bytes)
27 +
28 + # Verify attribute delegation works before pickling
29 + assert wrapper.width == original_image.width
30 + assert wrapper.height == original_image.height
31 + assert wrapper.mode == original_image.mode
32 +
33 + # Pickle and unpickle (this would cause RecursionError before the fix)
34 + pickled = pickle.dumps(wrapper)
35 + unpickled = pickle.loads(pickled)
36 +
37 + # Verify the unpickled object works correctly
38 + assert unpickled.original_bytes == original_bytes
39 + assert unpickled.media.width == original_image.width
40 + assert unpickled.media.height == original_image.height
41 +
42 + # Verify attribute delegation works after unpickling
43 + assert unpickled.width == original_image.width
44 + assert unpickled.height == original_image.height
45 + assert unpickled.mode == original_image.mode
tests/multimodal/media/test_image.py new
+133
@@ -0,0 +1,133 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from pathlib import Path
4 +
5 +import numpy as np
6 +import pytest
7 +from PIL import Image
8 +
9 +from vllm.multimodal.media import ImageMediaIO
10 +
11 +pytestmark = pytest.mark.cpu_test
12 +
13 +ASSETS_DIR = Path(__file__).parent.parent / "assets"
14 +assert ASSETS_DIR.exists()
15 +
16 +
17 +def test_image_media_io_rgba_custom_background(tmp_path):
18 + """Test RGBA to RGB conversion with custom background colors."""
19 + # Create a simple RGBA image with transparent and opaque pixels
20 + rgba_image = Image.new("RGBA", (10, 10), (255, 0, 0, 255)) # Red with full opacity
21 +
22 + # Make top-left quadrant transparent
23 + for i in range(5):
24 + for j in range(5):
25 + rgba_image.putpixel((i, j), (0, 0, 0, 0)) # Fully transparent
26 +
27 + # Save the test image to tmp_path
28 + test_image_path = tmp_path / "test_rgba.png"
29 + rgba_image.save(test_image_path)
30 +
31 + # Test 1: Default white background (backward compatibility)
32 + image_io_default = ImageMediaIO()
33 + converted_default = image_io_default.load_file(test_image_path)
34 + default_numpy = np.array(converted_default)
35 +
36 + # Check transparent pixels are white
37 + assert default_numpy[0][0][0] == 255 # R
38 + assert default_numpy[0][0][1] == 255 # G
39 + assert default_numpy[0][0][2] == 255 # B
40 + # Check opaque pixels remain red
41 + assert default_numpy[5][5][0] == 255 # R
42 + assert default_numpy[5][5][1] == 0 # G
43 + assert default_numpy[5][5][2] == 0 # B
44 +
45 + # Test 2: Custom black background via kwargs
46 + image_io_black = ImageMediaIO(rgba_background_color=(0, 0, 0))
47 + converted_black = image_io_black.load_file(test_image_path)
48 + black_numpy = np.array(converted_black)
49 +
50 + # Check transparent pixels are black
51 + assert black_numpy[0][0][0] == 0 # R
52 + assert black_numpy[0][0][1] == 0 # G
53 + assert black_numpy[0][0][2] == 0 # B
54 + # Check opaque pixels remain red
55 + assert black_numpy[5][5][0] == 255 # R
56 + assert black_numpy[5][5][1] == 0 # G
57 + assert black_numpy[5][5][2] == 0 # B
58 +
59 + # Test 3: Custom blue background via kwargs (as list)
60 + image_io_blue = ImageMediaIO(rgba_background_color=[0, 0, 255])
61 + converted_blue = image_io_blue.load_file(test_image_path)
62 + blue_numpy = np.array(converted_blue)
63 +
64 + # Check transparent pixels are blue
65 + assert blue_numpy[0][0][0] == 0 # R
66 + assert blue_numpy[0][0][1] == 0 # G
67 + assert blue_numpy[0][0][2] == 255 # B
68 +
69 + # Test 4: Test with load_bytes method
70 + with open(test_image_path, "rb") as f:
71 + image_data = f.read()
72 +
73 + image_io_green = ImageMediaIO(rgba_background_color=(0, 255, 0))
74 + converted_green = image_io_green.load_bytes(image_data)
75 + green_numpy = np.array(converted_green)
76 +
77 + # Check transparent pixels are green
78 + assert green_numpy[0][0][0] == 0 # R
79 + assert green_numpy[0][0][1] == 255 # G
80 + assert green_numpy[0][0][2] == 0 # B
81 +
82 +
83 +def test_image_media_io_rgba_background_color_validation():
84 + """Test that invalid rgba_background_color values are properly rejected."""
85 +
86 + # Test invalid types
87 + with pytest.raises(
88 + ValueError, match="rgba_background_color must be a list or tuple"
89 + ):
90 + ImageMediaIO(rgba_background_color="255,255,255")
91 +
92 + with pytest.raises(
93 + ValueError, match="rgba_background_color must be a list or tuple"
94 + ):
95 + ImageMediaIO(rgba_background_color=255)
96 +
97 + # Test wrong number of elements
98 + with pytest.raises(
99 + ValueError, match="rgba_background_color must be a list or tuple"
100 + ):
101 + ImageMediaIO(rgba_background_color=(255, 255))
102 +
103 + with pytest.raises(
104 + ValueError, match="rgba_background_color must be a list or tuple"
105 + ):
106 + ImageMediaIO(rgba_background_color=(255, 255, 255, 255))
107 +
108 + # Test non-integer values
109 + with pytest.raises(
110 + ValueError, match="rgba_background_color must be a list or tuple"
111 + ):
112 + ImageMediaIO(rgba_background_color=(255.0, 255.0, 255.0))
113 +
114 + with pytest.raises(
115 + ValueError, match="rgba_background_color must be a list or tuple"
116 + ):
117 + ImageMediaIO(rgba_background_color=(255, "255", 255))
118 +
119 + # Test out of range values
120 + with pytest.raises(
121 + ValueError, match="rgba_background_color must be a list or tuple"
122 + ):
123 + ImageMediaIO(rgba_background_color=(256, 255, 255))
124 +
125 + with pytest.raises(
126 + ValueError, match="rgba_background_color must be a list or tuple"
127 + ):
128 + ImageMediaIO(rgba_background_color=(255, -1, 255))
129 +
130 + # Test that valid values work
131 + ImageMediaIO(rgba_background_color=(0, 0, 0)) # Should not raise
132 + ImageMediaIO(rgba_background_color=[255, 255, 255]) # Should not raise
133 + ImageMediaIO(rgba_background_color=(128, 128, 128)) # Should not raise
tests/multimodal/media/test_video.py new
+237
@@ -0,0 +1,237 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from pathlib import Path
4 +
5 +import numpy as np
6 +import numpy.typing as npt
7 +import pytest
8 +from PIL import Image
9 +
10 +from vllm.assets.base import get_vllm_public_assets
11 +from vllm.assets.video import video_to_ndarrays, video_to_pil_images_list
12 +from vllm.multimodal.media import ImageMediaIO, VideoMediaIO
13 +from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader
14 +
15 +from ..utils import cosine_similarity, create_video_from_image, normalize_image
16 +
17 +pytestmark = pytest.mark.cpu_test
18 +
19 +ASSETS_DIR = Path(__file__).parent.parent / "assets"
20 +assert ASSETS_DIR.exists()
21 +
22 +
23 +@VIDEO_LOADER_REGISTRY.register("assert_10_frames_1_fps")
24 +class Assert10Frames1FPSVideoLoader(VideoLoader):
25 + @classmethod
26 + def load_bytes(
27 + cls, data: bytes, num_frames: int = -1, fps: float = -1.0, **kwargs
28 + ) -> npt.NDArray:
29 + assert num_frames == 10, "bad num_frames"
30 + assert fps == 1.0, "bad fps"
31 + return FAKE_OUTPUT_2
32 +
33 +
34 +def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
35 + with monkeypatch.context() as m:
36 + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "assert_10_frames_1_fps")
37 + imageio = ImageMediaIO()
38 +
39 + # Verify that different args pass/fail assertions as expected.
40 + videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0})
41 + _ = videoio.load_bytes(b"test")
42 +
43 + videoio = VideoMediaIO(
44 + imageio, **{"num_frames": 10, "fps": 1.0, "not_used": "not_used"}
45 + )
46 + _ = videoio.load_bytes(b"test")
47 +
48 + with pytest.raises(AssertionError, match="bad num_frames"):
49 + videoio = VideoMediaIO(imageio, **{})
50 + _ = videoio.load_bytes(b"test")
51 +
52 + with pytest.raises(AssertionError, match="bad num_frames"):
53 + videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0})
54 + _ = videoio.load_bytes(b"test")
55 +
56 + with pytest.raises(AssertionError, match="bad fps"):
57 + videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0})
58 + _ = videoio.load_bytes(b"test")
59 +
60 +
61 +@pytest.mark.parametrize("is_color", [True, False])
62 +@pytest.mark.parametrize("fourcc, ext", [("mp4v", "mp4"), ("XVID", "avi")])
63 +def test_opencv_video_io_colorspace(tmp_path, is_color: bool, fourcc: str, ext: str):
64 + """
65 + Test all functions that use OpenCV for video I/O return RGB format.
66 + Both RGB and grayscale videos are tested.
67 + """
68 + image_path = get_vllm_public_assets(
69 + filename="stop_sign.jpg", s3_prefix="vision_model_images"
70 + )
71 + image = Image.open(image_path)
72 +
73 + if not is_color:
74 + image_path = f"{tmp_path}/test_grayscale_image.png"
75 + image = image.convert("L")
76 + image.save(image_path)
77 + # Convert to gray RGB for comparison
78 + image = image.convert("RGB")
79 + video_path = f"{tmp_path}/test_RGB_video.{ext}"
80 + create_video_from_image(
81 + image_path,
82 + video_path,
83 + num_frames=2,
84 + is_color=is_color,
85 + fourcc=fourcc,
86 + )
87 +
88 + frames = video_to_ndarrays(video_path)
89 + for frame in frames:
90 + sim = cosine_similarity(
91 + normalize_image(np.array(frame)), normalize_image(np.array(image))
92 + )
93 + assert np.sum(np.isnan(sim)) / sim.size < 0.001
94 + assert np.nanmean(sim) > 0.99
95 +
96 + pil_frames = video_to_pil_images_list(video_path)
97 + for frame in pil_frames:
98 + sim = cosine_similarity(
99 + normalize_image(np.array(frame)), normalize_image(np.array(image))
100 + )
101 + assert np.sum(np.isnan(sim)) / sim.size < 0.001
102 + assert np.nanmean(sim) > 0.99
103 +
104 + io_frames, _ = VideoMediaIO(ImageMediaIO()).load_file(Path(video_path))
105 + for frame in io_frames:
106 + sim = cosine_similarity(
107 + normalize_image(np.array(frame)), normalize_image(np.array(image))
108 + )
109 + assert np.sum(np.isnan(sim)) / sim.size < 0.001
110 + assert np.nanmean(sim) > 0.99
111 +
112 +
113 +NUM_FRAMES = 10
114 +FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3)
115 +FAKE_OUTPUT_2 = np.random.rand(NUM_FRAMES, 1280, 720, 3)
116 +
117 +
118 +@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_1")
119 +class TestVideoBackendOverride1(VideoLoader):
120 + """Test loader that returns FAKE_OUTPUT_1 to verify backend selection."""
121 +
122 + @classmethod
123 + def load_bytes(
124 + cls, data: bytes, num_frames: int = -1, **kwargs
125 + ) -> tuple[npt.NDArray, dict]:
126 + return FAKE_OUTPUT_1, {"video_backend": "test_video_backend_override_1"}
127 +
128 +
129 +@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_2")
130 +class TestVideoBackendOverride2(VideoLoader):
131 + """Test loader that returns FAKE_OUTPUT_2 to verify backend selection."""
132 +
133 + @classmethod
134 + def load_bytes(
135 + cls, data: bytes, num_frames: int = -1, **kwargs
136 + ) -> tuple[npt.NDArray, dict]:
137 + return FAKE_OUTPUT_2, {"video_backend": "test_video_backend_override_2"}
138 +
139 +
140 +def test_video_media_io_backend_kwarg_override(monkeypatch: pytest.MonkeyPatch):
141 + """
142 + Test that video_backend kwarg can override the VLLM_VIDEO_LOADER_BACKEND
143 + environment variable.
144 +
145 + This allows users to dynamically select a different video backend
146 + via --media-io-kwargs without changing the global env var, which is
147 + useful when plugins set a default backend but a specific request
148 + needs a different one.
149 + """
150 + with monkeypatch.context() as m:
151 + # Set the env var to one backend
152 + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_1")
153 +
154 + imageio = ImageMediaIO()
155 +
156 + # Without video_backend kwarg, should use env var backend
157 + videoio_default = VideoMediaIO(imageio, num_frames=10)
158 + frames_default, metadata_default = videoio_default.load_bytes(b"test")
159 + np.testing.assert_array_equal(frames_default, FAKE_OUTPUT_1)
160 + assert metadata_default["video_backend"] == "test_video_backend_override_1"
161 +
162 + # With video_backend kwarg, should override env var
163 + videoio_override = VideoMediaIO(
164 + imageio, num_frames=10, video_backend="test_video_backend_override_2"
165 + )
166 + frames_override, metadata_override = videoio_override.load_bytes(b"test")
167 + np.testing.assert_array_equal(frames_override, FAKE_OUTPUT_2)
168 + assert metadata_override["video_backend"] == "test_video_backend_override_2"
169 +
170 +
171 +def test_video_media_io_backend_kwarg_not_passed_to_loader(
172 + monkeypatch: pytest.MonkeyPatch,
173 +):
174 + """
175 + Test that video_backend kwarg is consumed by VideoMediaIO and NOT passed
176 + through to the underlying video loader's load_bytes method.
177 +
178 + This ensures the kwarg is properly popped from kwargs before forwarding.
179 + """
180 +
181 + @VIDEO_LOADER_REGISTRY.register("test_reject_video_backend_kwarg")
182 + class RejectVideoBackendKwargLoader(VideoLoader):
183 + """Test loader that fails if video_backend is passed through."""
184 +
185 + @classmethod
186 + def load_bytes(
187 + cls, data: bytes, num_frames: int = -1, **kwargs
188 + ) -> tuple[npt.NDArray, dict]:
189 + # This should never receive video_backend in kwargs
190 + if "video_backend" in kwargs:
191 + raise AssertionError(
192 + "video_backend should be consumed by VideoMediaIO, "
193 + "not passed to loader"
194 + )
195 + return FAKE_OUTPUT_1, {"received_kwargs": list(kwargs.keys())}
196 +
197 + with monkeypatch.context() as m:
198 + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_reject_video_backend_kwarg")
199 +
200 + imageio = ImageMediaIO()
201 +
202 + # Even when video_backend is provided, it should NOT be passed to loader
203 + videoio = VideoMediaIO(
204 + imageio,
205 + num_frames=10,
206 + video_backend="test_reject_video_backend_kwarg",
207 + other_kwarg="should_pass_through",
208 + )
209 +
210 + # This should NOT raise AssertionError
211 + frames, metadata = videoio.load_bytes(b"test")
212 + np.testing.assert_array_equal(frames, FAKE_OUTPUT_1)
213 + # Verify other kwargs are still passed through
214 + assert "other_kwarg" in metadata["received_kwargs"]
215 +
216 +
217 +def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch):
218 + """
219 + Test that when video_backend kwarg is None or not provided,
220 + VideoMediaIO falls back to VLLM_VIDEO_LOADER_BACKEND env var.
221 + """
222 + with monkeypatch.context() as m:
223 + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_2")
224 +
225 + imageio = ImageMediaIO()
226 +
227 + # Explicit None should fall back to env var
228 + videoio_none = VideoMediaIO(imageio, num_frames=10, video_backend=None)
229 + frames_none, metadata_none = videoio_none.load_bytes(b"test")
230 + np.testing.assert_array_equal(frames_none, FAKE_OUTPUT_2)
231 + assert metadata_none["video_backend"] == "test_video_backend_override_2"
232 +
233 + # Not providing video_backend should also fall back to env var
234 + videoio_missing = VideoMediaIO(imageio, num_frames=10)
235 + frames_missing, metadata_missing = videoio_missing.load_bytes(b"test")
236 + np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2)
237 + assert metadata_missing["video_backend"] == "test_video_backend_override_2"
tests/multimodal/test_audio.py
-56
@@ -1,8 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 # test_audio.py
4 -import base64
5 -from pathlib import Path
4 from unittest.mock import patch
5
6 import numpy as np
@@ -12,7 +10,6 @@ import torch
10 from vllm.multimodal.audio import (
11 MONO_AUDIO_SPEC,
12 PASSTHROUGH_AUDIO_SPEC,
15 - AudioMediaIO,
13 AudioResampler,
14 AudioSpec,
15 ChannelReduction,
@@ -92,59 +89,6 @@ def test_audio_resampler_no_target_sr(dummy_audio):
89 resampler.resample(dummy_audio, orig_sr=44100)
90
91
95 -@pytest.fixture
96 -def dummy_audio_bytes():
97 - return b"FAKEAUDIOBYTES"
98 -
99 -
100 -def test_audio_media_io_load_bytes(dummy_audio_bytes):
101 - audio_io = AudioMediaIO()
102 - with patch("vllm.multimodal.audio.librosa.load") as mock_load:
103 - mock_load.return_value = (np.array([0.1, 0.2]), 16000)
104 - out = audio_io.load_bytes(dummy_audio_bytes)
105 - mock_load.assert_called_once()
106 - assert isinstance(out[0], np.ndarray)
107 - assert out[1] == 16000
108 -
109 -
110 -def test_audio_media_io_load_base64(dummy_audio_bytes):
111 - audio_io = AudioMediaIO()
112 - encoded = base64.b64encode(dummy_audio_bytes).decode("utf-8")
113 - with patch.object(AudioMediaIO, "load_bytes") as mock_load_bytes:
114 - mock_load_bytes.return_value = (np.array([0.1, 0.2]), 16000)
115 - out = audio_io.load_base64("audio/wav", encoded)
116 - mock_load_bytes.assert_called_once()
117 - assert isinstance(out[0], np.ndarray)
118 - assert out[1] == 16000
119 -
120 -
121 -def test_audio_media_io_load_file():
122 - audio_io = AudioMediaIO()
123 - path = Path("/fake/path.wav")
124 - with patch("vllm.multimodal.audio.librosa.load") as mock_load:
125 - mock_load.return_value = (np.array([0.1, 0.2]), 16000)
126 - out = audio_io.load_file(path)
127 - mock_load.assert_called_once_with(path, sr=None)
128 - assert isinstance(out[0], np.ndarray)
129 - assert out[1] == 16000
130 -
131 -
132 -def test_audio_media_io_encode_base64(dummy_audio):
133 - audio_io = AudioMediaIO()
134 - media = (dummy_audio, 16000)
135 - with patch("vllm.multimodal.audio.soundfile.write") as mock_write:
136 -
137 - def write_to_buffer(buffer, *_args, **_kwargs):
138 - buffer.write(b"dummy_wav_data")
139 -
140 - mock_write.side_effect = write_to_buffer
141 -
142 - out = audio_io.encode_base64(media)
143 - decoded = base64.b64decode(out)
144 - assert decoded == b"dummy_wav_data"
145 - mock_write.assert_called_once()
146 -
147 -
92 # ============================================================
93 # Tests for normalize_audio function
94 # ============================================================
tests/multimodal/test_image.py
+1 -153
@@ -1,14 +1,12 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import pickle
3 from pathlib import Path
4
5 import numpy as np
6 import pytest
7 from PIL import Image, ImageChops
8
10 -from vllm.multimodal.base import MediaWithBytes
11 -from vllm.multimodal.image import ImageMediaIO, convert_image_mode
9 +from vllm.multimodal.image import convert_image_mode
10
11 pytestmark = pytest.mark.cpu_test
12
@@ -40,153 +38,3 @@ def test_rgba_to_rgb():
38 assert converted_image_numpy[i][j][0] == 255
39 assert converted_image_numpy[i][j][1] == 255
40 assert converted_image_numpy[i][j][2] == 255
43 -
44 -
45 -def test_rgba_to_rgb_custom_background(tmp_path):
46 - """Test RGBA to RGB conversion with custom background colors."""
47 - # Create a simple RGBA image with transparent and opaque pixels
48 - rgba_image = Image.new("RGBA", (10, 10), (255, 0, 0, 255)) # Red with full opacity
49 -
50 - # Make top-left quadrant transparent
51 - for i in range(5):
52 - for j in range(5):
53 - rgba_image.putpixel((i, j), (0, 0, 0, 0)) # Fully transparent
54 -
55 - # Save the test image to tmp_path
56 - test_image_path = tmp_path / "test_rgba.png"
57 - rgba_image.save(test_image_path)
58 -
59 - # Test 1: Default white background (backward compatibility)
60 - image_io_default = ImageMediaIO()
61 - converted_default = image_io_default.load_file(test_image_path)
62 - default_numpy = np.array(converted_default)
63 -
64 - # Check transparent pixels are white
65 - assert default_numpy[0][0][0] == 255 # R
66 - assert default_numpy[0][0][1] == 255 # G
67 - assert default_numpy[0][0][2] == 255 # B
68 - # Check opaque pixels remain red
69 - assert default_numpy[5][5][0] == 255 # R
70 - assert default_numpy[5][5][1] == 0 # G
71 - assert default_numpy[5][5][2] == 0 # B
72 -
73 - # Test 2: Custom black background via kwargs
74 - image_io_black = ImageMediaIO(rgba_background_color=(0, 0, 0))
75 - converted_black = image_io_black.load_file(test_image_path)
76 - black_numpy = np.array(converted_black)
77 -
78 - # Check transparent pixels are black
79 - assert black_numpy[0][0][0] == 0 # R
80 - assert black_numpy[0][0][1] == 0 # G
81 - assert black_numpy[0][0][2] == 0 # B
82 - # Check opaque pixels remain red
83 - assert black_numpy[5][5][0] == 255 # R
84 - assert black_numpy[5][5][1] == 0 # G
85 - assert black_numpy[5][5][2] == 0 # B
86 -
87 - # Test 3: Custom blue background via kwargs (as list)
88 - image_io_blue = ImageMediaIO(rgba_background_color=[0, 0, 255])
89 - converted_blue = image_io_blue.load_file(test_image_path)
90 - blue_numpy = np.array(converted_blue)
91 -
92 - # Check transparent pixels are blue
93 - assert blue_numpy[0][0][0] == 0 # R
94 - assert blue_numpy[0][0][1] == 0 # G
95 - assert blue_numpy[0][0][2] == 255 # B
96 -
97 - # Test 4: Test with load_bytes method
98 - with open(test_image_path, "rb") as f:
99 - image_data = f.read()
100 -
101 - image_io_green = ImageMediaIO(rgba_background_color=(0, 255, 0))
102 - converted_green = image_io_green.load_bytes(image_data)
103 - green_numpy = np.array(converted_green)
104 -
105 - # Check transparent pixels are green
106 - assert green_numpy[0][0][0] == 0 # R
107 - assert green_numpy[0][0][1] == 255 # G
108 - assert green_numpy[0][0][2] == 0 # B
109 -
110 -
111 -def test_rgba_background_color_validation():
112 - """Test that invalid rgba_background_color values are properly rejected."""
113 -
114 - # Test invalid types
115 - with pytest.raises(
116 - ValueError, match="rgba_background_color must be a list or tuple"
117 - ):
118 - ImageMediaIO(rgba_background_color="255,255,255")
119 -
120 - with pytest.raises(
121 - ValueError, match="rgba_background_color must be a list or tuple"
122 - ):
123 - ImageMediaIO(rgba_background_color=255)
124 -
125 - # Test wrong number of elements
126 - with pytest.raises(
127 - ValueError, match="rgba_background_color must be a list or tuple"
128 - ):
129 - ImageMediaIO(rgba_background_color=(255, 255))
130 -
131 - with pytest.raises(
132 - ValueError, match="rgba_background_color must be a list or tuple"
133 - ):
134 - ImageMediaIO(rgba_background_color=(255, 255, 255, 255))
135 -
136 - # Test non-integer values
137 - with pytest.raises(
138 - ValueError, match="rgba_background_color must be a list or tuple"
139 - ):
140 - ImageMediaIO(rgba_background_color=(255.0, 255.0, 255.0))
141 -
142 - with pytest.raises(
143 - ValueError, match="rgba_background_color must be a list or tuple"
144 - ):
145 - ImageMediaIO(rgba_background_color=(255, "255", 255))
146 -
147 - # Test out of range values
148 - with pytest.raises(
149 - ValueError, match="rgba_background_color must be a list or tuple"
150 - ):
151 - ImageMediaIO(rgba_background_color=(256, 255, 255))
152 -
153 - with pytest.raises(
154 - ValueError, match="rgba_background_color must be a list or tuple"
155 - ):
156 - ImageMediaIO(rgba_background_color=(255, -1, 255))
157 -
158 - # Test that valid values work
159 - ImageMediaIO(rgba_background_color=(0, 0, 0)) # Should not raise
160 - ImageMediaIO(rgba_background_color=[255, 255, 255]) # Should not raise
161 - ImageMediaIO(rgba_background_color=(128, 128, 128)) # Should not raise
162 -
163 -
164 -def test_media_with_bytes_pickle_roundtrip():
165 - """Regression test for pickle/unpickle of MediaWithBytes.
166 -
167 - Verifies that MediaWithBytes can be pickled and unpickled without
168 - RecursionError. See: https://github.com/vllm-project/vllm/issues/30818
169 - """
170 - original_image = Image.open(ASSETS_DIR / "image1.png").convert("RGB")
171 - original_bytes = b"test_bytes_data"
172 -
173 - wrapper = MediaWithBytes(media=original_image, original_bytes=original_bytes)
174 -
175 - # Verify attribute delegation works before pickling
176 - assert wrapper.width == original_image.width
177 - assert wrapper.height == original_image.height
178 - assert wrapper.mode == original_image.mode
179 -
180 - # Pickle and unpickle (this would cause RecursionError before the fix)
181 - pickled = pickle.dumps(wrapper)
182 - unpickled = pickle.loads(pickled)
183 -
184 - # Verify the unpickled object works correctly
185 - assert unpickled.original_bytes == original_bytes
186 - assert unpickled.media.width == original_image.width
187 - assert unpickled.media.height == original_image.height
188 -
189 - # Verify attribute delegation works after unpickling
190 - assert unpickled.width == original_image.width
191 - assert unpickled.height == original_image.height
192 - assert unpickled.mode == original_image.mode
tests/multimodal/test_video.py
+3 -220
@@ -1,24 +1,19 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -import tempfile
4 from pathlib import Path
5
6 import numpy as np
7 import numpy.typing as npt
8 import pytest
10 -from PIL import Image
9
12 -from vllm.assets.base import get_vllm_public_assets
13 -from vllm.assets.video import video_to_ndarrays, video_to_pil_images_list
14 -from vllm.multimodal.image import ImageMediaIO
15 -from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader, VideoMediaIO
16 -
17 -from .utils import cosine_similarity, create_video_from_image, normalize_image
10 +from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader
11
12 pytestmark = pytest.mark.cpu_test
13
14 ASSETS_DIR = Path(__file__).parent / "assets"
15 +assert ASSETS_DIR.exists()
16 +
17 NUM_FRAMES = 10
18 FAKE_OUTPUT_1 = np.random.rand(NUM_FRAMES, 1280, 720, 3)
19 FAKE_OUTPUT_2 = np.random.rand(NUM_FRAMES, 1280, 720, 3)
@@ -53,96 +48,6 @@ def test_video_loader_type_doesnt_exist():
48 VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
49
50
56 -@VIDEO_LOADER_REGISTRY.register("assert_10_frames_1_fps")
57 -class Assert10Frames1FPSVideoLoader(VideoLoader):
58 - @classmethod
59 - def load_bytes(
60 - cls, data: bytes, num_frames: int = -1, fps: float = -1.0, **kwargs
61 - ) -> npt.NDArray:
62 - assert num_frames == 10, "bad num_frames"
63 - assert fps == 1.0, "bad fps"
64 - return FAKE_OUTPUT_2
65 -
66 -
67 -def test_video_media_io_kwargs(monkeypatch: pytest.MonkeyPatch):
68 - with monkeypatch.context() as m:
69 - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "assert_10_frames_1_fps")
70 - imageio = ImageMediaIO()
71 -
72 - # Verify that different args pass/fail assertions as expected.
73 - videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 1.0})
74 - _ = videoio.load_bytes(b"test")
75 -
76 - videoio = VideoMediaIO(
77 - imageio, **{"num_frames": 10, "fps": 1.0, "not_used": "not_used"}
78 - )
79 - _ = videoio.load_bytes(b"test")
80 -
81 - with pytest.raises(AssertionError, match="bad num_frames"):
82 - videoio = VideoMediaIO(imageio, **{})
83 - _ = videoio.load_bytes(b"test")
84 -
85 - with pytest.raises(AssertionError, match="bad num_frames"):
86 - videoio = VideoMediaIO(imageio, **{"num_frames": 9, "fps": 1.0})
87 - _ = videoio.load_bytes(b"test")
88 -
89 - with pytest.raises(AssertionError, match="bad fps"):
90 - videoio = VideoMediaIO(imageio, **{"num_frames": 10, "fps": 2.0})
91 - _ = videoio.load_bytes(b"test")
92 -
93 -
94 -@pytest.mark.parametrize("is_color", [True, False])
95 -@pytest.mark.parametrize("fourcc, ext", [("mp4v", "mp4"), ("XVID", "avi")])
96 -def test_opencv_video_io_colorspace(is_color: bool, fourcc: str, ext: str):
97 - """
98 - Test all functions that use OpenCV for video I/O return RGB format.
99 - Both RGB and grayscale videos are tested.
100 - """
101 - image_path = get_vllm_public_assets(
102 - filename="stop_sign.jpg", s3_prefix="vision_model_images"
103 - )
104 - image = Image.open(image_path)
105 - with tempfile.TemporaryDirectory() as tmpdir:
106 - if not is_color:
107 - image_path = f"{tmpdir}/test_grayscale_image.png"
108 - image = image.convert("L")
109 - image.save(image_path)
110 - # Convert to gray RGB for comparison
111 - image = image.convert("RGB")
112 - video_path = f"{tmpdir}/test_RGB_video.{ext}"
113 - create_video_from_image(
114 - image_path,
115 - video_path,
116 - num_frames=2,
117 - is_color=is_color,
118 - fourcc=fourcc,
119 - )
120 -
121 - frames = video_to_ndarrays(video_path)
122 - for frame in frames:
123 - sim = cosine_similarity(
124 - normalize_image(np.array(frame)), normalize_image(np.array(image))
125 - )
126 - assert np.sum(np.isnan(sim)) / sim.size < 0.001
127 - assert np.nanmean(sim) > 0.99
128 -
129 - pil_frames = video_to_pil_images_list(video_path)
130 - for frame in pil_frames:
131 - sim = cosine_similarity(
132 - normalize_image(np.array(frame)), normalize_image(np.array(image))
133 - )
134 - assert np.sum(np.isnan(sim)) / sim.size < 0.001
135 - assert np.nanmean(sim) > 0.99
136 -
137 - io_frames, _ = VideoMediaIO(ImageMediaIO()).load_file(Path(video_path))
138 - for frame in io_frames:
139 - sim = cosine_similarity(
140 - normalize_image(np.array(frame)), normalize_image(np.array(image))
141 - )
142 - assert np.sum(np.isnan(sim)) / sim.size < 0.001
143 - assert np.nanmean(sim) > 0.99
144 -
145 -
51 def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
52 """
53 Regression test for handling videos with broken frames.
@@ -179,128 +84,6 @@ def test_video_backend_handles_broken_frames(monkeypatch: pytest.MonkeyPatch):
84 )
85
86
182 -@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_1")
183 -class TestVideoBackendOverride1(VideoLoader):
184 - """Test loader that returns FAKE_OUTPUT_1 to verify backend selection."""
185 -
186 - @classmethod
187 - def load_bytes(
188 - cls, data: bytes, num_frames: int = -1, **kwargs
189 - ) -> tuple[npt.NDArray, dict]:
190 - return FAKE_OUTPUT_1, {"video_backend": "test_video_backend_override_1"}
191 -
192 -
193 -@VIDEO_LOADER_REGISTRY.register("test_video_backend_override_2")
194 -class TestVideoBackendOverride2(VideoLoader):
195 - """Test loader that returns FAKE_OUTPUT_2 to verify backend selection."""
196 -
197 - @classmethod
198 - def load_bytes(
199 - cls, data: bytes, num_frames: int = -1, **kwargs
200 - ) -> tuple[npt.NDArray, dict]:
201 - return FAKE_OUTPUT_2, {"video_backend": "test_video_backend_override_2"}
202 -
203 -
204 -def test_video_media_io_backend_kwarg_override(monkeypatch: pytest.MonkeyPatch):
205 - """
206 - Test that video_backend kwarg can override the VLLM_VIDEO_LOADER_BACKEND
207 - environment variable.
208 -
209 - This allows users to dynamically select a different video backend
210 - via --media-io-kwargs without changing the global env var, which is
211 - useful when plugins set a default backend but a specific request
212 - needs a different one.
213 - """
214 - with monkeypatch.context() as m:
215 - # Set the env var to one backend
216 - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_1")
217 -
218 - imageio = ImageMediaIO()
219 -
220 - # Without video_backend kwarg, should use env var backend
221 - videoio_default = VideoMediaIO(imageio, num_frames=10)
222 - frames_default, metadata_default = videoio_default.load_bytes(b"test")
223 - np.testing.assert_array_equal(frames_default, FAKE_OUTPUT_1)
224 - assert metadata_default["video_backend"] == "test_video_backend_override_1"
225 -
226 - # With video_backend kwarg, should override env var
227 - videoio_override = VideoMediaIO(
228 - imageio, num_frames=10, video_backend="test_video_backend_override_2"
229 - )
230 - frames_override, metadata_override = videoio_override.load_bytes(b"test")
231 - np.testing.assert_array_equal(frames_override, FAKE_OUTPUT_2)
232 - assert metadata_override["video_backend"] == "test_video_backend_override_2"
233 -
234 -
235 -def test_video_media_io_backend_kwarg_not_passed_to_loader(
236 - monkeypatch: pytest.MonkeyPatch,
237 -):
238 - """
239 - Test that video_backend kwarg is consumed by VideoMediaIO and NOT passed
240 - through to the underlying video loader's load_bytes method.
241 -
242 - This ensures the kwarg is properly popped from kwargs before forwarding.
243 - """
244 -
245 - @VIDEO_LOADER_REGISTRY.register("test_reject_video_backend_kwarg")
246 - class RejectVideoBackendKwargLoader(VideoLoader):
247 - """Test loader that fails if video_backend is passed through."""
248 -
249 - @classmethod
250 - def load_bytes(
251 - cls, data: bytes, num_frames: int = -1, **kwargs
252 - ) -> tuple[npt.NDArray, dict]:
253 - # This should never receive video_backend in kwargs
254 - if "video_backend" in kwargs:
255 - raise AssertionError(
256 - "video_backend should be consumed by VideoMediaIO, "
257 - "not passed to loader"
258 - )
259 - return FAKE_OUTPUT_1, {"received_kwargs": list(kwargs.keys())}
260 -
261 - with monkeypatch.context() as m:
262 - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_reject_video_backend_kwarg")
263 -
264 - imageio = ImageMediaIO()
265 -
266 - # Even when video_backend is provided, it should NOT be passed to loader
267 - videoio = VideoMediaIO(
268 - imageio,
269 - num_frames=10,
270 - video_backend="test_reject_video_backend_kwarg",
271 - other_kwarg="should_pass_through",
272 - )
273 -
274 - # This should NOT raise AssertionError
275 - frames, metadata = videoio.load_bytes(b"test")
276 - np.testing.assert_array_equal(frames, FAKE_OUTPUT_1)
277 - # Verify other kwargs are still passed through
278 - assert "other_kwarg" in metadata["received_kwargs"]
279 -
280 -
281 -def test_video_media_io_backend_env_var_fallback(monkeypatch: pytest.MonkeyPatch):
282 - """
283 - Test that when video_backend kwarg is None or not provided,
284 - VideoMediaIO falls back to VLLM_VIDEO_LOADER_BACKEND env var.
285 - """
286 - with monkeypatch.context() as m:
287 - m.setenv("VLLM_VIDEO_LOADER_BACKEND", "test_video_backend_override_2")
288 -
289 - imageio = ImageMediaIO()
290 -
291 - # Explicit None should fall back to env var
292 - videoio_none = VideoMediaIO(imageio, num_frames=10, video_backend=None)
293 - frames_none, metadata_none = videoio_none.load_bytes(b"test")
294 - np.testing.assert_array_equal(frames_none, FAKE_OUTPUT_2)
295 - assert metadata_none["video_backend"] == "test_video_backend_override_2"
296 -
297 - # Not providing video_backend should also fall back to env var
298 - videoio_missing = VideoMediaIO(imageio, num_frames=10)
299 - frames_missing, metadata_missing = videoio_missing.load_bytes(b"test")
300 - np.testing.assert_array_equal(frames_missing, FAKE_OUTPUT_2)
301 - assert metadata_missing["video_backend"] == "test_video_backend_override_2"
302 -
303 -
87 # ============================================================================
88 # Frame Recovery Tests
89 # ============================================================================
tools/pre_commit/check_pickle_imports.py
+1 -1
@@ -27,7 +27,7 @@ ALLOWED_FILES = {
27 "vllm/distributed/device_communicators/shm_broadcast.py",
28 "vllm/distributed/device_communicators/shm_object_storage.py",
29 "vllm/utils/hashing.py",
30 - "tests/multimodal/test_image.py",
30 + "tests/multimodal/media/test_base.py",
31 "tests/tokenizers_/test_hf.py",
32 "tests/utils_/test_hashing.py",
33 "benchmarks/kernels/graph_machete_bench.py",
vllm/multimodal/audio.py
-77
@@ -1,32 +1,20 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import base64
3 from dataclasses import dataclass
4 from enum import Enum
6 -from io import BytesIO
7 -from pathlib import Path
5 from typing import Literal
6
7 import numpy as np
8 import numpy.typing as npt
12 -import pybase64
9 import torch
10
11 from vllm.utils.import_utils import PlaceholderModule
16 -from vllm.utils.serial_utils import tensor2base64
17 -
18 -from .base import MediaIO
12
13 try:
14 import librosa
15 except ImportError:
16 librosa = PlaceholderModule("librosa") # type: ignore[assignment]
17
25 -try:
26 - import soundfile
27 -except ImportError:
28 - soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
29 -
18
19 try:
20 import scipy.signal as scipy_signal
@@ -220,68 +208,3 @@ class AudioResampler:
208 f"Invalid resampling method: {self.method}. "
209 "Supported methods are 'librosa' and 'scipy'."
210 )
223 -
224 -
225 -class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
226 - def __init__(self, **kwargs) -> None:
227 - super().__init__()
228 -
229 - # `kwargs` contains custom arguments from
230 - # --media-io-kwargs for this modality.
231 - # They can be passed to the underlying
232 - # media loaders (e.g. custom implementations)
233 - # for flexible control.
234 - self.kwargs = kwargs
235 -
236 - def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
237 - return librosa.load(BytesIO(data), sr=None)
238 -
239 - def load_base64(
240 - self,
241 - media_type: str,
242 - data: str,
243 - ) -> tuple[npt.NDArray, float]:
244 - return self.load_bytes(base64.b64decode(data))
245 -
246 - def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
247 - return librosa.load(filepath, sr=None)
248 -
249 - def encode_base64(
250 - self,
251 - media: tuple[npt.NDArray, int],
252 - *,
253 - audio_format: str = "WAV",
254 - ) -> str:
255 - audio, sr = media
256 -
257 - with BytesIO() as buffer:
258 - soundfile.write(buffer, audio, sr, format=audio_format)
259 - data = buffer.getvalue()
260 -
261 - return base64.b64encode(data).decode("utf-8")
262 -
263 -
264 -class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
265 - def __init__(self) -> None:
266 - super().__init__()
267 -
268 - def load_bytes(self, data: bytes) -> torch.Tensor:
269 - buffer = BytesIO(data)
270 - # Enable sparse tensor integrity checks to prevent out-of-bounds
271 - # writes from maliciously crafted tensors
272 - with torch.sparse.check_sparse_tensor_invariants():
273 - tensor = torch.load(buffer, weights_only=True)
274 - return tensor.to_dense()
275 -
276 - def load_base64(self, media_type: str, data: str) -> torch.Tensor:
277 - return self.load_bytes(pybase64.b64decode(data, validate=True))
278 -
279 - def load_file(self, filepath: Path) -> torch.Tensor:
280 - # Enable sparse tensor integrity checks to prevent out-of-bounds
281 - # writes from maliciously crafted tensors
282 - with torch.sparse.check_sparse_tensor_invariants():
283 - tensor = torch.load(filepath, weights_only=True)
284 - return tensor.to_dense()
285 -
286 - def encode_base64(self, media: torch.Tensor) -> str:
287 - return tensor2base64(media)
vllm/multimodal/hasher.py
+1 -1
@@ -12,7 +12,7 @@ from PIL import Image
12
13 from vllm.logger import init_logger
14
15 -from .base import MediaWithBytes
15 +from .media import MediaWithBytes
16
17 logger = init_logger(__name__)
18
vllm/multimodal/image.py
-119
@@ -1,19 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 -from io import BytesIO
5 -from pathlib import Path
6 -
7 -import pybase64
8 -import torch
4 from PIL import Image
5
11 -from vllm.logger import init_logger
12 -
13 -from .base import MediaIO, MediaWithBytes
14 -
15 -logger = init_logger(__file__)
16 -
6
7 def rescale_image_size(
8 image: Image.Image, size_factor: float, transpose: int = -1
@@ -45,111 +34,3 @@ def convert_image_mode(image: Image.Image, to_mode: str):
34 return rgba_to_rgb(image)
35 else:
36 return image.convert(to_mode)
48 -
49 -
50 -class ImageMediaIO(MediaIO[Image.Image]):
51 - def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
52 - super().__init__()
53 -
54 - self.image_mode = image_mode
55 - # `kwargs` contains custom arguments from
56 - # --media-io-kwargs for this modality.
57 - # They can be passed to the underlying
58 - # media loaders (e.g. custom implementations)
59 - # for flexible control.
60 - self.kwargs = kwargs
61 -
62 - # Extract RGBA background color from kwargs if provided
63 - # Default to white background for backward compatibility
64 - rgba_bg = kwargs.get("rgba_background_color", (255, 255, 255))
65 - # Convert list to tuple for consistency
66 - if isinstance(rgba_bg, list):
67 - rgba_bg = tuple(rgba_bg)
68 -
69 - # Validate rgba_background_color format
70 - if not (
71 - isinstance(rgba_bg, tuple)
72 - and len(rgba_bg) == 3
73 - and all(isinstance(c, int) and 0 <= c <= 255 for c in rgba_bg)
74 - ):
75 - raise ValueError(
76 - "rgba_background_color must be a list or tuple of 3 integers "
77 - "in the range [0, 255]."
78 - )
79 - self.rgba_background_color = rgba_bg
80 -
81 - def _convert_image_mode(
82 - self, image: Image.Image | MediaWithBytes[Image.Image]
83 - ) -> Image.Image:
84 - """Convert image mode with custom background color."""
85 - if isinstance(image, MediaWithBytes):
86 - image = image.media
87 - if image.mode == self.image_mode:
88 - return image
89 - elif image.mode == "RGBA" and self.image_mode == "RGB":
90 - return rgba_to_rgb(image, self.rgba_background_color)
91 - else:
92 - return convert_image_mode(image, self.image_mode)
93 -
94 - def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
95 - image = Image.open(BytesIO(data))
96 - return MediaWithBytes(self._convert_image_mode(image), data)
97 -
98 - def load_base64(self, media_type: str, data: str) -> MediaWithBytes[Image.Image]:
99 - return self.load_bytes(pybase64.b64decode(data, validate=True))
100 -
101 - def load_file(self, filepath: Path) -> MediaWithBytes[Image.Image]:
102 - with open(filepath, "rb") as f:
103 - data = f.read()
104 - image = Image.open(BytesIO(data))
105 - return MediaWithBytes(self._convert_image_mode(image), data)
106 -
107 - def encode_base64(
108 - self,
109 - media: Image.Image,
110 - *,
111 - image_format: str | None = None,
112 - ) -> str:
113 - if image_format is None:
114 - logger.warning_once(
115 - "The default format of `ImageMediaIO.encode_base64` will be changed "
116 - 'from "JPEG" to "PNG" in v0.15 to avoid lossy compression. '
117 - "To continue using the old default, "
118 - 'pass `format="JPEG"` explicitly to silence this warning.'
119 - )
120 - image_format = "JPEG"
121 -
122 - image = media
123 -
124 - with BytesIO() as buffer:
125 - image = self._convert_image_mode(image)
126 - image.save(buffer, image_format)
127 - data = buffer.getvalue()
128 -
129 - return pybase64.b64encode(data).decode("utf-8")
130 -
131 -
132 -class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
133 - def __init__(self) -> None:
134 - super().__init__()
135 -
136 - def load_bytes(self, data: bytes) -> torch.Tensor:
137 - buffer = BytesIO(data)
138 - # Enable sparse tensor integrity checks to prevent out-of-bounds
139 - # writes from maliciously crafted tensors
140 - with torch.sparse.check_sparse_tensor_invariants():
141 - tensor = torch.load(buffer, weights_only=True)
142 - return tensor.to_dense()
143 -
144 - def load_base64(self, media_type: str, data: str) -> torch.Tensor:
145 - return self.load_bytes(pybase64.b64decode(data, validate=True))
146 -
147 - def load_file(self, filepath: Path) -> torch.Tensor:
148 - # Enable sparse tensor integrity checks to prevent out-of-bounds
149 - # writes from maliciously crafted tensors
150 - with torch.sparse.check_sparse_tensor_invariants():
151 - tensor = torch.load(filepath, weights_only=True)
152 - return tensor.to_dense()
153 -
154 - def encode_base64(self, media: torch.Tensor) -> str:
155 - return pybase64.b64encode(media.numpy()).decode("utf-8")
vllm/multimodal/inputs.py
+1 -1
@@ -32,7 +32,7 @@ if TYPE_CHECKING:
32 from PIL.Image import Image
33 from transformers.feature_extraction_utils import BatchFeature
34
35 - from .base import MediaWithBytes
35 + from .media import MediaWithBytes
36 else:
37 torch = LazyLoader("torch", globals(), "torch")
38
vllm/multimodal/media/__init__.py new
+16
@@ -0,0 +1,16 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +from .audio import AudioEmbeddingMediaIO, AudioMediaIO
4 +from .base import MediaIO, MediaWithBytes
5 +from .image import ImageEmbeddingMediaIO, ImageMediaIO
6 +from .video import VideoMediaIO
7 +
8 +__all__ = [
9 + "MediaIO",
10 + "MediaWithBytes",
11 + "AudioEmbeddingMediaIO",
12 + "AudioMediaIO",
13 + "ImageEmbeddingMediaIO",
14 + "ImageMediaIO",
15 + "VideoMediaIO",
16 +]
vllm/multimodal/media/audio.py new
+89
@@ -0,0 +1,89 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import base64
4 +from io import BytesIO
5 +from pathlib import Path
6 +
7 +import numpy.typing as npt
8 +import pybase64
9 +import torch
10 +
11 +from vllm.utils.import_utils import PlaceholderModule
12 +from vllm.utils.serial_utils import tensor2base64
13 +
14 +from .base import MediaIO
15 +
16 +try:
17 + import librosa
18 +except ImportError:
19 + librosa = PlaceholderModule("librosa") # type: ignore[assignment]
20 +
21 +try:
22 + import soundfile
23 +except ImportError:
24 + soundfile = PlaceholderModule("soundfile") # type: ignore[assignment]
25 +
26 +
27 +class AudioMediaIO(MediaIO[tuple[npt.NDArray, float]]):
28 + def __init__(self, **kwargs) -> None:
29 + super().__init__()
30 +
31 + # `kwargs` contains custom arguments from
32 + # --media-io-kwargs for this modality.
33 + # They can be passed to the underlying
34 + # media loaders (e.g. custom implementations)
35 + # for flexible control.
36 + self.kwargs = kwargs
37 +
38 + def load_bytes(self, data: bytes) -> tuple[npt.NDArray, float]:
39 + return librosa.load(BytesIO(data), sr=None)
40 +
41 + def load_base64(
42 + self,
43 + media_type: str,
44 + data: str,
45 + ) -> tuple[npt.NDArray, float]:
46 + return self.load_bytes(base64.b64decode(data))
47 +
48 + def load_file(self, filepath: Path) -> tuple[npt.NDArray, float]:
49 + return librosa.load(filepath, sr=None)
50 +
51 + def encode_base64(
52 + self,
53 + media: tuple[npt.NDArray, int],
54 + *,
55 + audio_format: str = "WAV",
56 + ) -> str:
57 + audio, sr = media
58 +
59 + with BytesIO() as buffer:
60 + soundfile.write(buffer, audio, sr, format=audio_format)
61 + data = buffer.getvalue()
62 +
63 + return base64.b64encode(data).decode("utf-8")
64 +
65 +
66 +class AudioEmbeddingMediaIO(MediaIO[torch.Tensor]):
67 + def __init__(self) -> None:
68 + super().__init__()
69 +
70 + def load_bytes(self, data: bytes) -> torch.Tensor:
71 + buffer = BytesIO(data)
72 + # Enable sparse tensor integrity checks to prevent out-of-bounds
73 + # writes from maliciously crafted tensors
74 + with torch.sparse.check_sparse_tensor_invariants():
75 + tensor = torch.load(buffer, weights_only=True)
76 + return tensor.to_dense()
77 +
78 + def load_base64(self, media_type: str, data: str) -> torch.Tensor:
79 + return self.load_bytes(pybase64.b64decode(data, validate=True))
80 +
81 + def load_file(self, filepath: Path) -> torch.Tensor:
82 + # Enable sparse tensor integrity checks to prevent out-of-bounds
83 + # writes from maliciously crafted tensors
84 + with torch.sparse.check_sparse_tensor_invariants():
85 + tensor = torch.load(filepath, weights_only=True)
86 + return tensor.to_dense()
87 +
88 + def encode_base64(self, media: torch.Tensor) -> str:
89 + return tensor2base64(media)
vllm/multimodal/media/base.py renamed
+7 -6
@@ -4,7 +4,7 @@
4 from abc import ABC, abstractmethod
5 from dataclasses import dataclass
6 from pathlib import Path
7 -from typing import Generic, TypeVar
7 +from typing import Any, Generic, TypeVar
8
9 import numpy as np
10
@@ -32,13 +32,14 @@ class MediaWithBytes(Generic[_T]):
32 """Allow np.array(obj) to return np.array(obj.media)."""
33 return np.array(self.media, *args, **kwargs)
34
35 + def __getstate__(self):
36 + return self.__dict__.copy()
37 +
38 + def __setstate__(self, state: dict[str, Any]):
39 + self.__dict__.update(state)
40 +
41 def __getattr__(self, name: str):
42 """Delegate attribute access to the underlying media object."""
37 - # Guard against recursion during unpickling when media isn't set yet.
38 - # pickle creates objects without calling __init__, so self.media may
39 - # not exist when __getattr__ is called for methods like __setstate__.
40 - if "media" not in self.__dict__:
41 - raise AttributeError(name)
43 return getattr(self.media, name)
44
45
vllm/multimodal/media/image.py new
+124
@@ -0,0 +1,124 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from io import BytesIO
5 +from pathlib import Path
6 +
7 +import pybase64
8 +import torch
9 +from PIL import Image
10 +
11 +from vllm.logger import init_logger
12 +
13 +from ..image import convert_image_mode, rgba_to_rgb
14 +from .base import MediaIO, MediaWithBytes
15 +
16 +logger = init_logger(__file__)
17 +
18 +
19 +class ImageMediaIO(MediaIO[Image.Image]):
20 + def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
21 + super().__init__()
22 +
23 + self.image_mode = image_mode
24 + # `kwargs` contains custom arguments from
25 + # --media-io-kwargs for this modality.
26 + # They can be passed to the underlying
27 + # media loaders (e.g. custom implementations)
28 + # for flexible control.
29 + self.kwargs = kwargs
30 +
31 + # Extract RGBA background color from kwargs if provided
32 + # Default to white background for backward compatibility
33 + rgba_bg = kwargs.get("rgba_background_color", (255, 255, 255))
34 + # Convert list to tuple for consistency
35 + if isinstance(rgba_bg, list):
36 + rgba_bg = tuple(rgba_bg)
37 +
38 + # Validate rgba_background_color format
39 + if not (
40 + isinstance(rgba_bg, tuple)
41 + and len(rgba_bg) == 3
42 + and all(isinstance(c, int) and 0 <= c <= 255 for c in rgba_bg)
43 + ):
44 + raise ValueError(
45 + "rgba_background_color must be a list or tuple of 3 integers "
46 + "in the range [0, 255]."
47 + )
48 + self.rgba_background_color = rgba_bg
49 +
50 + def _convert_image_mode(
51 + self, image: Image.Image | MediaWithBytes[Image.Image]
52 + ) -> Image.Image:
53 + """Convert image mode with custom background color."""
54 + if isinstance(image, MediaWithBytes):
55 + image = image.media
56 + if image.mode == self.image_mode:
57 + return image
58 + elif image.mode == "RGBA" and self.image_mode == "RGB":
59 + return rgba_to_rgb(image, self.rgba_background_color)
60 + else:
61 + return convert_image_mode(image, self.image_mode)
62 +
63 + def load_bytes(self, data: bytes) -> MediaWithBytes[Image.Image]:
64 + image = Image.open(BytesIO(data))
65 + return MediaWithBytes(self._convert_image_mode(image), data)
66 +
67 + def load_base64(self, media_type: str, data: str) -> MediaWithBytes[Image.Image]:
68 + return self.load_bytes(pybase64.b64decode(data, validate=True))
69 +
70 + def load_file(self, filepath: Path) -> MediaWithBytes[Image.Image]:
71 + with open(filepath, "rb") as f:
72 + data = f.read()
73 + image = Image.open(BytesIO(data))
74 + return MediaWithBytes(self._convert_image_mode(image), data)
75 +
76 + def encode_base64(
77 + self,
78 + media: Image.Image,
79 + *,
80 + image_format: str | None = None,
81 + ) -> str:
82 + if image_format is None:
83 + logger.warning_once(
84 + "The default format of `ImageMediaIO.encode_base64` will be changed "
85 + 'from "JPEG" to "PNG" in v0.15 to avoid lossy compression. '
86 + "To continue using the old default, "
87 + 'pass `format="JPEG"` explicitly to silence this warning.'
88 + )
89 + image_format = "JPEG"
90 +
91 + image = media
92 +
93 + with BytesIO() as buffer:
94 + image = self._convert_image_mode(image)
95 + image.save(buffer, image_format)
96 + data = buffer.getvalue()
97 +
98 + return pybase64.b64encode(data).decode("utf-8")
99 +
100 +
101 +class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
102 + def __init__(self) -> None:
103 + super().__init__()
104 +
105 + def load_bytes(self, data: bytes) -> torch.Tensor:
106 + buffer = BytesIO(data)
107 + # Enable sparse tensor integrity checks to prevent out-of-bounds
108 + # writes from maliciously crafted tensors
109 + with torch.sparse.check_sparse_tensor_invariants():
110 + tensor = torch.load(buffer, weights_only=True)
111 + return tensor.to_dense()
112 +
113 + def load_base64(self, media_type: str, data: str) -> torch.Tensor:
114 + return self.load_bytes(pybase64.b64decode(data, validate=True))
115 +
116 + def load_file(self, filepath: Path) -> torch.Tensor:
117 + # Enable sparse tensor integrity checks to prevent out-of-bounds
118 + # writes from maliciously crafted tensors
119 + with torch.sparse.check_sparse_tensor_invariants():
120 + tensor = torch.load(filepath, weights_only=True)
121 + return tensor.to_dense()
122 +
123 + def encode_base64(self, media: torch.Tensor) -> str:
124 + return pybase64.b64encode(media.numpy()).decode("utf-8")
vllm/multimodal/media/video.py new
+89
@@ -0,0 +1,89 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +import base64
4 +from functools import partial
5 +from pathlib import Path
6 +from typing import Any
7 +
8 +import numpy as np
9 +import numpy.typing as npt
10 +from PIL import Image
11 +
12 +from vllm import envs
13 +
14 +from ..video import VIDEO_LOADER_REGISTRY
15 +from .base import MediaIO
16 +from .image import ImageMediaIO
17 +
18 +
19 +class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
20 + def __init__(
21 + self,
22 + image_io: ImageMediaIO,
23 + num_frames: int = 32,
24 + **kwargs,
25 + ) -> None:
26 + super().__init__()
27 +
28 + self.image_io = image_io
29 + self.num_frames = num_frames
30 + # `kwargs` contains custom arguments from
31 + # --media-io-kwargs for this modality.
32 + # They can be passed to the underlying
33 + # media loaders (e.g. custom implementations)
34 + # for flexible control.
35 +
36 + # Allow per-request override of video backend via kwargs.
37 + # This enables users to specify a different backend than the
38 + # global VLLM_VIDEO_LOADER_BACKEND env var, e.g.:
39 + # --media-io-kwargs '{"video": {"video_backend": "torchcodec"}}'
40 + video_loader_backend = (
41 + kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
42 + )
43 + self.kwargs = kwargs
44 + self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
45 +
46 + def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
47 + return self.video_loader.load_bytes(
48 + data, num_frames=self.num_frames, **self.kwargs
49 + )
50 +
51 + def load_base64(
52 + self, media_type: str, data: str
53 + ) -> tuple[npt.NDArray, dict[str, Any]]:
54 + if media_type.lower() == "video/jpeg":
55 + load_frame = partial(
56 + self.image_io.load_base64,
57 + "image/jpeg",
58 + )
59 +
60 + return np.stack(
61 + [np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
62 + ), {}
63 +
64 + return self.load_bytes(base64.b64decode(data))
65 +
66 + def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]:
67 + with filepath.open("rb") as f:
68 + data = f.read()
69 +
70 + return self.load_bytes(data)
71 +
72 + def encode_base64(
73 + self,
74 + media: npt.NDArray,
75 + *,
76 + video_format: str = "JPEG",
77 + ) -> str:
78 + video = media
79 +
80 + if video_format == "JPEG":
81 + encode_frame = partial(
82 + self.image_io.encode_base64,
83 + image_format=video_format,
84 + )
85 +
86 + return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
87 +
88 + msg = "Only JPEG format is supported for now."
89 + raise NotImplementedError(msg)
vllm/multimodal/parse.py
+1 -1
@@ -23,7 +23,6 @@ from vllm.utils.collection_utils import is_list_of
23 from vllm.utils.import_utils import LazyLoader
24
25 from .audio import AudioResampler, AudioSpec, normalize_audio
26 -from .base import MediaWithBytes
26 from .inputs import (
27 AudioItem,
28 HfAudioItem,
@@ -36,6 +35,7 @@ from .inputs import (
35 MultiModalKwargsItems,
36 VideoItem,
37 )
38 +from .media import MediaWithBytes
39
40 _T = TypeVar("_T")
41 _I = TypeVar("_I")
vllm/multimodal/utils.py
+8 -4
@@ -22,10 +22,14 @@ from vllm.connections import HTTPConnection, global_http_connection
22 from vllm.logger import init_logger
23 from vllm.utils.registry import ExtensionManager
24
25 -from .audio import AudioEmbeddingMediaIO, AudioMediaIO
26 -from .base import MediaIO
27 -from .image import ImageEmbeddingMediaIO, ImageMediaIO
28 -from .video import VideoMediaIO
25 +from .media import (
26 + AudioEmbeddingMediaIO,
27 + AudioMediaIO,
28 + ImageEmbeddingMediaIO,
29 + ImageMediaIO,
30 + MediaIO,
31 + VideoMediaIO,
32 +)
33
34 if TYPE_CHECKING:
35 from .inputs import (
vllm/multimodal/video.py
-81
@@ -1,27 +1,19 @@
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import base64
3 import math
4 from abc import abstractmethod
6 -from functools import partial
5 from io import BytesIO
8 -from pathlib import Path
6 from typing import TYPE_CHECKING, Any, cast
7
8 import numpy as np
9 import numpy.typing as npt
13 -from PIL import Image
10
11 if TYPE_CHECKING:
12 import cv2
13
18 -from vllm import envs
14 from vllm.logger import init_logger
15 from vllm.utils.registry import ExtensionManager
16
22 -from .base import MediaIO
23 -from .image import ImageMediaIO
24 -
17 logger = init_logger(__name__)
18
19
@@ -755,76 +747,3 @@ class Molmo2VideoBackend(VideoLoader):
747 **kwargs,
748 )
749 return out
758 -
759 -
760 -class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
761 - def __init__(
762 - self,
763 - image_io: ImageMediaIO,
764 - num_frames: int = 32,
765 - **kwargs,
766 - ) -> None:
767 - super().__init__()
768 -
769 - self.image_io = image_io
770 - self.num_frames = num_frames
771 - # `kwargs` contains custom arguments from
772 - # --media-io-kwargs for this modality.
773 - # They can be passed to the underlying
774 - # media loaders (e.g. custom implementations)
775 - # for flexible control.
776 -
777 - # Allow per-request override of video backend via kwargs.
778 - # This enables users to specify a different backend than the
779 - # global VLLM_VIDEO_LOADER_BACKEND env var, e.g.:
780 - # --media-io-kwargs '{"video": {"video_backend": "torchcodec"}}'
781 - video_loader_backend = (
782 - kwargs.pop("video_backend", None) or envs.VLLM_VIDEO_LOADER_BACKEND
783 - )
784 - self.kwargs = kwargs
785 - self.video_loader = VIDEO_LOADER_REGISTRY.load(video_loader_backend)
786 -
787 - def load_bytes(self, data: bytes) -> tuple[npt.NDArray, dict[str, Any]]:
788 - return self.video_loader.load_bytes(
789 - data, num_frames=self.num_frames, **self.kwargs
790 - )
791 -
792 - def load_base64(
793 - self, media_type: str, data: str
794 - ) -> tuple[npt.NDArray, dict[str, Any]]:
795 - if media_type.lower() == "video/jpeg":
796 - load_frame = partial(
797 - self.image_io.load_base64,
798 - "image/jpeg",
799 - )
800 -
801 - return np.stack(
802 - [np.asarray(load_frame(frame_data)) for frame_data in data.split(",")]
803 - ), {}
804 -
805 - return self.load_bytes(base64.b64decode(data))
806 -
807 - def load_file(self, filepath: Path) -> tuple[npt.NDArray, dict[str, Any]]:
808 - with filepath.open("rb") as f:
809 - data = f.read()
810 -
811 - return self.load_bytes(data)
812 -
813 - def encode_base64(
814 - self,
815 - media: npt.NDArray,
816 - *,
817 - video_format: str = "JPEG",
818 - ) -> str:
819 - video = media
820 -
821 - if video_format == "JPEG":
822 - encode_frame = partial(
823 - self.image_io.encode_base64,
824 - image_format=video_format,
825 - )
826 -
827 - return ",".join(encode_frame(Image.fromarray(frame)) for frame in video)
828 -
829 - msg = "Only JPEG format is supported for now."
830 - raise NotImplementedError(msg)