[Multimodal] Allow keeping original image mode for ImageIO (#49159)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Isotr0py committed
Jul 20, 2026 at 21:42 UTC
15cb8e140dffd9d2a33737c083fcf1d696e1d1f6
5 files changed
+77
-15
tests/multimodal/media/test_connector.py
+29
@@ -6,6 +6,7 @@ import mimetypes
6
import os
7
import shutil
8
import time
9
+from io import BytesIO
10
from tempfile import NamedTemporaryFile, TemporaryDirectory
11
12
import aiohttp
@@ -111,6 +112,34 @@ async def test_fetch_image_base64(
112
assert _image_equals(data_image_sync, data_image_async)
113
114
115
+@pytest.mark.asyncio
116
+async def test_fetch_image_keep_original_mode():
117
+ """media_io_kwargs can disable the default RGB conversion."""
118
+ # RGBA image: opaque black pixel on a fully transparent background
119
+ rgba_image = Image.new("RGBA", (4, 4), (0, 0, 0, 0))
120
+ rgba_image.putpixel((2, 2), (0, 0, 0, 255))
121
+ buffer = BytesIO()
122
+ rgba_image.save(buffer, "PNG")
123
+ data_url = (
124
+ f"data:image/png;base64,{base64.b64encode(buffer.getvalue()).decode('utf-8')}"
125
+ )
126
+
127
+ # Default behavior: RGBA is composited onto a white background
128
+ default_image = MediaConnector().fetch_image(data_url)
129
+ assert default_image.mode == "RGB"
130
+ assert default_image.getpixel((0, 0)) == (255, 255, 255)
131
+ assert default_image.getpixel((2, 2)) == (0, 0, 0)
132
+
133
+ # image_mode=None via media_io_kwargs: original mode is preserved
134
+ connector = MediaConnector(media_io_kwargs={"image": {"image_mode": None}})
135
+ image_sync = connector.fetch_image(data_url)
136
+ image_async = await connector.fetch_image_async(data_url)
137
+ for image in (image_sync, image_async):
138
+ assert image.mode == "RGBA"
139
+ assert image.getpixel((0, 0)) == (0, 0, 0, 0)
140
+ assert image.getpixel((2, 2)) == (0, 0, 0, 255)
141
+
142
+
143
@pytest.mark.asyncio
144
@pytest.mark.parametrize("image_url", TEST_IMAGE_ASSETS, indirect=True)
145
async def test_fetch_image_local_files(image_url: str):
tests/multimodal/media/test_image.py
+23
@@ -80,6 +80,29 @@ def test_image_media_io_rgba_custom_background(tmp_path):
80
assert green_numpy[0][0][2] == 0 # B
81
82
83
+def test_image_media_io_no_mode_conversion(tmp_path):
84
+ """image_mode=None skips conversion and preserves the original mode."""
85
+ # RGBA image: opaque black pixel on a fully transparent background
86
+ rgba_image = Image.new("RGBA", (10, 10), (0, 0, 0, 0))
87
+ rgba_image.putpixel((5, 5), (0, 0, 0, 255))
88
+ test_image_path = tmp_path / "test_rgba.png"
89
+ rgba_image.save(test_image_path)
90
+
91
+ # Default behavior: RGBA is composited onto a white background
92
+ image_io_default = ImageMediaIO()
93
+ converted_default = image_io_default.load_file(test_image_path)
94
+ assert converted_default.media.mode == "RGB"
95
+ assert converted_default.media.getpixel((0, 0)) == (255, 255, 255)
96
+ assert converted_default.media.getpixel((5, 5)) == (0, 0, 0)
97
+
98
+ # image_mode=None: original mode and alpha channel are preserved
99
+ image_io_keep = ImageMediaIO(image_mode=None)
100
+ converted_keep = image_io_keep.load_file(test_image_path)
101
+ assert converted_keep.media.mode == "RGBA"
102
+ assert converted_keep.media.getpixel((0, 0)) == (0, 0, 0, 0)
103
+ assert converted_keep.media.getpixel((5, 5)) == (0, 0, 0, 255)
104
+
105
+
106
def test_image_media_io_rgba_background_color_validation():
107
"""Test that invalid rgba_background_color values are properly rejected."""
108
vllm/multimodal/media/connector.py
+17
-11
@@ -476,15 +476,17 @@ class MediaConnector:
476
self,
477
image_url: str,
478
*,
479
- image_mode: str = "RGB",
479
+ image_mode: str | None = "RGB",
480
) -> Image.Image:
481
"""
482
Load a PIL image from an HTTP or base64 data URL.
483
484
- By default, the image is converted into RGB format.
484
+ By default, the image is converted into RGB format. Set
485
+ `media_io_kwargs={"image": {"image_mode": None}}` to keep the
486
+ original image mode (e.g. preserving the alpha channel).
487
"""
488
image_io = ImageMediaIO(
487
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
489
+ **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
490
)
491
492
try:
@@ -501,15 +503,17 @@ class MediaConnector:
503
self,
504
image_url: str,
505
*,
504
- image_mode: str = "RGB",
506
+ image_mode: str | None = "RGB",
507
) -> Image.Image:
508
"""
509
Asynchronously load a PIL image from an HTTP or base64 data URL.
510
509
- By default, the image is converted into RGB format.
511
+ By default, the image is converted into RGB format. Set
512
+ `media_io_kwargs={"image": {"image_mode": None}}` to keep the
513
+ original image mode (e.g. preserving the alpha channel).
514
"""
515
image_io = ImageMediaIO(
512
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
516
+ **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
517
)
518
519
try:
@@ -526,14 +530,14 @@ class MediaConnector:
530
self,
531
video_url: str,
532
*,
529
- image_mode: str = "RGB",
533
+ image_mode: str | None = "RGB",
534
video_processor: str | None = None,
535
) -> tuple[npt.NDArray, dict[str, Any]]:
536
"""
537
Load video from an HTTP or base64 data URL.
538
"""
539
image_io = ImageMediaIO(
536
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
540
+ **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
541
)
542
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
543
if "video_backend" not in video_io_kwargs and (
@@ -552,16 +556,18 @@ class MediaConnector:
556
self,
557
video_url: str,
558
*,
555
- image_mode: str = "RGB",
559
+ image_mode: str | None = "RGB",
560
video_processor: str | None = None,
561
) -> tuple[npt.NDArray, dict[str, Any]]:
562
"""
563
Asynchronously load video from an HTTP or base64 data URL.
564
561
- By default, the image is converted into RGB format.
565
+ By default, the image is converted into RGB format. Set
566
+ `media_io_kwargs={"image": {"image_mode": None}}` to keep the
567
+ original image mode (e.g. preserving the alpha channel).
568
"""
569
image_io = ImageMediaIO(
564
- image_mode=image_mode, **self.media_io_kwargs.get("image", {})
570
+ **({"image_mode": image_mode} | self.media_io_kwargs.get("image", {}))
571
)
572
video_io_kwargs = dict(self.media_io_kwargs.get("video", {}))
573
if "video_backend" not in video_io_kwargs and (
vllm/multimodal/media/image.py
+4
-2
@@ -25,9 +25,11 @@ class ImageMediaIO(MediaIO[Image.Image]):
25
error handling.
26
"""
27
28
- def __init__(self, image_mode: str = "RGB", **kwargs) -> None:
28
+ def __init__(self, image_mode: str | None = "RGB", **kwargs) -> None:
29
super().__init__()
30
31
+ # Target mode for loaded images; `None` keeps the original mode
32
+ # (i.e. no conversion, alpha channel is preserved as-is).
33
self.image_mode = image_mode
34
# `kwargs` contains custom arguments from
35
# --media-io-kwargs for this modality, merged with
@@ -62,7 +64,7 @@ class ImageMediaIO(MediaIO[Image.Image]):
64
"""Convert image mode with custom background color."""
65
if isinstance(image, MediaWithBytes):
66
image = image.media
65
- if image.mode == self.image_mode:
67
+ if self.image_mode is None or image.mode == self.image_mode:
68
return image
69
elif image.mode == "RGBA" and self.image_mode == "RGB":
70
return rgba_to_rgb(image, self.rgba_background_color)
vllm/multimodal/utils.py
+4
-2
@@ -58,13 +58,14 @@ def encode_audio_url(
58
def encode_image_base64(
59
image: Image.Image,
60
*,
61
- image_mode: str = "RGB",
61
+ image_mode: str | None = "RGB",
62
format: str = "PNG",
63
) -> str:
64
"""
65
Encode a pillow image to base64 format.
66
67
By default, the image is converted into RGB format before being encoded.
68
+ Pass `image_mode=None` to keep the original image mode.
69
"""
70
image_io = ImageMediaIO(image_mode=image_mode)
71
return image_io.encode_base64(image, image_format=format)
@@ -73,13 +74,14 @@ def encode_image_base64(
74
def encode_image_url(
75
image: Image.Image,
76
*,
76
- image_mode: str = "RGB",
77
+ image_mode: str | None = "RGB",
78
format: str = "PNG",
79
) -> str:
80
"""
81
Encode a pillow image as a data URL.
82
83
By default, the image is converted into RGB format before being encoded.
84
+ Pass `image_mode=None` to keep the original image mode.
85
"""
86
image_b64 = encode_image_base64(image, image_mode=image_mode, format=format)
87
mimetype = mimetypes.types_map.get("." + format.lower(), "image")