@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0a6d305e0f

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 22:35 UTC

feat(multimodal): Add customizable background color for RGBA to RGB conversion (#22052)

Signed-off-by: Jinheng Li <ahengljh@gmail.com> Co-authored-by: Jinheng Li <ahengljh@gmail.com>

Gamhang committed Aug 1, 2025 at 21:07 UTC 0a6d305e0f7b63b06c87bb1f7564ae8d148a3311

3 files changed +190 -6

docs/features/multimodal_inputs.md

+44

@@ -172,6 +172,36 @@ Multi-image input can be extended to perform video captioning. We show this with

172 print(generated_text)

173 ```

174

175 +#### Custom RGBA Background Color

176 +

177 +When loading RGBA images (images with transparency), vLLM converts them to RGB format. By default, transparent pixels are replaced with white background. You can customize this background color using the `rgba_background_color` parameter in `media_io_kwargs`.

178 +

179 +??? code

180 +

181 + ```python

182 + from vllm import LLM

183 +

184 + # Default white background (no configuration needed)

185 + llm = LLM(model="llava-hf/llava-1.5-7b-hf")

186 +

187 + # Custom black background for dark theme

188 + llm = LLM(

189 + model="llava-hf/llava-1.5-7b-hf",

190 + media_io_kwargs={"image": {"rgba_background_color": [0, 0, 0]}}

191 + )

192 +

193 + # Custom brand color background (e.g., blue)

194 + llm = LLM(

195 + model="llava-hf/llava-1.5-7b-hf",

196 + media_io_kwargs={"image": {"rgba_background_color": [0, 0, 255]}}

197 + )

198 + ```

199 +

200 +!!! note

201 + - The `rgba_background_color` accepts RGB values as a list `[R, G, B]` or tuple `(R, G, B)` where each value is 0-255

202 + - This setting only affects RGBA images with transparency; RGB images are unchanged

203 + - If not specified, the default white background `(255, 255, 255)` is used for backward compatibility

204 +

205 ### Video Inputs

206

207 You can pass a list of NumPy arrays directly to the `'video'` field of the multi-modal dictionary

@@ -478,6 +508,20 @@ Full example: <gh-file:examples/online_serving/openai_chat_completion_client_for

508 export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>

509 ```

510

511 +#### Custom RGBA Background Color

512 +

513 +To use a custom background color for RGBA images, pass the `rgba_background_color` parameter via `--media-io-kwargs`:

514 +

515 +```bash

516 +# Example: Black background for dark theme

517 +vllm serve llava-hf/llava-1.5-7b-hf \

518 + --media-io-kwargs '{"image": {"rgba_background_color": [0, 0, 0]}}'

519 +

520 +# Example: Custom gray background

521 +vllm serve llava-hf/llava-1.5-7b-hf \

522 + --media-io-kwargs '{"image": {"rgba_background_color": [128, 128, 128]}}'

523 +```

524 +

525 ### Audio Inputs

526

527 Audio input is supported according to [OpenAI Audio API](https://platform.openai.com/docs/guides/audio?audio-generation-quickstart-example=audio-in).

tests/multimodal/test_image.py

+114 -1

@@ -3,9 +3,10 @@

3 from pathlib import Path

5 import numpy as np

6 +import pytest

7 from PIL import Image, ImageChops

8 -from vllm.multimodal.image import convert_image_mode

9 +from vllm.multimodal.image import ImageMediaIO, convert_image_mode

11 ASSETS_DIR = Path(__file__).parent / "assets"

12 assert ASSETS_DIR.exists()

@@ -35,3 +36,115 @@ def test_rgba_to_rgb():

36 assert converted_image_numpy[i][j][0] == 255

37 assert converted_image_numpy[i][j][1] == 255

38 assert converted_image_numpy[i][j][2] == 255

39 +

40 +

41 +def test_rgba_to_rgb_custom_background(tmp_path):

42 + """Test RGBA to RGB conversion with custom background colors."""

43 + # Create a simple RGBA image with transparent and opaque pixels

44 + rgba_image = Image.new("RGBA", (10, 10),

45 + (255, 0, 0, 255)) # Red with full opacity

46 +

47 + # Make top-left quadrant transparent

48 + for i in range(5):

49 + for j in range(5):

50 + rgba_image.putpixel((i, j), (0, 0, 0, 0)) # Fully transparent

51 +

52 + # Save the test image to tmp_path

53 + test_image_path = tmp_path / "test_rgba.png"

54 + rgba_image.save(test_image_path)

55 +

56 + # Test 1: Default white background (backward compatibility)

57 + image_io_default = ImageMediaIO()

58 + converted_default = image_io_default.load_file(test_image_path)

59 + default_numpy = np.array(converted_default)

60 +

61 + # Check transparent pixels are white

62 + assert default_numpy[0][0][0] == 255 # R

63 + assert default_numpy[0][0][1] == 255 # G

64 + assert default_numpy[0][0][2] == 255 # B

65 + # Check opaque pixels remain red

66 + assert default_numpy[5][5][0] == 255 # R

67 + assert default_numpy[5][5][1] == 0 # G

68 + assert default_numpy[5][5][2] == 0 # B

69 +

70 + # Test 2: Custom black background via kwargs

71 + image_io_black = ImageMediaIO(rgba_background_color=(0, 0, 0))

72 + converted_black = image_io_black.load_file(test_image_path)

73 + black_numpy = np.array(converted_black)

74 +

75 + # Check transparent pixels are black

76 + assert black_numpy[0][0][0] == 0 # R

77 + assert black_numpy[0][0][1] == 0 # G

78 + assert black_numpy[0][0][2] == 0 # B

79 + # Check opaque pixels remain red

80 + assert black_numpy[5][5][0] == 255 # R

81 + assert black_numpy[5][5][1] == 0 # G

82 + assert black_numpy[5][5][2] == 0 # B

83 +

84 + # Test 3: Custom blue background via kwargs (as list)

85 + image_io_blue = ImageMediaIO(rgba_background_color=[0, 0, 255])

86 + converted_blue = image_io_blue.load_file(test_image_path)

87 + blue_numpy = np.array(converted_blue)

88 +

89 + # Check transparent pixels are blue

90 + assert blue_numpy[0][0][0] == 0 # R

91 + assert blue_numpy[0][0][1] == 0 # G

92 + assert blue_numpy[0][0][2] == 255 # B

93 +

94 + # Test 4: Test with load_bytes method

95 + with open(test_image_path, 'rb') as f:

96 + image_data = f.read()

97 +

98 + image_io_green = ImageMediaIO(rgba_background_color=(0, 255, 0))

99 + converted_green = image_io_green.load_bytes(image_data)

100 + green_numpy = np.array(converted_green)

101 +

102 + # Check transparent pixels are green

103 + assert green_numpy[0][0][0] == 0 # R

104 + assert green_numpy[0][0][1] == 255 # G

105 + assert green_numpy[0][0][2] == 0 # B

106 +

107 +

108 +def test_rgba_background_color_validation():

109 + """Test that invalid rgba_background_color values are properly rejected."""

110 +

111 + # Test invalid types

112 + with pytest.raises(ValueError,

113 + match="rgba_background_color must be a list or tuple"):

114 + ImageMediaIO(rgba_background_color="255,255,255")

115 +

116 + with pytest.raises(ValueError,

117 + match="rgba_background_color must be a list or tuple"):

118 + ImageMediaIO(rgba_background_color=255)

119 +

120 + # Test wrong number of elements

121 + with pytest.raises(ValueError,

122 + match="rgba_background_color must be a list or tuple"):

123 + ImageMediaIO(rgba_background_color=(255, 255))

124 +

125 + with pytest.raises(ValueError,

126 + match="rgba_background_color must be a list or tuple"):

127 + ImageMediaIO(rgba_background_color=(255, 255, 255, 255))

128 +

129 + # Test non-integer values

130 + with pytest.raises(ValueError,

131 + match="rgba_background_color must be a list or tuple"):

132 + ImageMediaIO(rgba_background_color=(255.0, 255.0, 255.0))

133 +

134 + with pytest.raises(ValueError,

135 + match="rgba_background_color must be a list or tuple"):

136 + ImageMediaIO(rgba_background_color=(255, "255", 255))

137 +

138 + # Test out of range values

139 + with pytest.raises(ValueError,

140 + match="rgba_background_color must be a list or tuple"):

141 + ImageMediaIO(rgba_background_color=(256, 255, 255))

142 +

143 + with pytest.raises(ValueError,

144 + match="rgba_background_color must be a list or tuple"):

145 + ImageMediaIO(rgba_background_color=(255, -1, 255))

146 +

147 + # Test that valid values work

148 + ImageMediaIO(rgba_background_color=(0, 0, 0)) # Should not raise

149 + ImageMediaIO(rgba_background_color=[255, 255, 255]) # Should not raise

150 + ImageMediaIO(rgba_background_color=(128, 128, 128)) # Should not raise

vllm/multimodal/image.py

+32 -5

@@ -3,6 +3,7 @@

4 from io import BytesIO

5 from pathlib import Path

6 +from typing import Union

8 import pybase64

9 import torch

@@ -23,9 +24,10 @@ def rescale_image_size(image: Image.Image,

24 return image

26 -# TODO: Support customizable background color to fill in.

27 def rgba_to_rgb(

28 - image: Image.Image, background_color=(255, 255, 255)) -> Image.Image:

28 + image: Image.Image,

29 + background_color: Union[tuple[int, int, int], list[int]] = (255, 255, 255)

30 +) -> Image.Image:

31 """Convert an RGBA image to RGB with filled background color."""

32 assert image.mode == "RGBA"

33 converted = Image.new("RGB", image.size, background_color)

@@ -55,10 +57,35 @@ class ImageMediaIO(MediaIO[Image.Image]):

57 # for flexible control.

58 self.kwargs = kwargs

60 + # Extract RGBA background color from kwargs if provided

61 + # Default to white background for backward compatibility

62 + rgba_bg = kwargs.get('rgba_background_color', (255, 255, 255))

63 + # Convert list to tuple for consistency

64 + if isinstance(rgba_bg, list):

65 + rgba_bg = tuple(rgba_bg)

66 +

67 + # Validate rgba_background_color format

68 + if not (isinstance(rgba_bg, tuple) and len(rgba_bg) == 3

69 + and all(isinstance(c, int) and 0 <= c <= 255

70 + for c in rgba_bg)):

71 + raise ValueError(

72 + "rgba_background_color must be a list or tuple of 3 integers "

73 + "in the range [0, 255].")

74 + self.rgba_background_color = rgba_bg

75 +

76 + def _convert_image_mode(self, image: Image.Image) -> Image.Image:

77 + """Convert image mode with custom background color."""

78 + if image.mode == self.image_mode:

79 + return image

80 + elif image.mode == "RGBA" and self.image_mode == "RGB":

81 + return rgba_to_rgb(image, self.rgba_background_color)

82 + else:

83 + return convert_image_mode(image, self.image_mode)

84 +

85 def load_bytes(self, data: bytes) -> Image.Image:

86 image = Image.open(BytesIO(data))

87 image.load()

61 - return convert_image_mode(image, self.image_mode)

88 + return self._convert_image_mode(image)

90 def load_base64(self, media_type: str, data: str) -> Image.Image:

91 return self.load_bytes(pybase64.b64decode(data, validate=True))

@@ -66,7 +93,7 @@ class ImageMediaIO(MediaIO[Image.Image]):

93 def load_file(self, filepath: Path) -> Image.Image:

94 image = Image.open(filepath)

95 image.load()

69 - return convert_image_mode(image, self.image_mode)

96 + return self._convert_image_mode(image)

98 def encode_base64(

99 self,

@@ -77,7 +104,7 @@ class ImageMediaIO(MediaIO[Image.Image]):

104 image = media

105

106 with BytesIO() as buffer:

80 - image = convert_image_mode(image, self.image_mode)

107 + image = self._convert_image_mode(image)

108 image.save(buffer, image_format)

109 data = buffer.getvalue()

110