04eb88dc80 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/benchmark_dataset.py

+2 -1

@@ -35,6 +35,7 @@ from transformers import PreTrainedTokenizerBase

35 from vllm.lora.request import LoRARequest

36 from vllm.lora.utils import get_adapter_absolute_path

37 from vllm.multimodal import MultiModalDataDict

38 +from vllm.multimodal.image import convert_image_mode

39 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer

40

41 logger = logging.getLogger(__name__)

@@ -257,7 +258,7 @@ def process_image(image: Any) -> Mapping[str, Any]:

258 if isinstance(image, dict) and "bytes" in image:

259 image = Image.open(BytesIO(image["bytes"]))

260 if isinstance(image, Image.Image):

260 - image = image.convert("RGB")

261 + image = convert_image_mode(image, "RGB")

262 with io.BytesIO() as image_data:

263 image.save(image_data, format="JPEG")

264 image_base64 = base64.b64encode(image_data.getvalue()).decode("utf-8")

examples/offline_inference/qwen2_5_omni/only_thinker.py

+3 -1

@@ -11,6 +11,7 @@ from vllm import LLM, SamplingParams

11 from vllm.assets.audio import AudioAsset

12 from vllm.assets.image import ImageAsset

13 from vllm.assets.video import VideoAsset

14 +from vllm.multimodal.image import convert_image_mode

15 from vllm.utils import FlexibleArgumentParser

16

17

@@ -45,7 +46,8 @@ def get_mixed_modalities_query() -> QueryResult:

46 "audio":

47 AudioAsset("mary_had_lamb").audio_and_sample_rate,

48 "image":

48 - ImageAsset("cherry_blossom").pil_image.convert("RGB"),

49 + convert_image_mode(

50 + ImageAsset("cherry_blossom").pil_image, "RGB"),

51 "video":

52 VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,

53 },

examples/offline_inference/vision_language.py

+3 -2

@@ -19,6 +19,7 @@ from vllm import LLM, EngineArgs, SamplingParams

19 from vllm.assets.image import ImageAsset

20 from vllm.assets.video import VideoAsset

21 from vllm.lora.request import LoRARequest

22 +from vllm.multimodal.image import convert_image_mode

23 from vllm.utils import FlexibleArgumentParser

24

25

@@ -1096,8 +1097,8 @@ def get_multi_modal_input(args):

1097 """

1098 if args.modality == "image":

1099 # Input image and question

1099 - image = ImageAsset("cherry_blossom") \

1100 - .pil_image.convert("RGB")

1100 + image = convert_image_mode(

1101 + ImageAsset("cherry_blossom").pil_image, "RGB")

1102 img_questions = [

1103 "What is the content of this image?",

1104 "Describe the content of this image in detail.",

tests/models/multimodal/generation/test_interleaved.py

+4 -2

@@ -4,6 +4,7 @@ import pytest

4

5 from vllm.assets.image import ImageAsset

6 from vllm.assets.video import VideoAsset

7 +from vllm.multimodal.image import convert_image_mode

8

9 models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]

10

@@ -26,8 +27,9 @@ def test_models(vllm_runner, model, dtype: str, max_tokens: int) -> None:

27 give the same result.

28 """

29

29 - image_cherry = ImageAsset("cherry_blossom").pil_image.convert("RGB")

30 - image_stop = ImageAsset("stop_sign").pil_image.convert("RGB")

30 + image_cherry = convert_image_mode(

31 + ImageAsset("cherry_blossom").pil_image, "RGB")

32 + image_stop = convert_image_mode(ImageAsset("stop_sign").pil_image, "RGB")

33 images = [image_cherry, image_stop]

34 video = VideoAsset(name="baby_reading", num_frames=16).np_ndarrays

35

tests/models/multimodal/generation/test_phi4mm.py

+2 -2

@@ -12,7 +12,7 @@ from transformers import AutoTokenizer

12

13 from vllm.assets.image import ImageAsset

14 from vllm.lora.request import LoRARequest

15 -from vllm.multimodal.image import rescale_image_size

15 +from vllm.multimodal.image import convert_image_mode, rescale_image_size

16 from vllm.platforms import current_platform

17 from vllm.sequence import SampleLogprobs

18

@@ -267,7 +267,7 @@ def test_vision_speech_models(hf_runner, vllm_runner, model, dtype: str,

267

268 # use the example speech question so that the model outputs are reasonable

269 audio = librosa.load(speech_question, sr=None)

270 - image = ImageAsset("cherry_blossom").pil_image.convert("RGB")

270 + image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")

271

272 inputs_vision_speech = [

273 (

tests/models/test_oot_registration.py

+2 -1

@@ -4,6 +4,7 @@ import pytest

4

5 from vllm import LLM, SamplingParams

6 from vllm.assets.image import ImageAsset

7 +from vllm.multimodal.image import convert_image_mode

8

9 from ..utils import create_new_process_for_each_test

10

@@ -58,7 +59,7 @@ def test_oot_registration_embedding(

59 assert all(v == 0 for v in output.outputs.embedding)

60

61

61 -image = ImageAsset("cherry_blossom").pil_image.convert("RGB")

62 +image = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")

63

64

65 @create_new_process_for_each_test()

tests/multimodal/assets/rgba.png

Binary files /dev/null and b/tests/multimodal/assets/rgba.png differ

tests/multimodal/test_image.py new

+36

@@ -0,0 +1,36 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +from pathlib import Path

3 +

4 +import numpy as np

5 +from PIL import Image, ImageChops

6 +

7 +from vllm.multimodal.image import convert_image_mode

8 +

9 +ASSETS_DIR = Path(__file__).parent / "assets"

10 +assert ASSETS_DIR.exists()

11 +

12 +

13 +def test_rgb_to_rgb():

14 + # Start with an RGB image.

15 + original_image = Image.open(ASSETS_DIR / "image1.png").convert("RGB")

16 + converted_image = convert_image_mode(original_image, "RGB")

17 +

18 + # RGB to RGB should be a no-op.

19 + diff = ImageChops.difference(original_image, converted_image)

20 + assert diff.getbbox() is None

21 +

22 +

23 +def test_rgba_to_rgb():

24 + original_image = Image.open(ASSETS_DIR / "rgba.png")

25 + original_image_numpy = np.array(original_image)

26 +

27 + converted_image = convert_image_mode(original_image, "RGB")

28 + converted_image_numpy = np.array(converted_image)

29 +

30 + for i in range(original_image_numpy.shape[0]):

31 + for j in range(original_image_numpy.shape[1]):

32 + # Verify that all transparent pixels are converted to white.

33 + if original_image_numpy[i][j][3] == 0:

34 + assert converted_image_numpy[i][j][0] == 255

35 + assert converted_image_numpy[i][j][1] == 255

36 + assert converted_image_numpy[i][j][2] == 255

tests/multimodal/test_utils.py

+2 -1

@@ -10,6 +10,7 @@ import numpy as np

10 import pytest

11 from PIL import Image, ImageChops

12

13 +from vllm.multimodal.image import convert_image_mode

14 from vllm.multimodal.inputs import PlaceholderRange

15 from vllm.multimodal.utils import (MediaConnector,

16 merge_and_sort_multimodal_metadata)

@@ -53,7 +54,7 @@ def get_supported_suffixes() -> tuple[str, ...]:

54

55

56 def _image_equals(a: Image.Image, b: Image.Image) -> bool:

56 - return (np.asarray(a) == np.asarray(b.convert(a.mode))).all()

57 + return (np.asarray(a) == np.asarray(convert_image_mode(b, a.mode))).all()

58

59

60 @pytest.mark.asyncio

vllm/benchmarks/datasets.py

+2 -2

@@ -13,7 +13,6 @@ generation. Supported dataset types include:

13 TODO: Implement CustomDataset to parse a JSON file and convert its contents into

14 SampleRequest instances, similar to the approach used in ShareGPT.

15 """

16 -

16 import base64

17 import io

18 import json

@@ -33,6 +32,7 @@ from transformers import PreTrainedTokenizerBase

32 from vllm.lora.request import LoRARequest

33 from vllm.lora.utils import get_adapter_absolute_path

34 from vllm.multimodal import MultiModalDataDict

35 +from vllm.multimodal.image import convert_image_mode

36 from vllm.transformers_utils.tokenizer import AnyTokenizer, get_lora_tokenizer

37

38 logger = logging.getLogger(__name__)

@@ -259,7 +259,7 @@ def process_image(image: Any) -> Mapping[str, Any]:

259 if isinstance(image, dict) and 'bytes' in image:

260 image = Image.open(BytesIO(image['bytes']))

261 if isinstance(image, Image.Image):

262 - image = image.convert("RGB")

262 + image = convert_image_mode(image, "RGB")

263 with io.BytesIO() as image_data:

264 image.save(image_data, format="JPEG")

265 image_base64 = base64.b64encode(

vllm/model_executor/models/internvl.py

+2 -1

@@ -23,6 +23,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,

23 InternVisionPatchModel)

24 from vllm.model_executor.sampling_metadata import SamplingMetadata

25 from vllm.multimodal import MULTIMODAL_REGISTRY

26 +from vllm.multimodal.image import convert_image_mode

27 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

28 MultiModalKwargs, NestedTensors)

29 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

@@ -77,7 +78,7 @@ InternVLImageInputs = Union[InternVLImagePixelInputs,

78 def build_transform(input_size: int):

79 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD

80 return T.Compose([

80 - T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),

81 + T.Lambda(lambda img: convert_image_mode(img, 'RGB')),

82 T.Resize((input_size, input_size),

83 interpolation=T.InterpolationMode.BICUBIC),

84 T.ToTensor(),

vllm/model_executor/models/skyworkr1v.py

+2 -1

@@ -24,6 +24,7 @@ from vllm.model_executor.models.intern_vit import (InternVisionModel,

24 InternVisionPatchModel)

25 from vllm.model_executor.sampling_metadata import SamplingMetadata

26 from vllm.multimodal import MULTIMODAL_REGISTRY

27 +from vllm.multimodal.image import convert_image_mode

28 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

29 MultiModalKwargs, NestedTensors)

30 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,

@@ -78,7 +79,7 @@ SkyworkR1VImageInputs = Union[SkyworkR1VImagePixelInputs,

79 def build_transform(input_size: int):

80 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD

81 return T.Compose([

81 - T.Lambda(lambda img: img.convert('RGB') if img.mode != 'RGB' else img),

82 + T.Lambda(lambda img: convert_image_mode(img, 'RGB')),

83 T.Resize((input_size, input_size),

84 interpolation=T.InterpolationMode.BICUBIC),

85 T.ToTensor(),

vllm/multimodal/hasher.py

+3 -1

@@ -10,6 +10,7 @@ from blake3 import blake3

10 from PIL import Image

11

12 from vllm.logger import init_logger

13 +from vllm.multimodal.image import convert_image_mode

14

15 if TYPE_CHECKING:

16 from vllm.inputs import TokensPrompt

@@ -35,7 +36,8 @@ class MultiModalHasher:

36 return np.array(obj).tobytes()

37

38 if isinstance(obj, Image.Image):

38 - return cls.item_to_bytes("image", np.array(obj.convert("RGBA")))

39 + return cls.item_to_bytes("image",

40 + np.array(convert_image_mode(obj, "RGBA")))

41 if isinstance(obj, torch.Tensor):

42 return cls.item_to_bytes("tensor", obj.numpy())

43 if isinstance(obj, np.ndarray):

vllm/multimodal/image.py

+22 -3

@@ -22,6 +22,25 @@ def rescale_image_size(image: Image.Image,

22 return image

23

24

25 +# TODO: Support customizable background color to fill in.

26 +def rgba_to_rgb(

27 + image: Image.Image, background_color=(255, 255, 255)) -> Image.Image:

28 + """Convert an RGBA image to RGB with filled background color."""

29 + assert image.mode == "RGBA"

30 + converted = Image.new("RGB", image.size, background_color)

31 + converted.paste(image, mask=image.split()[3]) # 3 is the alpha channel

32 + return converted

33 +

34 +

35 +def convert_image_mode(image: Image.Image, to_mode: str):

36 + if image.mode == to_mode:

37 + return image

38 + elif image.mode == "RGBA" and to_mode == "RGB":

39 + return rgba_to_rgb(image)

40 + else:

41 + return image.convert(to_mode)

42 +

43 +

44 class ImageMediaIO(MediaIO[Image.Image]):

45

46 def __init__(self, *, image_mode: str = "RGB") -> None:

@@ -32,7 +51,7 @@ class ImageMediaIO(MediaIO[Image.Image]):

51 def load_bytes(self, data: bytes) -> Image.Image:

52 image = Image.open(BytesIO(data))

53 image.load()

35 - return image.convert(self.image_mode)

54 + return convert_image_mode(image, self.image_mode)

55

56 def load_base64(self, media_type: str, data: str) -> Image.Image:

57 return self.load_bytes(base64.b64decode(data))

@@ -40,7 +59,7 @@ class ImageMediaIO(MediaIO[Image.Image]):

59 def load_file(self, filepath: Path) -> Image.Image:

60 image = Image.open(filepath)

61 image.load()

43 - return image.convert(self.image_mode)

62 + return convert_image_mode(image, self.image_mode)

63

64 def encode_base64(

65 self,

@@ -51,7 +70,7 @@ class ImageMediaIO(MediaIO[Image.Image]):

70 image = media

71

72 with BytesIO() as buffer:

54 - image = image.convert(self.image_mode)

73 + image = convert_image_mode(image, self.image_mode)

74 image.save(buffer, image_format)

75 data = buffer.getvalue()

76

vllm/transformers_utils/processors/ovis.py

+4 -2

@@ -33,6 +33,8 @@ from transformers.processing_utils import (ProcessingKwargs, ProcessorMixin,

33 Unpack)

34 from transformers.tokenization_utils_base import PreTokenizedInput, TextInput

35

36 +from vllm.multimodal.image import convert_image_mode

37 +

38 __all__ = ['OvisProcessor']

39 IGNORE_ID = -100

40

@@ -361,8 +363,8 @@ class OvisProcessor(ProcessorMixin):

363 # pick the partition with maximum covering_ratio and break the tie using #sub_images

364 return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]

365

364 - if convert_to_rgb and image.mode != 'RGB':

365 - image = image.convert('RGB')

366 + if convert_to_rgb:

367 + image = convert_image_mode(image, 'RGB')

368

369

370 sides = self.get_image_size()

Re-submit: Fix: Proper RGBA -> RGB conversion for PIL images. (#18569)