@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 98d6682cd1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[VLM] Remove `image_input_type` from VLM config (#5852)

Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

xwjiang2010 committed Jul 2, 2024 at 00:57 UTC 98d6682cd1f27fa48bf915d3fd3e1eb1ee3014c4
35 files changed +325 -747
.buildkite/download-images.sh
-4
@@ -8,10 +8,6 @@ set -o pipefail
8 # aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9 mkdir -p images
10 cd images
11 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_pixel_values.pt
12 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_image_features.pt
13 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_pixel_values.pt
14 -wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_image_features.pt
11 wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
12 wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
13
docs/requirements-docs.txt
+4 -12
@@ -1,13 +1,5 @@
1 -sphinx == 6.2.1
2 -sphinx-book-theme == 1.0.1
3 -sphinx-copybutton == 0.5.2
4 -myst-parser == 2.0.0
1 +sphinx==6.2.1
2 +sphinx-book-theme==1.0.1
3 +sphinx-copybutton==0.5.2
4 +myst-parser==2.0.0
5 sphinx-argparse
6 -
7 -# packages to install to build the documentation
8 -pydantic
9 --f https://download.pytorch.org/whl/cpu
10 -torch
11 -py-cpuinfo
12 -transformers
13 -openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
docs/source/dev/multimodal/multimodal_index.rst
+5 -3
@@ -9,8 +9,10 @@ vLLM provides experimental support for multi-modal models through the :mod:`vllm
9 which allows you to pass in multi-modal input alongside text and token prompts.
10
11 By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model,
12 -you must decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_dummy_data <MultiModalRegistry.register_dummy_data>`,
13 -as well as :meth:`MULTIMODAL_REGISTRY.register_input <MultiModalRegistry.register_input>` for each modality type to support.
12 +you must decorate the model class with :meth:`InputRegistry.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`,
13 +as well as :meth:`MULTIMODAL_REGISTRY.register_input_mapper <MultiModalRegistry.register_input_mapper>` for each modality type to support.
14 +
15 +# TODO: Add more instructions on how to do that once embeddings is in.
16
17 Module Contents
18 +++++++++++++++
@@ -29,7 +31,7 @@ Registry
31 Base Classes
32 ------------
33
32 -.. autoclass:: vllm.multimodal.MultiModalData
34 +.. autoclass:: vllm.multimodal.MultiModalDataDict
35 :members:
36 :show-inheritance:
37
docs/source/models/vlm.rst
+7 -4
@@ -36,7 +36,6 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
36
37 llm = LLM(
38 model="llava-hf/llava-1.5-7b-hf",
39 - image_input_type="pixel_values",
39 image_token_id=32000,
40 image_input_shape="1,3,336,336",
41 image_feature_size=576,
@@ -49,7 +48,12 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
48 To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
49
50 * ``prompt``: The prompt should have a number of ``<image>`` tokens equal to ``image_feature_size``.
52 -* ``multi_modal_data``: This should be an instance of :class:`~vllm.multimodal.image.ImagePixelData` or :class:`~vllm.multimodal.image.ImageFeatureData`.
51 +* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
52 +
53 +.. note::
54 +
55 + ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
56 + :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
57
58 .. code-block:: python
59
@@ -61,7 +65,7 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
65
66 outputs = llm.generate({
67 "prompt": prompt,
64 - "multi_modal_data": ImagePixelData(image),
68 + "multi_modal_data": {"image": image},
69 })
70
71 for o in outputs:
@@ -93,7 +97,6 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
97
98 python -m vllm.entrypoints.openai.api_server \
99 --model llava-hf/llava-1.5-7b-hf \
96 - --image-input-type pixel_values \
100 --image-token-id 32000 \
101 --image-input-shape 1,3,336,336 \
102 --image-feature-size 576 \
examples/llava_example.py
+8 -48
@@ -1,38 +1,32 @@
1 -import argparse
1 import os
2 import subprocess
3
5 -import torch
4 from PIL import Image
5
6 from vllm import LLM
9 -from vllm.multimodal.image import ImageFeatureData, ImagePixelData
7
8 # The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9 # You can use `.buildkite/download-images.sh` to download them
10
11
15 -def run_llava_pixel_values(*, disable_image_processor: bool = False):
12 +def run_llava():
13 llm = LLM(
14 model="llava-hf/llava-1.5-7b-hf",
18 - image_input_type="pixel_values",
15 image_token_id=32000,
16 image_input_shape="1,3,336,336",
17 image_feature_size=576,
22 - disable_image_processor=disable_image_processor,
18 )
19
20 prompt = "<image>" * 576 + (
21 "\nUSER: What is the content of this image?\nASSISTANT:")
22
28 - if disable_image_processor:
29 - image = torch.load("images/stop_sign_pixel_values.pt")
30 - else:
31 - image = Image.open("images/stop_sign.jpg")
23 + image = Image.open("images/stop_sign.jpg")
24
25 outputs = llm.generate({
26 "prompt": prompt,
35 - "multi_modal_data": ImagePixelData(image),
27 + "multi_modal_data": {
28 + "image": image
29 + },
30 })
31
32 for o in outputs:
@@ -40,45 +34,11 @@ def run_llava_pixel_values(*, disable_image_processor: bool = False):
34 print(generated_text)
35
36
43 -def run_llava_image_features():
44 - llm = LLM(
45 - model="llava-hf/llava-1.5-7b-hf",
46 - image_input_type="image_features",
47 - image_token_id=32000,
48 - image_input_shape="1,576,1024",
49 - image_feature_size=576,
50 - )
51 -
52 - prompt = "<image>" * 576 + (
53 - "\nUSER: What is the content of this image?\nASSISTANT:")
54 -
55 - image: torch.Tensor = torch.load("images/stop_sign_image_features.pt")
56 -
57 - outputs = llm.generate({
58 - "prompt": prompt,
59 - "multi_modal_data": ImageFeatureData(image),
60 - })
61 -
62 - for o in outputs:
63 - generated_text = o.outputs[0].text
64 - print(generated_text)
65 -
66 -
67 -def main(args):
68 - if args.type == "pixel_values":
69 - run_llava_pixel_values()
70 - else:
71 - run_llava_image_features()
37 +def main():
38 + run_llava()
39
40
41 if __name__ == "__main__":
75 - parser = argparse.ArgumentParser(description="Demo on Llava")
76 - parser.add_argument("--type",
77 - type=str,
78 - choices=["pixel_values", "image_features"],
79 - default="pixel_values",
80 - help="image input type")
81 - args = parser.parse_args()
42 # Download from s3
43 s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
44 local_directory = "images"
@@ -95,4 +55,4 @@ if __name__ == "__main__":
55 local_directory,
56 "--no-sign-request",
57 ])
98 - main(args)
58 + main()
examples/llava_next_example.py
+35 -26
@@ -4,35 +4,44 @@ import requests
4 from PIL import Image
5
6 from vllm import LLM, SamplingParams
7 -from vllm.multimodal.image import ImagePixelData
7
8 # Dynamic image input is currently not supported and therefore
9 # a fixed image input shape and its corresponding feature size is required.
10 # See https://github.com/vllm-project/vllm/pull/4199 for the complete
11 # configuration matrix.
12
14 -llm = LLM(
15 - model="llava-hf/llava-v1.6-mistral-7b-hf",
16 - image_input_type="pixel_values",
17 - image_token_id=32000,
18 - image_input_shape="1,3,336,336",
19 - image_feature_size=1176,
20 -)
21 -
22 -prompt = "[INST] " + "<image>" * 1176 + "\nWhat is shown in this image? [/INST]"
23 -url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
24 -image = Image.open(BytesIO(requests.get(url).content))
25 -sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=100)
26 -
27 -outputs = llm.generate(
28 - {
29 - "prompt": prompt,
30 - "multi_modal_data": ImagePixelData(image),
31 - },
32 - sampling_params=sampling_params)
33 -
34 -generated_text = ""
35 -for o in outputs:
36 - generated_text += o.outputs[0].text
37 -
38 -print(f"LLM output:{generated_text}")
13 +
14 +def run_llava_next():
15 + llm = LLM(
16 + model="llava-hf/llava-v1.6-mistral-7b-hf",
17 + image_token_id=32000,
18 + image_input_shape="1,3,336,336",
19 + image_feature_size=1176,
20 + )
21 +
22 + prompt = "[INST] " + "<image>" * 1176 + (
23 + "\nWhat is shown in this image? [/INST]")
24 + url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
25 + image = Image.open(BytesIO(requests.get(url).content))
26 + sampling_params = SamplingParams(temperature=0.8,
27 + top_p=0.95,
28 + max_tokens=100)
29 +
30 + outputs = llm.generate(
31 + {
32 + "prompt": prompt,
33 + "multi_modal_data": {
34 + "image": image
35 + }
36 + },
37 + sampling_params=sampling_params)
38 +
39 + generated_text = ""
40 + for o in outputs:
41 + generated_text += o.outputs[0].text
42 +
43 + print(f"LLM output:{generated_text}")
44 +
45 +
46 +if __name__ == "__main__":
47 + run_llava_next()
examples/openai_vision_api_client.py
-1
@@ -3,7 +3,6 @@
3 Launch the vLLM server with the following command:
4 python -m vllm.entrypoints.openai.api_server \
5 --model llava-hf/llava-1.5-7b-hf \
6 - --image-input-type pixel_values \
6 --image-token-id 32000 \
7 --image-input-shape 1,3,336,336 \
8 --image-feature-size 576 \
examples/phi3v_example.py
+3 -3
@@ -4,7 +4,6 @@ import subprocess
4 from PIL import Image
5
6 from vllm import LLM, SamplingParams
7 -from vllm.multimodal.image import ImagePixelData
7
8
9 def run_phi3v():
@@ -17,7 +16,6 @@ def run_phi3v():
16 llm = LLM(
17 model=model_path,
18 trust_remote_code=True,
20 - image_input_type="pixel_values",
19 image_token_id=32044,
20 image_input_shape="1,3,1008,1344",
21 image_feature_size=1921,
@@ -35,7 +33,9 @@ def run_phi3v():
33 outputs = llm.generate(
34 {
35 "prompt": prompt,
38 - "multi_modal_data": ImagePixelData(image),
36 + "multi_modal_data": {
37 + "image": image
38 + },
39 },
40 sampling_params=sampling_params)
41 for o in outputs:
tests/conftest.py
+8 -30
@@ -17,19 +17,17 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
17 AutoTokenizer, BatchEncoding)
18
19 from vllm import LLM, SamplingParams
20 -from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
20 +from vllm.config import TokenizerPoolConfig
21 from vllm.distributed import (destroy_distributed_environment,
22 destroy_model_parallel)
23 from vllm.inputs import TextPrompt
24 from vllm.logger import init_logger
25 +from vllm.sequence import SampleLogprobs
26 +from vllm.utils import cuda_device_count_stateless, is_cpu
27
28 if TYPE_CHECKING:
27 - from vllm.multimodal import MultiModalData
28 -else:
29 # it will call torch.cuda.device_count()
30 - MultiModalData = None
31 -from vllm.sequence import SampleLogprobs
32 -from vllm.utils import cuda_device_count_stateless, is_cpu
30 + from vllm.multimodal import MultiModalDataDict
31
32 logger = init_logger(__name__)
33
@@ -51,14 +49,6 @@ def _read_prompts(filename: str) -> List[str]:
49 class ImageAsset:
50 name: Literal["stop_sign", "cherry_blossom"]
51
54 - @cached_property
55 - def pixel_values(self) -> torch.Tensor:
56 - return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
57 -
58 - @cached_property
59 - def image_features(self) -> torch.Tensor:
60 - return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
61 -
52 @cached_property
53 def pil_image(self) -> Image.Image:
54 return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
@@ -66,20 +56,8 @@ class ImageAsset:
56 def for_hf(self) -> Image.Image:
57 return self.pil_image
58
69 - def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
70 - # don't put this import at the top level
71 - # it will call torch.cuda.device_count()
72 - from vllm.multimodal.image import ImageFeatureData # noqa: F401
73 - from vllm.multimodal.image import ImagePixelData
74 - image_input_type = vision_config.image_input_type
75 - ImageInputType = VisionLanguageConfig.ImageInputType
76 -
77 - if image_input_type == ImageInputType.IMAGE_FEATURES:
78 - return ImageFeatureData(self.image_features)
79 - if image_input_type == ImageInputType.PIXEL_VALUES:
80 - return ImagePixelData(self.pil_image)
81 -
82 - raise NotImplementedError
59 + def for_vllm(self) -> Dict[str, Any]:
60 + return {"image": self.pil_image}
61
62
63 class _ImageAssetPrompts(TypedDict):
@@ -453,7 +431,7 @@ class VllmRunner:
431 self,
432 prompts: List[str],
433 sampling_params: SamplingParams,
456 - images: Optional[List[MultiModalData]] = None,
434 + images: Optional[List["MultiModalDataDict"]] = None,
435 ) -> List[Tuple[List[List[int]], List[str]]]:
436 if images is not None:
437 assert len(prompts) == len(images)
@@ -502,7 +480,7 @@ class VllmRunner:
480 self,
481 prompts: List[str],
482 max_tokens: int,
505 - images: Optional[List[MultiModalData]] = None,
483 + images: Optional[List["MultiModalDataDict"]] = None,
484 ) -> List[Tuple[List[int], str]]:
485 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
486 outputs = self.generate(prompts, greedy_params, images=images)
tests/entrypoints/openai/test_vision.py
-2
@@ -39,8 +39,6 @@ def server():
39 "--max-model-len",
40 "4096",
41 "--enforce-eager",
42 - "--image-input-type",
43 - "pixel_values",
42 "--image-token-id",
43 "32000",
44 "--image-input-shape",
tests/models/test_llava.py
+8 -14
@@ -25,17 +25,11 @@ def iter_llava_configs(model_name: str):
25 }
26
27 for (h, w), f in image_hw_to_feature_size.items():
28 - for input_type, input_shape in [
29 - (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
30 - (VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
31 - ]:
32 - yield (model_name,
33 - VisionLanguageConfig(image_input_type=input_type,
34 - image_feature_size=f,
35 - image_token_id=32000,
36 - image_input_shape=input_shape,
37 - image_processor=model_name,
38 - image_processor_revision=None))
28 + input_shape = (1, 3, h, w)
29 + yield (model_name,
30 + VisionLanguageConfig(image_feature_size=f,
31 + image_token_id=32000,
32 + image_input_shape=input_shape))
33
34
35 model_and_vl_config = [
@@ -81,8 +75,8 @@ def run_test(
75
76 All the image fixtures for the test is under tests/images.
77 For huggingface runner, we provide the PIL images as input.
84 - For vllm runner, we provide MultiModalData objects and corresponding
85 - vision language config as input.
78 + For vllm runner, we provide MultiModalDataDict objects
79 + and corresponding vision language config as input.
80 Note, the text input is also adjusted to abide by vllm contract.
81 The text output is sanitized to be able to compare with hf.
82 """
@@ -104,7 +98,7 @@ def run_test(
98 # NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
99 # we must put it inside the vllm_runner context manager
100 # i.e. after creating vLLM instance.
107 - vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
101 + vllm_images = [asset.for_vllm() for asset in image_assets]
102
103 vllm_image_prompts = [
104 p.replace("<image>", "<image>" * vlm_config.image_feature_size)
tests/models/test_llava_next.py
+10 -13
@@ -33,16 +33,13 @@ def iter_llava_next_configs(model_name: str):
33 }
34
35 for (h, w), f in image_hw_to_feature_size.items():
36 - for input_type, input_shape in [
37 - (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
38 - ]:
39 - yield (model_name,
40 - VisionLanguageConfig(image_input_type=input_type,
41 - image_feature_size=f,
42 - image_token_id=32000,
43 - image_input_shape=input_shape,
44 - image_processor=model_name,
45 - image_processor_revision=None))
36 + input_shape = (1, 3, h, w)
37 + yield (model_name,
38 + VisionLanguageConfig(
39 + image_feature_size=f,
40 + image_token_id=32000,
41 + image_input_shape=input_shape,
42 + ))
43
44
45 model_and_vl_config = [
@@ -85,14 +82,14 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
82
83 All the image fixtures for the test is under tests/images.
84 For huggingface runner, we provide the PIL images as input.
88 - For vllm runner, we provide MultiModalData objects and corresponding
89 - vision language config as input.
85 + For vllm runner, we provide MultiModalDataDict objects
86 + and corresponding vision language config as input.
87 Note, the text input is also adjusted to abide by vllm contract.
88 The text output is sanitized to be able to compare with hf.
89 """
90 model_id, vlm_config = model_and_config
91 hf_images = [asset.for_hf() for asset in image_assets]
95 - vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
92 + vllm_images = [asset.for_vllm() for asset in image_assets]
93
94 with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
95 hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_phi3v.py
+8 -13
@@ -27,16 +27,11 @@ def iter_phi3v_configs(model_name: str):
27 }
28
29 for (h, w), f in image_hw_to_feature_size.items():
30 - for input_type, input_shape in [
31 - (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
32 - ]:
33 - yield (model_name,
34 - VisionLanguageConfig(image_input_type=input_type,
35 - image_feature_size=f,
36 - image_token_id=32044,
37 - image_input_shape=input_shape,
38 - image_processor=model_name,
39 - image_processor_revision=None))
30 + input_shape = (1, 3, h, w)
31 + yield (model_name,
32 + VisionLanguageConfig(image_feature_size=f,
33 + image_token_id=32044,
34 + image_input_shape=input_shape))
35
36
37 model_and_vl_config = [
@@ -89,8 +84,8 @@ def run_test(
84
85 All the image fixtures for the test is under tests/images.
86 For huggingface runner, we provide the PIL images as input.
92 - For vllm runner, we provide MultiModalData objects and corresponding
93 - vision language config as input.
87 + For vllm runner, we provide MultiModalDataDict objects
88 + and corresponding vision language config as input.
89 Note, the text input is also adjusted to abide by vllm contract.
90 The text output is sanitized to be able to compare with hf.
91 """
@@ -113,7 +108,7 @@ def run_test(
108 # we must put it inside the vllm_runner context manager
109 # i.e. after creating vLLM instance.
110
116 - vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
111 + vllm_images = [asset.for_vllm() for asset in image_assets]
112
113 vllm_image_prompts = [
114 p.replace("<|image_1|>",
tests/multimodal/test_mapper.py
+6 -34
@@ -2,9 +2,8 @@ import numpy as np
2 import pytest
3 from transformers import CLIPImageProcessor, LlavaNextImageProcessor
4
5 -from vllm.config import ModelConfig, VisionLanguageConfig
5 +from vllm.config import ModelConfig
6 from vllm.multimodal import MULTIMODAL_REGISTRY
7 -from vllm.multimodal.image import ImagePixelData
7
8 from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
9
@@ -12,7 +11,6 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
11 @pytest.mark.parametrize("dtype", ["half", "float"])
12 def test_clip_image_processor(image_assets, dtype):
13 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
15 - IMAGE_HEIGHT = IMAGE_WIDTH = 560
14
15 hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
16 assert isinstance(hf_processor, CLIPImageProcessor)
@@ -25,14 +23,6 @@ def test_clip_image_processor(image_assets, dtype):
23 seed=0,
24 dtype=dtype,
25 revision=None,
28 - multimodal_config=VisionLanguageConfig(
29 - image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
30 - image_token_id=32000,
31 - image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
32 - image_feature_size=576,
33 - image_processor=MODEL_NAME,
34 - image_processor_revision=None,
35 - ),
26 )
27
28 for asset in image_assets:
@@ -42,7 +32,7 @@ def test_clip_image_processor(image_assets, dtype):
32 ).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
33 vllm_result = MULTIMODAL_REGISTRY.map_input(
34 model_config,
45 - ImagePixelData(asset.pil_image),
35 + {"image": asset.pil_image},
36 )
37
38 assert hf_result.keys() == vllm_result.keys()
@@ -60,7 +50,6 @@ def test_clip_image_processor(image_assets, dtype):
50 @pytest.mark.parametrize("dtype", ["half", "float"])
51 def test_llava_next_image_processor(image_assets, dtype):
52 MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
63 - IMAGE_HEIGHT = IMAGE_WIDTH = 560
53
54 hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
55 assert isinstance(hf_processor, LlavaNextImageProcessor)
@@ -73,14 +62,6 @@ def test_llava_next_image_processor(image_assets, dtype):
62 seed=0,
63 dtype=dtype,
64 revision=None,
76 - multimodal_config=VisionLanguageConfig(
77 - image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
78 - image_token_id=64000,
79 - image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
80 - image_feature_size=2928,
81 - image_processor=MODEL_NAME,
82 - image_processor_revision=None,
83 - ),
65 )
66
67 for asset in image_assets:
@@ -90,7 +71,7 @@ def test_llava_next_image_processor(image_assets, dtype):
71 ).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
72 vllm_result = MULTIMODAL_REGISTRY.map_input(
73 model_config,
93 - ImagePixelData(asset.pil_image),
74 + {"image": asset.pil_image},
75 )
76
77 assert hf_result.keys() == vllm_result.keys()
@@ -107,7 +88,6 @@ def test_llava_next_image_processor(image_assets, dtype):
88 @pytest.mark.parametrize("dtype", ["float"])
89 def test_image_pixel_types(image_assets, dtype):
90 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
110 - IMAGE_HEIGHT = IMAGE_WIDTH = 560
91
92 model_config = ModelConfig(
93 model=MODEL_NAME,
@@ -117,23 +97,15 @@ def test_image_pixel_types(image_assets, dtype):
97 seed=0,
98 dtype=dtype,
99 revision=None,
120 - multimodal_config=VisionLanguageConfig(
121 - image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
122 - image_token_id=32000,
123 - image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
124 - image_feature_size=576,
125 - image_processor=MODEL_NAME,
126 - image_processor_revision=None,
127 - ))
128 -
100 + )
101 for asset in image_assets:
102 image_result = MULTIMODAL_REGISTRY.map_input(
103 model_config,
132 - ImagePixelData(asset.pil_image),
104 + {"image": asset.pil_image},
105 )
106 tensor_result = MULTIMODAL_REGISTRY.map_input(
107 model_config,
136 - ImagePixelData(asset.pixel_values),
108 + {"image": asset.pil_image},
109 )
110
111 assert image_result.keys() == tensor_result.keys()
tests/spec_decode/e2e/conftest.py
+2 -2
@@ -11,7 +11,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
11 from vllm.engine.async_llm_engine import AsyncLLMEngine
12 from vllm.lora.request import LoRARequest
13 from vllm.model_executor.utils import set_random_seed
14 -from vllm.multimodal import MultiModalData
14 +from vllm.multimodal import MultiModalDataDict
15 from vllm.outputs import RequestOutput
16 from vllm.sampling_params import SamplingParams
17 from vllm.sequence import Logprob
@@ -91,7 +91,7 @@ class AsyncLLM:
91 prompt_token_ids: Optional[List[List[int]]] = None,
92 use_tqdm: bool = True,
93 lora_request: Optional[LoRARequest] = None,
94 - multi_modal_data: Optional[MultiModalData] = None,
94 + multi_modal_data: Optional[MultiModalDataDict] = None,
95 ) -> List[RequestOutput]:
96
97 if prompts is None:
tests/tokenization/test_image_processor.py deleted
-20
@@ -1,20 +0,0 @@
1 -import pytest
2 -from transformers.image_processing_utils import BaseImageProcessor
3 -
4 -from vllm.transformers_utils.image_processor import get_image_processor
5 -
6 -IMAGE_PROCESSOR_NAMES = [
7 - "llava-hf/llava-1.5-7b-hf",
8 - "llava-hf/llava-v1.6-34b-hf",
9 -]
10 -
11 -
12 -@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
13 -def test_image_processor_revision(processor_name: str):
14 - # Assume that "main" branch always exists
15 - image_processor = get_image_processor(processor_name, revision="main")
16 - assert isinstance(image_processor, BaseImageProcessor)
17 -
18 - # Assume that "never" branch always does not exist
19 - with pytest.raises(OSError, match='not a valid git identifier'):
20 - get_image_processor(processor_name, revision="never")
vllm/config.py
+1 -33
@@ -1250,28 +1250,11 @@ class LoRAConfig:
1250 raise ValueError("LoRA is not supported with chunked prefill yet.")
1251
1252
1253 +# TODO: To be replaced by MultiModalConfig.
1254 @dataclass
1255 class VisionLanguageConfig:
1256 """Configs the input data format and how models should run for
1257 vision language models."""
1257 -
1258 - class ImageInputType(enum.Enum):
1259 - """Image input type into the vision language model.
1260 -
1261 - An image roughly goes through the following transformation:
1262 - Raw image --> pixel values --> image features --> image embeddings.
1263 -
1264 - The difference between different image input types is where the
1265 - image encoder (pixel values --> image features) is run.
1266 - Different image input types also correspond to different tensor shapes.
1267 -
1268 - For example, for Llava, PIXEL_VALUES: (1, 3, 336, 336).
1269 - IMAGE_FEATURES: (1, 576, 1024).
1270 - """
1271 - PIXEL_VALUES = enum.auto()
1272 - IMAGE_FEATURES = enum.auto()
1273 -
1274 - image_input_type: ImageInputType
1258 # The input id corresponding to image token.
1259 image_token_id: int
1260 # Used for running `run_prefill_max_token`.
@@ -1279,19 +1262,6 @@ class VisionLanguageConfig:
1262 # worst case scenario (biggest supported resolution).
1263 image_input_shape: tuple
1264 image_feature_size: int
1282 - # The image processor to load from HuggingFace
1283 - image_processor: Optional[str]
1284 - image_processor_revision: Optional[str]
1285 -
1286 - @classmethod
1287 - def get_image_input_enum_type(cls, value: str) -> ImageInputType:
1288 - """Get the image input type from a string."""
1289 - try:
1290 - return cls.ImageInputType[value.upper()]
1291 - except KeyError as e:
1292 - raise ValueError(f"{value} is not a valid choice. "
1293 - f"Expecting to choose from "
1294 - f"{[x.name for x in cls.ImageInputType]}.") from e
1265
1266 #TODO(ywang96): make this a cached property once we refactor the
1267 # VisionLanguageConfig class.
@@ -1318,8 +1288,6 @@ class VisionLanguageConfig:
1288 else:
1289 result[f.name] = value
1290
1321 - result["disable_image_processor"] = self.image_processor is None
1322 -
1291 return result
1292
1293
vllm/engine/arg_utils.py
+4 -52
@@ -1,7 +1,6 @@
1 import argparse
2 import dataclasses
3 import json
4 -import warnings
4 from dataclasses import dataclass
5 from typing import List, Optional, Tuple, Union
6
@@ -80,13 +79,9 @@ class EngineArgs:
79 preemption_mode: Optional[str] = None
80
81 # Related to Vision-language models such as llava
83 - image_input_type: Optional[str] = None
82 image_token_id: Optional[int] = None
83 image_input_shape: Optional[str] = None
84 image_feature_size: Optional[int] = None
87 - image_processor: Optional[str] = None
88 - image_processor_revision: Optional[str] = None
89 - disable_image_processor: bool = False
85
86 scheduler_delay_factor: float = 0.0
87 enable_chunked_prefill: bool = False
@@ -114,14 +109,6 @@ class EngineArgs:
109 @staticmethod
110 def add_cli_args_for_vlm(
111 parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
117 - parser.add_argument('--image-input-type',
118 - type=nullable_str,
119 - default=None,
120 - choices=[
121 - t.name.lower()
122 - for t in VisionLanguageConfig.ImageInputType
123 - ],
124 - help=('The image input type passed into vLLM.'))
112 parser.add_argument('--image-token-id',
113 type=int,
114 default=None,
@@ -137,24 +124,6 @@ class EngineArgs:
124 type=int,
125 default=None,
126 help=('The image feature size along the context dimension.'))
140 - parser.add_argument(
141 - '--image-processor',
142 - type=str,
143 - default=EngineArgs.image_processor,
144 - help='Name or path of the huggingface image processor to use. '
145 - 'If unspecified, model name or path will be used.')
146 - parser.add_argument(
147 - '--image-processor-revision',
148 - type=str,
149 - default=None,
150 - help='Revision of the huggingface image processor version to use. '
151 - 'It can be a branch name, a tag name, or a commit id. '
152 - 'If unspecified, will use the default version.')
153 - parser.add_argument(
154 - '--disable-image-processor',
155 - action='store_true',
156 - help='Disables the use of image processor, even if one is defined '
157 - 'for the model on huggingface.')
127
128 return parser
129
@@ -679,33 +648,16 @@ class EngineArgs:
648 raise ValueError(
649 "BitsAndBytes load format and QLoRA adapter only support "
650 f"'bitsandbytes' quantization, but got {self.quantization}")
682 - if self.image_input_type:
683 - if (not self.image_token_id or not self.image_input_shape
684 - or not self.image_feature_size):
651 + if self.image_token_id is not None:
652 + if (not self.image_input_shape or not self.image_feature_size):
653 raise ValueError(
686 - 'Specify `image_token_id`, `image_input_shape` and '
687 - '`image_feature_size` together with `image_input_type`.')
688 -
689 - if self.image_processor is None:
690 - self.image_processor = self.model
691 - if self.disable_image_processor:
692 - if self.image_processor != self.model:
693 - warnings.warn(
694 - "You've specified an image processor "
695 - f"({self.image_processor}) but also disabled "
696 - "it via `--disable-image-processor`.",
697 - stacklevel=2)
698 -
699 - self.image_processor = None
654 + 'Specify `image_input_shape` and '
655 + '`image_feature_size` together with `image_token_id`.')
656
657 vision_language_config = VisionLanguageConfig(
702 - image_input_type=VisionLanguageConfig.
703 - get_image_input_enum_type(self.image_input_type),
658 image_token_id=self.image_token_id,
659 image_input_shape=str_to_int_tuple(self.image_input_shape),
660 image_feature_size=self.image_feature_size,
707 - image_processor=self.image_processor,
708 - image_processor_revision=self.image_processor_revision,
661 )
662 else:
663 vision_language_config = None
vllm/entrypoints/openai/api_server.py
-9
@@ -213,15 +213,6 @@ if __name__ == "__main__":
213
214 engine_args = AsyncEngineArgs.from_cli_args(args)
215
216 - # Enforce pixel values as image input type for vision language models
217 - # when serving with API server
218 - if engine_args.image_input_type is not None and \
219 - engine_args.image_input_type.upper() != "PIXEL_VALUES":
220 - raise ValueError(
221 - f"Invalid image_input_type: {engine_args.image_input_type}. "
222 - "Only --image-input-type 'pixel_values' is supported for serving "
223 - "vision language models with the vLLM API server.")
224 -
216 engine = AsyncLLMEngine.from_engine_args(
217 engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
218
vllm/entrypoints/openai/serving_chat.py
+29 -36
@@ -26,7 +26,7 @@ from vllm.inputs import PromptInputs
26 from vllm.logger import init_logger
27 from vllm.model_executor.guided_decoding import (
28 get_guided_decoding_logits_processor)
29 -from vllm.multimodal.image import ImagePixelData
29 +from vllm.multimodal import MultiModalDataDict
30 from vllm.multimodal.utils import (async_get_and_parse_image,
31 get_full_image_text_prompt)
32 from vllm.outputs import RequestOutput
@@ -47,7 +47,7 @@ class ConversationMessage(TypedDict):
47 @dataclass(frozen=True)
48 class ChatMessageParseResult:
49 messages: List[ConversationMessage]
50 - image_futures: List[Awaitable[ImagePixelData]] = field(
50 + mm_futures: List[Awaitable[MultiModalDataDict]] = field(
51 default_factory=list)
52
53
@@ -103,7 +103,7 @@ class OpenAIServingChat(OpenAIServing):
103 parts: Iterable[ChatCompletionContentPartParam],
104 ) -> ChatMessageParseResult:
105 texts: List[str] = []
106 - image_futures: List[Awaitable[ImagePixelData]] = []
106 + mm_futures: List[Awaitable[MultiModalDataDict]] = []
107
108 vlm_config: Optional[VisionLanguageConfig] = getattr(
109 self.engine.engine, "vision_language_config", None)
@@ -113,39 +113,34 @@ class OpenAIServingChat(OpenAIServing):
113 part_type = part["type"]
114 if part_type == "text":
115 text = cast(ChatCompletionContentPartTextParam, part)["text"]
116 -
116 texts.append(text)
117 elif part_type == "image_url":
118 if vlm_config is None:
119 raise ValueError(
120 "'image_url' input is not supported as the loaded "
121 "model is not multimodal.")
122 + assert self.tokenizer is not None
123 + image_url = cast(ChatCompletionContentPartImageParam,
124 + part)["image_url"]
125
124 - elif len(image_futures) == 0:
125 - assert self.tokenizer is not None
126 - image_url = cast(ChatCompletionContentPartImageParam,
127 - part)["image_url"]
128 -
129 - if image_url.get("detail", "auto") != "auto":
130 - logger.warning(
131 - "'image_url.detail' is currently not supported and "
132 - "will be ignored.")
133 -
134 - image_future = async_get_and_parse_image(image_url["url"])
135 - image_futures.append(image_future)
126 + if image_url.get("detail", "auto") != "auto":
127 + logger.warning(
128 + "'image_url.detail' is currently not supported and "
129 + "will be ignored.")
130
137 - else:
138 - raise NotImplementedError(
139 - "Multiple 'image_url' input is currently not supported."
140 - )
131 + mm_future = async_get_and_parse_image(image_url["url"])
132 + mm_futures.append(mm_future)
133
134 else:
135 raise NotImplementedError(f"Unknown part type: {part_type}")
136
137 text_prompt = "\n".join(texts)
138
147 - if vlm_config is not None and len(image_futures):
139 + if vlm_config is not None and len(mm_futures):
140
141 + assert len(
142 + mm_futures
143 + ) == 1, "Multiple 'image_url' input is currently not supported."
144 (image_token_prompt,
145 image_token_str) = vlm_config.get_image_token_text(self.tokenizer)
146
@@ -171,8 +166,7 @@ class OpenAIServingChat(OpenAIServing):
166 else:
167 messages = [ConversationMessage(role=role, content=text_prompt)]
168
174 - return ChatMessageParseResult(messages=messages,
175 - image_futures=image_futures)
169 + return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
170
171 def _parse_chat_message_content(
172 self,
@@ -182,10 +176,10 @@ class OpenAIServingChat(OpenAIServing):
176 content = message.get("content")
177
178 if content is None:
185 - return ChatMessageParseResult(messages=[], image_futures=[])
179 + return ChatMessageParseResult(messages=[], mm_futures=[])
180 if isinstance(content, str):
181 messages = [ConversationMessage(role=role, content=content)]
188 - return ChatMessageParseResult(messages=messages, image_futures=[])
182 + return ChatMessageParseResult(messages=messages, mm_futures=[])
183
184 return self._parse_chat_message_content_parts(role, content)
185
@@ -210,13 +204,13 @@ class OpenAIServingChat(OpenAIServing):
204
205 try:
206 conversation: List[ConversationMessage] = []
213 - image_futures: List[Awaitable[ImagePixelData]] = []
207 + mm_futures: List[Awaitable[MultiModalDataDict]] = []
208
209 for msg in request.messages:
210 chat_parsed_result = self._parse_chat_message_content(msg)
211
212 conversation.extend(chat_parsed_result.messages)
219 - image_futures.extend(chat_parsed_result.image_futures)
213 + mm_futures.extend(chat_parsed_result.mm_futures)
214
215 tool_dicts = None if request.tools is None else [
216 tool.model_dump() for tool in request.tools
@@ -235,15 +229,14 @@ class OpenAIServingChat(OpenAIServing):
229 logger.error("Error in applying chat template from request: %s", e)
230 return self.create_error_response(str(e))
231
238 - # Fetch image data
239 - image_data: Optional[ImagePixelData] = None
232 + mm_data: Optional[MultiModalDataDict] = None
233 try:
241 - if len(image_futures):
242 - # since we support only single image currently
243 - assert len(image_futures) == 1
244 - image_data = await image_futures[0]
234 + if len(mm_futures):
235 + # since we support only single mm data currently
236 + assert len(mm_futures) == 1
237 + mm_data = await mm_futures[0]
238 except Exception as e:
246 - logger.error("Error in loading image data: %s", e)
239 + logger.error("Error in loading multi-modal data: %s", e)
240 return self.create_error_response(str(e))
241
242 request_id = f"cmpl-{random_uuid()}"
@@ -274,8 +267,8 @@ class OpenAIServingChat(OpenAIServing):
267 "prompt": prompt_text,
268 "prompt_token_ids": prompt_ids,
269 }
277 - if image_data is not None:
278 - inputs["multi_modal_data"] = image_data
270 + if mm_data is not None:
271 + inputs["multi_modal_data"] = mm_data
272
273 is_tracing_enabled = await self.engine.is_tracing_enabled()
274 trace_headers = None
vllm/inputs/data.py
+5 -6
@@ -4,7 +4,7 @@ from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
4 from typing_extensions import NotRequired
5
6 if TYPE_CHECKING:
7 - from vllm.multimodal import MultiModalData
7 + from vllm.multimodal import MultiModalDataDict
8
9
10 class ParsedText(TypedDict):
@@ -72,7 +72,7 @@ class TextPrompt(TypedDict):
72 prompt: str
73 """The input text to be tokenized before passing to the model."""
74
75 - multi_modal_data: NotRequired["MultiModalData"]
75 + multi_modal_data: NotRequired["MultiModalDataDict"]
76 """
77 Optional multi-modal data to pass to the model,
78 if the model supports it.
@@ -85,7 +85,7 @@ class TokensPrompt(TypedDict):
85 prompt_token_ids: List[int]
86 """A list of token IDs to pass to the model."""
87
88 - multi_modal_data: NotRequired["MultiModalData"]
88 + multi_modal_data: NotRequired["MultiModalDataDict"]
89 """
90 Optional multi-modal data to pass to the model,
91 if the model supports it.
@@ -103,7 +103,7 @@ class TextTokensPrompt(TypedDict):
103 prompt_token_ids: List[int]
104 """The token IDs of the prompt."""
105
106 - multi_modal_data: NotRequired["MultiModalData"]
106 + multi_modal_data: NotRequired["MultiModalDataDict"]
107 """
108 Optional multi-modal data to pass to the model,
109 if the model supports it.
@@ -128,7 +128,6 @@ class LLMInputs(TypedDict):
128 The inputs in :class:`~vllm.LLMEngine` before they are
129 passed to the model executor.
130 """
131 -
131 prompt_token_ids: List[int]
132 """The token IDs of the prompt."""
133
@@ -137,7 +136,7 @@ class LLMInputs(TypedDict):
136 The original prompt text corresponding to the token IDs, if available.
137 """
138
140 - multi_modal_data: NotRequired[Optional["MultiModalData"]]
139 + multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
140 """
141 Optional multi-modal data to pass to the model,
142 if the model supports it.
vllm/inputs/registry.py
+4 -3
@@ -12,7 +12,7 @@ from .data import LLMInputs
12
13 if TYPE_CHECKING:
14 from vllm.config import ModelConfig, VisionLanguageConfig
15 - from vllm.multimodal import MultiModalData
15 + from vllm.multimodal import MultiModalDataDict
16 from vllm.sequence import SequenceData
17
18 logger = init_logger(__name__)
@@ -66,7 +66,8 @@ class InputContext:
66 N = TypeVar("N", bound=Type[nn.Module])
67
68 DummyDataFactory = Callable[[InputContext, int],
69 - Tuple["SequenceData", Optional["MultiModalData"]]]
69 + Tuple["SequenceData",
70 + Optional["MultiModalDataDict"]]]
71 """
72 Create dummy data to be inputted into the model.
73
@@ -94,7 +95,7 @@ class InputRegistry:
95 self,
96 ctx: InputContext,
97 seq_len: int,
97 - ) -> Tuple["SequenceData", Optional["MultiModalData"]]:
98 + ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
99 """
100 The default dummy data factory represents the longest possible text
101 that can be inputted to the model.
vllm/model_executor/model_loader/loader.py
+2 -3
@@ -84,9 +84,8 @@ def _get_model_initialization_kwargs(
84
85 if supports_vision(model_class):
86 if vlm_config is None:
87 - raise ValueError("Provide `image_input_type` and other vision "
88 - "related configurations through LLM entrypoint "
89 - "or engine arguments.")
87 + raise ValueError("Provide vision related configurations "
88 + "through LLM entrypoint or engine arguments.")
89
90 extra_kwargs["vlm_config"] = vlm_config
91
vllm/model_executor/models/clip.py
+2 -18
@@ -12,7 +12,6 @@ from vllm.model_executor.layers.activation import get_act_fn
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 RowParallelLinear)
14 from vllm.model_executor.layers.quantization import QuantizationConfig
15 -from vllm.multimodal.image import ImageFeatureData, ImagePixelData
15 from vllm.sequence import SequenceData
16
17
@@ -49,7 +48,7 @@ def dummy_seq_data_for_clip(
48 return SequenceData(token_ids)
49
50
52 -def dummy_pixel_data_for_clip(
51 +def dummy_image_for_clip(
52 hf_config: CLIPVisionConfig,
53 *,
54 image_width_override: Optional[int] = None,
@@ -62,22 +61,7 @@ def dummy_pixel_data_for_clip(
61 height = image_height_override
62
63 image = Image.new("RGB", (width, height), color=0)
65 - return ImagePixelData(image)
66 -
67 -
68 -def dummy_feature_data_for_clip(
69 - hf_config: CLIPVisionConfig,
70 - *,
71 - image_feature_size_override: Optional[int] = None,
72 -):
73 - if image_feature_size_override is None:
74 - image_feature_size = get_clip_image_feature_size(hf_config)
75 - else:
76 - image_feature_size = image_feature_size_override
77 -
78 - values = torch.zeros((1, image_feature_size, hf_config.hidden_size),
79 - dtype=torch.float16)
80 - return ImageFeatureData(values)
64 + return {"image": image}
65
66
67 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/llava.py
+19 -83
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
3 import torch
4 import torch.nn as nn
@@ -17,11 +17,10 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17 from vllm.model_executor.models.clip import CLIPVisionModel
18 from vllm.model_executor.models.llama import LlamaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
20 +from vllm.multimodal import MULTIMODAL_REGISTRY
21 from vllm.sequence import SamplerOutput
22
23 -from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
24 - dummy_seq_data_for_clip)
23 +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
24 from .interfaces import SupportsVision
25
26 _KEYS_TO_MODIFY_MAPPING = {
@@ -76,17 +75,10 @@ class LlavaImagePixelInputs(TypedDict):
75 """Shape: (batch_size, num_channels, height, width)"""
76
77
79 -class LlavaImageFeatureInputs(TypedDict):
80 - type: Literal["image_features"]
81 - data: torch.Tensor
82 - """Shape: (batch_size, image_feature_size, hidden_size)"""
83 -
84 -
85 -LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
78 +LlavaImageInputs = LlavaImagePixelInputs
79
80
81 def dummy_data_for_llava(ctx: InputContext, seq_len: int):
89 - multimodal_config = ctx.get_multimodal_config()
82 hf_config = ctx.get_hf_config(LlavaConfig)
83 vision_config = hf_config.vision_config
84
@@ -97,22 +89,14 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
89 image_token_id=hf_config.image_token_index,
90 )
91
100 - image_input_type = multimodal_config.image_input_type
101 - ImageInputType = VisionLanguageConfig.ImageInputType
102 - mm_data: MultiModalData
103 - if image_input_type == ImageInputType.PIXEL_VALUES:
104 - mm_data = dummy_pixel_data_for_clip(vision_config)
105 - elif image_input_type == ImageInputType.IMAGE_FEATURES:
106 - mm_data = dummy_feature_data_for_clip(vision_config)
107 -
92 + mm_data = dummy_image_for_clip(vision_config)
93 return seq_data, mm_data
94
95 msg = f"Unsupported vision config: {type(vision_config)}"
96 raise NotImplementedError(msg)
97
98
114 -@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
115 -@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
99 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
100 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
101 class LlavaForConditionalGeneration(nn.Module, SupportsVision):
102
@@ -126,11 +110,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
110 self.config = config
111 self.vlm_config = vlm_config
112
129 - if self.vlm_config.image_input_type == (
130 - VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
131 - self.vision_tower = CLIPVisionModel(config.vision_config)
132 - else:
133 - self.vision_tower = None
113 + # TODO: Optionally initializes this for supporting embeddings.
114 + self.vision_tower = CLIPVisionModel(config.vision_config)
115
116 self.multi_modal_projector = LlavaMultiModalProjector(
117 vision_hidden_size=config.vision_config.hidden_size,
@@ -165,44 +146,18 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
146 def _parse_and_validate_image_input(
147 self, **kwargs: object) -> Optional[LlavaImageInputs]:
148 pixel_values = kwargs.pop("pixel_values", None)
168 - image_features = kwargs.pop("image_features", None)
169 -
170 - expected_input_type = self.vlm_config.image_input_type
171 - ImageInputType = VisionLanguageConfig.ImageInputType
172 -
173 - if expected_input_type == ImageInputType.PIXEL_VALUES:
174 - if image_features is not None:
175 - raise ValueError(
176 - "Expected pixel values but got image features")
177 - if pixel_values is None:
178 - return None
179 -
180 - if not isinstance(pixel_values, torch.Tensor):
181 - raise ValueError("Incorrect type of pixel values. "
182 - f"Got type: {type(pixel_values)}")
149
184 - return LlavaImagePixelInputs(
185 - type="pixel_values",
186 - data=self._validate_image_data(pixel_values),
187 - )
150 + if pixel_values is None:
151 + return None
152
189 - if expected_input_type == ImageInputType.IMAGE_FEATURES:
190 - if pixel_values is not None:
191 - raise ValueError(
192 - "Expected image features but got pixel values")
193 - if image_features is None:
194 - return None
153 + if not isinstance(pixel_values, torch.Tensor):
154 + raise ValueError("Incorrect type of pixel values. "
155 + f"Got type: {type(pixel_values)}")
156
196 - if not isinstance(image_features, torch.Tensor):
197 - raise ValueError("Incorrect type of image features. "
198 - f"Got type: {type(image_features)}")
199 -
200 - return LlavaImageFeatureInputs(
201 - type="image_features",
202 - data=self._validate_image_data(image_features),
203 - )
204 -
205 - return None
157 + return LlavaImagePixelInputs(
158 + type="pixel_values",
159 + data=self._validate_image_data(pixel_values),
160 + )
161
162 def _select_image_features(self, image_features: torch.Tensor, *,
163 strategy: str) -> torch.Tensor:
@@ -237,12 +192,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
192
193 def _process_image_input(self,
194 image_input: LlavaImageInputs) -> torch.Tensor:
240 - if image_input["type"] == "pixel_values":
241 - assert self.vision_tower is not None
242 - image_features = self._process_image_pixels(image_input)
243 - else:
244 - image_features = image_input["data"]
245 -
195 + assert self.vision_tower is not None
196 + image_features = self._process_image_pixels(image_input)
197 return self.multi_modal_projector(image_features)
198
199 def forward(
@@ -273,25 +224,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
224 This way, the `positions` and `attn_metadata` are consistent
225 with the `input_ids`.
226
276 - This model has two modes of image inputs:
277 - `PIXEL_VALUES` and `IMAGE_FEATURES`.
278 -
227 Args:
228 input_ids: Flattened (concatenated) input_ids corresponding to a
229 batch.
230 pixel_values: The pixels in each input image.
283 - Expects a batch with shape `[1, 3, 336, 336]`.
284 - (Only applicable to `PIXEL_VALUES` mode)
285 - image_features: The image features for each input image outputted by
286 - the vision tower before passing to the multi-modal projector.
287 - Expects a batch with shape `[1, 576, 1024]`.
288 - (Only applicable to `IMAGE_FEATURES` mode)
289 -
290 - See also:
291 - Each input maps to huggingface implementation, as follows:
292 -
293 - - `pixel_values`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava/modeling_llava.py#L360
294 - - `image_features`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava/modeling_llava.py#L437
231 """
232 image_input = self._parse_and_validate_image_input(**kwargs)
233
vllm/model_executor/models/llava_next.py
+40 -86
@@ -1,8 +1,8 @@
1 -from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
2 - Union)
1 +from typing import Dict, Iterable, List, Literal, Optional, Tuple, TypedDict
2
3 import torch
4 import torch.nn as nn
5 +from PIL import Image
6 from transformers import CLIPVisionConfig, LlavaNextConfig
7 from transformers.models.llava_next.modeling_llava_next import (
8 get_anyres_image_grid_shape, unpad_image)
@@ -21,12 +21,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.model_executor.models.clip import CLIPVisionModel
22 from vllm.model_executor.models.llama import LlamaModel
23 from vllm.model_executor.sampling_metadata import SamplingMetadata
24 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
25 -from vllm.multimodal.image import ImagePixelData
24 +from vllm.multimodal import MULTIMODAL_REGISTRY
25 from vllm.sequence import SamplerOutput
26
28 -from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
29 - dummy_seq_data_for_clip, get_clip_patch_grid_length)
27 +from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
28 + get_clip_patch_grid_length)
29 from .interfaces import SupportsVision
30 from .llava import LlavaMultiModalProjector, merge_vision_embeddings
31
@@ -47,17 +46,7 @@ class LlavaNextImagePixelInputs(TypedDict):
46 """Shape: (batch_size, 2)"""
47
48
50 -class LlavaNextImageFeatureInputs(TypedDict):
51 - type: Literal["image_features"]
52 - data: torch.Tensor
53 - """Shape: (batch_size, 1 + num_patches, image_feature_size, hidden_size)"""
54 -
55 - image_sizes: NotRequired[torch.Tensor]
56 - """Shape: (batch_size, 2)"""
57 -
58 -
59 -LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
60 - LlavaNextImageFeatureInputs]
49 +LlavaNextImageInputs = LlavaNextImagePixelInputs
50
51
52 def _get_llava_next_num_unpadded_features(
@@ -138,20 +127,11 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
127 image_feature_size_override=image_feature_size,
128 )
129
141 - image_input_type = multimodal_config.image_input_type
142 - ImageInputType = VisionLanguageConfig.ImageInputType
143 - mm_data: MultiModalData
144 - if image_input_type == ImageInputType.PIXEL_VALUES:
145 - mm_data = dummy_pixel_data_for_clip(
146 - vision_config,
147 - image_width_override=dummy_width,
148 - image_height_override=dummy_height,
149 - )
150 - elif image_input_type == ImageInputType.IMAGE_FEATURES:
151 - mm_data = dummy_feature_data_for_clip(
152 - vision_config,
153 - image_feature_size_override=image_feature_size,
154 - )
130 + mm_data = dummy_image_for_clip(
131 + vision_config,
132 + image_width_override=dummy_width,
133 + image_height_override=dummy_height,
134 + )
135
136 return seq_data, mm_data
137
@@ -159,32 +139,26 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
139 raise NotImplementedError(msg)
140
141
162 -def _pixel_mapper(ctx: InputContext,
163 - data: ImagePixelData) -> Dict[str, torch.Tensor]:
164 - image = data.image
142 +def _pixel_mapper(ctx: InputContext, image: object) -> Dict[str, torch.Tensor]:
143
166 - if isinstance(image, torch.Tensor):
167 - pixel_values = image.to(ctx.model_config.dtype)
168 - batch_size, _, _, h, w = pixel_values.shape
169 - image_sizes = torch.tensor([(w, h) for _ in range(batch_size)])
144 + if isinstance(image, Image.Image):
145
171 - return {"pixel_values": pixel_values, "image_sizes": image_sizes}
146 + # Temporary patch before dynamic number of image tokens is supported
147 + _, _, h, w = ctx.get_multimodal_config().image_input_shape
148 + if (w, h) != (image.width, image.height):
149 + logger.warning(
150 + "Dynamic image shape is currently not supported. "
151 + "Resizing input image to (%d, %d).", w, h)
152
173 - # Temporary patch before dynamic number of image tokens is supported
174 - _, _, h, w = ctx.get_multimodal_config().image_input_shape
175 - if (w, h) != (image.width, image.height):
176 - logger.warning(
177 - "Dynamic image shape is currently not supported. "
178 - "Resizing input image to (%d, %d).", w, h)
153 + image = image.resize((w, h))
154
180 - data.image = image.resize((w, h))
155 + return MULTIMODAL_REGISTRY._get_plugin("image") \
156 + ._default_input_mapper(ctx, image)
157
182 - return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
183 - ._default_input_mapper(ctx, data)
158 + raise TypeError(f"Invalid type for 'image': {type(image)}")
159
160
186 -@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
187 -@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_pixel_mapper)
161 +@MULTIMODAL_REGISTRY.register_image_input_mapper(_pixel_mapper)
162 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
163 class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
164
@@ -198,11 +172,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
172 self.config = config
173 self.vlm_config = vlm_config
174
201 - if self.vlm_config.image_input_type == (
202 - VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
203 - self.vision_tower = CLIPVisionModel(config=config.vision_config)
204 - else:
205 - raise TypeError("Image features are not supported by LLaVA-NeXT")
175 + self.vision_tower = CLIPVisionModel(config=config.vision_config)
176
177 self.multi_modal_projector = LlavaMultiModalProjector(
178 vision_hidden_size=config.vision_config.hidden_size,
@@ -255,36 +225,23 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
225 self, **kwargs: object) -> Optional[LlavaNextImageInputs]:
226 pixel_values = kwargs.pop("pixel_values", None)
227 image_sizes = kwargs.pop("image_sizes", None)
258 - image_features = kwargs.pop("image_features", None)
259 -
260 - expected_input_type = self.vlm_config.image_input_type
261 - ImageInputType = VisionLanguageConfig.ImageInputType
262 -
263 - if expected_input_type == ImageInputType.PIXEL_VALUES:
264 - if image_features is not None:
265 - raise ValueError(
266 - "Expected pixel values but got image features")
267 - if pixel_values is None:
268 - return None
228
270 - if not isinstance(pixel_values, torch.Tensor):
271 - raise ValueError("Incorrect type of pixel values. "
272 - f"Got type: {type(pixel_values)}")
229 + if pixel_values is None or image_sizes is None:
230 + return None
231
274 - if not isinstance(image_sizes, torch.Tensor):
275 - raise ValueError("Incorrect type of image sizes. "
276 - f"Got type: {type(image_sizes)}")
232 + if not isinstance(pixel_values, torch.Tensor):
233 + raise ValueError("Incorrect type of pixel values. "
234 + f"Got type: {type(pixel_values)}")
235
278 - return LlavaNextImagePixelInputs(
279 - type="pixel_values",
280 - data=self._validate_image_pixels(pixel_values),
281 - image_sizes=self._validate_image_sizes(image_sizes),
282 - )
236 + if not isinstance(image_sizes, torch.Tensor):
237 + raise ValueError("Incorrect type of image sizes. "
238 + f"Got type: {type(image_sizes)}")
239
284 - assert expected_input_type != ImageInputType.IMAGE_FEATURES, (
285 - "Failed to validate this at initialization time")
286 -
287 - return None
240 + return LlavaNextImagePixelInputs(
241 + type="pixel_values",
242 + data=self._validate_image_pixels(pixel_values),
243 + image_sizes=self._validate_image_sizes(image_sizes),
244 + )
245
246 def _select_image_features(self, image_features: torch.Tensor, *,
247 strategy: str) -> torch.Tensor:
@@ -391,11 +348,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
348
349 def _process_image_input(
350 self, image_input: LlavaNextImageInputs) -> torch.Tensor:
394 - if image_input["type"] == "pixel_values":
395 - assert self.vision_tower is not None
396 - image_features = self._process_image_pixels(image_input)
397 - else:
398 - image_features = image_input["data"]
351 + assert self.vision_tower is not None
352 + image_features = self._process_image_pixels(image_input)
353
354 patch_embeddings = self.multi_modal_projector(image_features)
355
vllm/model_executor/models/phi3v.py
+8 -17
@@ -35,10 +35,9 @@ from vllm.model_executor.models.clip import CLIPVisionModel
35 from vllm.model_executor.models.llama import LlamaModel
36 from vllm.model_executor.sampling_metadata import SamplingMetadata
37 from vllm.multimodal import MULTIMODAL_REGISTRY
38 -from vllm.multimodal.image import ImagePixelData
38 from vllm.sequence import SamplerOutput
39
41 -from .clip import dummy_pixel_data_for_clip, dummy_seq_data_for_clip
40 +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
41 from .interfaces import SupportsVision
42
43 logger = init_logger(__name__)
@@ -286,7 +285,7 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
285 image_token_id=32044,
286 image_feature_size_override=image_feature_size,
287 )
289 - mm_data = dummy_pixel_data_for_clip(
288 + mm_data = dummy_image_for_clip(
289 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
290 image_width_override=dummy_width,
291 image_height_override=dummy_height,
@@ -331,8 +330,7 @@ def _calc_hd_transform_size(*, width: int, height: int, hd_num: int = 16):
330
331
332 def _image_processor(ctx: InputContext,
334 - data: ImagePixelData) -> Dict[str, torch.Tensor]:
335 - image = data.image
333 + image: object) -> Dict[str, torch.Tensor]:
334
335 if isinstance(image, Image.Image):
336 # Temporary patch before dynamic number of image tokens is supported
@@ -343,13 +341,14 @@ def _image_processor(ctx: InputContext,
341 "Dynamic image shape is currently not supported. "
342 "Resizing input image to (%d, %d).", w, h)
343
346 - data.image = image.resize((w, h))
344 + image = image.resize((w, h))
345
348 - return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
349 - ._default_input_mapper(ctx, data)
346 + return MULTIMODAL_REGISTRY._get_plugin("image") \
347 + ._default_input_mapper(ctx, image)
348 + raise TypeError(f"Invalid type for 'image': {type(image)}")
349
350
352 -@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_image_processor)
351 +@MULTIMODAL_REGISTRY.register_image_input_mapper(_image_processor)
352 @INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
353 class Phi3VForCausalLM(nn.Module, SupportsVision):
354
@@ -375,14 +374,6 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
374 pixel_values = kwargs.pop("pixel_values", None)
375 image_sizes = kwargs.pop("image_sizes", None)
376
378 - expected_input_type = self.vlm_config.image_input_type
379 - ImageInputType = VisionLanguageConfig.ImageInputType
380 -
381 - if expected_input_type != ImageInputType.PIXEL_VALUES:
382 - raise ValueError(
383 - f"Unexpected image input type: {expected_input_type}."
384 - "Phi3v only support pixel_values input currently.")
385 -
377 if pixel_values is not None and image_sizes is not None:
378 return Phi3VImagePixelInputs(type="pixel_values",
379 data=pixel_values,
vllm/multimodal/__init__.py
+5 -3
@@ -1,4 +1,4 @@
1 -from .base import MultiModalData, MultiModalPlugin
1 +from .base import MultiModalDataDict, MultiModalPlugin
2 from .registry import MultiModalRegistry
3
4 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -11,6 +11,8 @@ See also:
11 """
12
13 __all__ = [
14 - "MultiModalData", "MultiModalPlugin", "MULTIMODAL_REGISTRY",
15 - "MultiModalRegistry"
14 + "MultiModalPlugin",
15 + "MULTIMODAL_REGISTRY",
16 + "MultiModalRegistry",
17 + "MultiModalDataDict",
18 ]
vllm/multimodal/base.py
+25 -28
@@ -1,6 +1,6 @@
1 from abc import ABC, abstractmethod
2 -from typing import (TYPE_CHECKING, Callable, Dict, Generic, Optional, Type,
3 - TypeVar)
2 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Type,
3 + TypedDict, TypeVar, Union)
4
5 from vllm.config import ModelConfig
6 from vllm.inputs import InputContext
@@ -8,38 +8,35 @@ from vllm.logger import init_logger
8
9 if TYPE_CHECKING:
10 import torch
11 + from PIL import Image
12 from torch import nn
13
14 logger = init_logger(__name__)
15
16 +N = TypeVar("N", bound=Type["nn.Module"])
17
16 -class MultiModalData:
17 - """
18 - Base class that contains multi-modal data.
19 -
20 - To add a new modality, add a new file under ``multimodal`` directory.
18
22 - In this new file, subclass :class:`~MultiModalData` and
23 - :class:`~MultiModalPlugin`.
19 +class MultiModalDataBuiltins(TypedDict, total=False):
20 + image: "Image.Image"
21
25 - Finally, register the new plugin to
26 - :const:`vllm.multimodal.MULTIMODAL_REGISTRY`.
27 - This enables models to call :meth:`MultiModalRegistry.map_input` for
28 - the new modality.
29 - """
30 - pass
22
23 +MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
24 +"""
25 +A dictionary containing an item for each modality type to input.
26
33 -D = TypeVar("D", bound=MultiModalData)
34 -N = TypeVar("N", bound=Type["nn.Module"])
27 +The data belonging to each modality is converted into keyword arguments
28 +to the model by the corresponding mapper. By default, the mapper of
29 +the corresponding plugin with the same modality key is applied.
30 +"""
31
36 -MultiModalInputMapper = Callable[[InputContext, D], Dict[str, "torch.Tensor"]]
32 +MultiModalInputMapper = Callable[[InputContext, object], Dict[str,
33 + "torch.Tensor"]]
34 """Return a dictionary to be passed as keyword arguments to
35 :meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
36 and processors in HuggingFace Transformers."""
37
38
42 -class MultiModalPlugin(ABC, Generic[D]):
39 +class MultiModalPlugin(ABC):
40 """
41 Base class that defines data processing logic for a specific modality.
42
@@ -52,19 +49,18 @@ class MultiModalPlugin(ABC, Generic[D]):
49
50 def __init__(self) -> None:
51 self._input_mappers: Dict[Type["nn.Module"],
55 - MultiModalInputMapper[D]] = {}
52 + MultiModalInputMapper] = {}
53
54 @abstractmethod
58 - def get_data_type(self) -> Type[D]:
55 + def get_data_key(self) -> str:
56 """
60 - Get the modality (subclass of :class:`~MultiModalData`) served by
61 - this plugin.
57 + Get the data key corresponding to the modality.
58 """
59 raise NotImplementedError
60
61 @abstractmethod
62 def _default_input_mapper(self, ctx: InputContext,
67 - data: D) -> Dict[str, "torch.Tensor"]:
63 + data: object) -> Dict[str, "torch.Tensor"]:
64 """Return a dictionary to be passed as keyword arguments to
65 :meth:`~torch.nn.Module.forward`. This is similar in concept to
66 tokenizers and processors in HuggingFace Transformers.
@@ -73,11 +69,10 @@ class MultiModalPlugin(ABC, Generic[D]):
69
70 def register_input_mapper(
71 self,
76 - mapper: Optional[MultiModalInputMapper[D]] = None,
72 + mapper: Optional[MultiModalInputMapper] = None,
73 ):
74 """
75 Register an input mapper to a model class.
80 -
76 When the model receives input data that matches the modality served by
77 this plugin (see :meth:`get_data_type`), the provided function is
78 invoked to transform the data into a dictionary of model inputs.
@@ -102,11 +97,13 @@ class MultiModalPlugin(ABC, Generic[D]):
97 return wrapper
98
99 def map_input(self, model_config: ModelConfig,
105 - data: D) -> Dict[str, "torch.Tensor"]:
100 + data: object) -> Dict[str, "torch.Tensor"]:
101 """
107 - Apply an input mapper to a :class:`~MultiModalData` instance passed
102 + Apply an input mapper to a data passed
103 to the model, transforming the data into a dictionary of model inputs.
104
105 + If the data is not something that the mapper expects, throws TypeError.
106 +
107 The model is identified by ``model_config``.
108
109 TODO: Add guide [ref: PR #5276]
vllm/multimodal/image.py
+12 -81
@@ -1,5 +1,5 @@
1 from functools import lru_cache
2 -from typing import Dict, Type, Union
2 +from typing import Dict
3
4 import torch
5 from PIL import Image
@@ -9,105 +9,36 @@ from vllm.inputs.registry import InputContext
9 from vllm.logger import init_logger
10 from vllm.transformers_utils.image_processor import get_image_processor
11
12 -from .base import MultiModalData, MultiModalPlugin
12 +from .base import MultiModalPlugin
13
14 logger = init_logger(__name__)
15
16 cached_get_image_processor = lru_cache(get_image_processor)
17
18
19 -class ImagePixelData(MultiModalData):
20 - """
21 - The pixel data of an image. Can be one of:
19 +class ImagePlugin(MultiModalPlugin):
20
23 - - :class:`PIL.Image.Image`: An image object. Requires that a HuggingFace
24 - processor is available to the model.
25 - - :class:`torch.Tensor`: The raw pixel data which is passed to the model
26 - without additional pre-processing.
27 - """
28 -
29 - def __init__(self, image: Union[Image.Image, torch.Tensor]) -> None:
30 - if isinstance(image, Image.Image):
31 - # So that this class can be created inside the Image context manager
32 - image.load()
33 -
34 - self.image = image
35 -
36 - def __repr__(self) -> str:
37 - image = self.image
38 - if isinstance(image, Image.Image):
39 - return f"{type(self).__name__}(image={image})"
40 -
41 - return (f"{type(self).__name__}(image=torch.Tensor(shape="
42 - f"{image.shape}, dtype={image.dtype}))")
43 -
44 -
45 -class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
46 -
47 - def get_data_type(self) -> Type[ImagePixelData]:
48 - return ImagePixelData
21 + def get_data_key(self) -> str:
22 + return "image"
23
24 def _get_hf_image_processor(self, model_config: ModelConfig):
51 - vlm_config = model_config.multimodal_config
52 - if vlm_config is None or vlm_config.image_processor is None:
53 - return None
54 -
25 return cached_get_image_processor(
56 - vlm_config.image_processor,
57 - trust_remote_code=model_config.trust_remote_code,
58 - revision=vlm_config.image_processor_revision,
59 - )
26 + model_config.model,
27 + trust_remote_code=model_config.trust_remote_code)
28
29 def _default_input_mapper(self, ctx: InputContext,
62 - data: ImagePixelData) -> Dict[str, torch.Tensor]:
30 + data: object) -> Dict[str, torch.Tensor]:
31 model_config = ctx.model_config
64 - image = data.image
65 -
66 - if isinstance(image, Image.Image):
32 + if isinstance(data, Image.Image):
33 image_processor = self._get_hf_image_processor(model_config)
34 if image_processor is None:
35 raise RuntimeError("No HuggingFace processor is available"
36 "to process the image object")
37 try:
72 - return image_processor.preprocess(image, return_tensors="pt") \
38 + return image_processor.preprocess(data, return_tensors="pt") \
39 .to(model_config.dtype).data
40 except Exception:
75 - logger.error("Failed to process image (%s)", image)
41 + logger.error("Failed to process image (%s)", data)
42 raise
77 - elif isinstance(image, torch.Tensor):
78 - pixel_values = image.to(model_config.dtype)
79 -
80 - return {"pixel_values": pixel_values}
81 -
82 - raise TypeError(f"Invalid image type: {type(image)}")
83 -
84 -
85 -class ImageFeatureData(MultiModalData):
86 - """
87 - The feature vector of an image, passed directly to the model.
88 -
89 - This should be the output of the vision tower.
90 - """
91 -
92 - def __init__(self, image_features: torch.Tensor) -> None:
93 - self.image_features = image_features
94 -
95 - def __repr__(self) -> str:
96 - image_features = self.image_features
97 -
98 - return (f"{type(self).__name__}(image_features=torch.Tensor(shape="
99 - f"{image_features.shape}, dtype={image_features.dtype}))")
100 -
101 -
102 -class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
103 -
104 - def get_data_type(self) -> Type[ImageFeatureData]:
105 - return ImageFeatureData
106 -
107 - def _default_input_mapper(
108 - self, ctx: InputContext,
109 - data: ImageFeatureData) -> Dict[str, torch.Tensor]:
110 - model_config = ctx.model_config
111 - image_features = data.image_features.to(model_config.dtype)
43
113 - return {"image_features": image_features}
44 + raise TypeError(f"Invalid type for 'image': {type(data)}")
vllm/multimodal/registry.py
+52 -44
@@ -1,18 +1,16 @@
1 import functools
2 -from typing import Any, Optional, Sequence, Type, TypeVar
2 +from typing import Optional, Sequence, Type, TypeVar
3
4 from torch import nn
5
6 from vllm.config import ModelConfig
7 from vllm.logger import init_logger
8
9 -from .base import MultiModalData, MultiModalInputMapper, MultiModalPlugin
10 -from .image import (ImageFeatureData, ImageFeaturePlugin, ImagePixelData,
11 - ImagePixelPlugin)
9 +from .base import MultiModalDataDict, MultiModalInputMapper, MultiModalPlugin
10 +from .image import ImagePlugin
11
12 logger = init_logger(__name__)
13
15 -D = TypeVar("D", bound=MultiModalData)
14 N = TypeVar("N", bound=Type[nn.Module])
15
16
@@ -20,81 +18,91 @@ class MultiModalRegistry:
18 """
19 A registry to dispatch data processing
20 according to its modality and the target model.
21 +
22 + The registry handles both external and internal data input.
23 """
24
25 - DEFAULT_PLUGINS = (ImageFeaturePlugin(), ImagePixelPlugin())
25 + DEFAULT_PLUGINS = (ImagePlugin(), )
26
27 def __init__(
28 - self,
29 - *,
30 - plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS,
31 - ) -> None:
32 - self._plugins_by_data_type = {p.get_data_type(): p for p in plugins}
28 + self,
29 + *,
30 + plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
31 + self._plugins = {p.get_data_key(): p for p in plugins}
32
34 - def register_plugin(self, plugin: MultiModalPlugin[Any]) -> None:
35 - data_type = plugin.get_data_type()
33 + def register_plugin(self, plugin: MultiModalPlugin) -> None:
34 + data_type_key = plugin.get_data_key()
35
37 - if data_type in self._plugins_by_data_type:
36 + if data_type_key in self._plugins:
37 logger.warning(
38 "A plugin is already registered for data type %s, "
40 - "and will be overwritten by the new plugin %s.", data_type,
39 + "and will be overwritten by the new plugin %s.", data_type_key,
40 plugin)
41
43 - self._plugins_by_data_type[data_type] = plugin
42 + self._plugins[data_type_key] = plugin
43
45 - def _get_plugin_for_data_type(self, data_type: Type[MultiModalData]):
46 - for typ in data_type.mro():
47 - plugin = self._plugins_by_data_type.get(typ)
48 - if plugin is not None:
49 - return plugin
44 + def _get_plugin(self, data_type_key: str):
45 + plugin = self._plugins.get(data_type_key)
46 + if plugin is not None:
47 + return plugin
48
51 - msg = f"Unknown multi-modal data type: {data_type}"
49 + msg = f"Unknown multi-modal data type: {data_type_key}"
50 raise NotImplementedError(msg)
51
54 - def register_input_mapper(
52 + def register_image_input_mapper(
53 self,
56 - data_type: Type[D],
57 - mapper: Optional[MultiModalInputMapper[D]] = None,
54 + mapper: Optional[MultiModalInputMapper] = None,
55 ):
56 """
60 - Register an input mapper for a specific modality to a model class.
57 + Register an input mapper for image data to a model class.
58
59 See :meth:`MultiModalPlugin.register_input_mapper` for more details.
60 """
64 - return self._get_plugin_for_data_type(data_type) \
65 - .register_input_mapper(mapper)
61 + return self.register_input_mapper("image", mapper)
62 +
63 + def _process_input(self, key: str, value: object,
64 + model_config: ModelConfig):
65 + plugin = self._plugins.get(key)
66 + if plugin:
67 + return plugin.map_input(model_config, value)
68 + msg = f"Unknown multi-modal data type: {key}"
69 + raise NotImplementedError(msg)
70
67 - def register_image_pixel_input_mapper(
71 + def register_input_mapper(
72 self,
69 - mapper: Optional[MultiModalInputMapper[ImagePixelData]] = None,
73 + data_type: str,
74 + mapper: Optional[MultiModalInputMapper] = None,
75 ):
76 """
72 - Register an input mapper for image pixel data to a model class.
77 + Register an input mapper for a specific modality to a model class.
78
79 See :meth:`MultiModalPlugin.register_input_mapper` for more details.
80 """
76 - return self.register_input_mapper(ImagePixelData, mapper)
77 -
78 - def register_image_feature_input_mapper(
79 - self,
80 - mapper: Optional[MultiModalInputMapper[ImageFeatureData]] = None,
81 - ):
81 + plugin = self._plugins.get(data_type)
82 + if not plugin:
83 + msg = f"Unknown multi-modal data type: {data_type}"
84 + raise NotImplementedError(msg)
85 + return plugin.register_input_mapper(mapper)
86 +
87 + def register_image_input(self,
88 + mapper: Optional[MultiModalInputMapper] = None):
89 """
83 - Register an input mapper for image feature data to a model class.
90 + Register an input mapper for image pixel data to a model class.
91
92 See :meth:`MultiModalPlugin.register_input_mapper` for more details.
93 """
87 - return self.register_input_mapper(ImageFeatureData, mapper)
94 + return self.register_input_mapper("image", mapper)
95
89 - def map_input(self, model_config: ModelConfig, data: MultiModalData):
96 + def map_input(self, model_config: ModelConfig, data: MultiModalDataDict):
97 """
91 - Apply an input mapper to a :class:`~MultiModalData` instance passed
92 - to the model.
98 + Apply an input mapper to the data passed to the model.
99
100 See :meth:`MultiModalPlugin.map_input` for more details.
101 """
96 - return self._get_plugin_for_data_type(type(data)) \
97 - .map_input(model_config, data)
102 + result_list = [
103 + self._process_input(k, v, model_config) for k, v in data.items()
104 + ]
105 + return {k: v for d in result_list for k, v in d.items()}
106
107 def create_input_mapper(self, model_config: ModelConfig):
108 """
vllm/multimodal/utils.py
+7 -6
@@ -8,7 +8,7 @@ from PIL import Image
8
9 from vllm.config import ModelConfig
10 from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
11 -from vllm.multimodal.image import ImagePixelData
11 +from vllm.multimodal.base import MultiModalDataDict
12
13
14 class ImageFetchAiohttp:
@@ -53,14 +53,10 @@ class ImageFetchAiohttp:
53 "Invalid 'image_url': A valid 'image_url' must start "
54 "with either 'data:image' or 'http'.")
55
56 + image.load()
57 return image
58
59
59 -async def async_get_and_parse_image(image_url: str) -> ImagePixelData:
60 - with await ImageFetchAiohttp.fetch_image(image_url) as image:
61 - return ImagePixelData(image)
62 -
63 -
60 def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
61 """Encode a pillow image to base64 format."""
62
@@ -91,3 +87,8 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
87 raise ValueError(
88 f"Unsupported model type: {config.hf_config.model_type}")
89 return full_prompt
90 +
91 +
92 +async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
93 + image = await ImageFetchAiohttp.fetch_image(image_url)
94 + return {"image": image}
vllm/sequence.py
+5 -5
@@ -14,7 +14,7 @@ from vllm.sampling_params import SamplingParams
14
15 if TYPE_CHECKING:
16 from vllm.inputs import LLMInputs
17 - from vllm.multimodal import MultiModalData
17 + from vllm.multimodal import MultiModalDataDict
18 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
19
20
@@ -280,8 +280,8 @@ class Sequence:
280 return self.inputs["prompt_token_ids"]
281
282 @property
283 - def multi_modal_data(self) -> Optional["MultiModalData"]:
284 - return self.inputs.get("multi_modal_data")
283 + def multi_modal_data(self) -> "MultiModalDataDict":
284 + return self.inputs.get("multi_modal_data") or {}
285
286 @property
287 def lora_int_id(self) -> int:
@@ -457,7 +457,7 @@ class SequenceGroup:
457 return next(iter(self.seqs_dict.values())).prompt_token_ids
458
459 @property
460 - def multi_modal_data(self) -> Optional["MultiModalData"]:
460 + def multi_modal_data(self) -> Optional["MultiModalDataDict"]:
461 # All sequences in the group should have the same multi-modal data.
462 # We use the multi-modal data of an arbitrary sequence.
463 return next(iter(self.seqs_dict.values())).multi_modal_data
@@ -639,7 +639,7 @@ class SequenceGroupMetadata:
639 lora_request: Optional[LoRARequest] = None,
640 computed_block_nums: Optional[List[int]] = None,
641 state: Optional[SequenceGroupState] = None,
642 - multi_modal_data: Optional["MultiModalData"] = None,
642 + multi_modal_data: Optional["MultiModalDataDict"] = None,
643 encoder_seq_data: Optional[SequenceData] = None,
644 cross_block_table: Optional[List[int]] = None,
645 ) -> None:
vllm/transformers_utils/image_processor.py
-4
@@ -1,5 +1,3 @@
1 -from typing import Optional
2 -
1 from transformers import AutoImageProcessor
2 from transformers.image_processing_utils import BaseImageProcessor
3
@@ -12,7 +10,6 @@ def get_image_processor(
10 processor_name: str,
11 *args,
12 trust_remote_code: bool = False,
15 - revision: Optional[str] = None,
13 **kwargs,
14 ) -> BaseImageProcessor:
15 """Gets an image processor for the given model name via HuggingFace."""
@@ -21,7 +18,6 @@ def get_image_processor(
18 processor_name,
19 *args,
20 trust_remote_code=trust_remote_code,
24 - revision=revision,
21 **kwargs)
22 except ValueError as e:
23 # If the error pertains to the processor class not existing or not
vllm/worker/model_runner.py
+1 -1
@@ -504,7 +504,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
504 is not None else 1))
505
506 mm_data = seq_group_metadata.multi_modal_data
507 - if mm_data is not None:
507 + if mm_data:
508 # Process multi-modal data
509 mm_kwargs = self.multi_modal_input_mapper(mm_data)
510 for k, v in mm_kwargs.items():