A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[VLM] Remove `image_input_type` from VLM config (#5852)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
xwjiang2010 committed
Jul 2, 2024 at 00:57 UTC
98d6682cd1f27fa48bf915d3fd3e1eb1ee3014c4
35 files changed
+325
-747
.buildkite/download-images.sh
-4
@@ -8,10 +8,6 @@ set -o pipefail
8
# aws s3 sync s3://air-example-data-2/vllm_opensource_llava/ images/
9
mkdir -p images
10
cd images
11
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_pixel_values.pt
12
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign_image_features.pt
13
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_pixel_values.pt
14
-wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom_image_features.pt
11
wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/stop_sign.jpg
12
wget https://air-example-data-2.s3.us-west-2.amazonaws.com/vllm_opensource_llava/cherry_blossom.jpg
13
docs/requirements-docs.txt
+4
-12
@@ -1,13 +1,5 @@
1
-sphinx == 6.2.1
2
-sphinx-book-theme == 1.0.1
3
-sphinx-copybutton == 0.5.2
4
-myst-parser == 2.0.0
1
+sphinx==6.2.1
2
+sphinx-book-theme==1.0.1
3
+sphinx-copybutton==0.5.2
4
+myst-parser==2.0.0
5
sphinx-argparse
6
-
7
-# packages to install to build the documentation
8
-pydantic
9
--f https://download.pytorch.org/whl/cpu
10
-torch
11
-py-cpuinfo
12
-transformers
13
-openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
docs/source/dev/multimodal/multimodal_index.rst
+5
-3
@@ -9,8 +9,10 @@ vLLM provides experimental support for multi-modal models through the :mod:`vllm
9
which allows you to pass in multi-modal input alongside text and token prompts.
10
11
By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model,
12
-you must decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_dummy_data <MultiModalRegistry.register_dummy_data>`,
13
-as well as :meth:`MULTIMODAL_REGISTRY.register_input <MultiModalRegistry.register_input>` for each modality type to support.
12
+you must decorate the model class with :meth:`InputRegistry.register_dummy_data <vllm.inputs.registry.InputRegistry.register_dummy_data>`,
13
+as well as :meth:`MULTIMODAL_REGISTRY.register_input_mapper <MultiModalRegistry.register_input_mapper>` for each modality type to support.
14
+
15
+# TODO: Add more instructions on how to do that once embeddings is in.
16
17
Module Contents
18
+++++++++++++++
@@ -29,7 +31,7 @@ Registry
31
Base Classes
32
------------
33
32
-.. autoclass:: vllm.multimodal.MultiModalData
34
+.. autoclass:: vllm.multimodal.MultiModalDataDict
35
:members:
36
:show-inheritance:
37
docs/source/models/vlm.rst
+7
-4
@@ -36,7 +36,6 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
36
37
llm = LLM(
38
model="llava-hf/llava-1.5-7b-hf",
39
- image_input_type="pixel_values",
39
image_token_id=32000,
40
image_input_shape="1,3,336,336",
41
image_feature_size=576,
@@ -49,7 +48,12 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
48
To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
49
50
* ``prompt``: The prompt should have a number of ``<image>`` tokens equal to ``image_feature_size``.
52
-* ``multi_modal_data``: This should be an instance of :class:`~vllm.multimodal.image.ImagePixelData` or :class:`~vllm.multimodal.image.ImageFeatureData`.
51
+* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
52
+
53
+.. note::
54
+
55
+ ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
56
+ :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
57
58
.. code-block:: python
59
@@ -61,7 +65,7 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
65
66
outputs = llm.generate({
67
"prompt": prompt,
64
- "multi_modal_data": ImagePixelData(image),
68
+ "multi_modal_data": {"image": image},
69
})
70
71
for o in outputs:
@@ -93,7 +97,6 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
97
98
python -m vllm.entrypoints.openai.api_server \
99
--model llava-hf/llava-1.5-7b-hf \
96
- --image-input-type pixel_values \
100
--image-token-id 32000 \
101
--image-input-shape 1,3,336,336 \
102
--image-feature-size 576 \
examples/llava_example.py
+8
-48
@@ -1,38 +1,32 @@
1
-import argparse
1
import os
2
import subprocess
3
5
-import torch
4
from PIL import Image
5
6
from vllm import LLM
9
-from vllm.multimodal.image import ImageFeatureData, ImagePixelData
7
8
# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
9
# You can use `.buildkite/download-images.sh` to download them
10
11
15
-def run_llava_pixel_values(*, disable_image_processor: bool = False):
12
+def run_llava():
13
llm = LLM(
14
model="llava-hf/llava-1.5-7b-hf",
18
- image_input_type="pixel_values",
15
image_token_id=32000,
16
image_input_shape="1,3,336,336",
17
image_feature_size=576,
22
- disable_image_processor=disable_image_processor,
18
)
19
20
prompt = "<image>" * 576 + (
21
"\nUSER: What is the content of this image?\nASSISTANT:")
22
28
- if disable_image_processor:
29
- image = torch.load("images/stop_sign_pixel_values.pt")
30
- else:
31
- image = Image.open("images/stop_sign.jpg")
23
+ image = Image.open("images/stop_sign.jpg")
24
25
outputs = llm.generate({
26
"prompt": prompt,
35
- "multi_modal_data": ImagePixelData(image),
27
+ "multi_modal_data": {
28
+ "image": image
29
+ },
30
})
31
32
for o in outputs:
@@ -40,45 +34,11 @@ def run_llava_pixel_values(*, disable_image_processor: bool = False):
34
print(generated_text)
35
36
43
-def run_llava_image_features():
44
- llm = LLM(
45
- model="llava-hf/llava-1.5-7b-hf",
46
- image_input_type="image_features",
47
- image_token_id=32000,
48
- image_input_shape="1,576,1024",
49
- image_feature_size=576,
50
- )
51
-
52
- prompt = "<image>" * 576 + (
53
- "\nUSER: What is the content of this image?\nASSISTANT:")
54
-
55
- image: torch.Tensor = torch.load("images/stop_sign_image_features.pt")
56
-
57
- outputs = llm.generate({
58
- "prompt": prompt,
59
- "multi_modal_data": ImageFeatureData(image),
60
- })
61
-
62
- for o in outputs:
63
- generated_text = o.outputs[0].text
64
- print(generated_text)
65
-
66
-
67
-def main(args):
68
- if args.type == "pixel_values":
69
- run_llava_pixel_values()
70
- else:
71
- run_llava_image_features()
37
+def main():
38
+ run_llava()
39
40
41
if __name__ == "__main__":
75
- parser = argparse.ArgumentParser(description="Demo on Llava")
76
- parser.add_argument("--type",
77
- type=str,
78
- choices=["pixel_values", "image_features"],
79
- default="pixel_values",
80
- help="image input type")
81
- args = parser.parse_args()
42
# Download from s3
43
s3_bucket_path = "s3://air-example-data-2/vllm_opensource_llava/"
44
local_directory = "images"
@@ -95,4 +55,4 @@ if __name__ == "__main__":
55
local_directory,
56
"--no-sign-request",
57
])
98
- main(args)
58
+ main()
examples/llava_next_example.py
+35
-26
@@ -4,35 +4,44 @@ import requests
4
from PIL import Image
5
6
from vllm import LLM, SamplingParams
7
-from vllm.multimodal.image import ImagePixelData
7
8
# Dynamic image input is currently not supported and therefore
9
# a fixed image input shape and its corresponding feature size is required.
10
# See https://github.com/vllm-project/vllm/pull/4199 for the complete
11
# configuration matrix.
12
14
-llm = LLM(
15
- model="llava-hf/llava-v1.6-mistral-7b-hf",
16
- image_input_type="pixel_values",
17
- image_token_id=32000,
18
- image_input_shape="1,3,336,336",
19
- image_feature_size=1176,
20
-)
21
-
22
-prompt = "[INST] " + "<image>" * 1176 + "\nWhat is shown in this image? [/INST]"
23
-url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
24
-image = Image.open(BytesIO(requests.get(url).content))
25
-sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=100)
26
-
27
-outputs = llm.generate(
28
- {
29
- "prompt": prompt,
30
- "multi_modal_data": ImagePixelData(image),
31
- },
32
- sampling_params=sampling_params)
33
-
34
-generated_text = ""
35
-for o in outputs:
36
- generated_text += o.outputs[0].text
37
-
38
-print(f"LLM output:{generated_text}")
13
+
14
+def run_llava_next():
15
+ llm = LLM(
16
+ model="llava-hf/llava-v1.6-mistral-7b-hf",
17
+ image_token_id=32000,
18
+ image_input_shape="1,3,336,336",
19
+ image_feature_size=1176,
20
+ )
21
+
22
+ prompt = "[INST] " + "<image>" * 1176 + (
23
+ "\nWhat is shown in this image? [/INST]")
24
+ url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
25
+ image = Image.open(BytesIO(requests.get(url).content))
26
+ sampling_params = SamplingParams(temperature=0.8,
27
+ top_p=0.95,
28
+ max_tokens=100)
29
+
30
+ outputs = llm.generate(
31
+ {
32
+ "prompt": prompt,
33
+ "multi_modal_data": {
34
+ "image": image
35
+ }
36
+ },
37
+ sampling_params=sampling_params)
38
+
39
+ generated_text = ""
40
+ for o in outputs:
41
+ generated_text += o.outputs[0].text
42
+
43
+ print(f"LLM output:{generated_text}")
44
+
45
+
46
+if __name__ == "__main__":
47
+ run_llava_next()
examples/openai_vision_api_client.py
-1
@@ -3,7 +3,6 @@
3
Launch the vLLM server with the following command:
4
python -m vllm.entrypoints.openai.api_server \
5
--model llava-hf/llava-1.5-7b-hf \
6
- --image-input-type pixel_values \
6
--image-token-id 32000 \
7
--image-input-shape 1,3,336,336 \
8
--image-feature-size 576 \
examples/phi3v_example.py
+3
-3
@@ -4,7 +4,6 @@ import subprocess
4
from PIL import Image
5
6
from vllm import LLM, SamplingParams
7
-from vllm.multimodal.image import ImagePixelData
7
8
9
def run_phi3v():
@@ -17,7 +16,6 @@ def run_phi3v():
16
llm = LLM(
17
model=model_path,
18
trust_remote_code=True,
20
- image_input_type="pixel_values",
19
image_token_id=32044,
20
image_input_shape="1,3,1008,1344",
21
image_feature_size=1921,
@@ -35,7 +33,9 @@ def run_phi3v():
33
outputs = llm.generate(
34
{
35
"prompt": prompt,
38
- "multi_modal_data": ImagePixelData(image),
36
+ "multi_modal_data": {
37
+ "image": image
38
+ },
39
},
40
sampling_params=sampling_params)
41
for o in outputs:
tests/conftest.py
+8
-30
@@ -17,19 +17,17 @@ from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
17
AutoTokenizer, BatchEncoding)
18
19
from vllm import LLM, SamplingParams
20
-from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
20
+from vllm.config import TokenizerPoolConfig
21
from vllm.distributed import (destroy_distributed_environment,
22
destroy_model_parallel)
23
from vllm.inputs import TextPrompt
24
from vllm.logger import init_logger
25
+from vllm.sequence import SampleLogprobs
26
+from vllm.utils import cuda_device_count_stateless, is_cpu
27
28
if TYPE_CHECKING:
27
- from vllm.multimodal import MultiModalData
28
-else:
29
# it will call torch.cuda.device_count()
30
- MultiModalData = None
31
-from vllm.sequence import SampleLogprobs
32
-from vllm.utils import cuda_device_count_stateless, is_cpu
30
+ from vllm.multimodal import MultiModalDataDict
31
32
logger = init_logger(__name__)
33
@@ -51,14 +49,6 @@ def _read_prompts(filename: str) -> List[str]:
49
class ImageAsset:
50
name: Literal["stop_sign", "cherry_blossom"]
51
54
- @cached_property
55
- def pixel_values(self) -> torch.Tensor:
56
- return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
57
-
58
- @cached_property
59
- def image_features(self) -> torch.Tensor:
60
- return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
61
-
52
@cached_property
53
def pil_image(self) -> Image.Image:
54
return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
@@ -66,20 +56,8 @@ class ImageAsset:
56
def for_hf(self) -> Image.Image:
57
return self.pil_image
58
69
- def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
70
- # don't put this import at the top level
71
- # it will call torch.cuda.device_count()
72
- from vllm.multimodal.image import ImageFeatureData # noqa: F401
73
- from vllm.multimodal.image import ImagePixelData
74
- image_input_type = vision_config.image_input_type
75
- ImageInputType = VisionLanguageConfig.ImageInputType
76
-
77
- if image_input_type == ImageInputType.IMAGE_FEATURES:
78
- return ImageFeatureData(self.image_features)
79
- if image_input_type == ImageInputType.PIXEL_VALUES:
80
- return ImagePixelData(self.pil_image)
81
-
82
- raise NotImplementedError
59
+ def for_vllm(self) -> Dict[str, Any]:
60
+ return {"image": self.pil_image}
61
62
63
class _ImageAssetPrompts(TypedDict):
@@ -453,7 +431,7 @@ class VllmRunner:
431
self,
432
prompts: List[str],
433
sampling_params: SamplingParams,
456
- images: Optional[List[MultiModalData]] = None,
434
+ images: Optional[List["MultiModalDataDict"]] = None,
435
) -> List[Tuple[List[List[int]], List[str]]]:
436
if images is not None:
437
assert len(prompts) == len(images)
@@ -502,7 +480,7 @@ class VllmRunner:
480
self,
481
prompts: List[str],
482
max_tokens: int,
505
- images: Optional[List[MultiModalData]] = None,
483
+ images: Optional[List["MultiModalDataDict"]] = None,
484
) -> List[Tuple[List[int], str]]:
485
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
486
outputs = self.generate(prompts, greedy_params, images=images)
tests/entrypoints/openai/test_vision.py
-2
@@ -39,8 +39,6 @@ def server():
39
"--max-model-len",
40
"4096",
41
"--enforce-eager",
42
- "--image-input-type",
43
- "pixel_values",
42
"--image-token-id",
43
"32000",
44
"--image-input-shape",
tests/models/test_llava.py
+8
-14
@@ -25,17 +25,11 @@ def iter_llava_configs(model_name: str):
25
}
26
27
for (h, w), f in image_hw_to_feature_size.items():
28
- for input_type, input_shape in [
29
- (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
30
- (VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
31
- ]:
32
- yield (model_name,
33
- VisionLanguageConfig(image_input_type=input_type,
34
- image_feature_size=f,
35
- image_token_id=32000,
36
- image_input_shape=input_shape,
37
- image_processor=model_name,
38
- image_processor_revision=None))
28
+ input_shape = (1, 3, h, w)
29
+ yield (model_name,
30
+ VisionLanguageConfig(image_feature_size=f,
31
+ image_token_id=32000,
32
+ image_input_shape=input_shape))
33
34
35
model_and_vl_config = [
@@ -81,8 +75,8 @@ def run_test(
75
76
All the image fixtures for the test is under tests/images.
77
For huggingface runner, we provide the PIL images as input.
84
- For vllm runner, we provide MultiModalData objects and corresponding
85
- vision language config as input.
78
+ For vllm runner, we provide MultiModalDataDict objects
79
+ and corresponding vision language config as input.
80
Note, the text input is also adjusted to abide by vllm contract.
81
The text output is sanitized to be able to compare with hf.
82
"""
@@ -104,7 +98,7 @@ def run_test(
98
# NOTE: `asset.for_vllm` will call `torch.cuda.device_count()`
99
# we must put it inside the vllm_runner context manager
100
# i.e. after creating vLLM instance.
107
- vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
101
+ vllm_images = [asset.for_vllm() for asset in image_assets]
102
103
vllm_image_prompts = [
104
p.replace("<image>", "<image>" * vlm_config.image_feature_size)
tests/models/test_llava_next.py
+10
-13
@@ -33,16 +33,13 @@ def iter_llava_next_configs(model_name: str):
33
}
34
35
for (h, w), f in image_hw_to_feature_size.items():
36
- for input_type, input_shape in [
37
- (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
38
- ]:
39
- yield (model_name,
40
- VisionLanguageConfig(image_input_type=input_type,
41
- image_feature_size=f,
42
- image_token_id=32000,
43
- image_input_shape=input_shape,
44
- image_processor=model_name,
45
- image_processor_revision=None))
36
+ input_shape = (1, 3, h, w)
37
+ yield (model_name,
38
+ VisionLanguageConfig(
39
+ image_feature_size=f,
40
+ image_token_id=32000,
41
+ image_input_shape=input_shape,
42
+ ))
43
44
45
model_and_vl_config = [
@@ -85,14 +82,14 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
82
83
All the image fixtures for the test is under tests/images.
84
For huggingface runner, we provide the PIL images as input.
88
- For vllm runner, we provide MultiModalData objects and corresponding
89
- vision language config as input.
85
+ For vllm runner, we provide MultiModalDataDict objects
86
+ and corresponding vision language config as input.
87
Note, the text input is also adjusted to abide by vllm contract.
88
The text output is sanitized to be able to compare with hf.
89
"""
90
model_id, vlm_config = model_and_config
91
hf_images = [asset.for_hf() for asset in image_assets]
95
- vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
92
+ vllm_images = [asset.for_vllm() for asset in image_assets]
93
94
with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
95
hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_phi3v.py
+8
-13
@@ -27,16 +27,11 @@ def iter_phi3v_configs(model_name: str):
27
}
28
29
for (h, w), f in image_hw_to_feature_size.items():
30
- for input_type, input_shape in [
31
- (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
32
- ]:
33
- yield (model_name,
34
- VisionLanguageConfig(image_input_type=input_type,
35
- image_feature_size=f,
36
- image_token_id=32044,
37
- image_input_shape=input_shape,
38
- image_processor=model_name,
39
- image_processor_revision=None))
30
+ input_shape = (1, 3, h, w)
31
+ yield (model_name,
32
+ VisionLanguageConfig(image_feature_size=f,
33
+ image_token_id=32044,
34
+ image_input_shape=input_shape))
35
36
37
model_and_vl_config = [
@@ -89,8 +84,8 @@ def run_test(
84
85
All the image fixtures for the test is under tests/images.
86
For huggingface runner, we provide the PIL images as input.
92
- For vllm runner, we provide MultiModalData objects and corresponding
93
- vision language config as input.
87
+ For vllm runner, we provide MultiModalDataDict objects
88
+ and corresponding vision language config as input.
89
Note, the text input is also adjusted to abide by vllm contract.
90
The text output is sanitized to be able to compare with hf.
91
"""
@@ -113,7 +108,7 @@ def run_test(
108
# we must put it inside the vllm_runner context manager
109
# i.e. after creating vLLM instance.
110
116
- vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
111
+ vllm_images = [asset.for_vllm() for asset in image_assets]
112
113
vllm_image_prompts = [
114
p.replace("<|image_1|>",
tests/multimodal/test_mapper.py
+6
-34
@@ -2,9 +2,8 @@ import numpy as np
2
import pytest
3
from transformers import CLIPImageProcessor, LlavaNextImageProcessor
4
5
-from vllm.config import ModelConfig, VisionLanguageConfig
5
+from vllm.config import ModelConfig
6
from vllm.multimodal import MULTIMODAL_REGISTRY
7
-from vllm.multimodal.image import ImagePixelData
7
8
from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
9
@@ -12,7 +11,6 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
11
@pytest.mark.parametrize("dtype", ["half", "float"])
12
def test_clip_image_processor(image_assets, dtype):
13
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
15
- IMAGE_HEIGHT = IMAGE_WIDTH = 560
14
15
hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
16
assert isinstance(hf_processor, CLIPImageProcessor)
@@ -25,14 +23,6 @@ def test_clip_image_processor(image_assets, dtype):
23
seed=0,
24
dtype=dtype,
25
revision=None,
28
- multimodal_config=VisionLanguageConfig(
29
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
30
- image_token_id=32000,
31
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
32
- image_feature_size=576,
33
- image_processor=MODEL_NAME,
34
- image_processor_revision=None,
35
- ),
26
)
27
28
for asset in image_assets:
@@ -42,7 +32,7 @@ def test_clip_image_processor(image_assets, dtype):
32
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
33
vllm_result = MULTIMODAL_REGISTRY.map_input(
34
model_config,
45
- ImagePixelData(asset.pil_image),
35
+ {"image": asset.pil_image},
36
)
37
38
assert hf_result.keys() == vllm_result.keys()
@@ -60,7 +50,6 @@ def test_clip_image_processor(image_assets, dtype):
50
@pytest.mark.parametrize("dtype", ["half", "float"])
51
def test_llava_next_image_processor(image_assets, dtype):
52
MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
63
- IMAGE_HEIGHT = IMAGE_WIDTH = 560
53
54
hf_processor = LlavaNextImageProcessor.from_pretrained(MODEL_NAME)
55
assert isinstance(hf_processor, LlavaNextImageProcessor)
@@ -73,14 +62,6 @@ def test_llava_next_image_processor(image_assets, dtype):
62
seed=0,
63
dtype=dtype,
64
revision=None,
76
- multimodal_config=VisionLanguageConfig(
77
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
78
- image_token_id=64000,
79
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
80
- image_feature_size=2928,
81
- image_processor=MODEL_NAME,
82
- image_processor_revision=None,
83
- ),
65
)
66
67
for asset in image_assets:
@@ -90,7 +71,7 @@ def test_llava_next_image_processor(image_assets, dtype):
71
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
72
vllm_result = MULTIMODAL_REGISTRY.map_input(
73
model_config,
93
- ImagePixelData(asset.pil_image),
74
+ {"image": asset.pil_image},
75
)
76
77
assert hf_result.keys() == vllm_result.keys()
@@ -107,7 +88,6 @@ def test_llava_next_image_processor(image_assets, dtype):
88
@pytest.mark.parametrize("dtype", ["float"])
89
def test_image_pixel_types(image_assets, dtype):
90
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
110
- IMAGE_HEIGHT = IMAGE_WIDTH = 560
91
92
model_config = ModelConfig(
93
model=MODEL_NAME,
@@ -117,23 +97,15 @@ def test_image_pixel_types(image_assets, dtype):
97
seed=0,
98
dtype=dtype,
99
revision=None,
120
- multimodal_config=VisionLanguageConfig(
121
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
122
- image_token_id=32000,
123
- image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
124
- image_feature_size=576,
125
- image_processor=MODEL_NAME,
126
- image_processor_revision=None,
127
- ))
128
-
100
+ )
101
for asset in image_assets:
102
image_result = MULTIMODAL_REGISTRY.map_input(
103
model_config,
132
- ImagePixelData(asset.pil_image),
104
+ {"image": asset.pil_image},
105
)
106
tensor_result = MULTIMODAL_REGISTRY.map_input(
107
model_config,
136
- ImagePixelData(asset.pixel_values),
108
+ {"image": asset.pil_image},
109
)
110
111
assert image_result.keys() == tensor_result.keys()
tests/spec_decode/e2e/conftest.py
+2
-2
@@ -11,7 +11,7 @@ from vllm.engine.arg_utils import AsyncEngineArgs
11
from vllm.engine.async_llm_engine import AsyncLLMEngine
12
from vllm.lora.request import LoRARequest
13
from vllm.model_executor.utils import set_random_seed
14
-from vllm.multimodal import MultiModalData
14
+from vllm.multimodal import MultiModalDataDict
15
from vllm.outputs import RequestOutput
16
from vllm.sampling_params import SamplingParams
17
from vllm.sequence import Logprob
@@ -91,7 +91,7 @@ class AsyncLLM:
91
prompt_token_ids: Optional[List[List[int]]] = None,
92
use_tqdm: bool = True,
93
lora_request: Optional[LoRARequest] = None,
94
- multi_modal_data: Optional[MultiModalData] = None,
94
+ multi_modal_data: Optional[MultiModalDataDict] = None,
95
) -> List[RequestOutput]:
96
97
if prompts is None:
tests/tokenization/test_image_processor.py
deleted
-20
@@ -1,20 +0,0 @@
1
-import pytest
2
-from transformers.image_processing_utils import BaseImageProcessor
3
-
4
-from vllm.transformers_utils.image_processor import get_image_processor
5
-
6
-IMAGE_PROCESSOR_NAMES = [
7
- "llava-hf/llava-1.5-7b-hf",
8
- "llava-hf/llava-v1.6-34b-hf",
9
-]
10
-
11
-
12
-@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
13
-def test_image_processor_revision(processor_name: str):
14
- # Assume that "main" branch always exists
15
- image_processor = get_image_processor(processor_name, revision="main")
16
- assert isinstance(image_processor, BaseImageProcessor)
17
-
18
- # Assume that "never" branch always does not exist
19
- with pytest.raises(OSError, match='not a valid git identifier'):
20
- get_image_processor(processor_name, revision="never")
vllm/config.py
+1
-33
@@ -1250,28 +1250,11 @@ class LoRAConfig:
1250
raise ValueError("LoRA is not supported with chunked prefill yet.")
1251
1252
1253
+# TODO: To be replaced by MultiModalConfig.
1254
@dataclass
1255
class VisionLanguageConfig:
1256
"""Configs the input data format and how models should run for
1257
vision language models."""
1257
-
1258
- class ImageInputType(enum.Enum):
1259
- """Image input type into the vision language model.
1260
-
1261
- An image roughly goes through the following transformation:
1262
- Raw image --> pixel values --> image features --> image embeddings.
1263
-
1264
- The difference between different image input types is where the
1265
- image encoder (pixel values --> image features) is run.
1266
- Different image input types also correspond to different tensor shapes.
1267
-
1268
- For example, for Llava, PIXEL_VALUES: (1, 3, 336, 336).
1269
- IMAGE_FEATURES: (1, 576, 1024).
1270
- """
1271
- PIXEL_VALUES = enum.auto()
1272
- IMAGE_FEATURES = enum.auto()
1273
-
1274
- image_input_type: ImageInputType
1258
# The input id corresponding to image token.
1259
image_token_id: int
1260
# Used for running `run_prefill_max_token`.
@@ -1279,19 +1262,6 @@ class VisionLanguageConfig:
1262
# worst case scenario (biggest supported resolution).
1263
image_input_shape: tuple
1264
image_feature_size: int
1282
- # The image processor to load from HuggingFace
1283
- image_processor: Optional[str]
1284
- image_processor_revision: Optional[str]
1285
-
1286
- @classmethod
1287
- def get_image_input_enum_type(cls, value: str) -> ImageInputType:
1288
- """Get the image input type from a string."""
1289
- try:
1290
- return cls.ImageInputType[value.upper()]
1291
- except KeyError as e:
1292
- raise ValueError(f"{value} is not a valid choice. "
1293
- f"Expecting to choose from "
1294
- f"{[x.name for x in cls.ImageInputType]}.") from e
1265
1266
#TODO(ywang96): make this a cached property once we refactor the
1267
# VisionLanguageConfig class.
@@ -1318,8 +1288,6 @@ class VisionLanguageConfig:
1288
else:
1289
result[f.name] = value
1290
1321
- result["disable_image_processor"] = self.image_processor is None
1322
-
1291
return result
1292
1293
vllm/engine/arg_utils.py
+4
-52
@@ -1,7 +1,6 @@
1
import argparse
2
import dataclasses
3
import json
4
-import warnings
4
from dataclasses import dataclass
5
from typing import List, Optional, Tuple, Union
6
@@ -80,13 +79,9 @@ class EngineArgs:
79
preemption_mode: Optional[str] = None
80
81
# Related to Vision-language models such as llava
83
- image_input_type: Optional[str] = None
82
image_token_id: Optional[int] = None
83
image_input_shape: Optional[str] = None
84
image_feature_size: Optional[int] = None
87
- image_processor: Optional[str] = None
88
- image_processor_revision: Optional[str] = None
89
- disable_image_processor: bool = False
85
86
scheduler_delay_factor: float = 0.0
87
enable_chunked_prefill: bool = False
@@ -114,14 +109,6 @@ class EngineArgs:
109
@staticmethod
110
def add_cli_args_for_vlm(
111
parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
117
- parser.add_argument('--image-input-type',
118
- type=nullable_str,
119
- default=None,
120
- choices=[
121
- t.name.lower()
122
- for t in VisionLanguageConfig.ImageInputType
123
- ],
124
- help=('The image input type passed into vLLM.'))
112
parser.add_argument('--image-token-id',
113
type=int,
114
default=None,
@@ -137,24 +124,6 @@ class EngineArgs:
124
type=int,
125
default=None,
126
help=('The image feature size along the context dimension.'))
140
- parser.add_argument(
141
- '--image-processor',
142
- type=str,
143
- default=EngineArgs.image_processor,
144
- help='Name or path of the huggingface image processor to use. '
145
- 'If unspecified, model name or path will be used.')
146
- parser.add_argument(
147
- '--image-processor-revision',
148
- type=str,
149
- default=None,
150
- help='Revision of the huggingface image processor version to use. '
151
- 'It can be a branch name, a tag name, or a commit id. '
152
- 'If unspecified, will use the default version.')
153
- parser.add_argument(
154
- '--disable-image-processor',
155
- action='store_true',
156
- help='Disables the use of image processor, even if one is defined '
157
- 'for the model on huggingface.')
127
128
return parser
129
@@ -679,33 +648,16 @@ class EngineArgs:
648
raise ValueError(
649
"BitsAndBytes load format and QLoRA adapter only support "
650
f"'bitsandbytes' quantization, but got {self.quantization}")
682
- if self.image_input_type:
683
- if (not self.image_token_id or not self.image_input_shape
684
- or not self.image_feature_size):
651
+ if self.image_token_id is not None:
652
+ if (not self.image_input_shape or not self.image_feature_size):
653
raise ValueError(
686
- 'Specify `image_token_id`, `image_input_shape` and '
687
- '`image_feature_size` together with `image_input_type`.')
688
-
689
- if self.image_processor is None:
690
- self.image_processor = self.model
691
- if self.disable_image_processor:
692
- if self.image_processor != self.model:
693
- warnings.warn(
694
- "You've specified an image processor "
695
- f"({self.image_processor}) but also disabled "
696
- "it via `--disable-image-processor`.",
697
- stacklevel=2)
698
-
699
- self.image_processor = None
654
+ 'Specify `image_input_shape` and '
655
+ '`image_feature_size` together with `image_token_id`.')
656
657
vision_language_config = VisionLanguageConfig(
702
- image_input_type=VisionLanguageConfig.
703
- get_image_input_enum_type(self.image_input_type),
658
image_token_id=self.image_token_id,
659
image_input_shape=str_to_int_tuple(self.image_input_shape),
660
image_feature_size=self.image_feature_size,
707
- image_processor=self.image_processor,
708
- image_processor_revision=self.image_processor_revision,
661
)
662
else:
663
vision_language_config = None
vllm/entrypoints/openai/api_server.py
-9
@@ -213,15 +213,6 @@ if __name__ == "__main__":
213
214
engine_args = AsyncEngineArgs.from_cli_args(args)
215
216
- # Enforce pixel values as image input type for vision language models
217
- # when serving with API server
218
- if engine_args.image_input_type is not None and \
219
- engine_args.image_input_type.upper() != "PIXEL_VALUES":
220
- raise ValueError(
221
- f"Invalid image_input_type: {engine_args.image_input_type}. "
222
- "Only --image-input-type 'pixel_values' is supported for serving "
223
- "vision language models with the vLLM API server.")
224
-
216
engine = AsyncLLMEngine.from_engine_args(
217
engine_args, usage_context=UsageContext.OPENAI_API_SERVER)
218
vllm/entrypoints/openai/serving_chat.py
+29
-36
@@ -26,7 +26,7 @@ from vllm.inputs import PromptInputs
26
from vllm.logger import init_logger
27
from vllm.model_executor.guided_decoding import (
28
get_guided_decoding_logits_processor)
29
-from vllm.multimodal.image import ImagePixelData
29
+from vllm.multimodal import MultiModalDataDict
30
from vllm.multimodal.utils import (async_get_and_parse_image,
31
get_full_image_text_prompt)
32
from vllm.outputs import RequestOutput
@@ -47,7 +47,7 @@ class ConversationMessage(TypedDict):
47
@dataclass(frozen=True)
48
class ChatMessageParseResult:
49
messages: List[ConversationMessage]
50
- image_futures: List[Awaitable[ImagePixelData]] = field(
50
+ mm_futures: List[Awaitable[MultiModalDataDict]] = field(
51
default_factory=list)
52
53
@@ -103,7 +103,7 @@ class OpenAIServingChat(OpenAIServing):
103
parts: Iterable[ChatCompletionContentPartParam],
104
) -> ChatMessageParseResult:
105
texts: List[str] = []
106
- image_futures: List[Awaitable[ImagePixelData]] = []
106
+ mm_futures: List[Awaitable[MultiModalDataDict]] = []
107
108
vlm_config: Optional[VisionLanguageConfig] = getattr(
109
self.engine.engine, "vision_language_config", None)
@@ -113,39 +113,34 @@ class OpenAIServingChat(OpenAIServing):
113
part_type = part["type"]
114
if part_type == "text":
115
text = cast(ChatCompletionContentPartTextParam, part)["text"]
116
-
116
texts.append(text)
117
elif part_type == "image_url":
118
if vlm_config is None:
119
raise ValueError(
120
"'image_url' input is not supported as the loaded "
121
"model is not multimodal.")
122
+ assert self.tokenizer is not None
123
+ image_url = cast(ChatCompletionContentPartImageParam,
124
+ part)["image_url"]
125
124
- elif len(image_futures) == 0:
125
- assert self.tokenizer is not None
126
- image_url = cast(ChatCompletionContentPartImageParam,
127
- part)["image_url"]
128
-
129
- if image_url.get("detail", "auto") != "auto":
130
- logger.warning(
131
- "'image_url.detail' is currently not supported and "
132
- "will be ignored.")
133
-
134
- image_future = async_get_and_parse_image(image_url["url"])
135
- image_futures.append(image_future)
126
+ if image_url.get("detail", "auto") != "auto":
127
+ logger.warning(
128
+ "'image_url.detail' is currently not supported and "
129
+ "will be ignored.")
130
137
- else:
138
- raise NotImplementedError(
139
- "Multiple 'image_url' input is currently not supported."
140
- )
131
+ mm_future = async_get_and_parse_image(image_url["url"])
132
+ mm_futures.append(mm_future)
133
134
else:
135
raise NotImplementedError(f"Unknown part type: {part_type}")
136
137
text_prompt = "\n".join(texts)
138
147
- if vlm_config is not None and len(image_futures):
139
+ if vlm_config is not None and len(mm_futures):
140
141
+ assert len(
142
+ mm_futures
143
+ ) == 1, "Multiple 'image_url' input is currently not supported."
144
(image_token_prompt,
145
image_token_str) = vlm_config.get_image_token_text(self.tokenizer)
146
@@ -171,8 +166,7 @@ class OpenAIServingChat(OpenAIServing):
166
else:
167
messages = [ConversationMessage(role=role, content=text_prompt)]
168
174
- return ChatMessageParseResult(messages=messages,
175
- image_futures=image_futures)
169
+ return ChatMessageParseResult(messages=messages, mm_futures=mm_futures)
170
171
def _parse_chat_message_content(
172
self,
@@ -182,10 +176,10 @@ class OpenAIServingChat(OpenAIServing):
176
content = message.get("content")
177
178
if content is None:
185
- return ChatMessageParseResult(messages=[], image_futures=[])
179
+ return ChatMessageParseResult(messages=[], mm_futures=[])
180
if isinstance(content, str):
181
messages = [ConversationMessage(role=role, content=content)]
188
- return ChatMessageParseResult(messages=messages, image_futures=[])
182
+ return ChatMessageParseResult(messages=messages, mm_futures=[])
183
184
return self._parse_chat_message_content_parts(role, content)
185
@@ -210,13 +204,13 @@ class OpenAIServingChat(OpenAIServing):
204
205
try:
206
conversation: List[ConversationMessage] = []
213
- image_futures: List[Awaitable[ImagePixelData]] = []
207
+ mm_futures: List[Awaitable[MultiModalDataDict]] = []
208
209
for msg in request.messages:
210
chat_parsed_result = self._parse_chat_message_content(msg)
211
212
conversation.extend(chat_parsed_result.messages)
219
- image_futures.extend(chat_parsed_result.image_futures)
213
+ mm_futures.extend(chat_parsed_result.mm_futures)
214
215
tool_dicts = None if request.tools is None else [
216
tool.model_dump() for tool in request.tools
@@ -235,15 +229,14 @@ class OpenAIServingChat(OpenAIServing):
229
logger.error("Error in applying chat template from request: %s", e)
230
return self.create_error_response(str(e))
231
238
- # Fetch image data
239
- image_data: Optional[ImagePixelData] = None
232
+ mm_data: Optional[MultiModalDataDict] = None
233
try:
241
- if len(image_futures):
242
- # since we support only single image currently
243
- assert len(image_futures) == 1
244
- image_data = await image_futures[0]
234
+ if len(mm_futures):
235
+ # since we support only single mm data currently
236
+ assert len(mm_futures) == 1
237
+ mm_data = await mm_futures[0]
238
except Exception as e:
246
- logger.error("Error in loading image data: %s", e)
239
+ logger.error("Error in loading multi-modal data: %s", e)
240
return self.create_error_response(str(e))
241
242
request_id = f"cmpl-{random_uuid()}"
@@ -274,8 +267,8 @@ class OpenAIServingChat(OpenAIServing):
267
"prompt": prompt_text,
268
"prompt_token_ids": prompt_ids,
269
}
277
- if image_data is not None:
278
- inputs["multi_modal_data"] = image_data
270
+ if mm_data is not None:
271
+ inputs["multi_modal_data"] = mm_data
272
273
is_tracing_enabled = await self.engine.is_tracing_enabled()
274
trace_headers = None
vllm/inputs/data.py
+5
-6
@@ -4,7 +4,7 @@ from typing import (TYPE_CHECKING, List, Literal, Optional, Sequence,
4
from typing_extensions import NotRequired
5
6
if TYPE_CHECKING:
7
- from vllm.multimodal import MultiModalData
7
+ from vllm.multimodal import MultiModalDataDict
8
9
10
class ParsedText(TypedDict):
@@ -72,7 +72,7 @@ class TextPrompt(TypedDict):
72
prompt: str
73
"""The input text to be tokenized before passing to the model."""
74
75
- multi_modal_data: NotRequired["MultiModalData"]
75
+ multi_modal_data: NotRequired["MultiModalDataDict"]
76
"""
77
Optional multi-modal data to pass to the model,
78
if the model supports it.
@@ -85,7 +85,7 @@ class TokensPrompt(TypedDict):
85
prompt_token_ids: List[int]
86
"""A list of token IDs to pass to the model."""
87
88
- multi_modal_data: NotRequired["MultiModalData"]
88
+ multi_modal_data: NotRequired["MultiModalDataDict"]
89
"""
90
Optional multi-modal data to pass to the model,
91
if the model supports it.
@@ -103,7 +103,7 @@ class TextTokensPrompt(TypedDict):
103
prompt_token_ids: List[int]
104
"""The token IDs of the prompt."""
105
106
- multi_modal_data: NotRequired["MultiModalData"]
106
+ multi_modal_data: NotRequired["MultiModalDataDict"]
107
"""
108
Optional multi-modal data to pass to the model,
109
if the model supports it.
@@ -128,7 +128,6 @@ class LLMInputs(TypedDict):
128
The inputs in :class:`~vllm.LLMEngine` before they are
129
passed to the model executor.
130
"""
131
-
131
prompt_token_ids: List[int]
132
"""The token IDs of the prompt."""
133
@@ -137,7 +136,7 @@ class LLMInputs(TypedDict):
136
The original prompt text corresponding to the token IDs, if available.
137
"""
138
140
- multi_modal_data: NotRequired[Optional["MultiModalData"]]
139
+ multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
140
"""
141
Optional multi-modal data to pass to the model,
142
if the model supports it.
vllm/inputs/registry.py
+4
-3
@@ -12,7 +12,7 @@ from .data import LLMInputs
12
13
if TYPE_CHECKING:
14
from vllm.config import ModelConfig, VisionLanguageConfig
15
- from vllm.multimodal import MultiModalData
15
+ from vllm.multimodal import MultiModalDataDict
16
from vllm.sequence import SequenceData
17
18
logger = init_logger(__name__)
@@ -66,7 +66,8 @@ class InputContext:
66
N = TypeVar("N", bound=Type[nn.Module])
67
68
DummyDataFactory = Callable[[InputContext, int],
69
- Tuple["SequenceData", Optional["MultiModalData"]]]
69
+ Tuple["SequenceData",
70
+ Optional["MultiModalDataDict"]]]
71
"""
72
Create dummy data to be inputted into the model.
73
@@ -94,7 +95,7 @@ class InputRegistry:
95
self,
96
ctx: InputContext,
97
seq_len: int,
97
- ) -> Tuple["SequenceData", Optional["MultiModalData"]]:
98
+ ) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
99
"""
100
The default dummy data factory represents the longest possible text
101
that can be inputted to the model.
vllm/model_executor/model_loader/loader.py
+2
-3
@@ -84,9 +84,8 @@ def _get_model_initialization_kwargs(
84
85
if supports_vision(model_class):
86
if vlm_config is None:
87
- raise ValueError("Provide `image_input_type` and other vision "
88
- "related configurations through LLM entrypoint "
89
- "or engine arguments.")
87
+ raise ValueError("Provide vision related configurations "
88
+ "through LLM entrypoint or engine arguments.")
89
90
extra_kwargs["vlm_config"] = vlm_config
91
vllm/model_executor/models/clip.py
+2
-18
@@ -12,7 +12,6 @@ from vllm.model_executor.layers.activation import get_act_fn
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
RowParallelLinear)
14
from vllm.model_executor.layers.quantization import QuantizationConfig
15
-from vllm.multimodal.image import ImageFeatureData, ImagePixelData
15
from vllm.sequence import SequenceData
16
17
@@ -49,7 +48,7 @@ def dummy_seq_data_for_clip(
48
return SequenceData(token_ids)
49
50
52
-def dummy_pixel_data_for_clip(
51
+def dummy_image_for_clip(
52
hf_config: CLIPVisionConfig,
53
*,
54
image_width_override: Optional[int] = None,
@@ -62,22 +61,7 @@ def dummy_pixel_data_for_clip(
61
height = image_height_override
62
63
image = Image.new("RGB", (width, height), color=0)
65
- return ImagePixelData(image)
66
-
67
-
68
-def dummy_feature_data_for_clip(
69
- hf_config: CLIPVisionConfig,
70
- *,
71
- image_feature_size_override: Optional[int] = None,
72
-):
73
- if image_feature_size_override is None:
74
- image_feature_size = get_clip_image_feature_size(hf_config)
75
- else:
76
- image_feature_size = image_feature_size_override
77
-
78
- values = torch.zeros((1, image_feature_size, hf_config.hidden_size),
79
- dtype=torch.float16)
80
- return ImageFeatureData(values)
64
+ return {"image": image}
65
66
67
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
vllm/model_executor/models/llava.py
+19
-83
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
2
3
import torch
4
import torch.nn as nn
@@ -17,11 +17,10 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
17
from vllm.model_executor.models.clip import CLIPVisionModel
18
from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
20
+from vllm.multimodal import MULTIMODAL_REGISTRY
21
from vllm.sequence import SamplerOutput
22
23
-from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
24
- dummy_seq_data_for_clip)
23
+from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
24
from .interfaces import SupportsVision
25
26
_KEYS_TO_MODIFY_MAPPING = {
@@ -76,17 +75,10 @@ class LlavaImagePixelInputs(TypedDict):
75
"""Shape: (batch_size, num_channels, height, width)"""
76
77
79
-class LlavaImageFeatureInputs(TypedDict):
80
- type: Literal["image_features"]
81
- data: torch.Tensor
82
- """Shape: (batch_size, image_feature_size, hidden_size)"""
83
-
84
-
85
-LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
78
+LlavaImageInputs = LlavaImagePixelInputs
79
80
81
def dummy_data_for_llava(ctx: InputContext, seq_len: int):
89
- multimodal_config = ctx.get_multimodal_config()
82
hf_config = ctx.get_hf_config(LlavaConfig)
83
vision_config = hf_config.vision_config
84
@@ -97,22 +89,14 @@ def dummy_data_for_llava(ctx: InputContext, seq_len: int):
89
image_token_id=hf_config.image_token_index,
90
)
91
100
- image_input_type = multimodal_config.image_input_type
101
- ImageInputType = VisionLanguageConfig.ImageInputType
102
- mm_data: MultiModalData
103
- if image_input_type == ImageInputType.PIXEL_VALUES:
104
- mm_data = dummy_pixel_data_for_clip(vision_config)
105
- elif image_input_type == ImageInputType.IMAGE_FEATURES:
106
- mm_data = dummy_feature_data_for_clip(vision_config)
107
-
92
+ mm_data = dummy_image_for_clip(vision_config)
93
return seq_data, mm_data
94
95
msg = f"Unsupported vision config: {type(vision_config)}"
96
raise NotImplementedError(msg)
97
98
114
-@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
115
-@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper()
99
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
100
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava)
101
class LlavaForConditionalGeneration(nn.Module, SupportsVision):
102
@@ -126,11 +110,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
110
self.config = config
111
self.vlm_config = vlm_config
112
129
- if self.vlm_config.image_input_type == (
130
- VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
131
- self.vision_tower = CLIPVisionModel(config.vision_config)
132
- else:
133
- self.vision_tower = None
113
+ # TODO: Optionally initializes this for supporting embeddings.
114
+ self.vision_tower = CLIPVisionModel(config.vision_config)
115
116
self.multi_modal_projector = LlavaMultiModalProjector(
117
vision_hidden_size=config.vision_config.hidden_size,
@@ -165,44 +146,18 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
146
def _parse_and_validate_image_input(
147
self, **kwargs: object) -> Optional[LlavaImageInputs]:
148
pixel_values = kwargs.pop("pixel_values", None)
168
- image_features = kwargs.pop("image_features", None)
169
-
170
- expected_input_type = self.vlm_config.image_input_type
171
- ImageInputType = VisionLanguageConfig.ImageInputType
172
-
173
- if expected_input_type == ImageInputType.PIXEL_VALUES:
174
- if image_features is not None:
175
- raise ValueError(
176
- "Expected pixel values but got image features")
177
- if pixel_values is None:
178
- return None
179
-
180
- if not isinstance(pixel_values, torch.Tensor):
181
- raise ValueError("Incorrect type of pixel values. "
182
- f"Got type: {type(pixel_values)}")
149
184
- return LlavaImagePixelInputs(
185
- type="pixel_values",
186
- data=self._validate_image_data(pixel_values),
187
- )
150
+ if pixel_values is None:
151
+ return None
152
189
- if expected_input_type == ImageInputType.IMAGE_FEATURES:
190
- if pixel_values is not None:
191
- raise ValueError(
192
- "Expected image features but got pixel values")
193
- if image_features is None:
194
- return None
153
+ if not isinstance(pixel_values, torch.Tensor):
154
+ raise ValueError("Incorrect type of pixel values. "
155
+ f"Got type: {type(pixel_values)}")
156
196
- if not isinstance(image_features, torch.Tensor):
197
- raise ValueError("Incorrect type of image features. "
198
- f"Got type: {type(image_features)}")
199
-
200
- return LlavaImageFeatureInputs(
201
- type="image_features",
202
- data=self._validate_image_data(image_features),
203
- )
204
-
205
- return None
157
+ return LlavaImagePixelInputs(
158
+ type="pixel_values",
159
+ data=self._validate_image_data(pixel_values),
160
+ )
161
162
def _select_image_features(self, image_features: torch.Tensor, *,
163
strategy: str) -> torch.Tensor:
@@ -237,12 +192,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
192
193
def _process_image_input(self,
194
image_input: LlavaImageInputs) -> torch.Tensor:
240
- if image_input["type"] == "pixel_values":
241
- assert self.vision_tower is not None
242
- image_features = self._process_image_pixels(image_input)
243
- else:
244
- image_features = image_input["data"]
245
-
195
+ assert self.vision_tower is not None
196
+ image_features = self._process_image_pixels(image_input)
197
return self.multi_modal_projector(image_features)
198
199
def forward(
@@ -273,25 +224,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
224
This way, the `positions` and `attn_metadata` are consistent
225
with the `input_ids`.
226
276
- This model has two modes of image inputs:
277
- `PIXEL_VALUES` and `IMAGE_FEATURES`.
278
-
227
Args:
228
input_ids: Flattened (concatenated) input_ids corresponding to a
229
batch.
230
pixel_values: The pixels in each input image.
283
- Expects a batch with shape `[1, 3, 336, 336]`.
284
- (Only applicable to `PIXEL_VALUES` mode)
285
- image_features: The image features for each input image outputted by
286
- the vision tower before passing to the multi-modal projector.
287
- Expects a batch with shape `[1, 576, 1024]`.
288
- (Only applicable to `IMAGE_FEATURES` mode)
289
-
290
- See also:
291
- Each input maps to huggingface implementation, as follows:
292
-
293
- - `pixel_values`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava/modeling_llava.py#L360
294
- - `image_features`: https://github.com/huggingface/transformers/blob/v4.41.1/src/transformers/models/llava/modeling_llava.py#L437
231
"""
232
image_input = self._parse_and_validate_image_input(**kwargs)
233
vllm/model_executor/models/llava_next.py
+40
-86
@@ -1,8 +1,8 @@
1
-from typing import (Dict, Iterable, List, Literal, Optional, Tuple, TypedDict,
2
- Union)
1
+from typing import Dict, Iterable, List, Literal, Optional, Tuple, TypedDict
2
3
import torch
4
import torch.nn as nn
5
+from PIL import Image
6
from transformers import CLIPVisionConfig, LlavaNextConfig
7
from transformers.models.llava_next.modeling_llava_next import (
8
get_anyres_image_grid_shape, unpad_image)
@@ -21,12 +21,11 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
from vllm.model_executor.models.clip import CLIPVisionModel
22
from vllm.model_executor.models.llama import LlamaModel
23
from vllm.model_executor.sampling_metadata import SamplingMetadata
24
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalData
25
-from vllm.multimodal.image import ImagePixelData
24
+from vllm.multimodal import MULTIMODAL_REGISTRY
25
from vllm.sequence import SamplerOutput
26
28
-from .clip import (dummy_feature_data_for_clip, dummy_pixel_data_for_clip,
29
- dummy_seq_data_for_clip, get_clip_patch_grid_length)
27
+from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
28
+ get_clip_patch_grid_length)
29
from .interfaces import SupportsVision
30
from .llava import LlavaMultiModalProjector, merge_vision_embeddings
31
@@ -47,17 +46,7 @@ class LlavaNextImagePixelInputs(TypedDict):
46
"""Shape: (batch_size, 2)"""
47
48
50
-class LlavaNextImageFeatureInputs(TypedDict):
51
- type: Literal["image_features"]
52
- data: torch.Tensor
53
- """Shape: (batch_size, 1 + num_patches, image_feature_size, hidden_size)"""
54
-
55
- image_sizes: NotRequired[torch.Tensor]
56
- """Shape: (batch_size, 2)"""
57
-
58
-
59
-LlavaNextImageInputs = Union[LlavaNextImagePixelInputs,
60
- LlavaNextImageFeatureInputs]
49
+LlavaNextImageInputs = LlavaNextImagePixelInputs
50
51
52
def _get_llava_next_num_unpadded_features(
@@ -138,20 +127,11 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
127
image_feature_size_override=image_feature_size,
128
)
129
141
- image_input_type = multimodal_config.image_input_type
142
- ImageInputType = VisionLanguageConfig.ImageInputType
143
- mm_data: MultiModalData
144
- if image_input_type == ImageInputType.PIXEL_VALUES:
145
- mm_data = dummy_pixel_data_for_clip(
146
- vision_config,
147
- image_width_override=dummy_width,
148
- image_height_override=dummy_height,
149
- )
150
- elif image_input_type == ImageInputType.IMAGE_FEATURES:
151
- mm_data = dummy_feature_data_for_clip(
152
- vision_config,
153
- image_feature_size_override=image_feature_size,
154
- )
130
+ mm_data = dummy_image_for_clip(
131
+ vision_config,
132
+ image_width_override=dummy_width,
133
+ image_height_override=dummy_height,
134
+ )
135
136
return seq_data, mm_data
137
@@ -159,32 +139,26 @@ def dummy_data_for_llava_next(ctx: InputContext, seq_len: int):
139
raise NotImplementedError(msg)
140
141
162
-def _pixel_mapper(ctx: InputContext,
163
- data: ImagePixelData) -> Dict[str, torch.Tensor]:
164
- image = data.image
142
+def _pixel_mapper(ctx: InputContext, image: object) -> Dict[str, torch.Tensor]:
143
166
- if isinstance(image, torch.Tensor):
167
- pixel_values = image.to(ctx.model_config.dtype)
168
- batch_size, _, _, h, w = pixel_values.shape
169
- image_sizes = torch.tensor([(w, h) for _ in range(batch_size)])
144
+ if isinstance(image, Image.Image):
145
171
- return {"pixel_values": pixel_values, "image_sizes": image_sizes}
146
+ # Temporary patch before dynamic number of image tokens is supported
147
+ _, _, h, w = ctx.get_multimodal_config().image_input_shape
148
+ if (w, h) != (image.width, image.height):
149
+ logger.warning(
150
+ "Dynamic image shape is currently not supported. "
151
+ "Resizing input image to (%d, %d).", w, h)
152
173
- # Temporary patch before dynamic number of image tokens is supported
174
- _, _, h, w = ctx.get_multimodal_config().image_input_shape
175
- if (w, h) != (image.width, image.height):
176
- logger.warning(
177
- "Dynamic image shape is currently not supported. "
178
- "Resizing input image to (%d, %d).", w, h)
153
+ image = image.resize((w, h))
154
180
- data.image = image.resize((w, h))
155
+ return MULTIMODAL_REGISTRY._get_plugin("image") \
156
+ ._default_input_mapper(ctx, image)
157
182
- return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
183
- ._default_input_mapper(ctx, data)
158
+ raise TypeError(f"Invalid type for 'image': {type(image)}")
159
160
186
-@MULTIMODAL_REGISTRY.register_image_feature_input_mapper()
187
-@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_pixel_mapper)
161
+@MULTIMODAL_REGISTRY.register_image_input_mapper(_pixel_mapper)
162
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next)
163
class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
164
@@ -198,11 +172,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
172
self.config = config
173
self.vlm_config = vlm_config
174
201
- if self.vlm_config.image_input_type == (
202
- VisionLanguageConfig.ImageInputType.PIXEL_VALUES):
203
- self.vision_tower = CLIPVisionModel(config=config.vision_config)
204
- else:
205
- raise TypeError("Image features are not supported by LLaVA-NeXT")
175
+ self.vision_tower = CLIPVisionModel(config=config.vision_config)
176
177
self.multi_modal_projector = LlavaMultiModalProjector(
178
vision_hidden_size=config.vision_config.hidden_size,
@@ -255,36 +225,23 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
225
self, **kwargs: object) -> Optional[LlavaNextImageInputs]:
226
pixel_values = kwargs.pop("pixel_values", None)
227
image_sizes = kwargs.pop("image_sizes", None)
258
- image_features = kwargs.pop("image_features", None)
259
-
260
- expected_input_type = self.vlm_config.image_input_type
261
- ImageInputType = VisionLanguageConfig.ImageInputType
262
-
263
- if expected_input_type == ImageInputType.PIXEL_VALUES:
264
- if image_features is not None:
265
- raise ValueError(
266
- "Expected pixel values but got image features")
267
- if pixel_values is None:
268
- return None
228
270
- if not isinstance(pixel_values, torch.Tensor):
271
- raise ValueError("Incorrect type of pixel values. "
272
- f"Got type: {type(pixel_values)}")
229
+ if pixel_values is None or image_sizes is None:
230
+ return None
231
274
- if not isinstance(image_sizes, torch.Tensor):
275
- raise ValueError("Incorrect type of image sizes. "
276
- f"Got type: {type(image_sizes)}")
232
+ if not isinstance(pixel_values, torch.Tensor):
233
+ raise ValueError("Incorrect type of pixel values. "
234
+ f"Got type: {type(pixel_values)}")
235
278
- return LlavaNextImagePixelInputs(
279
- type="pixel_values",
280
- data=self._validate_image_pixels(pixel_values),
281
- image_sizes=self._validate_image_sizes(image_sizes),
282
- )
236
+ if not isinstance(image_sizes, torch.Tensor):
237
+ raise ValueError("Incorrect type of image sizes. "
238
+ f"Got type: {type(image_sizes)}")
239
284
- assert expected_input_type != ImageInputType.IMAGE_FEATURES, (
285
- "Failed to validate this at initialization time")
286
-
287
- return None
240
+ return LlavaNextImagePixelInputs(
241
+ type="pixel_values",
242
+ data=self._validate_image_pixels(pixel_values),
243
+ image_sizes=self._validate_image_sizes(image_sizes),
244
+ )
245
246
def _select_image_features(self, image_features: torch.Tensor, *,
247
strategy: str) -> torch.Tensor:
@@ -391,11 +348,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
348
349
def _process_image_input(
350
self, image_input: LlavaNextImageInputs) -> torch.Tensor:
394
- if image_input["type"] == "pixel_values":
395
- assert self.vision_tower is not None
396
- image_features = self._process_image_pixels(image_input)
397
- else:
398
- image_features = image_input["data"]
351
+ assert self.vision_tower is not None
352
+ image_features = self._process_image_pixels(image_input)
353
354
patch_embeddings = self.multi_modal_projector(image_features)
355
vllm/model_executor/models/phi3v.py
+8
-17
@@ -35,10 +35,9 @@ from vllm.model_executor.models.clip import CLIPVisionModel
35
from vllm.model_executor.models.llama import LlamaModel
36
from vllm.model_executor.sampling_metadata import SamplingMetadata
37
from vllm.multimodal import MULTIMODAL_REGISTRY
38
-from vllm.multimodal.image import ImagePixelData
38
from vllm.sequence import SamplerOutput
39
41
-from .clip import dummy_pixel_data_for_clip, dummy_seq_data_for_clip
40
+from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
41
from .interfaces import SupportsVision
42
43
logger = init_logger(__name__)
@@ -286,7 +285,7 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
285
image_token_id=32044,
286
image_feature_size_override=image_feature_size,
287
)
289
- mm_data = dummy_pixel_data_for_clip(
288
+ mm_data = dummy_image_for_clip(
289
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
290
image_width_override=dummy_width,
291
image_height_override=dummy_height,
@@ -331,8 +330,7 @@ def _calc_hd_transform_size(*, width: int, height: int, hd_num: int = 16):
330
331
332
def _image_processor(ctx: InputContext,
334
- data: ImagePixelData) -> Dict[str, torch.Tensor]:
335
- image = data.image
333
+ image: object) -> Dict[str, torch.Tensor]:
334
335
if isinstance(image, Image.Image):
336
# Temporary patch before dynamic number of image tokens is supported
@@ -343,13 +341,14 @@ def _image_processor(ctx: InputContext,
341
"Dynamic image shape is currently not supported. "
342
"Resizing input image to (%d, %d).", w, h)
343
346
- data.image = image.resize((w, h))
344
+ image = image.resize((w, h))
345
348
- return MULTIMODAL_REGISTRY._get_plugin_for_data_type(ImagePixelData) \
349
- ._default_input_mapper(ctx, data)
346
+ return MULTIMODAL_REGISTRY._get_plugin("image") \
347
+ ._default_input_mapper(ctx, image)
348
+ raise TypeError(f"Invalid type for 'image': {type(image)}")
349
350
352
-@MULTIMODAL_REGISTRY.register_image_pixel_input_mapper(_image_processor)
351
+@MULTIMODAL_REGISTRY.register_image_input_mapper(_image_processor)
352
@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
353
class Phi3VForCausalLM(nn.Module, SupportsVision):
354
@@ -375,14 +374,6 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
374
pixel_values = kwargs.pop("pixel_values", None)
375
image_sizes = kwargs.pop("image_sizes", None)
376
378
- expected_input_type = self.vlm_config.image_input_type
379
- ImageInputType = VisionLanguageConfig.ImageInputType
380
-
381
- if expected_input_type != ImageInputType.PIXEL_VALUES:
382
- raise ValueError(
383
- f"Unexpected image input type: {expected_input_type}."
384
- "Phi3v only support pixel_values input currently.")
385
-
377
if pixel_values is not None and image_sizes is not None:
378
return Phi3VImagePixelInputs(type="pixel_values",
379
data=pixel_values,
vllm/multimodal/__init__.py
+5
-3
@@ -1,4 +1,4 @@
1
-from .base import MultiModalData, MultiModalPlugin
1
+from .base import MultiModalDataDict, MultiModalPlugin
2
from .registry import MultiModalRegistry
3
4
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -11,6 +11,8 @@ See also:
11
"""
12
13
__all__ = [
14
- "MultiModalData", "MultiModalPlugin", "MULTIMODAL_REGISTRY",
15
- "MultiModalRegistry"
14
+ "MultiModalPlugin",
15
+ "MULTIMODAL_REGISTRY",
16
+ "MultiModalRegistry",
17
+ "MultiModalDataDict",
18
]
vllm/multimodal/base.py
+25
-28
@@ -1,6 +1,6 @@
1
from abc import ABC, abstractmethod
2
-from typing import (TYPE_CHECKING, Callable, Dict, Generic, Optional, Type,
3
- TypeVar)
2
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Type,
3
+ TypedDict, TypeVar, Union)
4
5
from vllm.config import ModelConfig
6
from vllm.inputs import InputContext
@@ -8,38 +8,35 @@ from vllm.logger import init_logger
8
9
if TYPE_CHECKING:
10
import torch
11
+ from PIL import Image
12
from torch import nn
13
14
logger = init_logger(__name__)
15
16
+N = TypeVar("N", bound=Type["nn.Module"])
17
16
-class MultiModalData:
17
- """
18
- Base class that contains multi-modal data.
19
-
20
- To add a new modality, add a new file under ``multimodal`` directory.
18
22
- In this new file, subclass :class:`~MultiModalData` and
23
- :class:`~MultiModalPlugin`.
19
+class MultiModalDataBuiltins(TypedDict, total=False):
20
+ image: "Image.Image"
21
25
- Finally, register the new plugin to
26
- :const:`vllm.multimodal.MULTIMODAL_REGISTRY`.
27
- This enables models to call :meth:`MultiModalRegistry.map_input` for
28
- the new modality.
29
- """
30
- pass
22
23
+MultiModalDataDict = Union[MultiModalDataBuiltins, Dict[str, Any]]
24
+"""
25
+A dictionary containing an item for each modality type to input.
26
33
-D = TypeVar("D", bound=MultiModalData)
34
-N = TypeVar("N", bound=Type["nn.Module"])
27
+The data belonging to each modality is converted into keyword arguments
28
+to the model by the corresponding mapper. By default, the mapper of
29
+the corresponding plugin with the same modality key is applied.
30
+"""
31
36
-MultiModalInputMapper = Callable[[InputContext, D], Dict[str, "torch.Tensor"]]
32
+MultiModalInputMapper = Callable[[InputContext, object], Dict[str,
33
+ "torch.Tensor"]]
34
"""Return a dictionary to be passed as keyword arguments to
35
:meth:`~torch.nn.Module.forward`. This is similar in concept to tokenizers
36
and processors in HuggingFace Transformers."""
37
38
42
-class MultiModalPlugin(ABC, Generic[D]):
39
+class MultiModalPlugin(ABC):
40
"""
41
Base class that defines data processing logic for a specific modality.
42
@@ -52,19 +49,18 @@ class MultiModalPlugin(ABC, Generic[D]):
49
50
def __init__(self) -> None:
51
self._input_mappers: Dict[Type["nn.Module"],
55
- MultiModalInputMapper[D]] = {}
52
+ MultiModalInputMapper] = {}
53
54
@abstractmethod
58
- def get_data_type(self) -> Type[D]:
55
+ def get_data_key(self) -> str:
56
"""
60
- Get the modality (subclass of :class:`~MultiModalData`) served by
61
- this plugin.
57
+ Get the data key corresponding to the modality.
58
"""
59
raise NotImplementedError
60
61
@abstractmethod
62
def _default_input_mapper(self, ctx: InputContext,
67
- data: D) -> Dict[str, "torch.Tensor"]:
63
+ data: object) -> Dict[str, "torch.Tensor"]:
64
"""Return a dictionary to be passed as keyword arguments to
65
:meth:`~torch.nn.Module.forward`. This is similar in concept to
66
tokenizers and processors in HuggingFace Transformers.
@@ -73,11 +69,10 @@ class MultiModalPlugin(ABC, Generic[D]):
69
70
def register_input_mapper(
71
self,
76
- mapper: Optional[MultiModalInputMapper[D]] = None,
72
+ mapper: Optional[MultiModalInputMapper] = None,
73
):
74
"""
75
Register an input mapper to a model class.
80
-
76
When the model receives input data that matches the modality served by
77
this plugin (see :meth:`get_data_type`), the provided function is
78
invoked to transform the data into a dictionary of model inputs.
@@ -102,11 +97,13 @@ class MultiModalPlugin(ABC, Generic[D]):
97
return wrapper
98
99
def map_input(self, model_config: ModelConfig,
105
- data: D) -> Dict[str, "torch.Tensor"]:
100
+ data: object) -> Dict[str, "torch.Tensor"]:
101
"""
107
- Apply an input mapper to a :class:`~MultiModalData` instance passed
102
+ Apply an input mapper to a data passed
103
to the model, transforming the data into a dictionary of model inputs.
104
105
+ If the data is not something that the mapper expects, throws TypeError.
106
+
107
The model is identified by ``model_config``.
108
109
TODO: Add guide [ref: PR #5276]
vllm/multimodal/image.py
+12
-81
@@ -1,5 +1,5 @@
1
from functools import lru_cache
2
-from typing import Dict, Type, Union
2
+from typing import Dict
3
4
import torch
5
from PIL import Image
@@ -9,105 +9,36 @@ from vllm.inputs.registry import InputContext
9
from vllm.logger import init_logger
10
from vllm.transformers_utils.image_processor import get_image_processor
11
12
-from .base import MultiModalData, MultiModalPlugin
12
+from .base import MultiModalPlugin
13
14
logger = init_logger(__name__)
15
16
cached_get_image_processor = lru_cache(get_image_processor)
17
18
19
-class ImagePixelData(MultiModalData):
20
- """
21
- The pixel data of an image. Can be one of:
19
+class ImagePlugin(MultiModalPlugin):
20
23
- - :class:`PIL.Image.Image`: An image object. Requires that a HuggingFace
24
- processor is available to the model.
25
- - :class:`torch.Tensor`: The raw pixel data which is passed to the model
26
- without additional pre-processing.
27
- """
28
-
29
- def __init__(self, image: Union[Image.Image, torch.Tensor]) -> None:
30
- if isinstance(image, Image.Image):
31
- # So that this class can be created inside the Image context manager
32
- image.load()
33
-
34
- self.image = image
35
-
36
- def __repr__(self) -> str:
37
- image = self.image
38
- if isinstance(image, Image.Image):
39
- return f"{type(self).__name__}(image={image})"
40
-
41
- return (f"{type(self).__name__}(image=torch.Tensor(shape="
42
- f"{image.shape}, dtype={image.dtype}))")
43
-
44
-
45
-class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
46
-
47
- def get_data_type(self) -> Type[ImagePixelData]:
48
- return ImagePixelData
21
+ def get_data_key(self) -> str:
22
+ return "image"
23
24
def _get_hf_image_processor(self, model_config: ModelConfig):
51
- vlm_config = model_config.multimodal_config
52
- if vlm_config is None or vlm_config.image_processor is None:
53
- return None
54
-
25
return cached_get_image_processor(
56
- vlm_config.image_processor,
57
- trust_remote_code=model_config.trust_remote_code,
58
- revision=vlm_config.image_processor_revision,
59
- )
26
+ model_config.model,
27
+ trust_remote_code=model_config.trust_remote_code)
28
29
def _default_input_mapper(self, ctx: InputContext,
62
- data: ImagePixelData) -> Dict[str, torch.Tensor]:
30
+ data: object) -> Dict[str, torch.Tensor]:
31
model_config = ctx.model_config
64
- image = data.image
65
-
66
- if isinstance(image, Image.Image):
32
+ if isinstance(data, Image.Image):
33
image_processor = self._get_hf_image_processor(model_config)
34
if image_processor is None:
35
raise RuntimeError("No HuggingFace processor is available"
36
"to process the image object")
37
try:
72
- return image_processor.preprocess(image, return_tensors="pt") \
38
+ return image_processor.preprocess(data, return_tensors="pt") \
39
.to(model_config.dtype).data
40
except Exception:
75
- logger.error("Failed to process image (%s)", image)
41
+ logger.error("Failed to process image (%s)", data)
42
raise
77
- elif isinstance(image, torch.Tensor):
78
- pixel_values = image.to(model_config.dtype)
79
-
80
- return {"pixel_values": pixel_values}
81
-
82
- raise TypeError(f"Invalid image type: {type(image)}")
83
-
84
-
85
-class ImageFeatureData(MultiModalData):
86
- """
87
- The feature vector of an image, passed directly to the model.
88
-
89
- This should be the output of the vision tower.
90
- """
91
-
92
- def __init__(self, image_features: torch.Tensor) -> None:
93
- self.image_features = image_features
94
-
95
- def __repr__(self) -> str:
96
- image_features = self.image_features
97
-
98
- return (f"{type(self).__name__}(image_features=torch.Tensor(shape="
99
- f"{image_features.shape}, dtype={image_features.dtype}))")
100
-
101
-
102
-class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
103
-
104
- def get_data_type(self) -> Type[ImageFeatureData]:
105
- return ImageFeatureData
106
-
107
- def _default_input_mapper(
108
- self, ctx: InputContext,
109
- data: ImageFeatureData) -> Dict[str, torch.Tensor]:
110
- model_config = ctx.model_config
111
- image_features = data.image_features.to(model_config.dtype)
43
113
- return {"image_features": image_features}
44
+ raise TypeError(f"Invalid type for 'image': {type(data)}")
vllm/multimodal/registry.py
+52
-44
@@ -1,18 +1,16 @@
1
import functools
2
-from typing import Any, Optional, Sequence, Type, TypeVar
2
+from typing import Optional, Sequence, Type, TypeVar
3
4
from torch import nn
5
6
from vllm.config import ModelConfig
7
from vllm.logger import init_logger
8
9
-from .base import MultiModalData, MultiModalInputMapper, MultiModalPlugin
10
-from .image import (ImageFeatureData, ImageFeaturePlugin, ImagePixelData,
11
- ImagePixelPlugin)
9
+from .base import MultiModalDataDict, MultiModalInputMapper, MultiModalPlugin
10
+from .image import ImagePlugin
11
12
logger = init_logger(__name__)
13
15
-D = TypeVar("D", bound=MultiModalData)
14
N = TypeVar("N", bound=Type[nn.Module])
15
16
@@ -20,81 +18,91 @@ class MultiModalRegistry:
18
"""
19
A registry to dispatch data processing
20
according to its modality and the target model.
21
+
22
+ The registry handles both external and internal data input.
23
"""
24
25
- DEFAULT_PLUGINS = (ImageFeaturePlugin(), ImagePixelPlugin())
25
+ DEFAULT_PLUGINS = (ImagePlugin(), )
26
27
def __init__(
28
- self,
29
- *,
30
- plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS,
31
- ) -> None:
32
- self._plugins_by_data_type = {p.get_data_type(): p for p in plugins}
28
+ self,
29
+ *,
30
+ plugins: Sequence[MultiModalPlugin] = DEFAULT_PLUGINS) -> None:
31
+ self._plugins = {p.get_data_key(): p for p in plugins}
32
34
- def register_plugin(self, plugin: MultiModalPlugin[Any]) -> None:
35
- data_type = plugin.get_data_type()
33
+ def register_plugin(self, plugin: MultiModalPlugin) -> None:
34
+ data_type_key = plugin.get_data_key()
35
37
- if data_type in self._plugins_by_data_type:
36
+ if data_type_key in self._plugins:
37
logger.warning(
38
"A plugin is already registered for data type %s, "
40
- "and will be overwritten by the new plugin %s.", data_type,
39
+ "and will be overwritten by the new plugin %s.", data_type_key,
40
plugin)
41
43
- self._plugins_by_data_type[data_type] = plugin
42
+ self._plugins[data_type_key] = plugin
43
45
- def _get_plugin_for_data_type(self, data_type: Type[MultiModalData]):
46
- for typ in data_type.mro():
47
- plugin = self._plugins_by_data_type.get(typ)
48
- if plugin is not None:
49
- return plugin
44
+ def _get_plugin(self, data_type_key: str):
45
+ plugin = self._plugins.get(data_type_key)
46
+ if plugin is not None:
47
+ return plugin
48
51
- msg = f"Unknown multi-modal data type: {data_type}"
49
+ msg = f"Unknown multi-modal data type: {data_type_key}"
50
raise NotImplementedError(msg)
51
54
- def register_input_mapper(
52
+ def register_image_input_mapper(
53
self,
56
- data_type: Type[D],
57
- mapper: Optional[MultiModalInputMapper[D]] = None,
54
+ mapper: Optional[MultiModalInputMapper] = None,
55
):
56
"""
60
- Register an input mapper for a specific modality to a model class.
57
+ Register an input mapper for image data to a model class.
58
59
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
60
"""
64
- return self._get_plugin_for_data_type(data_type) \
65
- .register_input_mapper(mapper)
61
+ return self.register_input_mapper("image", mapper)
62
+
63
+ def _process_input(self, key: str, value: object,
64
+ model_config: ModelConfig):
65
+ plugin = self._plugins.get(key)
66
+ if plugin:
67
+ return plugin.map_input(model_config, value)
68
+ msg = f"Unknown multi-modal data type: {key}"
69
+ raise NotImplementedError(msg)
70
67
- def register_image_pixel_input_mapper(
71
+ def register_input_mapper(
72
self,
69
- mapper: Optional[MultiModalInputMapper[ImagePixelData]] = None,
73
+ data_type: str,
74
+ mapper: Optional[MultiModalInputMapper] = None,
75
):
76
"""
72
- Register an input mapper for image pixel data to a model class.
77
+ Register an input mapper for a specific modality to a model class.
78
79
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
80
"""
76
- return self.register_input_mapper(ImagePixelData, mapper)
77
-
78
- def register_image_feature_input_mapper(
79
- self,
80
- mapper: Optional[MultiModalInputMapper[ImageFeatureData]] = None,
81
- ):
81
+ plugin = self._plugins.get(data_type)
82
+ if not plugin:
83
+ msg = f"Unknown multi-modal data type: {data_type}"
84
+ raise NotImplementedError(msg)
85
+ return plugin.register_input_mapper(mapper)
86
+
87
+ def register_image_input(self,
88
+ mapper: Optional[MultiModalInputMapper] = None):
89
"""
83
- Register an input mapper for image feature data to a model class.
90
+ Register an input mapper for image pixel data to a model class.
91
92
See :meth:`MultiModalPlugin.register_input_mapper` for more details.
93
"""
87
- return self.register_input_mapper(ImageFeatureData, mapper)
94
+ return self.register_input_mapper("image", mapper)
95
89
- def map_input(self, model_config: ModelConfig, data: MultiModalData):
96
+ def map_input(self, model_config: ModelConfig, data: MultiModalDataDict):
97
"""
91
- Apply an input mapper to a :class:`~MultiModalData` instance passed
92
- to the model.
98
+ Apply an input mapper to the data passed to the model.
99
100
See :meth:`MultiModalPlugin.map_input` for more details.
101
"""
96
- return self._get_plugin_for_data_type(type(data)) \
97
- .map_input(model_config, data)
102
+ result_list = [
103
+ self._process_input(k, v, model_config) for k, v in data.items()
104
+ ]
105
+ return {k: v for d in result_list for k, v in d.items()}
106
107
def create_input_mapper(self, model_config: ModelConfig):
108
"""
vllm/multimodal/utils.py
+7
-6
@@ -8,7 +8,7 @@ from PIL import Image
8
9
from vllm.config import ModelConfig
10
from vllm.envs import VLLM_IMAGE_FETCH_TIMEOUT
11
-from vllm.multimodal.image import ImagePixelData
11
+from vllm.multimodal.base import MultiModalDataDict
12
13
14
class ImageFetchAiohttp:
@@ -53,14 +53,10 @@ class ImageFetchAiohttp:
53
"Invalid 'image_url': A valid 'image_url' must start "
54
"with either 'data:image' or 'http'.")
55
56
+ image.load()
57
return image
58
59
59
-async def async_get_and_parse_image(image_url: str) -> ImagePixelData:
60
- with await ImageFetchAiohttp.fetch_image(image_url) as image:
61
- return ImagePixelData(image)
62
-
63
-
60
def encode_image_base64(image: Image.Image, format: str = 'JPEG') -> str:
61
"""Encode a pillow image to base64 format."""
62
@@ -91,3 +87,8 @@ def get_full_image_text_prompt(image_prompt: str, text_prompt: str,
87
raise ValueError(
88
f"Unsupported model type: {config.hf_config.model_type}")
89
return full_prompt
90
+
91
+
92
+async def async_get_and_parse_image(image_url: str) -> MultiModalDataDict:
93
+ image = await ImageFetchAiohttp.fetch_image(image_url)
94
+ return {"image": image}
vllm/sequence.py
+5
-5
@@ -14,7 +14,7 @@ from vllm.sampling_params import SamplingParams
14
15
if TYPE_CHECKING:
16
from vllm.inputs import LLMInputs
17
- from vllm.multimodal import MultiModalData
17
+ from vllm.multimodal import MultiModalDataDict
18
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
19
20
@@ -280,8 +280,8 @@ class Sequence:
280
return self.inputs["prompt_token_ids"]
281
282
@property
283
- def multi_modal_data(self) -> Optional["MultiModalData"]:
284
- return self.inputs.get("multi_modal_data")
283
+ def multi_modal_data(self) -> "MultiModalDataDict":
284
+ return self.inputs.get("multi_modal_data") or {}
285
286
@property
287
def lora_int_id(self) -> int:
@@ -457,7 +457,7 @@ class SequenceGroup:
457
return next(iter(self.seqs_dict.values())).prompt_token_ids
458
459
@property
460
- def multi_modal_data(self) -> Optional["MultiModalData"]:
460
+ def multi_modal_data(self) -> Optional["MultiModalDataDict"]:
461
# All sequences in the group should have the same multi-modal data.
462
# We use the multi-modal data of an arbitrary sequence.
463
return next(iter(self.seqs_dict.values())).multi_modal_data
@@ -639,7 +639,7 @@ class SequenceGroupMetadata:
639
lora_request: Optional[LoRARequest] = None,
640
computed_block_nums: Optional[List[int]] = None,
641
state: Optional[SequenceGroupState] = None,
642
- multi_modal_data: Optional["MultiModalData"] = None,
642
+ multi_modal_data: Optional["MultiModalDataDict"] = None,
643
encoder_seq_data: Optional[SequenceData] = None,
644
cross_block_table: Optional[List[int]] = None,
645
) -> None:
vllm/transformers_utils/image_processor.py
-4
@@ -1,5 +1,3 @@
1
-from typing import Optional
2
-
1
from transformers import AutoImageProcessor
2
from transformers.image_processing_utils import BaseImageProcessor
3
@@ -12,7 +10,6 @@ def get_image_processor(
10
processor_name: str,
11
*args,
12
trust_remote_code: bool = False,
15
- revision: Optional[str] = None,
13
**kwargs,
14
) -> BaseImageProcessor:
15
"""Gets an image processor for the given model name via HuggingFace."""
@@ -21,7 +18,6 @@ def get_image_processor(
18
processor_name,
19
*args,
20
trust_remote_code=trust_remote_code,
24
- revision=revision,
21
**kwargs)
22
except ValueError as e:
23
# If the error pertains to the processor class not existing or not
vllm/worker/model_runner.py
+1
-1
@@ -504,7 +504,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
504
is not None else 1))
505
506
mm_data = seq_group_metadata.multi_modal_data
507
- if mm_data is not None:
507
+ if mm_data:
508
# Process multi-modal data
509
mm_kwargs = self.multi_modal_input_mapper(mm_data)
510
for k, v in mm_kwargs.items():