A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Core] Support image processor (#4197)
Cyrus Leung committed
Jun 3, 2024 at 13:56 UTC
7a64d24aad69e4d2548aa0bf528d9fe63428ab01
29 files changed
+1042
-256
.github/workflows/mypy.yaml
+1
@@ -37,6 +37,7 @@ jobs:
37
mypy vllm/distributed --config-file pyproject.toml
38
mypy vllm/entrypoints --config-file pyproject.toml
39
mypy vllm/executor --config-file pyproject.toml
40
+ mypy vllm/multimodal --config-file pyproject.toml
41
mypy vllm/usage --config-file pyproject.toml
42
mypy vllm/*.py --config-file pyproject.toml
43
mypy vllm/transformers_utils --config-file pyproject.toml
docs/source/conf.py
+8
-6
@@ -90,6 +90,7 @@ autodoc_mock_imports = [
90
"sentencepiece",
91
"vllm.cuda_utils",
92
"vllm._C",
93
+ "PIL",
94
"numpy",
95
"tqdm",
96
"tensorizer",
@@ -116,12 +117,13 @@ class MockedClassDocumenter(autodoc.ClassDocumenter):
117
autodoc.ClassDocumenter = MockedClassDocumenter
118
119
intersphinx_mapping = {
119
- 'python': ('https://docs.python.org/3', None),
120
- 'typing_extensions':
121
- ('https://typing-extensions.readthedocs.io/en/latest', None),
122
- 'numpy': ('https://numpy.org/doc/stable', None),
123
- 'torch': ('https://pytorch.org/docs/stable', None),
124
- 'psutil': ('https://psutil.readthedocs.io/en/stable', None),
120
+ "python": ("https://docs.python.org/3", None),
121
+ "typing_extensions":
122
+ ("https://typing-extensions.readthedocs.io/en/latest", None),
123
+ "pillow": ("https://pillow.readthedocs.io/en/stable", None),
124
+ "numpy": ("https://numpy.org/doc/stable", None),
125
+ "torch": ("https://pytorch.org/docs/stable", None),
126
+ "psutil": ("https://psutil.readthedocs.io/en/stable", None),
127
}
128
129
autodoc_preserve_defaults = True
docs/source/dev/multimodal/multimodal_index.rst
new
+51
@@ -0,0 +1,51 @@
1
+Multi-Modality
2
+==============
3
+
4
+.. currentmodule:: vllm.multimodal
5
+
6
+vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
7
+
8
+:class:`vllm.inputs.PromptStrictInputs` accepts an additional attribute ``multi_modal_data``
9
+which allows you to pass in multi-modal input alongside text and token prompts.
10
+
11
+By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model,
12
+you must decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_dummy_data <MultiModalRegistry.register_dummy_data>`,
13
+as well as :meth:`MULTIMODAL_REGISTRY.register_input <MultiModalRegistry.register_input>` for each modality type to support.
14
+
15
+.. contents::
16
+ :local:
17
+ :backlinks: none
18
+
19
+Module Contents
20
++++++++++++++++
21
+
22
+.. automodule:: vllm.multimodal
23
+
24
+Registry
25
+--------
26
+
27
+.. data:: vllm.multimodal.MULTIMODAL_REGISTRY
28
+
29
+ The global :class:`MultiModalRegistry` which is used by model runners.
30
+
31
+.. autoclass:: vllm.multimodal.MultiModalRegistry
32
+ :members:
33
+ :show-inheritance:
34
+
35
+Base Classes
36
+------------
37
+
38
+.. autoclass:: vllm.multimodal.MultiModalData
39
+ :members:
40
+ :show-inheritance:
41
+
42
+.. autoclass:: vllm.multimodal.MultiModalPlugin
43
+ :members:
44
+ :show-inheritance:
45
+
46
+Image Classes
47
+-------------
48
+
49
+.. automodule:: vllm.multimodal.image
50
+ :members:
51
+ :show-inheritance:
docs/source/index.rst
+4
-2
@@ -88,6 +88,7 @@ Documentation
88
models/adding_model
89
models/engine_args
90
models/lora
91
+ models/vlm
92
models/performance
93
94
.. toctree::
@@ -99,17 +100,18 @@ Documentation
100
quantization/fp8_e4m3_kvcache
101
102
.. toctree::
102
- :maxdepth: 2
103
+ :maxdepth: 1
104
:caption: Developer Documentation
105
106
dev/sampling_params
107
dev/offline_inference/offline_index
108
dev/engine/engine_index
109
dev/kernel/paged_attention
110
+ dev/multimodal/multimodal_index
111
dev/dockerfile/dockerfile
112
113
.. toctree::
112
- :maxdepth: 2
114
+ :maxdepth: 1
115
:caption: Community
116
117
community/meetups
docs/source/models/supported_models.rst
+4
@@ -87,6 +87,10 @@ Alongside each architecture, we include some popular models that use it.
87
- LLaMA, Llama 2, Meta Llama 3, Vicuna, Alpaca, Yi
88
- :code:`meta-llama/Meta-Llama-3-8B-Instruct`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
89
- ✅︎
90
+ * - :code:`LlavaForConditionalGeneration`
91
+ - LLaVA-1.5
92
+ - :code:`llava-hf/llava-1.5-7b-hf`\*, :code:`llava-hf/llava-1.5-13b-hf`\*, etc.
93
+ -
94
* - :code:`MiniCPMForCausalLM`
95
- MiniCPM
96
- :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
docs/source/models/vlm.rst
new
+56
@@ -0,0 +1,56 @@
1
+.. _vlm:
2
+
3
+Using VLMs
4
+==========
5
+
6
+This document shows you how to run and serve Vision Language Models (VLMs) using vLLM.
7
+
8
+Engine Arguments
9
+----------------
10
+
11
+The following :ref:`engine arguments <engine_args>` are specific to VLMs:
12
+
13
+.. argparse::
14
+ :module: vllm.engine.arg_utils
15
+ :func: _vlm_engine_args_parser
16
+ :prog: -m vllm.entrypoints.openai.api_server
17
+ :nodefaultconst:
18
+
19
+Offline Batched Inference
20
+-------------------------
21
+
22
+To initialize a VLM, the aforementioned arguments must be passed to the ``LLM`` class for instantiating the engine.
23
+
24
+.. code-block:: python
25
+
26
+ llm = LLM(
27
+ model="llava-hf/llava-1.5-7b-hf",
28
+ image_input_type="pixel_values",
29
+ image_token_id=32000,
30
+ image_input_shape="1,3,336,336",
31
+ image_feature_size=576,
32
+ )
33
+
34
+For now, we only support a single image per text prompt. To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
35
+
36
+* ``prompt``: The prompt should have a number of ``<image>`` tokens equal to ``image_feature_size``.
37
+* ``multi_modal_data``: This should be an instance of :class:`~vllm.multimodal.image.ImagePixelData` or :class:`~vllm.multimodal.image.ImageFeatureData`.
38
+
39
+.. code-block:: python
40
+
41
+ prompt = "<image>" * 576 + (
42
+ "\nUSER: What is the content of this image?\nASSISTANT:")
43
+
44
+ # Load the image using PIL.Image
45
+ image = ...
46
+
47
+ outputs = llm.generate({
48
+ "prompt": prompt,
49
+ "multi_modal_data": ImagePixelData(image),
50
+ })
51
+
52
+ for o in outputs:
53
+ generated_text = o.outputs[0].text
54
+ print(generated_text)
55
+
56
+A code example can be found in `examples/llava_example.py <https://github.com/vllm-project/vllm/blob/main/examples/llava_example.py>`_.
examples/llava_example.py
+15
-14
@@ -3,33 +3,36 @@ import os
3
import subprocess
4
5
import torch
6
+from PIL import Image
7
8
from vllm import LLM
8
-from vllm.sequence import MultiModalData
9
+from vllm.multimodal.image import ImageFeatureData, ImagePixelData
10
11
# The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
12
+# You can use `.buildkite/download-images.sh` to download them
13
14
13
-def run_llava_pixel_values():
15
+def run_llava_pixel_values(*, disable_image_processor: bool = False):
16
llm = LLM(
17
model="llava-hf/llava-1.5-7b-hf",
18
image_input_type="pixel_values",
19
image_token_id=32000,
20
image_input_shape="1,3,336,336",
21
image_feature_size=576,
22
+ disable_image_processor=disable_image_processor,
23
)
24
25
prompt = "<image>" * 576 + (
26
"\nUSER: What is the content of this image?\nASSISTANT:")
27
25
- # This should be provided by another online or offline component.
26
- image = torch.load("images/stop_sign_pixel_values.pt")
28
+ if disable_image_processor:
29
+ image = torch.load("images/stop_sign_pixel_values.pt")
30
+ else:
31
+ image = Image.open("images/stop_sign.jpg")
32
33
outputs = llm.generate({
29
- "prompt":
30
- prompt,
31
- "multi_modal_data":
32
- MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
34
+ "prompt": prompt,
35
+ "multi_modal_data": ImagePixelData(image),
36
})
37
38
for o in outputs:
@@ -49,15 +52,13 @@ def run_llava_image_features():
52
prompt = "<image>" * 576 + (
53
"\nUSER: What is the content of this image?\nASSISTANT:")
54
52
- # This should be provided by another online or offline component.
53
- image = torch.load("images/stop_sign_image_features.pt")
55
+ image: torch.Tensor = torch.load("images/stop_sign_image_features.pt")
56
57
outputs = llm.generate({
56
- "prompt":
57
- prompt,
58
- "multi_modal_data":
59
- MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
58
+ "prompt": prompt,
59
+ "multi_modal_data": ImageFeatureData(image),
60
})
61
+
62
for o in outputs:
63
generated_text = o.outputs[0].text
64
print(generated_text)
format.sh
+1
@@ -101,6 +101,7 @@ mypy vllm/core --config-file pyproject.toml
101
mypy vllm/distributed --config-file pyproject.toml
102
mypy vllm/entrypoints --config-file pyproject.toml
103
mypy vllm/executor --config-file pyproject.toml
104
+mypy vllm/multimodal --config-file pyproject.toml
105
mypy vllm/usage --config-file pyproject.toml
106
mypy vllm/*.py --config-file pyproject.toml
107
mypy vllm/transformers_utils --config-file pyproject.toml
requirements-common.txt
+1
@@ -12,6 +12,7 @@ aiohttp
12
openai
13
uvicorn[standard]
14
pydantic >= 2.0 # Required for OpenAI server.
15
+pillow # Required for image processing
16
prometheus_client >= 0.18.0
17
prometheus-fastapi-instrumentator >= 7.0.0
18
tiktoken >= 0.6.0 # Required for DBRX tokenizer
requirements-dev.txt
-3
@@ -33,8 +33,5 @@ sentence-transformers # required for embedding
33
# Benchmarking
34
aiohttp
35
36
-# Multimodal
37
-pillow
38
-
36
# quantization
37
bitsandbytes==0.42.0
tests/conftest.py
+24
-21
@@ -15,7 +15,9 @@ from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
15
from vllm.distributed import destroy_model_parallel
16
from vllm.inputs import TextPrompt
17
from vllm.logger import init_logger
18
-from vllm.sequence import MultiModalData, SampleLogprobs
18
+from vllm.multimodal import MultiModalData
19
+from vllm.multimodal.image import ImageFeatureData, ImagePixelData
20
+from vllm.sequence import SampleLogprobs
21
22
logger = init_logger(__name__)
23
@@ -24,6 +26,7 @@ _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
26
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
27
28
# Multi modal related
29
+# You can use `.buildkite/download-images.sh` to download the assets
30
_PIXEL_VALUES_FILES = [
31
os.path.join(_TEST_DIR, "images", filename) for filename in
32
["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
@@ -89,17 +92,23 @@ def hf_images() -> List[Image.Image]:
92
93
94
@pytest.fixture()
92
-def vllm_images(request) -> "torch.Tensor":
95
+def vllm_images(request) -> List[MultiModalData]:
96
vision_language_config = request.getfixturevalue("model_and_config")[1]
94
- all_images = []
97
if vision_language_config.image_input_type == (
98
VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
97
- filenames = _IMAGE_FEATURES_FILES
99
+ return [
100
+ ImageFeatureData(torch.load(filename))
101
+ for filename in _IMAGE_FEATURES_FILES
102
+ ]
103
else:
99
- filenames = _PIXEL_VALUES_FILES
100
- for filename in filenames:
101
- all_images.append(torch.load(filename))
102
- return torch.concat(all_images, dim=0)
104
+ return [
105
+ ImagePixelData(Image.open(filename)) for filename in _IMAGE_FILES
106
+ ]
107
+
108
+
109
+@pytest.fixture()
110
+def vllm_image_tensors(request) -> List[torch.Tensor]:
111
+ return [torch.load(filename) for filename in _PIXEL_VALUES_FILES]
112
113
114
@pytest.fixture()
@@ -392,23 +401,17 @@ class VllmRunner:
401
self,
402
prompts: List[str],
403
sampling_params: SamplingParams,
395
- images: Optional[torch.Tensor] = None,
404
+ images: Optional[List[MultiModalData]] = None,
405
) -> List[Tuple[List[List[int]], List[str]]]:
406
if images is not None:
407
assert len(prompts) == len(images)
408
400
- prompt_inputs: List[TextPrompt] = []
401
- for i, prompt in enumerate(prompts):
402
- prompt = TextPrompt(prompt=prompt)
403
- if images is not None:
404
- prompt["multi_modal_data"] = MultiModalData(
405
- type=MultiModalData.Type.IMAGE,
406
- data=images[i:i + 1],
407
- )
408
-
409
- prompt_inputs.append(prompt)
409
+ inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
410
+ if images is not None:
411
+ for i, image in enumerate(images):
412
+ inputs[i]["multi_modal_data"] = image
413
411
- req_outputs = self.model.generate(prompt_inputs,
414
+ req_outputs = self.model.generate(inputs,
415
sampling_params=sampling_params)
416
417
outputs: List[Tuple[List[List[int]], List[str]]] = []
@@ -447,7 +450,7 @@ class VllmRunner:
450
self,
451
prompts: List[str],
452
max_tokens: int,
450
- images: Optional[torch.Tensor] = None,
453
+ images: Optional[List[MultiModalData]] = None,
454
) -> List[Tuple[List[int], str]]:
455
greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
456
outputs = self.generate(prompts, greedy_params, images=images)
tests/models/test_llava.py
+40
-20
@@ -1,7 +1,7 @@
1
import gc
2
from dataclasses import fields
3
from enum import Enum
4
-from typing import Dict, List, Tuple
4
+from typing import Any, Dict, List, Tuple
5
6
import pytest
7
import torch
@@ -9,36 +9,50 @@ from transformers import AutoTokenizer
9
10
from vllm.config import VisionLanguageConfig
11
12
+
13
+def iter_llava_configs(model_name: str):
14
+ image_hw_to_feature_size = {
15
+ (336, 336): 576,
16
+ }
17
+
18
+ for (h, w), f in image_hw_to_feature_size.items():
19
+ for input_type, input_shape in [
20
+ (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
21
+ (VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
22
+ ]:
23
+ yield (model_name,
24
+ VisionLanguageConfig(image_input_type=input_type,
25
+ image_feature_size=f,
26
+ image_token_id=32000,
27
+ image_input_shape=input_shape,
28
+ image_processor=model_name,
29
+ image_processor_revision=None))
30
+
31
+
32
model_and_vl_config = [
13
- ("llava-hf/llava-1.5-7b-hf",
14
- VisionLanguageConfig(
15
- image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
16
- image_feature_size=576,
17
- image_token_id=32000,
18
- image_input_shape=(1, 3, 336, 336))),
19
- ("llava-hf/llava-1.5-7b-hf",
20
- VisionLanguageConfig(
21
- image_input_type=VisionLanguageConfig.ImageInputType.IMAGE_FEATURES,
22
- image_feature_size=576,
23
- image_token_id=32000,
24
- image_input_shape=(1, 576, 1024)))
33
+ *iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
34
+ # Not enough memory
35
+ # *iter_llava_configs("llava-hf/llava-1.5-13b-hf"),
36
]
37
38
28
-def as_dict(vision_language_config: VisionLanguageConfig) -> Dict:
39
+def as_dict(vlm_config: VisionLanguageConfig) -> Dict[str, Any]:
40
"""Flatten vision language config to pure args.
41
42
Compatible with what llm entrypoint expects.
43
"""
44
result = {}
34
- for field in fields(vision_language_config):
35
- value = getattr(vision_language_config, field.name)
45
+ for field in fields(vlm_config):
46
+ value = getattr(vlm_config, field.name)
47
if isinstance(value, Enum):
48
result[field.name] = value.name.lower()
49
elif isinstance(value, tuple):
50
result[field.name] = ",".join([str(item) for item in value])
51
else:
52
result[field.name] = value
53
+
54
+ result["disable_image_processor"] = vlm_config.image_processor is None
55
+
56
return result
57
58
@@ -67,18 +81,19 @@ def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
81
@pytest.mark.parametrize("dtype", ["half"])
82
@pytest.mark.parametrize("max_tokens", [128])
83
def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
70
- vllm_image_prompts, vllm_images, model_and_config: tuple,
71
- dtype: str, max_tokens: int, worker_use_ray: bool) -> None:
84
+ vllm_image_prompts, vllm_images, model_and_config, dtype: str,
85
+ max_tokens: int, worker_use_ray: bool) -> None:
86
"""Inference result should be the same between hf and vllm.
87
88
All the image fixtures for the test is under tests/images.
75
- For huggingface runner, we provide the raw images as input.
76
- For vllm runner, we provide image tensors and corresponding
89
+ For huggingface runner, we provide the PIL images as input.
90
+ For vllm runner, we provide MultiModalData objects and corresponding
91
vision language config as input.
92
Note, the text input is also adjusted to abide by vllm contract.
93
The text output is sanitized to be able to compare with hf.
94
"""
95
model_id, vision_language_config = model_and_config
96
+
97
hf_model = hf_runner(model_id, dtype=dtype)
98
hf_outputs = hf_model.generate_greedy(hf_image_prompts,
99
max_tokens,
@@ -88,6 +103,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
103
vllm_model = vllm_runner(model_id,
104
dtype=dtype,
105
worker_use_ray=worker_use_ray,
106
+ enforce_eager=True,
107
**as_dict(vision_language_config))
108
vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
109
max_tokens,
@@ -105,3 +121,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
121
f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
122
assert hf_output_ids == vllm_output_ids, (
123
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
124
+
125
+
126
+# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
127
+# (Requires multiple GPUs)
tests/multimodal/__init__.py
tests/multimodal/test_processor.py
new
+98
@@ -0,0 +1,98 @@
1
+import numpy as np
2
+import pytest
3
+from transformers import CLIPImageProcessor
4
+
5
+from vllm.config import ModelConfig, VisionLanguageConfig
6
+from vllm.multimodal import MULTIMODAL_REGISTRY
7
+from vllm.multimodal.image import ImagePixelData
8
+
9
+
10
+@pytest.mark.parametrize("dtype", ["half", "bfloat16", "float"])
11
+def test_clip_image_processor(hf_images, dtype):
12
+ MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
13
+ IMAGE_HEIGHT = IMAGE_WIDTH = 33
14
+
15
+ hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
16
+ assert isinstance(hf_processor, CLIPImageProcessor)
17
+
18
+ model_config = ModelConfig(
19
+ model=MODEL_NAME,
20
+ tokenizer=MODEL_NAME,
21
+ tokenizer_mode="auto",
22
+ trust_remote_code=False,
23
+ seed=0,
24
+ dtype=dtype,
25
+ revision=None,
26
+ )
27
+ vlm_config = VisionLanguageConfig(
28
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
29
+ image_token_id=32000,
30
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
31
+ image_feature_size=576,
32
+ image_processor=MODEL_NAME,
33
+ image_processor_revision=None,
34
+ )
35
+
36
+ for image in hf_images:
37
+ hf_result = hf_processor.preprocess(
38
+ image,
39
+ return_tensors="np",
40
+ )
41
+ vllm_result = MULTIMODAL_REGISTRY.process_input(
42
+ ImagePixelData(image),
43
+ model_config=model_config,
44
+ vlm_config=vlm_config,
45
+ )
46
+
47
+ assert hf_result.keys() == vllm_result.keys()
48
+ for key, hf_arr in hf_result.items():
49
+ vllm_arr: np.ndarray = vllm_result[key].numpy()
50
+
51
+ assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
52
+ assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
53
+
54
+
55
+@pytest.mark.parametrize("dtype", ["float"])
56
+def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
57
+ MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
58
+ IMAGE_HEIGHT = IMAGE_WIDTH = 33
59
+
60
+ model_config = ModelConfig(
61
+ model=MODEL_NAME,
62
+ tokenizer=MODEL_NAME,
63
+ tokenizer_mode="auto",
64
+ trust_remote_code=False,
65
+ seed=0,
66
+ dtype=dtype,
67
+ revision=None,
68
+ )
69
+ vlm_config = VisionLanguageConfig(
70
+ image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
71
+ image_token_id=32000,
72
+ image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
73
+ image_feature_size=576,
74
+ image_processor=MODEL_NAME,
75
+ image_processor_revision=None,
76
+ )
77
+
78
+ for image, tensor in zip(hf_images, vllm_image_tensors):
79
+ image_result = MULTIMODAL_REGISTRY.process_input(
80
+ ImagePixelData(image),
81
+ model_config=model_config,
82
+ vlm_config=vlm_config,
83
+ )
84
+ tensor_result = MULTIMODAL_REGISTRY.process_input(
85
+ ImagePixelData(tensor),
86
+ model_config=model_config,
87
+ vlm_config=vlm_config,
88
+ )
89
+
90
+ assert image_result.keys() == tensor_result.keys()
91
+ for key, image_arr in image_result.items():
92
+ tensor_arr: np.ndarray = tensor_result[key].numpy()
93
+
94
+ assert image_arr.shape == tensor_arr.shape, f"Failed for key={key}"
95
+
96
+ # The examples in PR#3042 have slightly different preprocessing from
97
+ # HuggingFace's LlavaProcessor, causing the test to fail.
98
+ # assert np.allclose(image_arr, tensor_arr), f"Failed for key={key}"
tests/spec_decode/e2e/conftest.py
+2
-1
@@ -18,9 +18,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18
from vllm.engine.async_llm_engine import AsyncLLMEngine
19
from vllm.lora.request import LoRARequest
20
from vllm.model_executor.utils import set_random_seed
21
+from vllm.multimodal import MultiModalData
22
from vllm.outputs import RequestOutput
23
from vllm.sampling_params import SamplingParams
23
-from vllm.sequence import Logprob, MultiModalData
24
+from vllm.sequence import Logprob
25
from vllm.usage.usage_lib import UsageContext
26
from vllm.utils import Counter, random_uuid
27
tests/tokenization/test_image_processor.py
new
+20
@@ -0,0 +1,20 @@
1
+import pytest
2
+from transformers.image_processing_utils import BaseImageProcessor
3
+
4
+from vllm.transformers_utils.image_processor import get_image_processor
5
+
6
+IMAGE_PROCESSOR_NAMES = [
7
+ "llava-hf/llava-1.5-7b-hf",
8
+ "llava-hf/llava-v1.6-34b-hf",
9
+]
10
+
11
+
12
+@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
13
+def test_image_processor_revision(processor_name: str):
14
+ # Assume that "main" branch always exists
15
+ image_processor = get_image_processor(processor_name, revision="main")
16
+ assert isinstance(image_processor, BaseImageProcessor)
17
+
18
+ # Assume that "never" branch always does not exist
19
+ with pytest.raises(OSError, match='not a valid git identifier'):
20
+ get_image_processor(processor_name, revision="never")
vllm/config.py
+4
-2
@@ -1094,10 +1094,12 @@ class VisionLanguageConfig:
1094
# worst case scenario (biggest supported resolution).
1095
image_input_shape: tuple
1096
image_feature_size: int
1097
+ # The image processor to load from HuggingFace
1098
+ image_processor: Optional[str]
1099
+ image_processor_revision: Optional[str]
1100
1101
@classmethod
1099
- def get_image_input_enum_type(
1100
- cls, value: str) -> "VisionLanguageConfig.ImageInputType":
1102
+ def get_image_input_enum_type(cls, value: str) -> ImageInputType:
1103
"""Get the image input type from a string."""
1104
try:
1105
return cls.ImageInputType[value.upper()]
vllm/engine/arg_utils.py
+79
-29
@@ -1,6 +1,7 @@
1
import argparse
2
import dataclasses
3
import json
4
+import warnings
5
from dataclasses import dataclass
6
from typing import List, Optional, Tuple, Union
7
@@ -80,6 +81,10 @@ class EngineArgs:
81
image_token_id: Optional[int] = None
82
image_input_shape: Optional[str] = None
83
image_feature_size: Optional[int] = None
84
+ image_processor: Optional[str] = None
85
+ image_processor_revision: Optional[str] = None
86
+ disable_image_processor: bool = False
87
+
88
scheduler_delay_factor: float = 0.0
89
enable_chunked_prefill: bool = False
90
@@ -98,6 +103,53 @@ class EngineArgs:
103
if self.tokenizer is None:
104
self.tokenizer = self.model
105
106
+ @staticmethod
107
+ def add_cli_args_for_vlm(
108
+ parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
109
+ parser.add_argument('--image-input-type',
110
+ type=nullable_str,
111
+ default=None,
112
+ choices=[
113
+ t.name.lower()
114
+ for t in VisionLanguageConfig.ImageInputType
115
+ ],
116
+ help=('The image input type passed into vLLM.'))
117
+ parser.add_argument('--image-token-id',
118
+ type=int,
119
+ default=None,
120
+ help=('Input id for image token.'))
121
+ parser.add_argument(
122
+ '--image-input-shape',
123
+ type=nullable_str,
124
+ default=None,
125
+ help=('The biggest image input shape (worst for memory footprint) '
126
+ 'given an input type. Only used for vLLM\'s profile_run.'))
127
+ parser.add_argument(
128
+ '--image-feature-size',
129
+ type=int,
130
+ default=None,
131
+ help=('The image feature size along the context dimension.'))
132
+ parser.add_argument(
133
+ '--image-processor',
134
+ type=str,
135
+ default=EngineArgs.image_processor,
136
+ help='Name or path of the huggingface image processor to use. '
137
+ 'If unspecified, model name or path will be used.')
138
+ parser.add_argument(
139
+ '--image-processor-revision',
140
+ type=str,
141
+ default=None,
142
+ help='Revision of the huggingface image processor version to use. '
143
+ 'It can be a branch name, a tag name, or a commit id. '
144
+ 'If unspecified, will use the default version.')
145
+ parser.add_argument(
146
+ '--disable-image-processor',
147
+ action='store_true',
148
+ help='Disables the use of image processor, even if one is defined '
149
+ 'for the model on huggingface.')
150
+
151
+ return parser
152
+
153
@staticmethod
154
def add_cli_args(
155
parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
@@ -113,7 +165,8 @@ class EngineArgs:
165
'--tokenizer',
166
type=nullable_str,
167
default=EngineArgs.tokenizer,
116
- help='Name or path of the huggingface tokenizer to use.')
168
+ help='Name or path of the huggingface tokenizer to use. '
169
+ 'If unspecified, model name or path will be used.')
170
parser.add_argument(
171
'--skip-tokenizer-init',
172
action='store_true',
@@ -136,9 +189,9 @@ class EngineArgs:
189
'--tokenizer-revision',
190
type=nullable_str,
191
default=None,
139
- help='The specific tokenizer version to use. It can be a branch '
140
- 'name, a tag name, or a commit id. If unspecified, will use '
141
- 'the default version.')
192
+ help='Revision of the huggingface tokenizer to use. '
193
+ 'It can be a branch name, a tag name, or a commit id. '
194
+ 'If unspecified, will use the default version.')
195
parser.add_argument(
196
'--tokenizer-mode',
197
type=str,
@@ -445,31 +498,10 @@ class EngineArgs:
498
default=EngineArgs.device,
499
choices=["auto", "cuda", "neuron", "cpu"],
500
help='Device type for vLLM execution.')
501
+
502
# Related to Vision-language models such as llava
449
- parser.add_argument(
450
- '--image-input-type',
451
- type=nullable_str,
452
- default=None,
453
- choices=[
454
- t.name.lower() for t in VisionLanguageConfig.ImageInputType
455
- ],
456
- help=('The image input type passed into vLLM. '
457
- 'Should be one of "pixel_values" or "image_features".'))
458
- parser.add_argument('--image-token-id',
459
- type=int,
460
- default=None,
461
- help=('Input id for image token.'))
462
- parser.add_argument(
463
- '--image-input-shape',
464
- type=nullable_str,
465
- default=None,
466
- help=('The biggest image input shape (worst for memory footprint) '
467
- 'given an input type. Only used for vLLM\'s profile_run.'))
468
- parser.add_argument(
469
- '--image-feature-size',
470
- type=int,
471
- default=None,
472
- help=('The image feature size along the context dimension.'))
503
+ parser = EngineArgs.add_cli_args_for_vlm(parser)
504
+
505
parser.add_argument(
506
'--scheduler-delay-factor',
507
type=float,
@@ -488,7 +520,6 @@ class EngineArgs:
520
default=EngineArgs.speculative_model,
521
help=
522
'The name of the draft model to be used in speculative decoding.')
491
-
523
parser.add_argument(
524
'--num-speculative-tokens',
525
type=int,
@@ -666,12 +697,27 @@ class EngineArgs:
697
raise ValueError(
698
'Specify `image_token_id`, `image_input_shape` and '
699
'`image_feature_size` together with `image_input_type`.')
700
+
701
+ if self.image_processor is None:
702
+ self.image_processor = self.model
703
+ if self.disable_image_processor:
704
+ if self.image_processor != self.model:
705
+ warnings.warn(
706
+ "You've specified an image processor "
707
+ f"({self.image_processor}) but also disabled "
708
+ "it via `--disable-image-processor`.",
709
+ stacklevel=2)
710
+
711
+ self.image_processor = None
712
+
713
vision_language_config = VisionLanguageConfig(
714
image_input_type=VisionLanguageConfig.
715
get_image_input_enum_type(self.image_input_type),
716
image_token_id=self.image_token_id,
717
image_input_shape=str_to_int_tuple(self.image_input_shape),
718
image_feature_size=self.image_feature_size,
719
+ image_processor=self.image_processor,
720
+ image_processor_revision=self.image_processor_revision,
721
)
722
else:
723
vision_language_config = None
@@ -734,3 +780,7 @@ def _engine_args_parser():
780
def _async_engine_args_parser():
781
return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
782
async_args_only=True)
783
+
784
+
785
+def _vlm_engine_args_parser():
786
+ return EngineArgs.add_cli_args_for_vlm(argparse.ArgumentParser())
vllm/entrypoints/llm.py
+2
-23
@@ -14,7 +14,6 @@ from vllm.lora.request import LoRARequest
14
from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15
from vllm.pooling_params import PoolingParams
16
from vllm.sampling_params import SamplingParams
17
-from vllm.sequence import MultiModalData
17
from vllm.usage.usage_lib import UsageContext
18
from vllm.utils import Counter, deprecate_kwargs
19
@@ -164,7 +163,6 @@ class LLM:
163
prompt_token_ids: Optional[List[int]] = None,
164
use_tqdm: bool = True,
165
lora_request: Optional[LoRARequest] = None,
167
- multi_modal_data: Optional[MultiModalData] = None,
166
) -> List[RequestOutput]:
167
...
168
@@ -177,7 +175,6 @@ class LLM:
175
prompt_token_ids: Optional[List[List[int]]] = None,
176
use_tqdm: bool = True,
177
lora_request: Optional[LoRARequest] = None,
180
- multi_modal_data: Optional[MultiModalData] = None,
178
) -> List[RequestOutput]:
179
...
180
@@ -191,7 +188,6 @@ class LLM:
188
prompt_token_ids: List[int],
189
use_tqdm: bool = True,
190
lora_request: Optional[LoRARequest] = None,
194
- multi_modal_data: Optional[MultiModalData] = None,
191
) -> List[RequestOutput]:
192
...
193
@@ -205,7 +201,6 @@ class LLM:
201
prompt_token_ids: List[List[int]],
202
use_tqdm: bool = True,
203
lora_request: Optional[LoRARequest] = None,
208
- multi_modal_data: Optional[MultiModalData] = None,
204
) -> List[RequestOutput]:
205
...
206
@@ -217,7 +212,6 @@ class LLM:
212
prompt_token_ids: Union[List[int], List[List[int]]],
213
use_tqdm: bool = True,
214
lora_request: Optional[LoRARequest] = None,
220
- multi_modal_data: Optional[MultiModalData] = None,
215
) -> List[RequestOutput]:
216
...
217
@@ -236,7 +230,6 @@ class LLM:
230
231
@deprecate_kwargs("prompts",
232
"prompt_token_ids",
239
- "multi_modal_data",
233
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
234
additional_message="Please use the 'inputs' parameter "
235
"instead.")
@@ -249,7 +242,6 @@ class LLM:
242
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
243
use_tqdm: bool = True,
244
lora_request: Optional[LoRARequest] = None,
252
- multi_modal_data: Optional[MultiModalData] = None,
245
) -> List[RequestOutput]:
246
"""Generates the completions for the input prompts.
247
@@ -281,11 +273,10 @@ class LLM:
273
"LLM.generate() is only supported for generation models "
274
"(XForCausalLM).")
275
284
- if prompt_token_ids is not None or multi_modal_data is not None:
276
+ if prompt_token_ids is not None:
277
inputs = self._convert_v1_inputs(
278
prompts=cast(Optional[Union[str, List[str]]], prompts),
279
prompt_token_ids=prompt_token_ids,
288
- multi_modal_data=multi_modal_data,
280
)
281
else:
282
inputs = cast(
@@ -314,7 +305,6 @@ class LLM:
305
prompt_token_ids: Optional[List[int]] = None,
306
use_tqdm: bool = True,
307
lora_request: Optional[LoRARequest] = None,
317
- multi_modal_data: Optional[MultiModalData] = None,
308
) -> List[EmbeddingRequestOutput]:
309
...
310
@@ -327,7 +317,6 @@ class LLM:
317
prompt_token_ids: Optional[List[List[int]]] = None,
318
use_tqdm: bool = True,
319
lora_request: Optional[LoRARequest] = None,
330
- multi_modal_data: Optional[MultiModalData] = None,
320
) -> List[EmbeddingRequestOutput]:
321
...
322
@@ -341,7 +330,6 @@ class LLM:
330
prompt_token_ids: List[int],
331
use_tqdm: bool = True,
332
lora_request: Optional[LoRARequest] = None,
344
- multi_modal_data: Optional[MultiModalData] = None,
333
) -> List[EmbeddingRequestOutput]:
334
...
335
@@ -355,7 +343,6 @@ class LLM:
343
prompt_token_ids: List[List[int]],
344
use_tqdm: bool = True,
345
lora_request: Optional[LoRARequest] = None,
358
- multi_modal_data: Optional[MultiModalData] = None,
346
) -> List[EmbeddingRequestOutput]:
347
...
348
@@ -367,7 +354,6 @@ class LLM:
354
prompt_token_ids: Union[List[int], List[List[int]]],
355
use_tqdm: bool = True,
356
lora_request: Optional[LoRARequest] = None,
370
- multi_modal_data: Optional[MultiModalData] = None,
357
) -> List[EmbeddingRequestOutput]:
358
...
359
@@ -386,7 +372,6 @@ class LLM:
372
373
@deprecate_kwargs("prompts",
374
"prompt_token_ids",
389
- "multi_modal_data",
375
is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
376
additional_message="Please use the 'inputs' parameter "
377
"instead.")
@@ -399,7 +384,6 @@ class LLM:
384
prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
385
use_tqdm: bool = True,
386
lora_request: Optional[LoRARequest] = None,
402
- multi_modal_data: Optional[MultiModalData] = None,
387
) -> List[EmbeddingRequestOutput]:
388
"""Generates the completions for the input prompts.
389
@@ -430,11 +414,10 @@ class LLM:
414
"LLM.encode() is only supported for embedding models (XModel)."
415
)
416
433
- if prompt_token_ids is not None or multi_modal_data is not None:
417
+ if prompt_token_ids is not None:
418
inputs = self._convert_v1_inputs(
419
prompts=cast(Optional[Union[str, List[str]]], prompts),
420
prompt_token_ids=prompt_token_ids,
437
- multi_modal_data=multi_modal_data,
421
)
422
else:
423
inputs = cast(
@@ -459,7 +442,6 @@ class LLM:
442
self,
443
prompts: Optional[Union[str, List[str]]],
444
prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
462
- multi_modal_data: Optional[MultiModalData],
445
):
446
# skip_tokenizer_init is now checked in engine
447
@@ -499,9 +481,6 @@ class LLM:
481
else:
482
raise AssertionError
483
502
- if multi_modal_data is not None:
503
- item["multi_modal_data"] = multi_modal_data
504
-
484
inputs.append(item)
485
486
return inputs
vllm/model_executor/models/llava.py
+46
-27
@@ -17,6 +17,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18
from vllm.model_executor.models.llama import LlamaModel
19
from vllm.model_executor.sampling_metadata import SamplingMetadata
20
+from vllm.multimodal import MULTIMODAL_REGISTRY
21
+from vllm.multimodal.image import get_dummy_image_data
22
from vllm.sequence import SamplerOutput
23
24
from .vlm_base import VisionLanguageModelBase
@@ -82,6 +84,9 @@ class LlavaImageFeatureInputs(TypedDict):
84
LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
85
86
87
+@MULTIMODAL_REGISTRY.register_image_feature_input()
88
+@MULTIMODAL_REGISTRY.register_image_pixel_input()
89
+@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
90
class LlavaForConditionalGeneration(VisionLanguageModelBase):
91
92
def __init__(self,
@@ -131,30 +136,41 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
136
return data
137
138
def _parse_and_validate_image_input(
134
- self, data: object) -> Optional[LlavaImageInputs]:
139
+ self, **kwargs: object) -> Optional[LlavaImageInputs]:
140
+ pixel_values = kwargs.pop("pixel_values", None)
141
+ image_features = kwargs.pop("image_features", None)
142
+
143
expected_input_type = self.vision_language_config.image_input_type
144
ImageInputType = VisionLanguageConfig.ImageInputType
145
138
- if data is None:
139
- return None
140
-
146
if expected_input_type == ImageInputType.PIXEL_VALUES:
142
- if not isinstance(data, torch.Tensor):
143
- raise TypeError("Image pixel vector should be a tensor, "
144
- f"but received type: {type(data)}")
147
+ if image_features is not None:
148
+ raise ValueError(
149
+ "Expected pixel values but got image features")
150
+ if pixel_values is None:
151
+ return None
152
+
153
+ if not isinstance(pixel_values, torch.Tensor):
154
+ raise ValueError("Incorrect type of pixel values")
155
156
return LlavaImagePixelInputs(
157
type="pixel_values",
148
- data=self._validate_image_data(data),
158
+ data=self._validate_image_data(pixel_values),
159
)
150
- elif expected_input_type == ImageInputType.IMAGE_FEATURES:
151
- if not isinstance(data, torch.Tensor):
152
- raise TypeError("Image feature vector should be a tensor, "
153
- f"but received type: {type(data)}")
160
+
161
+ if expected_input_type == ImageInputType.IMAGE_FEATURES:
162
+ if pixel_values is not None:
163
+ raise ValueError(
164
+ "Expected image features but got pixel values")
165
+ if image_features is None:
166
+ return None
167
+
168
+ if not isinstance(image_features, torch.Tensor):
169
+ raise ValueError("Incorrect type of image features")
170
171
return LlavaImageFeatureInputs(
172
type="image_features",
157
- data=self._validate_image_data(data),
173
+ data=self._validate_image_data(image_features),
174
)
175
176
return None
@@ -201,12 +217,14 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
217
218
return self.multi_modal_projector(image_features)
219
204
- def forward(self,
205
- input_ids: torch.Tensor,
206
- positions: torch.Tensor,
207
- kv_caches: List[torch.Tensor],
208
- attn_metadata: AttentionMetadata,
209
- image_input: Optional[torch.Tensor] = None) -> SamplerOutput:
220
+ def forward(
221
+ self,
222
+ input_ids: torch.Tensor,
223
+ positions: torch.Tensor,
224
+ kv_caches: List[torch.Tensor],
225
+ attn_metadata: AttentionMetadata,
226
+ **kwargs: object,
227
+ ) -> SamplerOutput:
228
"""Run forward pass for Llava 1.5.
229
230
One key thing to understand is the `input_ids` already accounts for the
@@ -227,10 +245,10 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
245
This way, the `positions` and `attn_metadata` are consistent
246
with the `input_ids`.
247
230
- The model takes two types of image inputs:
248
+ The model takes two types of image inputs:
249
PIXEL_VALUES and IMAGE_FEATURES.
250
The following shows how each maps to huggingface implementation.
233
- PIXEL_VALUES:
251
+ PIXEL_VALUES:
252
- https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L353
253
IMAGE_FEATURES:
254
- https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L430
@@ -239,14 +257,15 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
257
Args:
258
input_ids: Flattened (concatenated) input_ids corresponding to a
259
batch.
242
- image_input: A batch of image inputs.
243
- For PIXEL_VALUES, expecting [1, 3, 336, 336].
244
- For IMAGE_FEATURES, expecting [1, 576, 1024].
260
+ pixel_values: For PIXEL_VALUES, expects a batch with shape
261
+ [1, 3, 336, 336].
262
+ image_features: For IMAGE_FEATURES, expects a batch with shape
263
+ [1, 576, 1024].
264
"""
246
- parsed_image_input = self._parse_and_validate_image_input(image_input)
265
+ image_input = self._parse_and_validate_image_input(**kwargs)
266
248
- if parsed_image_input is not None:
249
- vision_embeddings = self._process_image_input(parsed_image_input)
267
+ if image_input is not None:
268
+ vision_embeddings = self._process_image_input(image_input)
269
inputs_embeds = self.language_model.get_input_embeddings(input_ids)
270
271
inputs_embeds = _merge_vision_embeddings(
vllm/multimodal/__init__.py
new
+7
@@ -0,0 +1,7 @@
1
+from .base import MultiModalData, MultiModalPlugin
2
+from .registry import MULTIMODAL_REGISTRY, MultiModalRegistry
3
+
4
+__all__ = [
5
+ "MultiModalData", "MultiModalPlugin", "MULTIMODAL_REGISTRY",
6
+ "MultiModalRegistry"
7
+]
vllm/multimodal/base.py
new
+126
@@ -0,0 +1,126 @@
1
+from abc import ABC, abstractmethod
2
+from typing import (TYPE_CHECKING, Callable, Dict, Generic, Optional, Type,
3
+ TypeVar)
4
+
5
+from vllm.config import ModelConfig, VisionLanguageConfig
6
+from vllm.logger import init_logger
7
+
8
+if TYPE_CHECKING:
9
+ import torch
10
+ from torch import nn
11
+
12
+logger = init_logger(__name__)
13
+
14
+
15
+class MultiModalData:
16
+ """
17
+ Base class that contains multi-modal data.
18
+
19
+ To add a new modality, add a new file under ``multimodal`` directory.
20
+
21
+ In this new file, subclass :class:`~MultiModalData` and
22
+ :class:`~MultiModalPlugin`.
23
+
24
+ Finally, register the new plugin to
25
+ :const:`vllm.multimodal.MULTIMODAL_REGISTRY`.
26
+ This enables models to call :meth:`MultiModalRegistry.register_input` for
27
+ the new modality.
28
+ """
29
+ pass
30
+
31
+
32
+D = TypeVar("D", bound=MultiModalData)
33
+N = TypeVar("N", bound=Type["nn.Module"])
34
+
35
+MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
36
+ Dict[str, "torch.Tensor"]]
37
+"""Return a dictionary to be passed as keyword arguments to
38
+:meth:`torch.nn.Module.forward`. This is similar in concept to tokenizers
39
+and processors in HuggingFace Transformers."""
40
+
41
+
42
+class MultiModalPlugin(ABC, Generic[D]):
43
+ """
44
+ Base class that defines data processing logic for a specific modality.
45
+
46
+ In particular, we adopt a registry pattern to dispatch data processing
47
+ according to the model being used (considering that different models may
48
+ process the same data differently). This registry is in turn used by
49
+ :class:`~MultiModalRegistry` which acts at a higher level
50
+ (i.e., the modality of the data).
51
+ """
52
+
53
+ @classmethod
54
+ def get_model_cls(cls, model_config: ModelConfig) -> Type["nn.Module"]:
55
+ # Avoid circular import
56
+ from vllm.model_executor.model_loader import get_model_architecture
57
+
58
+ return get_model_architecture(model_config)[0]
59
+
60
+ def __init__(self) -> None:
61
+ self._input_processors: Dict[Type["nn.Module"],
62
+ MultiModalInputProcessor[D]] = {}
63
+
64
+ @abstractmethod
65
+ def get_data_type(self) -> Type[D]:
66
+ """
67
+ Get the modality (subclass of :class:`~MultiModalData`) served by
68
+ this plugin.
69
+ """
70
+ raise NotImplementedError
71
+
72
+ @abstractmethod
73
+ def _default_input_processor(
74
+ self, data: D, model_config: ModelConfig,
75
+ vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
76
+ """Return a dictionary to be passed as keyword arguments to
77
+ :meth:`torch.nn.Module.forward`. This is similar in concept to
78
+ tokenizers and processors in HuggingFace Transformers.
79
+ """
80
+ raise NotImplementedError
81
+
82
+ def register_input_processor(self,
83
+ processor: Optional[
84
+ MultiModalInputProcessor[D]] = None):
85
+ """
86
+ Register an input processor to a model class.
87
+
88
+ When the model receives input data that matches the modality served by
89
+ this plugin (see :meth:`get_data_type`), the provided input processor is
90
+ applied to preprocess the data. If `None` is provided, then the default
91
+ input processor is applied instead.
92
+ """
93
+
94
+ def wrapper(model_cls: N) -> N:
95
+ if model_cls in self._input_processors:
96
+ logger.warning(
97
+ "Model class %s already has an input processor "
98
+ "registered to %s. It is overwritten by the new one.",
99
+ model_cls, self)
100
+
101
+ self._input_processors[model_cls] = processor \
102
+ or self._default_input_processor
103
+
104
+ return model_cls
105
+
106
+ return wrapper
107
+
108
+ def process_input(
109
+ self, data: D, model_config: ModelConfig,
110
+ vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
111
+ """
112
+ Apply an input processor to a :class:`~MultiModalData` instance passed
113
+ to the model.
114
+
115
+ The model is identified by ``model_config``. ``vlm_config`` is
116
+ for compatibility purposes and may be merged into ``model_config``
117
+ in the near future.
118
+ """
119
+ model_cls = self.get_model_cls(model_config)
120
+
121
+ processor = self._input_processors.get(model_cls)
122
+ if processor is None:
123
+ raise KeyError(f"No input processor in {self} is registered for "
124
+ f"model class {model_cls.__name__}.")
125
+
126
+ return processor(data, model_config, vlm_config)
vllm/multimodal/image.py
new
+141
@@ -0,0 +1,141 @@
1
+from typing import Dict, Tuple, Type, Union
2
+
3
+import torch
4
+from PIL import Image
5
+
6
+from vllm.config import ModelConfig, VisionLanguageConfig
7
+from vllm.logger import init_logger
8
+from vllm.sequence import SequenceData
9
+from vllm.transformers_utils.image_processor import cached_get_image_processor
10
+
11
+from .base import MultiModalData, MultiModalPlugin
12
+
13
+logger = init_logger(__name__)
14
+
15
+
16
+def _get_dummy_seq_data(seq_len: int,
17
+ vlm_config: VisionLanguageConfig) -> SequenceData:
18
+ # NOTE: We assume that <image> token is repeated `image_feature_size` times
19
+ # and then concatenated with the text prompt
20
+ # TODO: Enable other ways of inserting the image into the prompt
21
+
22
+ token_ids = [vlm_config.image_token_id] * vlm_config.image_feature_size
23
+ token_ids += [0] * (seq_len - vlm_config.image_feature_size)
24
+
25
+ return SequenceData(token_ids)
26
+
27
+
28
+def _get_dummy_values(vlm_config: VisionLanguageConfig) -> torch.Tensor:
29
+ if vlm_config.image_processor is None:
30
+ values_dtype = torch.float16
31
+ else:
32
+ values_dtype = torch.uint8
33
+
34
+ return torch.zeros(vlm_config.image_input_shape, dtype=values_dtype)
35
+
36
+
37
+def get_dummy_image_data(
38
+ seq_len: int,
39
+ model_config: ModelConfig,
40
+ vlm_config: VisionLanguageConfig,
41
+) -> Tuple[SequenceData, MultiModalData]:
42
+ """Standard dummy data factory for image data (to be used in
43
+ :meth:`vlm.multimodal.MultiModalRegistry.register_dummy_data`)."""
44
+ seq_data = _get_dummy_seq_data(seq_len, vlm_config)
45
+ values = _get_dummy_values(vlm_config)
46
+
47
+ config_input_type = vlm_config.image_input_type
48
+ ImageInputType = VisionLanguageConfig.ImageInputType
49
+
50
+ fake_mm_data: MultiModalData
51
+ if config_input_type == ImageInputType.PIXEL_VALUES:
52
+ fake_mm_data = ImagePixelData(values)
53
+ elif config_input_type == ImageInputType.IMAGE_FEATURES:
54
+ fake_mm_data = ImageFeatureData(values)
55
+ else:
56
+ raise NotImplementedError
57
+
58
+ return seq_data, fake_mm_data
59
+
60
+
61
+class ImagePixelData(MultiModalData):
62
+ """
63
+ The pixel data of an image. Can be one of:
64
+
65
+ - :class:``PIL.Image``: An image object. Requires that a HuggingFace
66
+ processor is available to the model.
67
+ - :class:``torch.Tensor``: The raw pixel data which is passed to the model
68
+ without additional pre-processing.
69
+ """
70
+
71
+ def __init__(self, image: Union[Image.Image, torch.Tensor]) -> None:
72
+ if isinstance(image, Image.Image):
73
+ # So that this class can be created inside the Image context manager
74
+ image.load()
75
+
76
+ self.image = image
77
+
78
+
79
+class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
80
+
81
+ def get_data_type(self) -> Type[ImagePixelData]:
82
+ return ImagePixelData
83
+
84
+ def _get_hf_image_processor(self, model_config: ModelConfig,
85
+ vlm_config: VisionLanguageConfig):
86
+ if vlm_config is None or vlm_config.image_processor is None:
87
+ return None
88
+
89
+ return cached_get_image_processor(
90
+ vlm_config.image_processor,
91
+ trust_remote_code=model_config.trust_remote_code,
92
+ revision=vlm_config.image_processor_revision,
93
+ )
94
+
95
+ def _default_input_processor(
96
+ self, data: ImagePixelData, model_config: ModelConfig,
97
+ vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
98
+ image = data.image
99
+ image_processor = self._get_hf_image_processor(model_config,
100
+ vlm_config)
101
+
102
+ if isinstance(image, Image.Image):
103
+ if image_processor is None:
104
+ raise RuntimeError("No HuggingFace processor is available"
105
+ "to process the image object")
106
+ try:
107
+ return image_processor.preprocess(image, return_tensors="pt") \
108
+ .to(model_config.dtype).data
109
+ except Exception:
110
+ logger.error("Failed to process image (%s)", image)
111
+ raise
112
+ elif isinstance(image, torch.Tensor):
113
+ pixel_values = image.to(model_config.dtype)
114
+
115
+ return {"pixel_values": pixel_values}
116
+
117
+ raise TypeError(f"Invalid image type: {type(image)}")
118
+
119
+
120
+class ImageFeatureData(MultiModalData):
121
+ """
122
+ The feature vector of an image, passed directly to the model.
123
+
124
+ This should be the output of the vision tower.
125
+ """
126
+
127
+ def __init__(self, image_features: torch.Tensor) -> None:
128
+ self.image_features = image_features
129
+
130
+
131
+class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
132
+
133
+ def get_data_type(self) -> Type[ImageFeatureData]:
134
+ return ImageFeatureData
135
+
136
+ def _default_input_processor(
137
+ self, data: ImageFeatureData, model_config: ModelConfig,
138
+ vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
139
+ image_features = data.image_features.to(model_config.dtype)
140
+
141
+ return {"image_features": image_features}
vllm/multimodal/registry.py
new
+156
@@ -0,0 +1,156 @@
1
+import functools
2
+from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence,
3
+ Tuple, Type, TypeVar)
4
+
5
+from vllm.config import ModelConfig, VisionLanguageConfig
6
+from vllm.logger import init_logger
7
+
8
+from .base import MultiModalData, MultiModalPlugin
9
+from .image import (ImageFeatureData, ImageFeaturePlugin, ImagePixelData,
10
+ ImagePixelPlugin)
11
+
12
+if TYPE_CHECKING:
13
+ import torch
14
+ from torch import nn
15
+
16
+ from vllm.sequence import SequenceData
17
+
18
+logger = init_logger(__name__)
19
+
20
+D = TypeVar("D", bound=MultiModalData)
21
+N = TypeVar("N", bound=Type["nn.Module"])
22
+
23
+MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
24
+ Dict[str, "torch.Tensor"]]
25
+MultiModalDummyFactory = Callable[[int, ModelConfig, VisionLanguageConfig],
26
+ Tuple["SequenceData", MultiModalData]]
27
+
28
+
29
+class MultiModalRegistry:
30
+ """
31
+ This registry is used by model runners to dispatch data processing
32
+ according to its modality and the target model.
33
+ """
34
+
35
+ DEFAULT_PLUGINS = (ImageFeaturePlugin(), ImagePixelPlugin())
36
+
37
+ def __init__(self,
38
+ *,
39
+ plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS
40
+ ) -> None:
41
+ self._plugins_by_data_type = {p.get_data_type(): p for p in plugins}
42
+ self._dummy_factories_by_model_type: Dict[Type["nn.Module"],
43
+ MultiModalDummyFactory] = {}
44
+
45
+ def register_plugin(self, plugin: MultiModalPlugin[Any]) -> None:
46
+ data_type = plugin.get_data_type()
47
+
48
+ if data_type in self._plugins_by_data_type:
49
+ logger.warning(
50
+ "A plugin is already registered for data type %s, "
51
+ "and will be overwritten by the new plugin %s.", data_type,
52
+ plugin)
53
+
54
+ self._plugins_by_data_type[data_type] = plugin
55
+
56
+ def _get_plugin_for_data_type(self, data_type: Type[MultiModalData]):
57
+ for typ in data_type.mro():
58
+ plugin = self._plugins_by_data_type.get(typ)
59
+ if plugin is not None:
60
+ return plugin
61
+
62
+ msg = f"Unknown multi-modal data type: {data_type}"
63
+ raise NotImplementedError(msg)
64
+
65
+ def register_dummy_data(self, factory: MultiModalDummyFactory):
66
+ """
67
+ Register a dummy data factory to a model class.
68
+
69
+ During memory profiling, the provided function is invoked to create
70
+ dummy data to be inputted into the model. The modality and shape of
71
+ the dummy data should be an upper bound of what the model would receive
72
+ at inference time.
73
+ """
74
+
75
+ def wrapper(model_cls: N) -> N:
76
+ if model_cls in self._dummy_factories_by_model_type:
77
+ logger.warning(
78
+ "Model class %s already has dummy data "
79
+ "registered to %s. It is overwritten by the new one.",
80
+ model_cls, self)
81
+
82
+ self._dummy_factories_by_model_type[model_cls] = factory
83
+
84
+ return model_cls
85
+
86
+ return wrapper
87
+
88
+ def dummy_data_for_profiling(self, seq_len: int, model_config: ModelConfig,
89
+ vlm_config: VisionLanguageConfig):
90
+ """Create dummy data for memory profiling."""
91
+ model_cls = MultiModalPlugin.get_model_cls(model_config)
92
+ dummy_factory = self._dummy_factories_by_model_type.get(model_cls)
93
+ if dummy_factory is None:
94
+ msg = f"No dummy data defined for model class: {model_cls}"
95
+ raise NotImplementedError(msg)
96
+
97
+ return dummy_factory(seq_len, model_config, vlm_config)
98
+
99
+ def register_input(
100
+ self,
101
+ data_type: Type[D],
102
+ processor: Optional[MultiModalInputProcessor[D]] = None):
103
+ """
104
+ Register an input processor for a specific modality to a model class.
105
+
106
+ See :meth:`MultiModalPlugin.register_input_processor` for more details.
107
+ """
108
+ return self._get_plugin_for_data_type(data_type) \
109
+ .register_input_processor(processor)
110
+
111
+ def register_image_pixel_input(
112
+ self,
113
+ processor: Optional[
114
+ MultiModalInputProcessor[ImagePixelData]] = None):
115
+ """
116
+ Register an input processor for image pixel data to a model class.
117
+
118
+ See :meth:`MultiModalPlugin.register_input_processor` for more details.
119
+ """
120
+ return self.register_input(ImagePixelData, processor)
121
+
122
+ def register_image_feature_input(
123
+ self,
124
+ processor: Optional[
125
+ MultiModalInputProcessor[ImageFeatureData]] = None):
126
+ """
127
+ Register an input processor for image feature data to a model class.
128
+
129
+ See :meth:`MultiModalPlugin.register_input_processor` for more details.
130
+ """
131
+ return self.register_input(ImageFeatureData, processor)
132
+
133
+ def process_input(self, data: MultiModalData, model_config: ModelConfig,
134
+ vlm_config: VisionLanguageConfig):
135
+ """
136
+ Apply an input processor to a :class:`~MultiModalData` instance passed
137
+ to the model.
138
+
139
+ See :meth:`MultiModalPlugin.process_input` for more details.
140
+ """
141
+ return self._get_plugin_for_data_type(type(data)) \
142
+ .process_input(data, model_config, vlm_config)
143
+
144
+ def create_input_processor(self, model_config: ModelConfig,
145
+ vlm_config: VisionLanguageConfig):
146
+ """
147
+ Create an input processor (see :meth:`process_input`) for a
148
+ specific model.
149
+ """
150
+ return functools.partial(self.process_input,
151
+ model_config=model_config,
152
+ vlm_config=vlm_config)
153
+
154
+
155
+MULTIMODAL_REGISTRY = MultiModalRegistry()
156
+"""The global :class:`~MultiModalRegistry` which is used by model runners."""
vllm/sequence.py
+7
-25
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
5
from dataclasses import dataclass, field
6
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8
+import torch
9
+
10
from vllm.block import LogicalTokenBlock
11
from vllm.inputs import LLMInputs
12
from vllm.lora.request import LoRARequest
@@ -12,8 +14,7 @@ from vllm.pooling_params import PoolingParams
14
from vllm.sampling_params import SamplingParams
15
16
if TYPE_CHECKING:
15
- import torch
16
-
17
+ from vllm.multimodal import MultiModalData
18
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
19
20
@@ -398,25 +399,6 @@ class SequenceGroupState:
399
generator: Optional = None # type: ignore
400
401
401
-class MultiModalData:
402
- """Multi modal request.
403
-
404
- Args:
405
- type: The data type.
406
- data: The actual data.
407
- The required shape and semantic meaning of it depends on the vision
408
- language config of the hosted model.
409
- See `VisionLanguageConfig` in `config.py`.
410
- """
411
-
412
- class Type(enum.Enum):
413
- IMAGE = enum.auto()
414
-
415
- def __init__(self, type: Type, data: "torch.Tensor"):
416
- self.type = type
417
- self.data = data
418
-
419
-
402
class SequenceGroup:
403
"""A group of sequences that are generated from the same prompt.
404
@@ -473,7 +455,7 @@ class SequenceGroup:
455
return next(iter(self.seqs_dict.values())).prompt_token_ids
456
457
@property
476
- def multi_modal_data(self) -> Optional[MultiModalData]:
458
+ def multi_modal_data(self) -> Optional["MultiModalData"]:
459
# All sequences in the group should have the same multi-modal data.
460
# We use the multi-modal data of an arbitrary sequence.
461
return next(iter(self.seqs_dict.values())).multi_modal_data
@@ -655,7 +637,7 @@ class SequenceGroupMetadata:
637
lora_request: Optional[LoRARequest] = None,
638
computed_block_nums: Optional[List[int]] = None,
639
state: Optional[SequenceGroupState] = None,
658
- multi_modal_data: Optional[MultiModalData] = None,
640
+ multi_modal_data: Optional["MultiModalData"] = None,
641
encoder_seq_data: Optional[SequenceData] = None,
642
cross_block_table: Optional[List[int]] = None,
643
) -> None:
@@ -798,13 +780,13 @@ class SamplerOutput:
780
outputs: List[CompletionSequenceGroupOutput]
781
782
# On-device tensor containing probabilities of each token.
801
- sampled_token_probs: Optional["torch.Tensor"] = None
783
+ sampled_token_probs: Optional[torch.Tensor] = None
784
785
# On-device tensor containing the logprobs of each token.
786
logprobs: Optional["torch.Tensor"] = None
787
788
# On-device tensor containing the sampled token ids.
807
- sampled_token_ids: Optional["torch.Tensor"] = None
789
+ sampled_token_ids: Optional[torch.Tensor] = None
790
791
# Spec decode metrics populated by workers.
792
spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
vllm/transformers_utils/image_processor.py
new
+45
@@ -0,0 +1,45 @@
1
+from functools import lru_cache
2
+from typing import Optional
3
+
4
+from transformers import AutoImageProcessor
5
+from transformers.image_processing_utils import BaseImageProcessor
6
+
7
+from vllm.logger import init_logger
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+def get_image_processor(
13
+ processor_name: str,
14
+ *args,
15
+ trust_remote_code: bool = False,
16
+ revision: Optional[str] = None,
17
+ **kwargs,
18
+) -> BaseImageProcessor:
19
+ """Gets an image processor for the given model name via HuggingFace."""
20
+ try:
21
+ processor: BaseImageProcessor = AutoImageProcessor.from_pretrained(
22
+ processor_name,
23
+ *args,
24
+ trust_remote_code=trust_remote_code,
25
+ revision=revision,
26
+ **kwargs)
27
+ except ValueError as e:
28
+ # If the error pertains to the processor class not existing or not
29
+ # currently being imported, suggest using the --trust-remote-code flag.
30
+ # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
31
+ if not trust_remote_code:
32
+ err_msg = (
33
+ "Failed to load the image processor. If the image processor is "
34
+ "a custom processor not yet available in the HuggingFace "
35
+ "transformers library, consider setting "
36
+ "`trust_remote_code=True` in LLM or using the "
37
+ "`--trust-remote-code` flag in the CLI.")
38
+ raise RuntimeError(err_msg) from e
39
+ else:
40
+ raise e
41
+
42
+ return processor
43
+
44
+
45
+cached_get_image_processor = lru_cache(get_image_processor)
vllm/worker/cpu_model_runner.py
+37
-20
@@ -1,4 +1,5 @@
1
-from typing import List, Optional, Tuple
1
+from collections import defaultdict
2
+from typing import Dict, List, Optional, Tuple
3
4
import torch
5
from torch import nn
@@ -11,6 +12,7 @@ from vllm.distributed import broadcast_tensor_dict
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
15
+from vllm.multimodal import MULTIMODAL_REGISTRY
16
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17
from vllm.utils import make_tensor_with_pad
18
@@ -63,6 +65,16 @@ class CPUModelRunner:
65
self.block_size,
66
)
67
68
+ # Create processor for multi-modal data
69
+ if self.vision_language_config is not None:
70
+ self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
71
+ .create_input_processor(
72
+ self.model_config,
73
+ self.vision_language_config,
74
+ )
75
+ else:
76
+ self.multi_modal_input_processor = None
77
+
78
# Lazy initialization.
79
self.model: nn.Module # Set after init_Model
80
@@ -80,14 +92,15 @@ class CPUModelRunner:
92
def _prepare_prompt(
93
self,
94
seq_group_metadata_list: List[SequenceGroupMetadata],
83
- ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
84
- Optional[torch.Tensor]]:
95
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int], Dict[
96
+ str, torch.Tensor]]:
97
assert len(seq_group_metadata_list) > 0
98
input_tokens: List[int] = []
99
input_positions: List[int] = []
100
slot_mapping: List[int] = []
101
seq_lens: List[int] = []
90
- multi_modal_input_list: List[torch.Tensor] = []
102
+ multi_modal_kwargs_list: Dict[str,
103
+ List[torch.Tensor]] = defaultdict(list)
104
105
for seq_group_metadata in seq_group_metadata_list:
106
assert seq_group_metadata.is_prompt
@@ -108,9 +121,17 @@ class CPUModelRunner:
121
# is always the first token in the sequence.
122
input_positions.extend(list(range(computed_len, seq_len)))
123
111
- if seq_group_metadata.multi_modal_data:
112
- multi_modal_input_list.append(
113
- seq_group_metadata.multi_modal_data.data)
124
+ mm_data = seq_group_metadata.multi_modal_data
125
+ if mm_data is not None:
126
+ # Process multi-modal data
127
+ if self.multi_modal_input_processor is None:
128
+ raise ValueError(
129
+ "Multi-modal inputs are only supported by "
130
+ "vision language models.")
131
+
132
+ mm_kwargs = self.multi_modal_input_processor(mm_data)
133
+ for k, v in mm_kwargs.items():
134
+ multi_modal_kwargs_list[k].append(v)
135
136
# Compute the slot mapping.
137
block_table = seq_group_metadata.block_tables[seq_id]
@@ -134,14 +155,10 @@ class CPUModelRunner:
155
slot = block_number * self.block_size + block_offset
156
slot_mapping.append(slot)
157
137
- if multi_modal_input_list:
138
- assert self.vision_language_config, (
139
- "Multi-modal inputs are only supported by "
140
- "vision language models.")
141
- multi_modal_input = torch.cat(multi_modal_input_list,
142
- dim=0).to(self.device)
143
- else:
144
- multi_modal_input = None
158
+ multi_modal_kwargs = {
159
+ k: torch.cat(v, dim=0).to(self.device)
160
+ for k, v in multi_modal_kwargs_list.items()
161
+ }
162
163
num_prompt_tokens = len(input_tokens)
164
@@ -167,7 +184,7 @@ class CPUModelRunner:
184
slot_mapping=slot_mapping,
185
)
186
return (input_tokens, input_positions, attn_metadata, seq_lens,
170
- multi_modal_input)
187
+ multi_modal_kwargs)
188
189
def _prepare_decode(
190
self,
@@ -257,8 +274,8 @@ class CPUModelRunner:
274
self,
275
seq_group_metadata_list: List[SequenceGroupMetadata],
276
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
260
- Optional[torch.Tensor]]:
261
- multi_modal_input = None
277
+ Optional[Dict[str, torch.Tensor]]]:
278
+ multi_modal_kwargs = None
279
if self.is_driver_worker:
280
# NOTE: We assume that all sequences in the group are all prompts or
281
# all decodes.
@@ -266,7 +283,7 @@ class CPUModelRunner:
283
# Prepare input tensors.
284
if is_prompt:
285
(input_tokens, input_positions, attn_metadata, seq_lens,
269
- multi_modal_input
286
+ multi_modal_kwargs
287
) = self._prepare_prompt(seq_group_metadata_list)
288
else:
289
(input_tokens, input_positions,
@@ -307,7 +324,7 @@ class CPUModelRunner:
324
)
325
326
return (input_tokens, input_positions, attn_metadata,
310
- sampling_metadata, multi_modal_input)
327
+ sampling_metadata, multi_modal_kwargs)
328
329
@torch.inference_mode()
330
def execute_model(
vllm/worker/embedding_model_runner.py
+5
-5
@@ -90,7 +90,7 @@ class EmbeddingModelRunner(ModelRunner):
90
self,
91
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
92
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
93
- Set[LoRARequest], LoRAMapping, torch.Tensor]:
93
+ Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
94
if self.is_driver_worker:
95
assert seq_group_metadata_list is not None
96
# Prepare input tensors.
@@ -102,7 +102,7 @@ class EmbeddingModelRunner(ModelRunner):
102
_,
103
lora_mapping,
104
lora_requests,
105
- multi_modal_input,
105
+ multi_modal_kwargs,
106
slot_mapping,
107
num_prefill_tokens,
108
num_decode_tokens,
@@ -117,7 +117,7 @@ class EmbeddingModelRunner(ModelRunner):
117
"input_positions": input_positions,
118
"lora_requests": lora_requests,
119
"lora_mapping": lora_mapping,
120
- "multi_modal_input": multi_modal_input,
120
+ "multi_modal_kwargs": multi_modal_kwargs,
121
"num_prefill_tokens": num_prefill_tokens,
122
"num_decode_tokens": num_decode_tokens,
123
"slot_mapping": slot_mapping,
@@ -132,7 +132,7 @@ class EmbeddingModelRunner(ModelRunner):
132
input_positions = metadata_dict.pop("input_positions")
133
lora_mapping = metadata_dict.pop("lora_mapping")
134
lora_requests = metadata_dict.pop("lora_requests")
135
- multi_modal_input = metadata_dict.pop("multi_modal_input")
135
+ multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
136
if metadata_dict:
137
attn_metadata = self.attn_backend.make_metadata(
138
**metadata_dict)
@@ -143,7 +143,7 @@ class EmbeddingModelRunner(ModelRunner):
143
prompt_lens=None)
144
145
return (input_tokens, input_positions, attn_metadata, pooling_metadata,
146
- lora_requests, lora_mapping, multi_modal_input)
146
+ lora_requests, lora_mapping, multi_modal_kwargs)
147
148
def _prepare_pooling(
149
self,
vllm/worker/model_runner.py
+62
-58
@@ -1,5 +1,6 @@
1
import time
2
import warnings
3
+from collections import defaultdict
4
from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
5
6
import numpy as np
@@ -18,9 +19,9 @@ from vllm.lora.request import LoRARequest
19
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
20
from vllm.model_executor import SamplingMetadata
21
from vllm.model_executor.model_loader import get_model
22
+from vllm.multimodal import MULTIMODAL_REGISTRY
23
from vllm.sampling_params import SamplingParams
22
-from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
23
- SequenceGroupMetadata)
24
+from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
25
from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
26
is_pin_memory_available, make_tensor_with_pad)
27
@@ -44,7 +45,7 @@ class ModelInput(NamedTuple):
45
query_lens: List[int]
46
lora_mapping: Optional[LoRAMapping]
47
lora_requests: Set[LoRARequest]
47
- multi_modal_input: Optional[torch.Tensor]
48
+ multi_modal_kwargs: Dict[str, torch.Tensor]
49
slot_mapping: torch.Tensor
50
num_prefill_tokens: int
51
num_decode_tokens: int
@@ -60,7 +61,7 @@ class ModelInput(NamedTuple):
61
query_lens=[],
62
lora_mapping=None,
63
lora_requests=set(),
63
- multi_modal_input=None,
64
+ multi_modal_kwargs={},
65
slot_mapping=torch.empty(0, device=device),
66
num_prefill_tokens=0,
67
num_decode_tokens=0,
@@ -122,6 +123,16 @@ class ModelRunner:
123
self.block_size,
124
)
125
126
+ # Create processor for multi-modal data
127
+ if self.vision_language_config is not None:
128
+ self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
129
+ .create_input_processor(
130
+ self.model_config,
131
+ self.vision_language_config,
132
+ )
133
+ else:
134
+ self.multi_modal_input_processor = None
135
+
136
# Lazy initialization
137
self.model: nn.Module # Set after load_model
138
# Set if the backend is flashinfer.
@@ -242,7 +253,8 @@ class ModelRunner:
253
context_lens: List[int] = []
254
query_lens: List[int] = []
255
block_tables: List[List[int]] = []
245
- multi_modal_input_list: List[torch.Tensor] = []
256
+ multi_modal_kwargs_list: Dict[str,
257
+ List[torch.Tensor]] = defaultdict(list)
258
decode_only = True
259
num_prefills = 0
260
num_prefill_tokens = 0
@@ -417,9 +429,17 @@ class ModelRunner:
429
and seq_group_metadata.sampling_params.prompt_logprobs
430
else 1))
431
420
- if seq_group_metadata.multi_modal_data:
421
- multi_modal_input_list.append(
422
- seq_group_metadata.multi_modal_data.data)
432
+ mm_data = seq_group_metadata.multi_modal_data
433
+ if mm_data is not None:
434
+ # Process multi-modal data
435
+ if self.multi_modal_input_processor is None:
436
+ raise ValueError(
437
+ "Multi-modal inputs are only supported by "
438
+ "vision language models.")
439
+
440
+ mm_kwargs = self.multi_modal_input_processor(mm_data)
441
+ for k, v in mm_kwargs.items():
442
+ multi_modal_kwargs_list[k].append(v)
443
444
if _is_block_tables_empty(seq_group_metadata.block_tables):
445
# During memory profiling, the block tables are not
@@ -508,16 +528,6 @@ class ModelRunner:
528
context_lens_tensor = torch.tensor(context_lens,
529
dtype=torch.int,
530
device=self.device)
511
-
512
- if multi_modal_input_list:
513
- assert self.vision_language_config, (
514
- "Multi-modal inputs are only supported by "
515
- "vision language models.")
516
- multi_modal_input = torch.cat(multi_modal_input_list,
517
- dim=0).to(self.device)
518
- else:
519
- multi_modal_input = None
520
-
531
query_lens_tensor = torch.tensor(query_lens,
532
dtype=torch.long,
533
device=self.device)
@@ -614,6 +624,11 @@ class ModelRunner:
624
else:
625
lora_mapping = None
626
627
+ multi_modal_kwargs = {
628
+ k: torch.cat(v, dim=0).to(self.device)
629
+ for k, v in multi_modal_kwargs_list.items()
630
+ }
631
+
632
return ModelInput(
633
input_tokens=input_tokens_tensor,
634
input_positions=input_positions_tensor,
@@ -622,7 +637,7 @@ class ModelRunner:
637
query_lens=query_lens,
638
lora_mapping=lora_mapping,
639
lora_requests=lora_requests,
625
- multi_modal_input=multi_modal_input,
640
+ multi_modal_kwargs=multi_modal_kwargs,
641
slot_mapping=slot_mapping_tensor,
642
num_prefill_tokens=num_prefill_tokens,
643
num_decode_tokens=num_decode_tokens,
@@ -633,7 +648,7 @@ class ModelRunner:
648
self,
649
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
650
) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
636
- Set[LoRARequest], LoRAMapping, torch.Tensor]:
651
+ Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
652
if self.is_driver_worker:
653
assert seq_group_metadata_list is not None
654
# Prepare input tensors.
@@ -645,7 +660,7 @@ class ModelRunner:
660
query_lens,
661
lora_mapping,
662
lora_requests,
648
- multi_modal_input,
663
+ multi_modal_kwargs,
664
slot_mapping,
665
num_prefill_tokens,
666
num_decode_tokens,
@@ -662,7 +677,7 @@ class ModelRunner:
677
sampling_metadata.selected_token_indices,
678
"lora_requests": lora_requests,
679
"lora_mapping": lora_mapping,
665
- "multi_modal_input": multi_modal_input,
680
+ "multi_modal_kwargs": multi_modal_kwargs,
681
"num_prefill_tokens": num_prefill_tokens,
682
"num_decode_tokens": num_decode_tokens,
683
"slot_mapping": slot_mapping,
@@ -679,7 +694,7 @@ class ModelRunner:
694
"selected_token_indices")
695
lora_mapping = metadata_dict.pop("lora_mapping")
696
lora_requests = metadata_dict.pop("lora_requests")
682
- multi_modal_input = metadata_dict.pop("multi_modal_input")
697
+ multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
698
if metadata_dict:
699
attn_metadata = self.attn_backend.make_metadata(
700
**metadata_dict)
@@ -694,7 +709,7 @@ class ModelRunner:
709
710
return (input_tokens, input_positions, attn_metadata,
711
sampling_metadata, lora_requests, lora_mapping,
697
- multi_modal_input)
712
+ multi_modal_kwargs)
713
714
@torch.inference_mode()
715
def execute_model(
@@ -703,7 +718,7 @@ class ModelRunner:
718
kv_caches: List[torch.Tensor],
719
) -> Optional[SamplerOutput]:
720
(input_tokens, input_positions, attn_metadata, sampling_metadata,
706
- lora_requests, lora_mapping, multi_modal_input
721
+ lora_requests, lora_mapping, multi_modal_kwargs
722
) = self.prepare_input_tensors(seq_group_metadata_list)
723
724
if self.lora_config:
@@ -717,15 +732,14 @@ class ModelRunner:
732
model_executable = self.graph_runners[graph_batch_size]
733
else:
734
model_executable = self.model
720
- execute_model_kwargs = {
721
- "input_ids": input_tokens,
722
- "positions": input_positions,
723
- "kv_caches": kv_caches,
724
- "attn_metadata": attn_metadata,
725
- }
726
- if self.vision_language_config:
727
- execute_model_kwargs.update({"image_input": multi_modal_input})
728
- hidden_states = model_executable(**execute_model_kwargs)
735
+
736
+ hidden_states = model_executable(
737
+ input_ids=input_tokens,
738
+ positions=input_positions,
739
+ kv_caches=kv_caches,
740
+ attn_metadata=attn_metadata,
741
+ **multi_modal_kwargs,
742
+ )
743
744
# Compute the logits.
745
logits = self.model.compute_logits(hidden_states, sampling_metadata)
@@ -781,16 +795,24 @@ class ModelRunner:
795
# To exercise the worst scenario for GPU memory consumption,
796
# the number of seqs (batch_size) is chosen to maximize the number
797
# of images processed.
784
- if self.vision_language_config:
798
+ model_config = self.model_config
799
+ vlm_config = self.vision_language_config
800
+
801
+ if vlm_config:
802
max_num_seqs = min(
803
max_num_seqs,
787
- int(max_num_batched_tokens /
788
- self.vision_language_config.image_feature_size))
804
+ int(max_num_batched_tokens / vlm_config.image_feature_size))
805
for group_id in range(max_num_seqs):
806
seq_len = (max_num_batched_tokens // max_num_seqs +
807
(group_id < max_num_batched_tokens % max_num_seqs))
792
- seq_data, fake_multi_modal_input = _prepare_fake_inputs(
793
- seq_len, self.vision_language_config)
808
+
809
+ if vlm_config is None:
810
+ seq_data = SequenceData([0] * seq_len)
811
+ dummy_multi_modal_data = None
812
+ else:
813
+ seq_data, dummy_multi_modal_data = MULTIMODAL_REGISTRY \
814
+ .dummy_data_for_profiling(seq_len, model_config, vlm_config)
815
+
816
seq = SequenceGroupMetadata(
817
request_id=str(group_id),
818
is_prompt=True,
@@ -799,7 +821,7 @@ class ModelRunner:
821
block_tables=None,
822
lora_request=dummy_lora_requests_per_seq[group_id]
823
if dummy_lora_requests_per_seq else None,
802
- multi_modal_data=fake_multi_modal_input,
824
+ multi_modal_data=dummy_multi_modal_data,
825
)
826
seqs.append(seq)
827
@@ -1034,24 +1056,6 @@ def _get_graph_batch_size(batch_size: int) -> int:
1056
_BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1057
1058
1037
-def _prepare_fake_inputs(
1038
- seq_len: int, vision_language_config: Optional[VisionLanguageConfig]):
1039
- """Prepare fake inputs for profile run."""
1040
- if vision_language_config:
1041
- prompt_tokens = [
1042
- vision_language_config.image_token_id
1043
- ] * vision_language_config.image_feature_size + [0] * (
1044
- seq_len - vision_language_config.image_feature_size)
1045
- fake_image_input = MultiModalData(
1046
- type=MultiModalData.Type.IMAGE,
1047
- data=torch.zeros(vision_language_config.image_input_shape,
1048
- dtype=torch.float16))
1049
- else:
1050
- prompt_tokens = [0] * seq_len
1051
- fake_image_input = None
1052
- return SequenceData(prompt_tokens), fake_image_input
1053
-
1054
-
1059
def _is_block_tables_empty(block_tables: Union[None, Dict]):
1060
"""
1061
Check if block_tables is None or a dictionary with all None values.