@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7a64d24aad
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Core] Support image processor (#4197)

Cyrus Leung committed Jun 3, 2024 at 13:56 UTC 7a64d24aad69e4d2548aa0bf528d9fe63428ab01
29 files changed +1042 -256
.github/workflows/mypy.yaml
+1
@@ -37,6 +37,7 @@ jobs:
37 mypy vllm/distributed --config-file pyproject.toml
38 mypy vllm/entrypoints --config-file pyproject.toml
39 mypy vllm/executor --config-file pyproject.toml
40 + mypy vllm/multimodal --config-file pyproject.toml
41 mypy vllm/usage --config-file pyproject.toml
42 mypy vllm/*.py --config-file pyproject.toml
43 mypy vllm/transformers_utils --config-file pyproject.toml
docs/source/conf.py
+8 -6
@@ -90,6 +90,7 @@ autodoc_mock_imports = [
90 "sentencepiece",
91 "vllm.cuda_utils",
92 "vllm._C",
93 + "PIL",
94 "numpy",
95 "tqdm",
96 "tensorizer",
@@ -116,12 +117,13 @@ class MockedClassDocumenter(autodoc.ClassDocumenter):
117 autodoc.ClassDocumenter = MockedClassDocumenter
118
119 intersphinx_mapping = {
119 - 'python': ('https://docs.python.org/3', None),
120 - 'typing_extensions':
121 - ('https://typing-extensions.readthedocs.io/en/latest', None),
122 - 'numpy': ('https://numpy.org/doc/stable', None),
123 - 'torch': ('https://pytorch.org/docs/stable', None),
124 - 'psutil': ('https://psutil.readthedocs.io/en/stable', None),
120 + "python": ("https://docs.python.org/3", None),
121 + "typing_extensions":
122 + ("https://typing-extensions.readthedocs.io/en/latest", None),
123 + "pillow": ("https://pillow.readthedocs.io/en/stable", None),
124 + "numpy": ("https://numpy.org/doc/stable", None),
125 + "torch": ("https://pytorch.org/docs/stable", None),
126 + "psutil": ("https://psutil.readthedocs.io/en/stable", None),
127 }
128
129 autodoc_preserve_defaults = True
docs/source/dev/multimodal/multimodal_index.rst new
+51
@@ -0,0 +1,51 @@
1 +Multi-Modality
2 +==============
3 +
4 +.. currentmodule:: vllm.multimodal
5 +
6 +vLLM provides experimental support for multi-modal models through the :mod:`vllm.multimodal` package.
7 +
8 +:class:`vllm.inputs.PromptStrictInputs` accepts an additional attribute ``multi_modal_data``
9 +which allows you to pass in multi-modal input alongside text and token prompts.
10 +
11 +By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model,
12 +you must decorate the model class with :meth:`MULTIMODAL_REGISTRY.register_dummy_data <MultiModalRegistry.register_dummy_data>`,
13 +as well as :meth:`MULTIMODAL_REGISTRY.register_input <MultiModalRegistry.register_input>` for each modality type to support.
14 +
15 +.. contents::
16 + :local:
17 + :backlinks: none
18 +
19 +Module Contents
20 ++++++++++++++++
21 +
22 +.. automodule:: vllm.multimodal
23 +
24 +Registry
25 +--------
26 +
27 +.. data:: vllm.multimodal.MULTIMODAL_REGISTRY
28 +
29 + The global :class:`MultiModalRegistry` which is used by model runners.
30 +
31 +.. autoclass:: vllm.multimodal.MultiModalRegistry
32 + :members:
33 + :show-inheritance:
34 +
35 +Base Classes
36 +------------
37 +
38 +.. autoclass:: vllm.multimodal.MultiModalData
39 + :members:
40 + :show-inheritance:
41 +
42 +.. autoclass:: vllm.multimodal.MultiModalPlugin
43 + :members:
44 + :show-inheritance:
45 +
46 +Image Classes
47 +-------------
48 +
49 +.. automodule:: vllm.multimodal.image
50 + :members:
51 + :show-inheritance:
docs/source/index.rst
+4 -2
@@ -88,6 +88,7 @@ Documentation
88 models/adding_model
89 models/engine_args
90 models/lora
91 + models/vlm
92 models/performance
93
94 .. toctree::
@@ -99,17 +100,18 @@ Documentation
100 quantization/fp8_e4m3_kvcache
101
102 .. toctree::
102 - :maxdepth: 2
103 + :maxdepth: 1
104 :caption: Developer Documentation
105
106 dev/sampling_params
107 dev/offline_inference/offline_index
108 dev/engine/engine_index
109 dev/kernel/paged_attention
110 + dev/multimodal/multimodal_index
111 dev/dockerfile/dockerfile
112
113 .. toctree::
112 - :maxdepth: 2
114 + :maxdepth: 1
115 :caption: Community
116
117 community/meetups
docs/source/models/supported_models.rst
+4
@@ -87,6 +87,10 @@ Alongside each architecture, we include some popular models that use it.
87 - LLaMA, Llama 2, Meta Llama 3, Vicuna, Alpaca, Yi
88 - :code:`meta-llama/Meta-Llama-3-8B-Instruct`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
89 - ✅︎
90 + * - :code:`LlavaForConditionalGeneration`
91 + - LLaVA-1.5
92 + - :code:`llava-hf/llava-1.5-7b-hf`\*, :code:`llava-hf/llava-1.5-13b-hf`\*, etc.
93 + -
94 * - :code:`MiniCPMForCausalLM`
95 - MiniCPM
96 - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
docs/source/models/vlm.rst new
+56
@@ -0,0 +1,56 @@
1 +.. _vlm:
2 +
3 +Using VLMs
4 +==========
5 +
6 +This document shows you how to run and serve Vision Language Models (VLMs) using vLLM.
7 +
8 +Engine Arguments
9 +----------------
10 +
11 +The following :ref:`engine arguments <engine_args>` are specific to VLMs:
12 +
13 +.. argparse::
14 + :module: vllm.engine.arg_utils
15 + :func: _vlm_engine_args_parser
16 + :prog: -m vllm.entrypoints.openai.api_server
17 + :nodefaultconst:
18 +
19 +Offline Batched Inference
20 +-------------------------
21 +
22 +To initialize a VLM, the aforementioned arguments must be passed to the ``LLM`` class for instantiating the engine.
23 +
24 +.. code-block:: python
25 +
26 + llm = LLM(
27 + model="llava-hf/llava-1.5-7b-hf",
28 + image_input_type="pixel_values",
29 + image_token_id=32000,
30 + image_input_shape="1,3,336,336",
31 + image_feature_size=576,
32 + )
33 +
34 +For now, we only support a single image per text prompt. To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
35 +
36 +* ``prompt``: The prompt should have a number of ``<image>`` tokens equal to ``image_feature_size``.
37 +* ``multi_modal_data``: This should be an instance of :class:`~vllm.multimodal.image.ImagePixelData` or :class:`~vllm.multimodal.image.ImageFeatureData`.
38 +
39 +.. code-block:: python
40 +
41 + prompt = "<image>" * 576 + (
42 + "\nUSER: What is the content of this image?\nASSISTANT:")
43 +
44 + # Load the image using PIL.Image
45 + image = ...
46 +
47 + outputs = llm.generate({
48 + "prompt": prompt,
49 + "multi_modal_data": ImagePixelData(image),
50 + })
51 +
52 + for o in outputs:
53 + generated_text = o.outputs[0].text
54 + print(generated_text)
55 +
56 +A code example can be found in `examples/llava_example.py <https://github.com/vllm-project/vllm/blob/main/examples/llava_example.py>`_.
examples/llava_example.py
+15 -14
@@ -3,33 +3,36 @@ import os
3 import subprocess
4
5 import torch
6 +from PIL import Image
7
8 from vllm import LLM
8 -from vllm.sequence import MultiModalData
9 +from vllm.multimodal.image import ImageFeatureData, ImagePixelData
10
11 # The assets are located at `s3://air-example-data-2/vllm_opensource_llava/`.
12 +# You can use `.buildkite/download-images.sh` to download them
13
14
13 -def run_llava_pixel_values():
15 +def run_llava_pixel_values(*, disable_image_processor: bool = False):
16 llm = LLM(
17 model="llava-hf/llava-1.5-7b-hf",
18 image_input_type="pixel_values",
19 image_token_id=32000,
20 image_input_shape="1,3,336,336",
21 image_feature_size=576,
22 + disable_image_processor=disable_image_processor,
23 )
24
25 prompt = "<image>" * 576 + (
26 "\nUSER: What is the content of this image?\nASSISTANT:")
27
25 - # This should be provided by another online or offline component.
26 - image = torch.load("images/stop_sign_pixel_values.pt")
28 + if disable_image_processor:
29 + image = torch.load("images/stop_sign_pixel_values.pt")
30 + else:
31 + image = Image.open("images/stop_sign.jpg")
32
33 outputs = llm.generate({
29 - "prompt":
30 - prompt,
31 - "multi_modal_data":
32 - MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
34 + "prompt": prompt,
35 + "multi_modal_data": ImagePixelData(image),
36 })
37
38 for o in outputs:
@@ -49,15 +52,13 @@ def run_llava_image_features():
52 prompt = "<image>" * 576 + (
53 "\nUSER: What is the content of this image?\nASSISTANT:")
54
52 - # This should be provided by another online or offline component.
53 - image = torch.load("images/stop_sign_image_features.pt")
55 + image: torch.Tensor = torch.load("images/stop_sign_image_features.pt")
56
57 outputs = llm.generate({
56 - "prompt":
57 - prompt,
58 - "multi_modal_data":
59 - MultiModalData(type=MultiModalData.Type.IMAGE, data=image),
58 + "prompt": prompt,
59 + "multi_modal_data": ImageFeatureData(image),
60 })
61 +
62 for o in outputs:
63 generated_text = o.outputs[0].text
64 print(generated_text)
format.sh
+1
@@ -101,6 +101,7 @@ mypy vllm/core --config-file pyproject.toml
101 mypy vllm/distributed --config-file pyproject.toml
102 mypy vllm/entrypoints --config-file pyproject.toml
103 mypy vllm/executor --config-file pyproject.toml
104 +mypy vllm/multimodal --config-file pyproject.toml
105 mypy vllm/usage --config-file pyproject.toml
106 mypy vllm/*.py --config-file pyproject.toml
107 mypy vllm/transformers_utils --config-file pyproject.toml
requirements-common.txt
+1
@@ -12,6 +12,7 @@ aiohttp
12 openai
13 uvicorn[standard]
14 pydantic >= 2.0 # Required for OpenAI server.
15 +pillow # Required for image processing
16 prometheus_client >= 0.18.0
17 prometheus-fastapi-instrumentator >= 7.0.0
18 tiktoken >= 0.6.0 # Required for DBRX tokenizer
requirements-dev.txt
-3
@@ -33,8 +33,5 @@ sentence-transformers # required for embedding
33 # Benchmarking
34 aiohttp
35
36 -# Multimodal
37 -pillow
38 -
36 # quantization
37 bitsandbytes==0.42.0
tests/conftest.py
+24 -21
@@ -15,7 +15,9 @@ from vllm.config import TokenizerPoolConfig, VisionLanguageConfig
15 from vllm.distributed import destroy_model_parallel
16 from vllm.inputs import TextPrompt
17 from vllm.logger import init_logger
18 -from vllm.sequence import MultiModalData, SampleLogprobs
18 +from vllm.multimodal import MultiModalData
19 +from vllm.multimodal.image import ImageFeatureData, ImagePixelData
20 +from vllm.sequence import SampleLogprobs
21
22 logger = init_logger(__name__)
23
@@ -24,6 +26,7 @@ _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
26 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
27
28 # Multi modal related
29 +# You can use `.buildkite/download-images.sh` to download the assets
30 _PIXEL_VALUES_FILES = [
31 os.path.join(_TEST_DIR, "images", filename) for filename in
32 ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
@@ -89,17 +92,23 @@ def hf_images() -> List[Image.Image]:
92
93
94 @pytest.fixture()
92 -def vllm_images(request) -> "torch.Tensor":
95 +def vllm_images(request) -> List[MultiModalData]:
96 vision_language_config = request.getfixturevalue("model_and_config")[1]
94 - all_images = []
97 if vision_language_config.image_input_type == (
98 VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
97 - filenames = _IMAGE_FEATURES_FILES
99 + return [
100 + ImageFeatureData(torch.load(filename))
101 + for filename in _IMAGE_FEATURES_FILES
102 + ]
103 else:
99 - filenames = _PIXEL_VALUES_FILES
100 - for filename in filenames:
101 - all_images.append(torch.load(filename))
102 - return torch.concat(all_images, dim=0)
104 + return [
105 + ImagePixelData(Image.open(filename)) for filename in _IMAGE_FILES
106 + ]
107 +
108 +
109 +@pytest.fixture()
110 +def vllm_image_tensors(request) -> List[torch.Tensor]:
111 + return [torch.load(filename) for filename in _PIXEL_VALUES_FILES]
112
113
114 @pytest.fixture()
@@ -392,23 +401,17 @@ class VllmRunner:
401 self,
402 prompts: List[str],
403 sampling_params: SamplingParams,
395 - images: Optional[torch.Tensor] = None,
404 + images: Optional[List[MultiModalData]] = None,
405 ) -> List[Tuple[List[List[int]], List[str]]]:
406 if images is not None:
407 assert len(prompts) == len(images)
408
400 - prompt_inputs: List[TextPrompt] = []
401 - for i, prompt in enumerate(prompts):
402 - prompt = TextPrompt(prompt=prompt)
403 - if images is not None:
404 - prompt["multi_modal_data"] = MultiModalData(
405 - type=MultiModalData.Type.IMAGE,
406 - data=images[i:i + 1],
407 - )
408 -
409 - prompt_inputs.append(prompt)
409 + inputs = [TextPrompt(prompt=prompt) for prompt in prompts]
410 + if images is not None:
411 + for i, image in enumerate(images):
412 + inputs[i]["multi_modal_data"] = image
413
411 - req_outputs = self.model.generate(prompt_inputs,
414 + req_outputs = self.model.generate(inputs,
415 sampling_params=sampling_params)
416
417 outputs: List[Tuple[List[List[int]], List[str]]] = []
@@ -447,7 +450,7 @@ class VllmRunner:
450 self,
451 prompts: List[str],
452 max_tokens: int,
450 - images: Optional[torch.Tensor] = None,
453 + images: Optional[List[MultiModalData]] = None,
454 ) -> List[Tuple[List[int], str]]:
455 greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
456 outputs = self.generate(prompts, greedy_params, images=images)
tests/models/test_llava.py
+40 -20
@@ -1,7 +1,7 @@
1 import gc
2 from dataclasses import fields
3 from enum import Enum
4 -from typing import Dict, List, Tuple
4 +from typing import Any, Dict, List, Tuple
5
6 import pytest
7 import torch
@@ -9,36 +9,50 @@ from transformers import AutoTokenizer
9
10 from vllm.config import VisionLanguageConfig
11
12 +
13 +def iter_llava_configs(model_name: str):
14 + image_hw_to_feature_size = {
15 + (336, 336): 576,
16 + }
17 +
18 + for (h, w), f in image_hw_to_feature_size.items():
19 + for input_type, input_shape in [
20 + (VisionLanguageConfig.ImageInputType.PIXEL_VALUES, (1, 3, h, w)),
21 + (VisionLanguageConfig.ImageInputType.IMAGE_FEATURES, (1, f, 1024)),
22 + ]:
23 + yield (model_name,
24 + VisionLanguageConfig(image_input_type=input_type,
25 + image_feature_size=f,
26 + image_token_id=32000,
27 + image_input_shape=input_shape,
28 + image_processor=model_name,
29 + image_processor_revision=None))
30 +
31 +
32 model_and_vl_config = [
13 - ("llava-hf/llava-1.5-7b-hf",
14 - VisionLanguageConfig(
15 - image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
16 - image_feature_size=576,
17 - image_token_id=32000,
18 - image_input_shape=(1, 3, 336, 336))),
19 - ("llava-hf/llava-1.5-7b-hf",
20 - VisionLanguageConfig(
21 - image_input_type=VisionLanguageConfig.ImageInputType.IMAGE_FEATURES,
22 - image_feature_size=576,
23 - image_token_id=32000,
24 - image_input_shape=(1, 576, 1024)))
33 + *iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
34 + # Not enough memory
35 + # *iter_llava_configs("llava-hf/llava-1.5-13b-hf"),
36 ]
37
38
28 -def as_dict(vision_language_config: VisionLanguageConfig) -> Dict:
39 +def as_dict(vlm_config: VisionLanguageConfig) -> Dict[str, Any]:
40 """Flatten vision language config to pure args.
41
42 Compatible with what llm entrypoint expects.
43 """
44 result = {}
34 - for field in fields(vision_language_config):
35 - value = getattr(vision_language_config, field.name)
45 + for field in fields(vlm_config):
46 + value = getattr(vlm_config, field.name)
47 if isinstance(value, Enum):
48 result[field.name] = value.name.lower()
49 elif isinstance(value, tuple):
50 result[field.name] = ",".join([str(item) for item in value])
51 else:
52 result[field.name] = value
53 +
54 + result["disable_image_processor"] = vlm_config.image_processor is None
55 +
56 return result
57
58
@@ -67,18 +81,19 @@ def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
81 @pytest.mark.parametrize("dtype", ["half"])
82 @pytest.mark.parametrize("max_tokens", [128])
83 def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
70 - vllm_image_prompts, vllm_images, model_and_config: tuple,
71 - dtype: str, max_tokens: int, worker_use_ray: bool) -> None:
84 + vllm_image_prompts, vllm_images, model_and_config, dtype: str,
85 + max_tokens: int, worker_use_ray: bool) -> None:
86 """Inference result should be the same between hf and vllm.
87
88 All the image fixtures for the test is under tests/images.
75 - For huggingface runner, we provide the raw images as input.
76 - For vllm runner, we provide image tensors and corresponding
89 + For huggingface runner, we provide the PIL images as input.
90 + For vllm runner, we provide MultiModalData objects and corresponding
91 vision language config as input.
92 Note, the text input is also adjusted to abide by vllm contract.
93 The text output is sanitized to be able to compare with hf.
94 """
95 model_id, vision_language_config = model_and_config
96 +
97 hf_model = hf_runner(model_id, dtype=dtype)
98 hf_outputs = hf_model.generate_greedy(hf_image_prompts,
99 max_tokens,
@@ -88,6 +103,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
103 vllm_model = vllm_runner(model_id,
104 dtype=dtype,
105 worker_use_ray=worker_use_ray,
106 + enforce_eager=True,
107 **as_dict(vision_language_config))
108 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
109 max_tokens,
@@ -105,3 +121,7 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
121 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
122 assert hf_output_ids == vllm_output_ids, (
123 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
124 +
125 +
126 +# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
127 +# (Requires multiple GPUs)
tests/multimodal/__init__.py
tests/multimodal/test_processor.py new
+98
@@ -0,0 +1,98 @@
1 +import numpy as np
2 +import pytest
3 +from transformers import CLIPImageProcessor
4 +
5 +from vllm.config import ModelConfig, VisionLanguageConfig
6 +from vllm.multimodal import MULTIMODAL_REGISTRY
7 +from vllm.multimodal.image import ImagePixelData
8 +
9 +
10 +@pytest.mark.parametrize("dtype", ["half", "bfloat16", "float"])
11 +def test_clip_image_processor(hf_images, dtype):
12 + MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
13 + IMAGE_HEIGHT = IMAGE_WIDTH = 33
14 +
15 + hf_processor = CLIPImageProcessor.from_pretrained(MODEL_NAME)
16 + assert isinstance(hf_processor, CLIPImageProcessor)
17 +
18 + model_config = ModelConfig(
19 + model=MODEL_NAME,
20 + tokenizer=MODEL_NAME,
21 + tokenizer_mode="auto",
22 + trust_remote_code=False,
23 + seed=0,
24 + dtype=dtype,
25 + revision=None,
26 + )
27 + vlm_config = VisionLanguageConfig(
28 + image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
29 + image_token_id=32000,
30 + image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
31 + image_feature_size=576,
32 + image_processor=MODEL_NAME,
33 + image_processor_revision=None,
34 + )
35 +
36 + for image in hf_images:
37 + hf_result = hf_processor.preprocess(
38 + image,
39 + return_tensors="np",
40 + )
41 + vllm_result = MULTIMODAL_REGISTRY.process_input(
42 + ImagePixelData(image),
43 + model_config=model_config,
44 + vlm_config=vlm_config,
45 + )
46 +
47 + assert hf_result.keys() == vllm_result.keys()
48 + for key, hf_arr in hf_result.items():
49 + vllm_arr: np.ndarray = vllm_result[key].numpy()
50 +
51 + assert hf_arr.shape == vllm_arr.shape, f"Failed for key={key}"
52 + assert np.allclose(hf_arr, vllm_arr), f"Failed for key={key}"
53 +
54 +
55 +@pytest.mark.parametrize("dtype", ["float"])
56 +def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
57 + MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
58 + IMAGE_HEIGHT = IMAGE_WIDTH = 33
59 +
60 + model_config = ModelConfig(
61 + model=MODEL_NAME,
62 + tokenizer=MODEL_NAME,
63 + tokenizer_mode="auto",
64 + trust_remote_code=False,
65 + seed=0,
66 + dtype=dtype,
67 + revision=None,
68 + )
69 + vlm_config = VisionLanguageConfig(
70 + image_input_type=VisionLanguageConfig.ImageInputType.PIXEL_VALUES,
71 + image_token_id=32000,
72 + image_input_shape=(1, 3, IMAGE_HEIGHT, IMAGE_WIDTH),
73 + image_feature_size=576,
74 + image_processor=MODEL_NAME,
75 + image_processor_revision=None,
76 + )
77 +
78 + for image, tensor in zip(hf_images, vllm_image_tensors):
79 + image_result = MULTIMODAL_REGISTRY.process_input(
80 + ImagePixelData(image),
81 + model_config=model_config,
82 + vlm_config=vlm_config,
83 + )
84 + tensor_result = MULTIMODAL_REGISTRY.process_input(
85 + ImagePixelData(tensor),
86 + model_config=model_config,
87 + vlm_config=vlm_config,
88 + )
89 +
90 + assert image_result.keys() == tensor_result.keys()
91 + for key, image_arr in image_result.items():
92 + tensor_arr: np.ndarray = tensor_result[key].numpy()
93 +
94 + assert image_arr.shape == tensor_arr.shape, f"Failed for key={key}"
95 +
96 + # The examples in PR#3042 have slightly different preprocessing from
97 + # HuggingFace's LlavaProcessor, causing the test to fail.
98 + # assert np.allclose(image_arr, tensor_arr), f"Failed for key={key}"
tests/spec_decode/e2e/conftest.py
+2 -1
@@ -18,9 +18,10 @@ from vllm.engine.arg_utils import AsyncEngineArgs
18 from vllm.engine.async_llm_engine import AsyncLLMEngine
19 from vllm.lora.request import LoRARequest
20 from vllm.model_executor.utils import set_random_seed
21 +from vllm.multimodal import MultiModalData
22 from vllm.outputs import RequestOutput
23 from vllm.sampling_params import SamplingParams
23 -from vllm.sequence import Logprob, MultiModalData
24 +from vllm.sequence import Logprob
25 from vllm.usage.usage_lib import UsageContext
26 from vllm.utils import Counter, random_uuid
27
tests/tokenization/test_image_processor.py new
+20
@@ -0,0 +1,20 @@
1 +import pytest
2 +from transformers.image_processing_utils import BaseImageProcessor
3 +
4 +from vllm.transformers_utils.image_processor import get_image_processor
5 +
6 +IMAGE_PROCESSOR_NAMES = [
7 + "llava-hf/llava-1.5-7b-hf",
8 + "llava-hf/llava-v1.6-34b-hf",
9 +]
10 +
11 +
12 +@pytest.mark.parametrize("processor_name", IMAGE_PROCESSOR_NAMES)
13 +def test_image_processor_revision(processor_name: str):
14 + # Assume that "main" branch always exists
15 + image_processor = get_image_processor(processor_name, revision="main")
16 + assert isinstance(image_processor, BaseImageProcessor)
17 +
18 + # Assume that "never" branch always does not exist
19 + with pytest.raises(OSError, match='not a valid git identifier'):
20 + get_image_processor(processor_name, revision="never")
vllm/config.py
+4 -2
@@ -1094,10 +1094,12 @@ class VisionLanguageConfig:
1094 # worst case scenario (biggest supported resolution).
1095 image_input_shape: tuple
1096 image_feature_size: int
1097 + # The image processor to load from HuggingFace
1098 + image_processor: Optional[str]
1099 + image_processor_revision: Optional[str]
1100
1101 @classmethod
1099 - def get_image_input_enum_type(
1100 - cls, value: str) -> "VisionLanguageConfig.ImageInputType":
1102 + def get_image_input_enum_type(cls, value: str) -> ImageInputType:
1103 """Get the image input type from a string."""
1104 try:
1105 return cls.ImageInputType[value.upper()]
vllm/engine/arg_utils.py
+79 -29
@@ -1,6 +1,7 @@
1 import argparse
2 import dataclasses
3 import json
4 +import warnings
5 from dataclasses import dataclass
6 from typing import List, Optional, Tuple, Union
7
@@ -80,6 +81,10 @@ class EngineArgs:
81 image_token_id: Optional[int] = None
82 image_input_shape: Optional[str] = None
83 image_feature_size: Optional[int] = None
84 + image_processor: Optional[str] = None
85 + image_processor_revision: Optional[str] = None
86 + disable_image_processor: bool = False
87 +
88 scheduler_delay_factor: float = 0.0
89 enable_chunked_prefill: bool = False
90
@@ -98,6 +103,53 @@ class EngineArgs:
103 if self.tokenizer is None:
104 self.tokenizer = self.model
105
106 + @staticmethod
107 + def add_cli_args_for_vlm(
108 + parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
109 + parser.add_argument('--image-input-type',
110 + type=nullable_str,
111 + default=None,
112 + choices=[
113 + t.name.lower()
114 + for t in VisionLanguageConfig.ImageInputType
115 + ],
116 + help=('The image input type passed into vLLM.'))
117 + parser.add_argument('--image-token-id',
118 + type=int,
119 + default=None,
120 + help=('Input id for image token.'))
121 + parser.add_argument(
122 + '--image-input-shape',
123 + type=nullable_str,
124 + default=None,
125 + help=('The biggest image input shape (worst for memory footprint) '
126 + 'given an input type. Only used for vLLM\'s profile_run.'))
127 + parser.add_argument(
128 + '--image-feature-size',
129 + type=int,
130 + default=None,
131 + help=('The image feature size along the context dimension.'))
132 + parser.add_argument(
133 + '--image-processor',
134 + type=str,
135 + default=EngineArgs.image_processor,
136 + help='Name or path of the huggingface image processor to use. '
137 + 'If unspecified, model name or path will be used.')
138 + parser.add_argument(
139 + '--image-processor-revision',
140 + type=str,
141 + default=None,
142 + help='Revision of the huggingface image processor version to use. '
143 + 'It can be a branch name, a tag name, or a commit id. '
144 + 'If unspecified, will use the default version.')
145 + parser.add_argument(
146 + '--disable-image-processor',
147 + action='store_true',
148 + help='Disables the use of image processor, even if one is defined '
149 + 'for the model on huggingface.')
150 +
151 + return parser
152 +
153 @staticmethod
154 def add_cli_args(
155 parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
@@ -113,7 +165,8 @@ class EngineArgs:
165 '--tokenizer',
166 type=nullable_str,
167 default=EngineArgs.tokenizer,
116 - help='Name or path of the huggingface tokenizer to use.')
168 + help='Name or path of the huggingface tokenizer to use. '
169 + 'If unspecified, model name or path will be used.')
170 parser.add_argument(
171 '--skip-tokenizer-init',
172 action='store_true',
@@ -136,9 +189,9 @@ class EngineArgs:
189 '--tokenizer-revision',
190 type=nullable_str,
191 default=None,
139 - help='The specific tokenizer version to use. It can be a branch '
140 - 'name, a tag name, or a commit id. If unspecified, will use '
141 - 'the default version.')
192 + help='Revision of the huggingface tokenizer to use. '
193 + 'It can be a branch name, a tag name, or a commit id. '
194 + 'If unspecified, will use the default version.')
195 parser.add_argument(
196 '--tokenizer-mode',
197 type=str,
@@ -445,31 +498,10 @@ class EngineArgs:
498 default=EngineArgs.device,
499 choices=["auto", "cuda", "neuron", "cpu"],
500 help='Device type for vLLM execution.')
501 +
502 # Related to Vision-language models such as llava
449 - parser.add_argument(
450 - '--image-input-type',
451 - type=nullable_str,
452 - default=None,
453 - choices=[
454 - t.name.lower() for t in VisionLanguageConfig.ImageInputType
455 - ],
456 - help=('The image input type passed into vLLM. '
457 - 'Should be one of "pixel_values" or "image_features".'))
458 - parser.add_argument('--image-token-id',
459 - type=int,
460 - default=None,
461 - help=('Input id for image token.'))
462 - parser.add_argument(
463 - '--image-input-shape',
464 - type=nullable_str,
465 - default=None,
466 - help=('The biggest image input shape (worst for memory footprint) '
467 - 'given an input type. Only used for vLLM\'s profile_run.'))
468 - parser.add_argument(
469 - '--image-feature-size',
470 - type=int,
471 - default=None,
472 - help=('The image feature size along the context dimension.'))
503 + parser = EngineArgs.add_cli_args_for_vlm(parser)
504 +
505 parser.add_argument(
506 '--scheduler-delay-factor',
507 type=float,
@@ -488,7 +520,6 @@ class EngineArgs:
520 default=EngineArgs.speculative_model,
521 help=
522 'The name of the draft model to be used in speculative decoding.')
491 -
523 parser.add_argument(
524 '--num-speculative-tokens',
525 type=int,
@@ -666,12 +697,27 @@ class EngineArgs:
697 raise ValueError(
698 'Specify `image_token_id`, `image_input_shape` and '
699 '`image_feature_size` together with `image_input_type`.')
700 +
701 + if self.image_processor is None:
702 + self.image_processor = self.model
703 + if self.disable_image_processor:
704 + if self.image_processor != self.model:
705 + warnings.warn(
706 + "You've specified an image processor "
707 + f"({self.image_processor}) but also disabled "
708 + "it via `--disable-image-processor`.",
709 + stacklevel=2)
710 +
711 + self.image_processor = None
712 +
713 vision_language_config = VisionLanguageConfig(
714 image_input_type=VisionLanguageConfig.
715 get_image_input_enum_type(self.image_input_type),
716 image_token_id=self.image_token_id,
717 image_input_shape=str_to_int_tuple(self.image_input_shape),
718 image_feature_size=self.image_feature_size,
719 + image_processor=self.image_processor,
720 + image_processor_revision=self.image_processor_revision,
721 )
722 else:
723 vision_language_config = None
@@ -734,3 +780,7 @@ def _engine_args_parser():
780 def _async_engine_args_parser():
781 return AsyncEngineArgs.add_cli_args(argparse.ArgumentParser(),
782 async_args_only=True)
783 +
784 +
785 +def _vlm_engine_args_parser():
786 + return EngineArgs.add_cli_args_for_vlm(argparse.ArgumentParser())
vllm/entrypoints/llm.py
+2 -23
@@ -14,7 +14,6 @@ from vllm.lora.request import LoRARequest
14 from vllm.outputs import EmbeddingRequestOutput, RequestOutput
15 from vllm.pooling_params import PoolingParams
16 from vllm.sampling_params import SamplingParams
17 -from vllm.sequence import MultiModalData
17 from vllm.usage.usage_lib import UsageContext
18 from vllm.utils import Counter, deprecate_kwargs
19
@@ -164,7 +163,6 @@ class LLM:
163 prompt_token_ids: Optional[List[int]] = None,
164 use_tqdm: bool = True,
165 lora_request: Optional[LoRARequest] = None,
167 - multi_modal_data: Optional[MultiModalData] = None,
166 ) -> List[RequestOutput]:
167 ...
168
@@ -177,7 +175,6 @@ class LLM:
175 prompt_token_ids: Optional[List[List[int]]] = None,
176 use_tqdm: bool = True,
177 lora_request: Optional[LoRARequest] = None,
180 - multi_modal_data: Optional[MultiModalData] = None,
178 ) -> List[RequestOutput]:
179 ...
180
@@ -191,7 +188,6 @@ class LLM:
188 prompt_token_ids: List[int],
189 use_tqdm: bool = True,
190 lora_request: Optional[LoRARequest] = None,
194 - multi_modal_data: Optional[MultiModalData] = None,
191 ) -> List[RequestOutput]:
192 ...
193
@@ -205,7 +201,6 @@ class LLM:
201 prompt_token_ids: List[List[int]],
202 use_tqdm: bool = True,
203 lora_request: Optional[LoRARequest] = None,
208 - multi_modal_data: Optional[MultiModalData] = None,
204 ) -> List[RequestOutput]:
205 ...
206
@@ -217,7 +212,6 @@ class LLM:
212 prompt_token_ids: Union[List[int], List[List[int]]],
213 use_tqdm: bool = True,
214 lora_request: Optional[LoRARequest] = None,
220 - multi_modal_data: Optional[MultiModalData] = None,
215 ) -> List[RequestOutput]:
216 ...
217
@@ -236,7 +230,6 @@ class LLM:
230
231 @deprecate_kwargs("prompts",
232 "prompt_token_ids",
239 - "multi_modal_data",
233 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
234 additional_message="Please use the 'inputs' parameter "
235 "instead.")
@@ -249,7 +242,6 @@ class LLM:
242 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
243 use_tqdm: bool = True,
244 lora_request: Optional[LoRARequest] = None,
252 - multi_modal_data: Optional[MultiModalData] = None,
245 ) -> List[RequestOutput]:
246 """Generates the completions for the input prompts.
247
@@ -281,11 +273,10 @@ class LLM:
273 "LLM.generate() is only supported for generation models "
274 "(XForCausalLM).")
275
284 - if prompt_token_ids is not None or multi_modal_data is not None:
276 + if prompt_token_ids is not None:
277 inputs = self._convert_v1_inputs(
278 prompts=cast(Optional[Union[str, List[str]]], prompts),
279 prompt_token_ids=prompt_token_ids,
288 - multi_modal_data=multi_modal_data,
280 )
281 else:
282 inputs = cast(
@@ -314,7 +305,6 @@ class LLM:
305 prompt_token_ids: Optional[List[int]] = None,
306 use_tqdm: bool = True,
307 lora_request: Optional[LoRARequest] = None,
317 - multi_modal_data: Optional[MultiModalData] = None,
308 ) -> List[EmbeddingRequestOutput]:
309 ...
310
@@ -327,7 +317,6 @@ class LLM:
317 prompt_token_ids: Optional[List[List[int]]] = None,
318 use_tqdm: bool = True,
319 lora_request: Optional[LoRARequest] = None,
330 - multi_modal_data: Optional[MultiModalData] = None,
320 ) -> List[EmbeddingRequestOutput]:
321 ...
322
@@ -341,7 +330,6 @@ class LLM:
330 prompt_token_ids: List[int],
331 use_tqdm: bool = True,
332 lora_request: Optional[LoRARequest] = None,
344 - multi_modal_data: Optional[MultiModalData] = None,
333 ) -> List[EmbeddingRequestOutput]:
334 ...
335
@@ -355,7 +343,6 @@ class LLM:
343 prompt_token_ids: List[List[int]],
344 use_tqdm: bool = True,
345 lora_request: Optional[LoRARequest] = None,
358 - multi_modal_data: Optional[MultiModalData] = None,
346 ) -> List[EmbeddingRequestOutput]:
347 ...
348
@@ -367,7 +354,6 @@ class LLM:
354 prompt_token_ids: Union[List[int], List[List[int]]],
355 use_tqdm: bool = True,
356 lora_request: Optional[LoRARequest] = None,
370 - multi_modal_data: Optional[MultiModalData] = None,
357 ) -> List[EmbeddingRequestOutput]:
358 ...
359
@@ -386,7 +372,6 @@ class LLM:
372
373 @deprecate_kwargs("prompts",
374 "prompt_token_ids",
389 - "multi_modal_data",
375 is_deprecated=lambda: LLM.DEPRECATE_LEGACY,
376 additional_message="Please use the 'inputs' parameter "
377 "instead.")
@@ -399,7 +384,6 @@ class LLM:
384 prompt_token_ids: Optional[Union[List[int], List[List[int]]]] = None,
385 use_tqdm: bool = True,
386 lora_request: Optional[LoRARequest] = None,
402 - multi_modal_data: Optional[MultiModalData] = None,
387 ) -> List[EmbeddingRequestOutput]:
388 """Generates the completions for the input prompts.
389
@@ -430,11 +414,10 @@ class LLM:
414 "LLM.encode() is only supported for embedding models (XModel)."
415 )
416
433 - if prompt_token_ids is not None or multi_modal_data is not None:
417 + if prompt_token_ids is not None:
418 inputs = self._convert_v1_inputs(
419 prompts=cast(Optional[Union[str, List[str]]], prompts),
420 prompt_token_ids=prompt_token_ids,
437 - multi_modal_data=multi_modal_data,
421 )
422 else:
423 inputs = cast(
@@ -459,7 +442,6 @@ class LLM:
442 self,
443 prompts: Optional[Union[str, List[str]]],
444 prompt_token_ids: Optional[Union[List[int], List[List[int]]]],
462 - multi_modal_data: Optional[MultiModalData],
445 ):
446 # skip_tokenizer_init is now checked in engine
447
@@ -499,9 +481,6 @@ class LLM:
481 else:
482 raise AssertionError
483
502 - if multi_modal_data is not None:
503 - item["multi_modal_data"] = multi_modal_data
504 -
484 inputs.append(item)
485
486 return inputs
vllm/model_executor/models/llava.py
+46 -27
@@ -17,6 +17,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
17 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
18 from vllm.model_executor.models.llama import LlamaModel
19 from vllm.model_executor.sampling_metadata import SamplingMetadata
20 +from vllm.multimodal import MULTIMODAL_REGISTRY
21 +from vllm.multimodal.image import get_dummy_image_data
22 from vllm.sequence import SamplerOutput
23
24 from .vlm_base import VisionLanguageModelBase
@@ -82,6 +84,9 @@ class LlavaImageFeatureInputs(TypedDict):
84 LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageFeatureInputs]
85
86
87 +@MULTIMODAL_REGISTRY.register_image_feature_input()
88 +@MULTIMODAL_REGISTRY.register_image_pixel_input()
89 +@MULTIMODAL_REGISTRY.register_dummy_data(get_dummy_image_data)
90 class LlavaForConditionalGeneration(VisionLanguageModelBase):
91
92 def __init__(self,
@@ -131,30 +136,41 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
136 return data
137
138 def _parse_and_validate_image_input(
134 - self, data: object) -> Optional[LlavaImageInputs]:
139 + self, **kwargs: object) -> Optional[LlavaImageInputs]:
140 + pixel_values = kwargs.pop("pixel_values", None)
141 + image_features = kwargs.pop("image_features", None)
142 +
143 expected_input_type = self.vision_language_config.image_input_type
144 ImageInputType = VisionLanguageConfig.ImageInputType
145
138 - if data is None:
139 - return None
140 -
146 if expected_input_type == ImageInputType.PIXEL_VALUES:
142 - if not isinstance(data, torch.Tensor):
143 - raise TypeError("Image pixel vector should be a tensor, "
144 - f"but received type: {type(data)}")
147 + if image_features is not None:
148 + raise ValueError(
149 + "Expected pixel values but got image features")
150 + if pixel_values is None:
151 + return None
152 +
153 + if not isinstance(pixel_values, torch.Tensor):
154 + raise ValueError("Incorrect type of pixel values")
155
156 return LlavaImagePixelInputs(
157 type="pixel_values",
148 - data=self._validate_image_data(data),
158 + data=self._validate_image_data(pixel_values),
159 )
150 - elif expected_input_type == ImageInputType.IMAGE_FEATURES:
151 - if not isinstance(data, torch.Tensor):
152 - raise TypeError("Image feature vector should be a tensor, "
153 - f"but received type: {type(data)}")
160 +
161 + if expected_input_type == ImageInputType.IMAGE_FEATURES:
162 + if pixel_values is not None:
163 + raise ValueError(
164 + "Expected image features but got pixel values")
165 + if image_features is None:
166 + return None
167 +
168 + if not isinstance(image_features, torch.Tensor):
169 + raise ValueError("Incorrect type of image features")
170
171 return LlavaImageFeatureInputs(
172 type="image_features",
157 - data=self._validate_image_data(data),
173 + data=self._validate_image_data(image_features),
174 )
175
176 return None
@@ -201,12 +217,14 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
217
218 return self.multi_modal_projector(image_features)
219
204 - def forward(self,
205 - input_ids: torch.Tensor,
206 - positions: torch.Tensor,
207 - kv_caches: List[torch.Tensor],
208 - attn_metadata: AttentionMetadata,
209 - image_input: Optional[torch.Tensor] = None) -> SamplerOutput:
220 + def forward(
221 + self,
222 + input_ids: torch.Tensor,
223 + positions: torch.Tensor,
224 + kv_caches: List[torch.Tensor],
225 + attn_metadata: AttentionMetadata,
226 + **kwargs: object,
227 + ) -> SamplerOutput:
228 """Run forward pass for Llava 1.5.
229
230 One key thing to understand is the `input_ids` already accounts for the
@@ -227,10 +245,10 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
245 This way, the `positions` and `attn_metadata` are consistent
246 with the `input_ids`.
247
230 - The model takes two types of image inputs:
248 + The model takes two types of image inputs:
249 PIXEL_VALUES and IMAGE_FEATURES.
250 The following shows how each maps to huggingface implementation.
233 - PIXEL_VALUES:
251 + PIXEL_VALUES:
252 - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L353
253 IMAGE_FEATURES:
254 - https://github.com/huggingface/transformers/blob/07bdbeb/src/transformers/models/llava/modeling_llava.py#L430
@@ -239,14 +257,15 @@ class LlavaForConditionalGeneration(VisionLanguageModelBase):
257 Args:
258 input_ids: Flattened (concatenated) input_ids corresponding to a
259 batch.
242 - image_input: A batch of image inputs.
243 - For PIXEL_VALUES, expecting [1, 3, 336, 336].
244 - For IMAGE_FEATURES, expecting [1, 576, 1024].
260 + pixel_values: For PIXEL_VALUES, expects a batch with shape
261 + [1, 3, 336, 336].
262 + image_features: For IMAGE_FEATURES, expects a batch with shape
263 + [1, 576, 1024].
264 """
246 - parsed_image_input = self._parse_and_validate_image_input(image_input)
265 + image_input = self._parse_and_validate_image_input(**kwargs)
266
248 - if parsed_image_input is not None:
249 - vision_embeddings = self._process_image_input(parsed_image_input)
267 + if image_input is not None:
268 + vision_embeddings = self._process_image_input(image_input)
269 inputs_embeds = self.language_model.get_input_embeddings(input_ids)
270
271 inputs_embeds = _merge_vision_embeddings(
vllm/multimodal/__init__.py new
+7
@@ -0,0 +1,7 @@
1 +from .base import MultiModalData, MultiModalPlugin
2 +from .registry import MULTIMODAL_REGISTRY, MultiModalRegistry
3 +
4 +__all__ = [
5 + "MultiModalData", "MultiModalPlugin", "MULTIMODAL_REGISTRY",
6 + "MultiModalRegistry"
7 +]
vllm/multimodal/base.py new
+126
@@ -0,0 +1,126 @@
1 +from abc import ABC, abstractmethod
2 +from typing import (TYPE_CHECKING, Callable, Dict, Generic, Optional, Type,
3 + TypeVar)
4 +
5 +from vllm.config import ModelConfig, VisionLanguageConfig
6 +from vllm.logger import init_logger
7 +
8 +if TYPE_CHECKING:
9 + import torch
10 + from torch import nn
11 +
12 +logger = init_logger(__name__)
13 +
14 +
15 +class MultiModalData:
16 + """
17 + Base class that contains multi-modal data.
18 +
19 + To add a new modality, add a new file under ``multimodal`` directory.
20 +
21 + In this new file, subclass :class:`~MultiModalData` and
22 + :class:`~MultiModalPlugin`.
23 +
24 + Finally, register the new plugin to
25 + :const:`vllm.multimodal.MULTIMODAL_REGISTRY`.
26 + This enables models to call :meth:`MultiModalRegistry.register_input` for
27 + the new modality.
28 + """
29 + pass
30 +
31 +
32 +D = TypeVar("D", bound=MultiModalData)
33 +N = TypeVar("N", bound=Type["nn.Module"])
34 +
35 +MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
36 + Dict[str, "torch.Tensor"]]
37 +"""Return a dictionary to be passed as keyword arguments to
38 +:meth:`torch.nn.Module.forward`. This is similar in concept to tokenizers
39 +and processors in HuggingFace Transformers."""
40 +
41 +
42 +class MultiModalPlugin(ABC, Generic[D]):
43 + """
44 + Base class that defines data processing logic for a specific modality.
45 +
46 + In particular, we adopt a registry pattern to dispatch data processing
47 + according to the model being used (considering that different models may
48 + process the same data differently). This registry is in turn used by
49 + :class:`~MultiModalRegistry` which acts at a higher level
50 + (i.e., the modality of the data).
51 + """
52 +
53 + @classmethod
54 + def get_model_cls(cls, model_config: ModelConfig) -> Type["nn.Module"]:
55 + # Avoid circular import
56 + from vllm.model_executor.model_loader import get_model_architecture
57 +
58 + return get_model_architecture(model_config)[0]
59 +
60 + def __init__(self) -> None:
61 + self._input_processors: Dict[Type["nn.Module"],
62 + MultiModalInputProcessor[D]] = {}
63 +
64 + @abstractmethod
65 + def get_data_type(self) -> Type[D]:
66 + """
67 + Get the modality (subclass of :class:`~MultiModalData`) served by
68 + this plugin.
69 + """
70 + raise NotImplementedError
71 +
72 + @abstractmethod
73 + def _default_input_processor(
74 + self, data: D, model_config: ModelConfig,
75 + vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
76 + """Return a dictionary to be passed as keyword arguments to
77 + :meth:`torch.nn.Module.forward`. This is similar in concept to
78 + tokenizers and processors in HuggingFace Transformers.
79 + """
80 + raise NotImplementedError
81 +
82 + def register_input_processor(self,
83 + processor: Optional[
84 + MultiModalInputProcessor[D]] = None):
85 + """
86 + Register an input processor to a model class.
87 +
88 + When the model receives input data that matches the modality served by
89 + this plugin (see :meth:`get_data_type`), the provided input processor is
90 + applied to preprocess the data. If `None` is provided, then the default
91 + input processor is applied instead.
92 + """
93 +
94 + def wrapper(model_cls: N) -> N:
95 + if model_cls in self._input_processors:
96 + logger.warning(
97 + "Model class %s already has an input processor "
98 + "registered to %s. It is overwritten by the new one.",
99 + model_cls, self)
100 +
101 + self._input_processors[model_cls] = processor \
102 + or self._default_input_processor
103 +
104 + return model_cls
105 +
106 + return wrapper
107 +
108 + def process_input(
109 + self, data: D, model_config: ModelConfig,
110 + vlm_config: VisionLanguageConfig) -> Dict[str, "torch.Tensor"]:
111 + """
112 + Apply an input processor to a :class:`~MultiModalData` instance passed
113 + to the model.
114 +
115 + The model is identified by ``model_config``. ``vlm_config`` is
116 + for compatibility purposes and may be merged into ``model_config``
117 + in the near future.
118 + """
119 + model_cls = self.get_model_cls(model_config)
120 +
121 + processor = self._input_processors.get(model_cls)
122 + if processor is None:
123 + raise KeyError(f"No input processor in {self} is registered for "
124 + f"model class {model_cls.__name__}.")
125 +
126 + return processor(data, model_config, vlm_config)
vllm/multimodal/image.py new
+141
@@ -0,0 +1,141 @@
1 +from typing import Dict, Tuple, Type, Union
2 +
3 +import torch
4 +from PIL import Image
5 +
6 +from vllm.config import ModelConfig, VisionLanguageConfig
7 +from vllm.logger import init_logger
8 +from vllm.sequence import SequenceData
9 +from vllm.transformers_utils.image_processor import cached_get_image_processor
10 +
11 +from .base import MultiModalData, MultiModalPlugin
12 +
13 +logger = init_logger(__name__)
14 +
15 +
16 +def _get_dummy_seq_data(seq_len: int,
17 + vlm_config: VisionLanguageConfig) -> SequenceData:
18 + # NOTE: We assume that <image> token is repeated `image_feature_size` times
19 + # and then concatenated with the text prompt
20 + # TODO: Enable other ways of inserting the image into the prompt
21 +
22 + token_ids = [vlm_config.image_token_id] * vlm_config.image_feature_size
23 + token_ids += [0] * (seq_len - vlm_config.image_feature_size)
24 +
25 + return SequenceData(token_ids)
26 +
27 +
28 +def _get_dummy_values(vlm_config: VisionLanguageConfig) -> torch.Tensor:
29 + if vlm_config.image_processor is None:
30 + values_dtype = torch.float16
31 + else:
32 + values_dtype = torch.uint8
33 +
34 + return torch.zeros(vlm_config.image_input_shape, dtype=values_dtype)
35 +
36 +
37 +def get_dummy_image_data(
38 + seq_len: int,
39 + model_config: ModelConfig,
40 + vlm_config: VisionLanguageConfig,
41 +) -> Tuple[SequenceData, MultiModalData]:
42 + """Standard dummy data factory for image data (to be used in
43 + :meth:`vlm.multimodal.MultiModalRegistry.register_dummy_data`)."""
44 + seq_data = _get_dummy_seq_data(seq_len, vlm_config)
45 + values = _get_dummy_values(vlm_config)
46 +
47 + config_input_type = vlm_config.image_input_type
48 + ImageInputType = VisionLanguageConfig.ImageInputType
49 +
50 + fake_mm_data: MultiModalData
51 + if config_input_type == ImageInputType.PIXEL_VALUES:
52 + fake_mm_data = ImagePixelData(values)
53 + elif config_input_type == ImageInputType.IMAGE_FEATURES:
54 + fake_mm_data = ImageFeatureData(values)
55 + else:
56 + raise NotImplementedError
57 +
58 + return seq_data, fake_mm_data
59 +
60 +
61 +class ImagePixelData(MultiModalData):
62 + """
63 + The pixel data of an image. Can be one of:
64 +
65 + - :class:``PIL.Image``: An image object. Requires that a HuggingFace
66 + processor is available to the model.
67 + - :class:``torch.Tensor``: The raw pixel data which is passed to the model
68 + without additional pre-processing.
69 + """
70 +
71 + def __init__(self, image: Union[Image.Image, torch.Tensor]) -> None:
72 + if isinstance(image, Image.Image):
73 + # So that this class can be created inside the Image context manager
74 + image.load()
75 +
76 + self.image = image
77 +
78 +
79 +class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
80 +
81 + def get_data_type(self) -> Type[ImagePixelData]:
82 + return ImagePixelData
83 +
84 + def _get_hf_image_processor(self, model_config: ModelConfig,
85 + vlm_config: VisionLanguageConfig):
86 + if vlm_config is None or vlm_config.image_processor is None:
87 + return None
88 +
89 + return cached_get_image_processor(
90 + vlm_config.image_processor,
91 + trust_remote_code=model_config.trust_remote_code,
92 + revision=vlm_config.image_processor_revision,
93 + )
94 +
95 + def _default_input_processor(
96 + self, data: ImagePixelData, model_config: ModelConfig,
97 + vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
98 + image = data.image
99 + image_processor = self._get_hf_image_processor(model_config,
100 + vlm_config)
101 +
102 + if isinstance(image, Image.Image):
103 + if image_processor is None:
104 + raise RuntimeError("No HuggingFace processor is available"
105 + "to process the image object")
106 + try:
107 + return image_processor.preprocess(image, return_tensors="pt") \
108 + .to(model_config.dtype).data
109 + except Exception:
110 + logger.error("Failed to process image (%s)", image)
111 + raise
112 + elif isinstance(image, torch.Tensor):
113 + pixel_values = image.to(model_config.dtype)
114 +
115 + return {"pixel_values": pixel_values}
116 +
117 + raise TypeError(f"Invalid image type: {type(image)}")
118 +
119 +
120 +class ImageFeatureData(MultiModalData):
121 + """
122 + The feature vector of an image, passed directly to the model.
123 +
124 + This should be the output of the vision tower.
125 + """
126 +
127 + def __init__(self, image_features: torch.Tensor) -> None:
128 + self.image_features = image_features
129 +
130 +
131 +class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
132 +
133 + def get_data_type(self) -> Type[ImageFeatureData]:
134 + return ImageFeatureData
135 +
136 + def _default_input_processor(
137 + self, data: ImageFeatureData, model_config: ModelConfig,
138 + vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
139 + image_features = data.image_features.to(model_config.dtype)
140 +
141 + return {"image_features": image_features}
vllm/multimodal/registry.py new
+156
@@ -0,0 +1,156 @@
1 +import functools
2 +from typing import (TYPE_CHECKING, Any, Callable, Dict, Optional, Sequence,
3 + Tuple, Type, TypeVar)
4 +
5 +from vllm.config import ModelConfig, VisionLanguageConfig
6 +from vllm.logger import init_logger
7 +
8 +from .base import MultiModalData, MultiModalPlugin
9 +from .image import (ImageFeatureData, ImageFeaturePlugin, ImagePixelData,
10 + ImagePixelPlugin)
11 +
12 +if TYPE_CHECKING:
13 + import torch
14 + from torch import nn
15 +
16 + from vllm.sequence import SequenceData
17 +
18 +logger = init_logger(__name__)
19 +
20 +D = TypeVar("D", bound=MultiModalData)
21 +N = TypeVar("N", bound=Type["nn.Module"])
22 +
23 +MultiModalInputProcessor = Callable[[D, ModelConfig, VisionLanguageConfig],
24 + Dict[str, "torch.Tensor"]]
25 +MultiModalDummyFactory = Callable[[int, ModelConfig, VisionLanguageConfig],
26 + Tuple["SequenceData", MultiModalData]]
27 +
28 +
29 +class MultiModalRegistry:
30 + """
31 + This registry is used by model runners to dispatch data processing
32 + according to its modality and the target model.
33 + """
34 +
35 + DEFAULT_PLUGINS = (ImageFeaturePlugin(), ImagePixelPlugin())
36 +
37 + def __init__(self,
38 + *,
39 + plugins: Sequence[MultiModalPlugin[Any]] = DEFAULT_PLUGINS
40 + ) -> None:
41 + self._plugins_by_data_type = {p.get_data_type(): p for p in plugins}
42 + self._dummy_factories_by_model_type: Dict[Type["nn.Module"],
43 + MultiModalDummyFactory] = {}
44 +
45 + def register_plugin(self, plugin: MultiModalPlugin[Any]) -> None:
46 + data_type = plugin.get_data_type()
47 +
48 + if data_type in self._plugins_by_data_type:
49 + logger.warning(
50 + "A plugin is already registered for data type %s, "
51 + "and will be overwritten by the new plugin %s.", data_type,
52 + plugin)
53 +
54 + self._plugins_by_data_type[data_type] = plugin
55 +
56 + def _get_plugin_for_data_type(self, data_type: Type[MultiModalData]):
57 + for typ in data_type.mro():
58 + plugin = self._plugins_by_data_type.get(typ)
59 + if plugin is not None:
60 + return plugin
61 +
62 + msg = f"Unknown multi-modal data type: {data_type}"
63 + raise NotImplementedError(msg)
64 +
65 + def register_dummy_data(self, factory: MultiModalDummyFactory):
66 + """
67 + Register a dummy data factory to a model class.
68 +
69 + During memory profiling, the provided function is invoked to create
70 + dummy data to be inputted into the model. The modality and shape of
71 + the dummy data should be an upper bound of what the model would receive
72 + at inference time.
73 + """
74 +
75 + def wrapper(model_cls: N) -> N:
76 + if model_cls in self._dummy_factories_by_model_type:
77 + logger.warning(
78 + "Model class %s already has dummy data "
79 + "registered to %s. It is overwritten by the new one.",
80 + model_cls, self)
81 +
82 + self._dummy_factories_by_model_type[model_cls] = factory
83 +
84 + return model_cls
85 +
86 + return wrapper
87 +
88 + def dummy_data_for_profiling(self, seq_len: int, model_config: ModelConfig,
89 + vlm_config: VisionLanguageConfig):
90 + """Create dummy data for memory profiling."""
91 + model_cls = MultiModalPlugin.get_model_cls(model_config)
92 + dummy_factory = self._dummy_factories_by_model_type.get(model_cls)
93 + if dummy_factory is None:
94 + msg = f"No dummy data defined for model class: {model_cls}"
95 + raise NotImplementedError(msg)
96 +
97 + return dummy_factory(seq_len, model_config, vlm_config)
98 +
99 + def register_input(
100 + self,
101 + data_type: Type[D],
102 + processor: Optional[MultiModalInputProcessor[D]] = None):
103 + """
104 + Register an input processor for a specific modality to a model class.
105 +
106 + See :meth:`MultiModalPlugin.register_input_processor` for more details.
107 + """
108 + return self._get_plugin_for_data_type(data_type) \
109 + .register_input_processor(processor)
110 +
111 + def register_image_pixel_input(
112 + self,
113 + processor: Optional[
114 + MultiModalInputProcessor[ImagePixelData]] = None):
115 + """
116 + Register an input processor for image pixel data to a model class.
117 +
118 + See :meth:`MultiModalPlugin.register_input_processor` for more details.
119 + """
120 + return self.register_input(ImagePixelData, processor)
121 +
122 + def register_image_feature_input(
123 + self,
124 + processor: Optional[
125 + MultiModalInputProcessor[ImageFeatureData]] = None):
126 + """
127 + Register an input processor for image feature data to a model class.
128 +
129 + See :meth:`MultiModalPlugin.register_input_processor` for more details.
130 + """
131 + return self.register_input(ImageFeatureData, processor)
132 +
133 + def process_input(self, data: MultiModalData, model_config: ModelConfig,
134 + vlm_config: VisionLanguageConfig):
135 + """
136 + Apply an input processor to a :class:`~MultiModalData` instance passed
137 + to the model.
138 +
139 + See :meth:`MultiModalPlugin.process_input` for more details.
140 + """
141 + return self._get_plugin_for_data_type(type(data)) \
142 + .process_input(data, model_config, vlm_config)
143 +
144 + def create_input_processor(self, model_config: ModelConfig,
145 + vlm_config: VisionLanguageConfig):
146 + """
147 + Create an input processor (see :meth:`process_input`) for a
148 + specific model.
149 + """
150 + return functools.partial(self.process_input,
151 + model_config=model_config,
152 + vlm_config=vlm_config)
153 +
154 +
155 +MULTIMODAL_REGISTRY = MultiModalRegistry()
156 +"""The global :class:`~MultiModalRegistry` which is used by model runners."""
vllm/sequence.py
+7 -25
@@ -5,6 +5,8 @@ from abc import ABC, abstractmethod
5 from dataclasses import dataclass, field
6 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8 +import torch
9 +
10 from vllm.block import LogicalTokenBlock
11 from vllm.inputs import LLMInputs
12 from vllm.lora.request import LoRARequest
@@ -12,8 +14,7 @@ from vllm.pooling_params import PoolingParams
14 from vllm.sampling_params import SamplingParams
15
16 if TYPE_CHECKING:
15 - import torch
16 -
17 + from vllm.multimodal import MultiModalData
18 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
19
20
@@ -398,25 +399,6 @@ class SequenceGroupState:
399 generator: Optional = None # type: ignore
400
401
401 -class MultiModalData:
402 - """Multi modal request.
403 -
404 - Args:
405 - type: The data type.
406 - data: The actual data.
407 - The required shape and semantic meaning of it depends on the vision
408 - language config of the hosted model.
409 - See `VisionLanguageConfig` in `config.py`.
410 - """
411 -
412 - class Type(enum.Enum):
413 - IMAGE = enum.auto()
414 -
415 - def __init__(self, type: Type, data: "torch.Tensor"):
416 - self.type = type
417 - self.data = data
418 -
419 -
402 class SequenceGroup:
403 """A group of sequences that are generated from the same prompt.
404
@@ -473,7 +455,7 @@ class SequenceGroup:
455 return next(iter(self.seqs_dict.values())).prompt_token_ids
456
457 @property
476 - def multi_modal_data(self) -> Optional[MultiModalData]:
458 + def multi_modal_data(self) -> Optional["MultiModalData"]:
459 # All sequences in the group should have the same multi-modal data.
460 # We use the multi-modal data of an arbitrary sequence.
461 return next(iter(self.seqs_dict.values())).multi_modal_data
@@ -655,7 +637,7 @@ class SequenceGroupMetadata:
637 lora_request: Optional[LoRARequest] = None,
638 computed_block_nums: Optional[List[int]] = None,
639 state: Optional[SequenceGroupState] = None,
658 - multi_modal_data: Optional[MultiModalData] = None,
640 + multi_modal_data: Optional["MultiModalData"] = None,
641 encoder_seq_data: Optional[SequenceData] = None,
642 cross_block_table: Optional[List[int]] = None,
643 ) -> None:
@@ -798,13 +780,13 @@ class SamplerOutput:
780 outputs: List[CompletionSequenceGroupOutput]
781
782 # On-device tensor containing probabilities of each token.
801 - sampled_token_probs: Optional["torch.Tensor"] = None
783 + sampled_token_probs: Optional[torch.Tensor] = None
784
785 # On-device tensor containing the logprobs of each token.
786 logprobs: Optional["torch.Tensor"] = None
787
788 # On-device tensor containing the sampled token ids.
807 - sampled_token_ids: Optional["torch.Tensor"] = None
789 + sampled_token_ids: Optional[torch.Tensor] = None
790
791 # Spec decode metrics populated by workers.
792 spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
vllm/transformers_utils/image_processor.py new
+45
@@ -0,0 +1,45 @@
1 +from functools import lru_cache
2 +from typing import Optional
3 +
4 +from transformers import AutoImageProcessor
5 +from transformers.image_processing_utils import BaseImageProcessor
6 +
7 +from vllm.logger import init_logger
8 +
9 +logger = init_logger(__name__)
10 +
11 +
12 +def get_image_processor(
13 + processor_name: str,
14 + *args,
15 + trust_remote_code: bool = False,
16 + revision: Optional[str] = None,
17 + **kwargs,
18 +) -> BaseImageProcessor:
19 + """Gets an image processor for the given model name via HuggingFace."""
20 + try:
21 + processor: BaseImageProcessor = AutoImageProcessor.from_pretrained(
22 + processor_name,
23 + *args,
24 + trust_remote_code=trust_remote_code,
25 + revision=revision,
26 + **kwargs)
27 + except ValueError as e:
28 + # If the error pertains to the processor class not existing or not
29 + # currently being imported, suggest using the --trust-remote-code flag.
30 + # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
31 + if not trust_remote_code:
32 + err_msg = (
33 + "Failed to load the image processor. If the image processor is "
34 + "a custom processor not yet available in the HuggingFace "
35 + "transformers library, consider setting "
36 + "`trust_remote_code=True` in LLM or using the "
37 + "`--trust-remote-code` flag in the CLI.")
38 + raise RuntimeError(err_msg) from e
39 + else:
40 + raise e
41 +
42 + return processor
43 +
44 +
45 +cached_get_image_processor = lru_cache(get_image_processor)
vllm/worker/cpu_model_runner.py
+37 -20
@@ -1,4 +1,5 @@
1 -from typing import List, Optional, Tuple
1 +from collections import defaultdict
2 +from typing import Dict, List, Optional, Tuple
3
4 import torch
5 from torch import nn
@@ -11,6 +12,7 @@ from vllm.distributed import broadcast_tensor_dict
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
15 +from vllm.multimodal import MULTIMODAL_REGISTRY
16 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
17 from vllm.utils import make_tensor_with_pad
18
@@ -63,6 +65,16 @@ class CPUModelRunner:
65 self.block_size,
66 )
67
68 + # Create processor for multi-modal data
69 + if self.vision_language_config is not None:
70 + self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
71 + .create_input_processor(
72 + self.model_config,
73 + self.vision_language_config,
74 + )
75 + else:
76 + self.multi_modal_input_processor = None
77 +
78 # Lazy initialization.
79 self.model: nn.Module # Set after init_Model
80
@@ -80,14 +92,15 @@ class CPUModelRunner:
92 def _prepare_prompt(
93 self,
94 seq_group_metadata_list: List[SequenceGroupMetadata],
83 - ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
84 - Optional[torch.Tensor]]:
95 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int], Dict[
96 + str, torch.Tensor]]:
97 assert len(seq_group_metadata_list) > 0
98 input_tokens: List[int] = []
99 input_positions: List[int] = []
100 slot_mapping: List[int] = []
101 seq_lens: List[int] = []
90 - multi_modal_input_list: List[torch.Tensor] = []
102 + multi_modal_kwargs_list: Dict[str,
103 + List[torch.Tensor]] = defaultdict(list)
104
105 for seq_group_metadata in seq_group_metadata_list:
106 assert seq_group_metadata.is_prompt
@@ -108,9 +121,17 @@ class CPUModelRunner:
121 # is always the first token in the sequence.
122 input_positions.extend(list(range(computed_len, seq_len)))
123
111 - if seq_group_metadata.multi_modal_data:
112 - multi_modal_input_list.append(
113 - seq_group_metadata.multi_modal_data.data)
124 + mm_data = seq_group_metadata.multi_modal_data
125 + if mm_data is not None:
126 + # Process multi-modal data
127 + if self.multi_modal_input_processor is None:
128 + raise ValueError(
129 + "Multi-modal inputs are only supported by "
130 + "vision language models.")
131 +
132 + mm_kwargs = self.multi_modal_input_processor(mm_data)
133 + for k, v in mm_kwargs.items():
134 + multi_modal_kwargs_list[k].append(v)
135
136 # Compute the slot mapping.
137 block_table = seq_group_metadata.block_tables[seq_id]
@@ -134,14 +155,10 @@ class CPUModelRunner:
155 slot = block_number * self.block_size + block_offset
156 slot_mapping.append(slot)
157
137 - if multi_modal_input_list:
138 - assert self.vision_language_config, (
139 - "Multi-modal inputs are only supported by "
140 - "vision language models.")
141 - multi_modal_input = torch.cat(multi_modal_input_list,
142 - dim=0).to(self.device)
143 - else:
144 - multi_modal_input = None
158 + multi_modal_kwargs = {
159 + k: torch.cat(v, dim=0).to(self.device)
160 + for k, v in multi_modal_kwargs_list.items()
161 + }
162
163 num_prompt_tokens = len(input_tokens)
164
@@ -167,7 +184,7 @@ class CPUModelRunner:
184 slot_mapping=slot_mapping,
185 )
186 return (input_tokens, input_positions, attn_metadata, seq_lens,
170 - multi_modal_input)
187 + multi_modal_kwargs)
188
189 def _prepare_decode(
190 self,
@@ -257,8 +274,8 @@ class CPUModelRunner:
274 self,
275 seq_group_metadata_list: List[SequenceGroupMetadata],
276 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
260 - Optional[torch.Tensor]]:
261 - multi_modal_input = None
277 + Optional[Dict[str, torch.Tensor]]]:
278 + multi_modal_kwargs = None
279 if self.is_driver_worker:
280 # NOTE: We assume that all sequences in the group are all prompts or
281 # all decodes.
@@ -266,7 +283,7 @@ class CPUModelRunner:
283 # Prepare input tensors.
284 if is_prompt:
285 (input_tokens, input_positions, attn_metadata, seq_lens,
269 - multi_modal_input
286 + multi_modal_kwargs
287 ) = self._prepare_prompt(seq_group_metadata_list)
288 else:
289 (input_tokens, input_positions,
@@ -307,7 +324,7 @@ class CPUModelRunner:
324 )
325
326 return (input_tokens, input_positions, attn_metadata,
310 - sampling_metadata, multi_modal_input)
327 + sampling_metadata, multi_modal_kwargs)
328
329 @torch.inference_mode()
330 def execute_model(
vllm/worker/embedding_model_runner.py
+5 -5
@@ -90,7 +90,7 @@ class EmbeddingModelRunner(ModelRunner):
90 self,
91 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
92 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
93 - Set[LoRARequest], LoRAMapping, torch.Tensor]:
93 + Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
94 if self.is_driver_worker:
95 assert seq_group_metadata_list is not None
96 # Prepare input tensors.
@@ -102,7 +102,7 @@ class EmbeddingModelRunner(ModelRunner):
102 _,
103 lora_mapping,
104 lora_requests,
105 - multi_modal_input,
105 + multi_modal_kwargs,
106 slot_mapping,
107 num_prefill_tokens,
108 num_decode_tokens,
@@ -117,7 +117,7 @@ class EmbeddingModelRunner(ModelRunner):
117 "input_positions": input_positions,
118 "lora_requests": lora_requests,
119 "lora_mapping": lora_mapping,
120 - "multi_modal_input": multi_modal_input,
120 + "multi_modal_kwargs": multi_modal_kwargs,
121 "num_prefill_tokens": num_prefill_tokens,
122 "num_decode_tokens": num_decode_tokens,
123 "slot_mapping": slot_mapping,
@@ -132,7 +132,7 @@ class EmbeddingModelRunner(ModelRunner):
132 input_positions = metadata_dict.pop("input_positions")
133 lora_mapping = metadata_dict.pop("lora_mapping")
134 lora_requests = metadata_dict.pop("lora_requests")
135 - multi_modal_input = metadata_dict.pop("multi_modal_input")
135 + multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
136 if metadata_dict:
137 attn_metadata = self.attn_backend.make_metadata(
138 **metadata_dict)
@@ -143,7 +143,7 @@ class EmbeddingModelRunner(ModelRunner):
143 prompt_lens=None)
144
145 return (input_tokens, input_positions, attn_metadata, pooling_metadata,
146 - lora_requests, lora_mapping, multi_modal_input)
146 + lora_requests, lora_mapping, multi_modal_kwargs)
147
148 def _prepare_pooling(
149 self,
vllm/worker/model_runner.py
+62 -58
@@ -1,5 +1,6 @@
1 import time
2 import warnings
3 +from collections import defaultdict
4 from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
5
6 import numpy as np
@@ -18,9 +19,9 @@ from vllm.lora.request import LoRARequest
19 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
20 from vllm.model_executor import SamplingMetadata
21 from vllm.model_executor.model_loader import get_model
22 +from vllm.multimodal import MULTIMODAL_REGISTRY
23 from vllm.sampling_params import SamplingParams
22 -from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
23 - SequenceGroupMetadata)
24 +from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
25 from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
26 is_pin_memory_available, make_tensor_with_pad)
27
@@ -44,7 +45,7 @@ class ModelInput(NamedTuple):
45 query_lens: List[int]
46 lora_mapping: Optional[LoRAMapping]
47 lora_requests: Set[LoRARequest]
47 - multi_modal_input: Optional[torch.Tensor]
48 + multi_modal_kwargs: Dict[str, torch.Tensor]
49 slot_mapping: torch.Tensor
50 num_prefill_tokens: int
51 num_decode_tokens: int
@@ -60,7 +61,7 @@ class ModelInput(NamedTuple):
61 query_lens=[],
62 lora_mapping=None,
63 lora_requests=set(),
63 - multi_modal_input=None,
64 + multi_modal_kwargs={},
65 slot_mapping=torch.empty(0, device=device),
66 num_prefill_tokens=0,
67 num_decode_tokens=0,
@@ -122,6 +123,16 @@ class ModelRunner:
123 self.block_size,
124 )
125
126 + # Create processor for multi-modal data
127 + if self.vision_language_config is not None:
128 + self.multi_modal_input_processor = MULTIMODAL_REGISTRY \
129 + .create_input_processor(
130 + self.model_config,
131 + self.vision_language_config,
132 + )
133 + else:
134 + self.multi_modal_input_processor = None
135 +
136 # Lazy initialization
137 self.model: nn.Module # Set after load_model
138 # Set if the backend is flashinfer.
@@ -242,7 +253,8 @@ class ModelRunner:
253 context_lens: List[int] = []
254 query_lens: List[int] = []
255 block_tables: List[List[int]] = []
245 - multi_modal_input_list: List[torch.Tensor] = []
256 + multi_modal_kwargs_list: Dict[str,
257 + List[torch.Tensor]] = defaultdict(list)
258 decode_only = True
259 num_prefills = 0
260 num_prefill_tokens = 0
@@ -417,9 +429,17 @@ class ModelRunner:
429 and seq_group_metadata.sampling_params.prompt_logprobs
430 else 1))
431
420 - if seq_group_metadata.multi_modal_data:
421 - multi_modal_input_list.append(
422 - seq_group_metadata.multi_modal_data.data)
432 + mm_data = seq_group_metadata.multi_modal_data
433 + if mm_data is not None:
434 + # Process multi-modal data
435 + if self.multi_modal_input_processor is None:
436 + raise ValueError(
437 + "Multi-modal inputs are only supported by "
438 + "vision language models.")
439 +
440 + mm_kwargs = self.multi_modal_input_processor(mm_data)
441 + for k, v in mm_kwargs.items():
442 + multi_modal_kwargs_list[k].append(v)
443
444 if _is_block_tables_empty(seq_group_metadata.block_tables):
445 # During memory profiling, the block tables are not
@@ -508,16 +528,6 @@ class ModelRunner:
528 context_lens_tensor = torch.tensor(context_lens,
529 dtype=torch.int,
530 device=self.device)
511 -
512 - if multi_modal_input_list:
513 - assert self.vision_language_config, (
514 - "Multi-modal inputs are only supported by "
515 - "vision language models.")
516 - multi_modal_input = torch.cat(multi_modal_input_list,
517 - dim=0).to(self.device)
518 - else:
519 - multi_modal_input = None
520 -
531 query_lens_tensor = torch.tensor(query_lens,
532 dtype=torch.long,
533 device=self.device)
@@ -614,6 +624,11 @@ class ModelRunner:
624 else:
625 lora_mapping = None
626
627 + multi_modal_kwargs = {
628 + k: torch.cat(v, dim=0).to(self.device)
629 + for k, v in multi_modal_kwargs_list.items()
630 + }
631 +
632 return ModelInput(
633 input_tokens=input_tokens_tensor,
634 input_positions=input_positions_tensor,
@@ -622,7 +637,7 @@ class ModelRunner:
637 query_lens=query_lens,
638 lora_mapping=lora_mapping,
639 lora_requests=lora_requests,
625 - multi_modal_input=multi_modal_input,
640 + multi_modal_kwargs=multi_modal_kwargs,
641 slot_mapping=slot_mapping_tensor,
642 num_prefill_tokens=num_prefill_tokens,
643 num_decode_tokens=num_decode_tokens,
@@ -633,7 +648,7 @@ class ModelRunner:
648 self,
649 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
650 ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
636 - Set[LoRARequest], LoRAMapping, torch.Tensor]:
651 + Set[LoRARequest], LoRAMapping, Dict[str, torch.Tensor]]:
652 if self.is_driver_worker:
653 assert seq_group_metadata_list is not None
654 # Prepare input tensors.
@@ -645,7 +660,7 @@ class ModelRunner:
660 query_lens,
661 lora_mapping,
662 lora_requests,
648 - multi_modal_input,
663 + multi_modal_kwargs,
664 slot_mapping,
665 num_prefill_tokens,
666 num_decode_tokens,
@@ -662,7 +677,7 @@ class ModelRunner:
677 sampling_metadata.selected_token_indices,
678 "lora_requests": lora_requests,
679 "lora_mapping": lora_mapping,
665 - "multi_modal_input": multi_modal_input,
680 + "multi_modal_kwargs": multi_modal_kwargs,
681 "num_prefill_tokens": num_prefill_tokens,
682 "num_decode_tokens": num_decode_tokens,
683 "slot_mapping": slot_mapping,
@@ -679,7 +694,7 @@ class ModelRunner:
694 "selected_token_indices")
695 lora_mapping = metadata_dict.pop("lora_mapping")
696 lora_requests = metadata_dict.pop("lora_requests")
682 - multi_modal_input = metadata_dict.pop("multi_modal_input")
697 + multi_modal_kwargs = metadata_dict.pop("multi_modal_kwargs")
698 if metadata_dict:
699 attn_metadata = self.attn_backend.make_metadata(
700 **metadata_dict)
@@ -694,7 +709,7 @@ class ModelRunner:
709
710 return (input_tokens, input_positions, attn_metadata,
711 sampling_metadata, lora_requests, lora_mapping,
697 - multi_modal_input)
712 + multi_modal_kwargs)
713
714 @torch.inference_mode()
715 def execute_model(
@@ -703,7 +718,7 @@ class ModelRunner:
718 kv_caches: List[torch.Tensor],
719 ) -> Optional[SamplerOutput]:
720 (input_tokens, input_positions, attn_metadata, sampling_metadata,
706 - lora_requests, lora_mapping, multi_modal_input
721 + lora_requests, lora_mapping, multi_modal_kwargs
722 ) = self.prepare_input_tensors(seq_group_metadata_list)
723
724 if self.lora_config:
@@ -717,15 +732,14 @@ class ModelRunner:
732 model_executable = self.graph_runners[graph_batch_size]
733 else:
734 model_executable = self.model
720 - execute_model_kwargs = {
721 - "input_ids": input_tokens,
722 - "positions": input_positions,
723 - "kv_caches": kv_caches,
724 - "attn_metadata": attn_metadata,
725 - }
726 - if self.vision_language_config:
727 - execute_model_kwargs.update({"image_input": multi_modal_input})
728 - hidden_states = model_executable(**execute_model_kwargs)
735 +
736 + hidden_states = model_executable(
737 + input_ids=input_tokens,
738 + positions=input_positions,
739 + kv_caches=kv_caches,
740 + attn_metadata=attn_metadata,
741 + **multi_modal_kwargs,
742 + )
743
744 # Compute the logits.
745 logits = self.model.compute_logits(hidden_states, sampling_metadata)
@@ -781,16 +795,24 @@ class ModelRunner:
795 # To exercise the worst scenario for GPU memory consumption,
796 # the number of seqs (batch_size) is chosen to maximize the number
797 # of images processed.
784 - if self.vision_language_config:
798 + model_config = self.model_config
799 + vlm_config = self.vision_language_config
800 +
801 + if vlm_config:
802 max_num_seqs = min(
803 max_num_seqs,
787 - int(max_num_batched_tokens /
788 - self.vision_language_config.image_feature_size))
804 + int(max_num_batched_tokens / vlm_config.image_feature_size))
805 for group_id in range(max_num_seqs):
806 seq_len = (max_num_batched_tokens // max_num_seqs +
807 (group_id < max_num_batched_tokens % max_num_seqs))
792 - seq_data, fake_multi_modal_input = _prepare_fake_inputs(
793 - seq_len, self.vision_language_config)
808 +
809 + if vlm_config is None:
810 + seq_data = SequenceData([0] * seq_len)
811 + dummy_multi_modal_data = None
812 + else:
813 + seq_data, dummy_multi_modal_data = MULTIMODAL_REGISTRY \
814 + .dummy_data_for_profiling(seq_len, model_config, vlm_config)
815 +
816 seq = SequenceGroupMetadata(
817 request_id=str(group_id),
818 is_prompt=True,
@@ -799,7 +821,7 @@ class ModelRunner:
821 block_tables=None,
822 lora_request=dummy_lora_requests_per_seq[group_id]
823 if dummy_lora_requests_per_seq else None,
802 - multi_modal_data=fake_multi_modal_input,
824 + multi_modal_data=dummy_multi_modal_data,
825 )
826 seqs.append(seq)
827
@@ -1034,24 +1056,6 @@ def _get_graph_batch_size(batch_size: int) -> int:
1056 _BATCH_SIZE_ALIGNMENT * _BATCH_SIZE_ALIGNMENT)
1057
1058
1037 -def _prepare_fake_inputs(
1038 - seq_len: int, vision_language_config: Optional[VisionLanguageConfig]):
1039 - """Prepare fake inputs for profile run."""
1040 - if vision_language_config:
1041 - prompt_tokens = [
1042 - vision_language_config.image_token_id
1043 - ] * vision_language_config.image_feature_size + [0] * (
1044 - seq_len - vision_language_config.image_feature_size)
1045 - fake_image_input = MultiModalData(
1046 - type=MultiModalData.Type.IMAGE,
1047 - data=torch.zeros(vision_language_config.image_input_shape,
1048 - dtype=torch.float16))
1049 - else:
1050 - prompt_tokens = [0] * seq_len
1051 - fake_image_input = None
1052 - return SequenceData(prompt_tokens), fake_image_input
1053 -
1054 -
1059 def _is_block_tables_empty(block_tables: Union[None, Dict]):
1060 """
1061 Check if block_tables is None or a dictionary with all None values.