@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d9e98f42e4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[vlm] Remove vision language config. (#6089)

Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>

xwjiang2010 committed Jul 3, 2024 at 15:14 UTC d9e98f42e434d1f1a0f8ceed363047060aca6262
43 files changed +372 -466
docs/source/dev/multimodal/multimodal_index.rst
+5
@@ -10,8 +10,13 @@ vLLM provides experimental support for multi-modal models through the :mod:`vllm
10 :class:`vllm.inputs.PromptStrictInputs` accepts an additional attribute ``multi_modal_data``
11 which allows you to pass in multi-modal input alongside text and token prompts.
12
13 +.. note::
14 + ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
15 + :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
16 +
17 By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model, please follow :ref:`the guide for adding a new multimodal model. <adding_a_new_multimodal_model>`.
18
19 +
20 # TODO: Add more instructions on how to do that once embeddings is in.
21
22 Guides
docs/source/models/vlm.rst
+39 -39
@@ -8,18 +8,6 @@ vLLM provides experimental support for Vision Language Models (VLMs). This docum
8 .. important::
9 We are actively iterating on VLM support. Expect breaking changes to VLM usage and development in upcoming releases without prior deprecation.
10
11 -Engine Arguments
12 -----------------
13 -
14 -The following :ref:`engine arguments <engine_args>` are specific to VLMs:
15 -
16 -.. argparse::
17 - :module: vllm.engine.arg_utils
18 - :func: _vlm_engine_args_parser
19 - :prog: -m vllm.entrypoints.openai.api_server
20 - :nodefaultconst:
21 -
22 -.. important::
11 Currently, the support for vision language models on vLLM has the following limitations:
12
13 * Only single image input is supported per text prompt.
@@ -33,20 +21,17 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
21
22 .. code-block:: python
23
36 - llm = LLM(
37 - model="llava-hf/llava-1.5-7b-hf",
38 - image_token_id=32000,
39 - image_input_shape="1,3,336,336",
40 - image_feature_size=576,
41 - )
24 + llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25
26 .. important::
44 - Currently, you have to specify ``image_feature_size`` to support memory profiling.
45 - To avoid OOM during runtime, you should set this to the maximum value supported by the model.
46 - The calculation of feature size is specific to the model. For more details, please refer to
47 - the function :code:`get_<model_name>_image_feature_size` inside the corresponding model file.
27 + We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28 + the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
29 + every model to perform profiling with.
30
49 - We will remove most of the vision-specific arguments in a future release as they can be inferred from the HuggingFace configuration.
31 + This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
32 + :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
33 + for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
34 + with a more accurate profiling strategy in the future.
35
36
37 To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
@@ -54,19 +39,15 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
39 * ``prompt``: The prompt should follow the format that is documented on HuggingFace.
40 * ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
41
57 -.. note::
58 -
59 - ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
60 - :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
61 -
42 .. code-block:: python
43
44 # Refer to the HuggingFace repo for the correct format to use
45 prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
46
47 # Load the image using PIL.Image
68 - image = ...
69 -
48 + image = PIL.Image.open(...)
49 +
50 + # Single prompt inference
51 outputs = llm.generate({
52 "prompt": prompt,
53 "multi_modal_data": {"image": image},
@@ -75,6 +56,26 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
56 for o in outputs:
57 generated_text = o.outputs[0].text
58 print(generated_text)
59 +
60 + # Batch inference
61 + image_1 = PIL.Image.open(...)
62 + image_2 = PIL.Image.open(...)
63 + outputs = llm.generate(
64 + [
65 + {
66 + "prompt": "USER: <image>\nWhat is the content of this image?\nASSISTANT:",
67 + "multi_modal_data": {"image": image_1},
68 + },
69 + {
70 + "prompt": "USER: <image>\nWhat's the color of this image?\nASSISTANT:",
71 + "multi_modal_data": {"image": image_2},
72 + }
73 + ]
74 + )
75 +
76 + for o in outputs:
77 + generated_text = o.outputs[0].text
78 + print(generated_text)
79
80 A code example can be found in `examples/llava_example.py <https://github.com/vllm-project/vllm/blob/main/examples/llava_example.py>`_.
81
@@ -99,18 +100,17 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
100
101 python -m vllm.entrypoints.openai.api_server \
102 --model llava-hf/llava-1.5-7b-hf \
102 - --image-token-id 32000 \
103 - --image-input-shape 1,3,336,336 \
104 - --image-feature-size 576 \
103 --chat-template template_llava.jinja
104
105 .. important::
108 - Currently, you have to specify ``image_feature_size`` to support memory profiling.
109 - To avoid OOM during runtime, you should set this to the maximum value supported by the model.
110 - The calculation of feature size is specific to the model. For more details, please refer to
111 - the function :code:`get_<model_name>_image_feature_size` inside the corresponding model file.
112 -
113 - We will remove most of the vision-specific arguments in a future release as they can be inferred from the HuggingFace configuration.
106 + We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
107 + the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
108 + every model to perform profiling with.
109 +
110 + This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
111 + :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
112 + for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
113 + with a more accurate profiling strategy in the future.
114
115 To consume the server, you can use the OpenAI client like in the example below:
116
examples/llava_example.py
+1 -6
@@ -10,12 +10,7 @@ from vllm import LLM
10
11
12 def run_llava():
13 - llm = LLM(
14 - model="llava-hf/llava-1.5-7b-hf",
15 - image_token_id=32000,
16 - image_input_shape="1,3,336,336",
17 - image_feature_size=576,
18 - )
13 + llm = LLM(model="llava-hf/llava-1.5-7b-hf")
14
15 prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
16
examples/llava_next_example.py
+1 -7
@@ -7,13 +7,7 @@ from vllm import LLM, SamplingParams
7
8
9 def run_llava_next():
10 - llm = LLM(
11 - model="llava-hf/llava-v1.6-mistral-7b-hf",
12 - image_token_id=32000,
13 - image_input_shape="1,3,336,336",
14 - # Use the maximum possible value for memory profiling
15 - image_feature_size=2928,
16 - )
10 + llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=4096)
11
12 prompt = "[INST] <image>\nWhat is shown in this image? [/INST]"
13 url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
examples/openai_vision_api_client.py
-3
@@ -3,9 +3,6 @@
3 Launch the vLLM server with the following command:
4 python -m vllm.entrypoints.openai.api_server \
5 --model llava-hf/llava-1.5-7b-hf \
6 - --image-token-id 32000 \
7 - --image-input-shape 1,3,336,336 \
8 - --image-feature-size 576 \
6 --chat-template template_llava.jinja
7 """
8 import base64
examples/phi3v_example.py
+2 -4
@@ -14,15 +14,13 @@ def run_phi3v():
14
15 # Note: The default setting of max_num_seqs (256) and
16 # max_model_len (128k) for this model may cause OOM.
17 + # You may lower either to run this example on lower-end GPUs.
18 +
19 # In this example, we override max_num_seqs to 5 while
20 # keeping the original context length of 128k.
21 llm = LLM(
22 model=model_path,
23 trust_remote_code=True,
22 - image_token_id=32044,
23 - image_input_shape="1,3,1008,1344",
24 - # Use the maximum possible value for memory profiling
25 - image_feature_size=2653,
24 max_num_seqs=5,
25 )
26
tests/distributed/test_multimodal_broadcast.py
+3 -3
@@ -20,9 +20,9 @@ from vllm.utils import cuda_device_count_stateless
20 model = os.environ["TEST_DIST_MODEL"]
21
22 if model.startswith("llava-hf/llava"):
23 - from ..models.test_llava import model_and_vl_config, run_test
23 + from ..models.test_llava import models, run_test
24 elif model.startswith("microsoft/Phi-3-vision"):
25 - from ..models.test_phi3v import model_and_vl_config, run_test
25 + from ..models.test_phi3v import models, run_test
26 else:
27 raise NotImplementedError(f"Unsupported model: {model}")
28
@@ -44,7 +44,7 @@ def test_models(hf_runner, vllm_runner, image_assets,
44 hf_runner,
45 vllm_runner,
46 image_assets,
47 - model_and_config=model_and_vl_config[0],
47 + model=models[0],
48 size_factors=[1.0],
49 dtype=dtype,
50 max_tokens=max_tokens,
tests/entrypoints/openai/test_vision.py
-6
@@ -39,12 +39,6 @@ def server(ray_ctx):
39 "--max-model-len",
40 "4096",
41 "--enforce-eager",
42 - "--image-token-id",
43 - "32000",
44 - "--image-input-shape",
45 - "1,3,336,336",
46 - "--image-feature-size",
47 - "576",
42 "--chat-template",
43 str(LLAVA_CHAT_TEMPLATE),
44 ])
tests/models/test_llava.py
+17 -43
@@ -3,7 +3,6 @@ from typing import List, Optional, Tuple, Type
3 import pytest
4 from transformers import AutoTokenizer
5
6 -from vllm.config import VisionLanguageConfig
6 from vllm.multimodal.utils import rescale_image_size
7 from vllm.sequence import SampleLogprobs
8
@@ -21,49 +20,27 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20 "USER: <image>\nWhat's in this image?\nASSISTANT:",
21 })
22
23 +IMAGE_TOKEN_ID = 32000
24
25 -def iter_llava_configs(model_name: str):
26 - image_hw_to_feature_size = {
27 - (336, 336): 576,
28 - }
29 -
30 - for (h, w), f in image_hw_to_feature_size.items():
31 - input_shape = (1, 3, h, w)
32 - yield (model_name,
33 - VisionLanguageConfig(image_feature_size=f,
34 - image_token_id=32000,
35 - image_input_shape=input_shape))
36 -
37 -
38 -model_and_vl_config = [
39 - *iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
40 -]
25 +models = ["llava-hf/llava-1.5-7b-hf"]
26
27
28 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
29 Optional[SampleLogprobs]],
45 - vlm_config: VisionLanguageConfig, model_id: str):
46 - """Sanitize vllm output to be comparable with hf output.
47 - The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
48 - x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
49 - It also reduces `output_str` from "<image><image>bla" to "bla".
50 - """
30 + model: str):
31 + """Sanitize vllm output to be comparable with hf output."""
32 output_ids, output_str, out_logprobs = vllm_output
52 - image_token_id = vlm_config.image_token_id
33
54 - tokenizer = AutoTokenizer.from_pretrained(model_id)
55 - image_token_str = tokenizer.decode(image_token_id)
34 + tokenizer = AutoTokenizer.from_pretrained(model)
35 eos_token_id = tokenizer.eos_token_id
36
37 hf_output_ids = [
38 token_id for idx, token_id in enumerate(output_ids)
60 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
39 + if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
40 ]
41
63 - hf_output_str = output_str \
64 - .replace(image_token_str * vlm_config.image_feature_size, "")
65 - assert hf_output_str[0] == " "
66 - hf_output_str = hf_output_str[1:]
42 + assert output_str[0] == " "
43 + hf_output_str = output_str[1:]
44 if hf_output_ids[-1] == eos_token_id:
45 hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
46
@@ -74,7 +51,7 @@ def run_test(
51 hf_runner: Type[HfRunner],
52 vllm_runner: Type[VllmRunner],
53 image_assets: _ImageAssets,
77 - model_and_config: Tuple[str, VisionLanguageConfig],
54 + model: str,
55 *,
56 size_factors: List[float],
57 dtype: str,
@@ -92,7 +69,6 @@ def run_test(
69 Note, the text input is also adjusted to abide by vllm contract.
70 The text output is sanitized to be able to compare with hf.
71 """
95 - model_id, vlm_config = model_and_config
72 images = [asset.pil_image for asset in image_assets]
73
74 inputs_per_image = [(
@@ -106,12 +82,11 @@ def run_test(
82 # will hurt multiprocessing backend with fork method (the default method).
83
84 # max_model_len should be greater than image_feature_size
109 - with vllm_runner(model_id,
85 + with vllm_runner(model,
86 dtype=dtype,
87 tensor_parallel_size=tensor_parallel_size,
88 distributed_executor_backend=distributed_executor_backend,
113 - enforce_eager=True,
114 - **vlm_config.as_cli_args_dict()) as vllm_model:
89 + enforce_eager=True) as vllm_model:
90 vllm_outputs_per_image = [
91 vllm_model.generate_greedy_logprobs(prompts,
92 max_tokens,
@@ -120,7 +95,7 @@ def run_test(
95 for prompts, images in inputs_per_image
96 ]
97
123 - with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
98 + with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
99 hf_outputs_per_image = [
100 hf_model.generate_greedy_logprobs_limit(prompts,
101 max_tokens,
@@ -136,7 +111,7 @@ def run_test(
111 check_logprobs_close(
112 outputs_0_lst=hf_outputs,
113 outputs_1_lst=[
139 - vllm_to_hf_output(vllm_output, vlm_config, model_id)
114 + vllm_to_hf_output(vllm_output, model)
115 for vllm_output in vllm_outputs
116 ],
117 name_0="hf",
@@ -144,7 +119,7 @@ def run_test(
119 )
120
121
147 -@pytest.mark.parametrize("model_and_config", model_and_vl_config)
122 +@pytest.mark.parametrize("model", models)
123 @pytest.mark.parametrize(
124 "size_factors",
125 [
@@ -161,14 +136,13 @@ def run_test(
136 @pytest.mark.parametrize("dtype", ["half"])
137 @pytest.mark.parametrize("max_tokens", [128])
138 @pytest.mark.parametrize("num_logprobs", [5])
164 -def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
165 - size_factors, dtype: str, max_tokens: int,
166 - num_logprobs: int) -> None:
139 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
140 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
141 run_test(
142 hf_runner,
143 vllm_runner,
144 image_assets,
171 - model_and_config,
145 + model,
146 size_factors=size_factors,
147 dtype=dtype,
148 max_tokens=max_tokens,
tests/models/test_llava_next.py
+13 -41
@@ -4,7 +4,6 @@ from typing import List, Optional, Tuple
4 import pytest
5 from transformers import AutoTokenizer
6
7 -from vllm.config import VisionLanguageConfig
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
9
@@ -27,46 +26,22 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
26 f"{_PREFACE} USER: <image>\nWhat's in this image? ASSISTANT:",
27 })
28
30 -
31 -def iter_llava_next_configs(model_name: str):
32 - # Need to use the max possible feature size for profile_run
33 - image_hw_to_feature_size = {
34 - (336, 336): 2928,
35 - }
36 -
37 - for (h, w), f in image_hw_to_feature_size.items():
38 - input_shape = (1, 3, h, w)
39 - yield (model_name,
40 - VisionLanguageConfig(
41 - image_feature_size=f,
42 - image_token_id=32000,
43 - image_input_shape=input_shape,
44 - ))
45 -
46 -
47 -model_and_vl_config = [
48 - *iter_llava_next_configs("llava-hf/llava-v1.6-vicuna-7b-hf"),
49 -]
29 +IMAGE_TOKEN_ID = 32000
30
31
32 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
33 Optional[SampleLogprobs]],
54 - vlm_config: VisionLanguageConfig, model_id: str):
55 - """Sanitize vllm output to be comparable with hf output.
56 - The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
57 - x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
58 - It also reduces `output_str` from "<image><image>bla" to "bla".
59 - """
34 + model: str):
35 + """Sanitize vllm output to be comparable with hf output."""
36 output_ids, output_str, out_logprobs = vllm_output
61 - image_token_id = vlm_config.image_token_id
37
63 - tokenizer = AutoTokenizer.from_pretrained(model_id)
64 - image_token_str = tokenizer.decode(image_token_id)
38 + tokenizer = AutoTokenizer.from_pretrained(model)
39 + image_token_str = tokenizer.decode(IMAGE_TOKEN_ID)
40 eos_token_id = tokenizer.eos_token_id
41
42 hf_output_ids = [
43 token_id for idx, token_id in enumerate(output_ids)
69 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44 + if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
45 ]
46
47 hf_output_str = re.sub(fr"({image_token_str})+", "", output_str)
@@ -78,7 +53,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
53 return hf_output_ids, hf_output_str, out_logprobs
54
55
81 -@pytest.mark.parametrize("model_and_config", model_and_vl_config)
56 +@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-vicuna-7b-hf"])
57 @pytest.mark.parametrize(
58 "size_factors",
59 [
@@ -95,9 +70,8 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
70 @pytest.mark.parametrize("dtype", ["half"])
71 @pytest.mark.parametrize("max_tokens", [128])
72 @pytest.mark.parametrize("num_logprobs", [5])
98 -def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
99 - size_factors, dtype: str, max_tokens: int,
100 - num_logprobs: int) -> None:
73 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
74 + dtype, max_tokens, num_logprobs) -> None:
75 """Inference result should be the same between hf and vllm.
76
77 All the image fixtures for the test is under tests/images.
@@ -107,7 +81,6 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
81 Note, the text input is also adjusted to abide by vllm contract.
82 The text output is sanitized to be able to compare with hf.
83 """
110 - model_id, vlm_config = model_and_config
84 images = [asset.pil_image for asset in image_assets]
85
86 inputs_per_image = [(
@@ -116,11 +89,10 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
89 ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
90
91 # max_model_len should be greater than image_feature_size
119 - with vllm_runner(model_id,
92 + with vllm_runner(model,
93 dtype=dtype,
94 max_model_len=4096,
122 - enforce_eager=True,
123 - **vlm_config.as_cli_args_dict()) as vllm_model:
95 + enforce_eager=True) as vllm_model:
96 vllm_outputs_per_image = [
97 vllm_model.generate_greedy_logprobs(prompts,
98 max_tokens,
@@ -129,7 +101,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
101 for prompts, images in inputs_per_image
102 ]
103
132 - with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
104 + with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
105 hf_outputs_per_image = [
106 hf_model.generate_greedy_logprobs_limit(prompts,
107 max_tokens,
@@ -145,7 +117,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
117 check_logprobs_close(
118 outputs_0_lst=hf_outputs,
119 outputs_1_lst=[
148 - vllm_to_hf_output(vllm_output, vlm_config, model_id)
120 + vllm_to_hf_output(vllm_output, model)
121 for vllm_output in vllm_outputs
122 ],
123 name_0="hf",
tests/models/test_phi3v.py
+15 -39
@@ -4,7 +4,6 @@ from typing import List, Optional, Tuple, Type
4 import pytest
5 from transformers import AutoTokenizer
6
7 -from vllm.config import VisionLanguageConfig
7 from vllm.multimodal.utils import rescale_image_size
8 from vllm.sequence import SampleLogprobs
9 from vllm.utils import is_cpu
@@ -23,35 +22,14 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
22 "<|user|>\n<|image_1|>\nWhat's in this image?<|end|>\n<|assistant|>\n",
23 })
24
26 -
27 -def iter_phi3v_configs(model_name: str):
28 - # Need to use the max possible feature size for profile_run
29 - image_hw_to_feature_size = {
30 - (1008, 1344): 2653,
31 - }
32 -
33 - for (h, w), f in image_hw_to_feature_size.items():
34 - input_shape = (1, 3, h, w)
35 - yield (model_name,
36 - VisionLanguageConfig(image_feature_size=f,
37 - image_token_id=32044,
38 - image_input_shape=input_shape))
39 -
40 -
41 -model_and_vl_config = [
42 - *iter_phi3v_configs("microsoft/Phi-3-vision-128k-instruct"),
43 -]
25 +models = ["microsoft/Phi-3-vision-128k-instruct"]
26
27
28 def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
29 Optional[SampleLogprobs]],
48 - vlm_config: VisionLanguageConfig, model_id: str):
49 - """Sanitize vllm output to be comparable with hf output.
50 - The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
51 - x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
52 - It also reduces `output_str` from "<image><image>bla" to "bla".
53 - """
54 - output_ids, output_str, out_logprobs = vllm_output
30 + model: str):
31 + """Sanitize vllm output to be comparable with hf output."""
32 + _, output_str, out_logprobs = vllm_output
33
34 output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
35 assert output_str_without_image[0] == " "
@@ -60,7 +38,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
38 hf_output_str = output_str_without_image.replace("<|user|>", "") \
39 .replace("<|end|>\n<|assistant|>", " ")
40
63 - tokenizer = AutoTokenizer.from_pretrained(model_id)
41 + tokenizer = AutoTokenizer.from_pretrained(model)
42 hf_output_ids = tokenizer.encode(output_str_without_image)
43 assert hf_output_ids[0] == 1
44 hf_output_ids = hf_output_ids[1:]
@@ -77,7 +55,7 @@ def run_test(
55 hf_runner: Type[HfRunner],
56 vllm_runner: Type[VllmRunner],
57 image_assets: _ImageAssets,
80 - model_and_config: Tuple[str, VisionLanguageConfig],
58 + model: str,
59 *,
60 size_factors: List[float],
61 dtype: str,
@@ -95,7 +73,6 @@ def run_test(
73 Note, the text input is also adjusted to abide by vllm contract.
74 The text output is sanitized to be able to compare with hf.
75 """
98 - model_id, vlm_config = model_and_config
76 images = [asset.pil_image for asset in image_assets]
77
78 inputs_per_image = [(
@@ -109,13 +86,13 @@ def run_test(
86 # will hurt multiprocessing backend with fork method (the default method).
87
88 # max_model_len should be greater than image_feature_size
112 - with vllm_runner(model_id,
89 + with vllm_runner(model,
90 max_model_len=4096,
91 + max_num_seqs=1,
92 dtype=dtype,
93 tensor_parallel_size=tensor_parallel_size,
94 distributed_executor_backend=distributed_executor_backend,
117 - enforce_eager=True,
118 - **vlm_config.as_cli_args_dict()) as vllm_model:
95 + enforce_eager=True) as vllm_model:
96 vllm_outputs_per_image = [
97 vllm_model.generate_greedy_logprobs(prompts,
98 max_tokens,
@@ -126,7 +103,7 @@ def run_test(
103
104 # use eager mode for hf runner, since phi3_v didn't work with flash_attn
105 hf_model_kwargs = {"_attn_implementation": "eager"}
129 - with hf_runner(model_id, dtype=dtype,
106 + with hf_runner(model, dtype=dtype,
107 model_kwargs=hf_model_kwargs) as hf_model:
108 eos_token_id = hf_model.processor.tokenizer.eos_token_id
109 hf_outputs_per_image = [
@@ -143,7 +120,7 @@ def run_test(
120 check_logprobs_close(
121 outputs_0_lst=hf_outputs,
122 outputs_1_lst=[
146 - vllm_to_hf_output(vllm_output, vlm_config, model_id)
123 + vllm_to_hf_output(vllm_output, model)
124 for vllm_output in vllm_outputs
125 ],
126 name_0="hf",
@@ -153,7 +130,7 @@ def run_test(
130
131 # Since we use _attn_implementation="eager" for hf_runner, there is more
132 # significant numerical difference. The basic `logprobs=5` fails to pass.
156 -@pytest.mark.parametrize("model_and_config", model_and_vl_config)
133 +@pytest.mark.parametrize("model", models)
134 @pytest.mark.parametrize(
135 "size_factors",
136 [
@@ -170,14 +147,13 @@ def run_test(
147 @pytest.mark.parametrize("dtype", [target_dtype])
148 @pytest.mark.parametrize("max_tokens", [128])
149 @pytest.mark.parametrize("num_logprobs", [10])
173 -def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
174 - size_factors, dtype: str, max_tokens: int,
175 - num_logprobs: int) -> None:
150 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
151 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
152 run_test(
153 hf_runner,
154 vllm_runner,
155 image_assets,
180 - model_and_config,
156 + model,
157 size_factors=size_factors,
158 dtype=dtype,
159 max_tokens=max_tokens,
vllm/config.py
+7 -31
@@ -1,8 +1,7 @@
1 import enum
2 import json
3 from dataclasses import dataclass, field, fields
4 -from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
5 - Union)
4 +from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
5
6 import torch
7 from transformers import PretrainedConfig
@@ -120,7 +119,7 @@ class ModelConfig:
119 disable_sliding_window: bool = False,
120 skip_tokenizer_init: bool = False,
121 served_model_name: Optional[Union[str, List[str]]] = None,
123 - multimodal_config: Optional["VisionLanguageConfig"] = None,
122 + multimodal_config: Optional["MultiModalConfig"] = None,
123 ) -> None:
124 self.model = model
125 self.tokenizer = tokenizer
@@ -1289,35 +1288,12 @@ class LoRAConfig:
1288 raise ValueError("LoRA is not supported with chunked prefill yet.")
1289
1290
1292 -# TODO: To be replaced by MultiModalConfig.
1291 @dataclass
1294 -class VisionLanguageConfig:
1292 +class MultiModalConfig:
1293 """Configs the input data format and how models should run for
1296 - vision language models."""
1297 - # The input id corresponding to image token.
1298 - image_token_id: int
1299 - # Used for running `run_prefill_max_token`.
1300 - # For models that support varying resolution, this corresponds to
1301 - # worst case scenario (biggest supported resolution).
1302 - image_input_shape: tuple
1303 - image_feature_size: int
1304 -
1305 - def as_cli_args_dict(self) -> Dict[str, Any]:
1306 - """Flatten vision language config to pure args.
1307 -
1308 - Compatible with what llm entrypoint expects.
1309 - """
1310 - result: Dict[str, Any] = {}
1311 - for f in fields(self):
1312 - value = getattr(self, f.name)
1313 - if isinstance(value, enum.Enum):
1314 - result[f.name] = value.name.lower()
1315 - elif isinstance(value, tuple):
1316 - result[f.name] = ",".join([str(item) for item in value])
1317 - else:
1318 - result[f.name] = value
1319 -
1320 - return result
1294 + multimodal models."""
1295 + # TODO: Add configs to init vision tower or not.
1296 + pass
1297
1298
1299 _STR_DTYPE_TO_TORCH_DTYPE = {
@@ -1541,7 +1517,7 @@ class EngineConfig:
1517 device_config: DeviceConfig
1518 load_config: LoadConfig
1519 lora_config: Optional[LoRAConfig]
1544 - vision_language_config: Optional[VisionLanguageConfig]
1520 + multimodal_config: Optional[MultiModalConfig]
1521 speculative_config: Optional[SpeculativeConfig]
1522 decoding_config: Optional[DecodingConfig]
1523 observability_config: Optional[ObservabilityConfig]
vllm/engine/arg_utils.py
+7 -52
@@ -6,11 +6,11 @@ from typing import List, Optional, Tuple, Union
6
7 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8 EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9 - ObservabilityConfig, ParallelConfig, SchedulerConfig,
10 - SpeculativeConfig, TokenizerPoolConfig,
11 - VisionLanguageConfig)
9 + MultiModalConfig, ObservabilityConfig, ParallelConfig,
10 + SchedulerConfig, SpeculativeConfig,
11 + TokenizerPoolConfig)
12 from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13 -from vllm.utils import FlexibleArgumentParser, str_to_int_tuple
13 +from vllm.utils import FlexibleArgumentParser
14
15
16 def nullable_str(val: str):
@@ -78,11 +78,6 @@ class EngineArgs:
78 model_loader_extra_config: Optional[dict] = None
79 preemption_mode: Optional[str] = None
80
81 - # Related to Vision-language models such as llava
82 - image_token_id: Optional[int] = None
83 - image_input_shape: Optional[str] = None
84 - image_feature_size: Optional[int] = None
85 -
81 scheduler_delay_factor: float = 0.0
82 enable_chunked_prefill: bool = False
83
@@ -106,27 +101,6 @@ class EngineArgs:
101 if self.tokenizer is None:
102 self.tokenizer = self.model
103
109 - @staticmethod
110 - def add_cli_args_for_vlm(
111 - parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
112 - parser.add_argument('--image-token-id',
113 - type=int,
114 - default=None,
115 - help=('Input id for image token.'))
116 - parser.add_argument(
117 - '--image-input-shape',
118 - type=nullable_str,
119 - default=None,
120 - help=('The biggest image input shape (worst for memory footprint) '
121 - 'given an input type. Only used for vLLM\'s profile_run.'))
122 - parser.add_argument(
123 - '--image-feature-size',
124 - type=int,
125 - default=None,
126 - help=('The image feature size along the context dimension.'))
127 -
128 - return parser
129 -
104 @staticmethod
105 def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
106 """Shared CLI arguments for vLLM engine."""
@@ -484,9 +458,6 @@ class EngineArgs:
458 ],
459 help='Device type for vLLM execution.')
460
487 - # Related to Vision-language models such as llava
488 - parser = EngineArgs.add_cli_args_for_vlm(parser)
489 -
461 parser.add_argument(
462 '--scheduler-delay-factor',
463 type=float,
@@ -648,19 +619,7 @@ class EngineArgs:
619 raise ValueError(
620 "BitsAndBytes load format and QLoRA adapter only support "
621 f"'bitsandbytes' quantization, but got {self.quantization}")
651 - if self.image_token_id is not None:
652 - if (not self.image_input_shape or not self.image_feature_size):
653 - raise ValueError(
654 - 'Specify `image_input_shape` and '
655 - '`image_feature_size` together with `image_token_id`.')
656 -
657 - vision_language_config = VisionLanguageConfig(
658 - image_token_id=self.image_token_id,
659 - image_input_shape=str_to_int_tuple(self.image_input_shape),
660 - image_feature_size=self.image_feature_size,
661 - )
662 - else:
663 - vision_language_config = None
622 + multimodal_config = MultiModalConfig()
623
624 device_config = DeviceConfig(device=self.device)
625 model_config = ModelConfig(
@@ -685,7 +644,7 @@ class EngineArgs:
644 disable_sliding_window=self.disable_sliding_window,
645 skip_tokenizer_init=self.skip_tokenizer_init,
646 served_model_name=self.served_model_name,
688 - multimodal_config=vision_language_config)
647 + multimodal_config=multimodal_config)
648 cache_config = CacheConfig(
649 block_size=self.block_size,
650 gpu_memory_utilization=self.gpu_memory_utilization,
@@ -787,7 +746,7 @@ class EngineArgs:
746 scheduler_config=scheduler_config,
747 device_config=device_config,
748 lora_config=lora_config,
790 - vision_language_config=vision_language_config,
749 + multimodal_config=multimodal_config,
750 speculative_config=speculative_config,
751 load_config=load_config,
752 decoding_config=decoding_config,
@@ -831,7 +790,3 @@ def _engine_args_parser():
790 def _async_engine_args_parser():
791 return AsyncEngineArgs.add_cli_args(FlexibleArgumentParser(),
792 async_args_only=True)
834 -
835 -
836 -def _vlm_engine_args_parser():
837 - return EngineArgs.add_cli_args_for_vlm(FlexibleArgumentParser())
vllm/engine/llm_engine.py
+8 -8
@@ -7,9 +7,9 @@ from typing import Set, Type, TypeVar, Union
7 from transformers import PreTrainedTokenizer
8
9 from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10 - LoRAConfig, ModelConfig, ObservabilityConfig,
11 - ParallelConfig, SchedulerConfig, SpeculativeConfig,
12 - VisionLanguageConfig)
10 + LoRAConfig, ModelConfig, MultiModalConfig,
11 + ObservabilityConfig, ParallelConfig, SchedulerConfig,
12 + SpeculativeConfig)
13 from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14 SchedulerOutputs)
15 from vllm.engine.arg_utils import EngineArgs
@@ -87,8 +87,8 @@ class LLMEngine:
87 scheduler_config: The configuration related to the request scheduler.
88 device_config: The configuration related to the device.
89 lora_config (Optional): The configuration related to serving multi-LoRA.
90 - vision_language_config (Optional): The configuration related to vision
91 - language models.
90 + multimodal_config (Optional): The configuration related to multimodal
91 + models.
92 speculative_config (Optional): The configuration related to speculative
93 decoding.
94 executor_class: The model executor class for managing distributed
@@ -157,7 +157,7 @@ class LLMEngine:
157 device_config: DeviceConfig,
158 load_config: LoadConfig,
159 lora_config: Optional[LoRAConfig],
160 - vision_language_config: Optional[VisionLanguageConfig],
160 + multimodal_config: Optional[MultiModalConfig],
161 speculative_config: Optional[SpeculativeConfig],
162 decoding_config: Optional[DecodingConfig],
163 observability_config: Optional[ObservabilityConfig],
@@ -215,7 +215,7 @@ class LLMEngine:
215 self.model_config = model_config
216 self.cache_config = cache_config
217 self.lora_config = lora_config
218 - self.vision_language_config = vision_language_config
218 + self.multimodal_config = multimodal_config
219 self.parallel_config = parallel_config
220 self.scheduler_config = scheduler_config
221 self.device_config = device_config
@@ -247,7 +247,7 @@ class LLMEngine:
247 scheduler_config=scheduler_config,
248 device_config=device_config,
249 lora_config=lora_config,
250 - vision_language_config=vision_language_config,
250 + multimodal_config=multimodal_config,
251 speculative_config=speculative_config,
252 load_config=load_config,
253 )
vllm/entrypoints/llm.py
+5
@@ -121,6 +121,11 @@ class LLM:
121 ) -> None:
122 if "disable_log_stats" not in kwargs:
123 kwargs["disable_log_stats"] = True
124 + removed_vision_keys = ("image_token_id", "image_feature_size",
125 + "image_input_shape", "image_input_type")
126 + if any(k in kwargs for k in removed_vision_keys):
127 + raise TypeError(
128 + "There is no need to pass vision-related arguments anymore.")
129 engine_args = EngineArgs(
130 model=model,
131 tokenizer=tokenizer,
vllm/entrypoints/openai/serving_chat.py
+5 -16
@@ -109,23 +109,12 @@ class OpenAIServingChat(OpenAIServing):
109 "paligemma"):
110 # These models do not use image tokens in the prompt
111 return None
112 + if model_type.startswith("llava"):
113 + return self.tokenizer.decode(
114 + self.model_config.hf_config.image_token_index)
115
113 - # The default behaviour assumes that the image token is
114 - # available to the tokenizer.
115 - # (Suitable for LLaVA, Idefics2, DeepSeek-VL)
116 - vlm_config = self.model_config.multimodal_config
117 - if vlm_config is None:
118 - raise ValueError(
119 - "'image_url' input is not supported as the loaded "
120 - "model is not multimodal.")
121 -
122 - image_token_id = vlm_config.image_token_id
123 - if vlm_config.image_token_id is None:
124 - raise ValueError(
125 - "'image_url' input is not supported as the loaded "
126 - "model does not specify an image token.")
127 -
128 - return self.tokenizer.decode(image_token_id)
116 + else:
117 + raise TypeError("Unknown model type: {model_type}")
118
119 # TODO: Let user specify how to insert image tokens into prompt
120 # (similar to chat template)
vllm/executor/cpu_executor.py
+1 -1
@@ -46,7 +46,7 @@ class CPUExecutor(ExecutorBase):
46 rank=0,
47 distributed_init_method=distributed_init_method,
48 lora_config=self.lora_config,
49 - vision_language_config=self.vision_language_config,
49 + multimodal_config=self.multimodal_config,
50 kv_cache_dtype=self.cache_config.cache_dtype,
51 is_driver_worker=True,
52 )
vllm/executor/executor_base.py
+6 -7
@@ -3,8 +3,8 @@ from abc import ABC, abstractmethod
3 from typing import List, Optional, Set, Tuple
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, SchedulerConfig,
7 - SpeculativeConfig, VisionLanguageConfig)
6 + ModelConfig, MultiModalConfig, ParallelConfig,
7 + SchedulerConfig, SpeculativeConfig)
8 from vllm.lora.request import LoRARequest
9 from vllm.sequence import ExecuteModelRequest, SamplerOutput
10
@@ -26,7 +26,7 @@ class ExecutorBase(ABC):
26 device_config: DeviceConfig,
27 load_config: LoadConfig,
28 lora_config: Optional[LoRAConfig],
29 - vision_language_config: Optional[VisionLanguageConfig],
29 + multimodal_config: Optional[MultiModalConfig],
30 speculative_config: Optional[SpeculativeConfig],
31 ) -> None:
32 self.model_config = model_config
@@ -36,7 +36,7 @@ class ExecutorBase(ABC):
36 self.parallel_config = parallel_config
37 self.scheduler_config = scheduler_config
38 self.device_config = device_config
39 - self.vision_language_config = vision_language_config
39 + self.multimodal_config = multimodal_config
40 self.speculative_config = speculative_config
41
42 self._init_executor()
@@ -120,7 +120,7 @@ class ExecutorAsyncBase(ExecutorBase):
120 device_config: DeviceConfig,
121 load_config: LoadConfig,
122 lora_config: Optional[LoRAConfig],
123 - vision_language_config: Optional[VisionLanguageConfig],
123 + multimodal_config: Optional[MultiModalConfig],
124 speculative_config: Optional[SpeculativeConfig],
125 ) -> None:
126 # This locks each pipeline parallel stage so multiple virtual engines
@@ -132,8 +132,7 @@ class ExecutorAsyncBase(ExecutorBase):
132
133 super().__init__(model_config, cache_config, parallel_config,
134 scheduler_config, device_config, load_config,
135 - lora_config, vision_language_config,
136 - speculative_config)
135 + lora_config, multimodal_config, speculative_config)
136
137 @abstractmethod
138 async def execute_model_async(
vllm/executor/gpu_executor.py
+1 -1
@@ -43,7 +43,7 @@ class GPUExecutor(ExecutorBase):
43 rank=rank,
44 distributed_init_method=distributed_init_method,
45 lora_config=self.lora_config,
46 - vision_language_config=self.vision_language_config,
46 + multimodal_config=self.multimodal_config,
47 speculative_config=self.speculative_config,
48 is_driver_worker=(not self.parallel_config)
49 or (rank % self.parallel_config.tensor_parallel_size == 0),
vllm/executor/openvino_executor.py
+1 -1
@@ -47,7 +47,7 @@ class OpenVINOExecutor(ExecutorBase):
47 rank=0,
48 distributed_init_method=distributed_init_method,
49 lora_config=self.lora_config,
50 - vision_language_config=self.vision_language_config,
50 + multimodal_config=self.multimodal_config,
51 kv_cache_dtype=self.cache_config.cache_dtype,
52 is_driver_worker=True,
53 )
vllm/executor/ray_xpu_executor.py
+5 -5
@@ -7,8 +7,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
7 Tuple, Union)
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - SpeculativeConfig, VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig, SpeculativeConfig)
12 from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13 DistributedGPUExecutor, DistributedGPUExecutorAsync)
14 from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -43,7 +43,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
43 device_config: DeviceConfig,
44 load_config: LoadConfig,
45 lora_config: Optional[LoRAConfig],
46 - vision_language_config: Optional[VisionLanguageConfig],
46 + multimodal_config: Optional[MultiModalConfig],
47 speculative_config: Optional[SpeculativeConfig],
48 ) -> None:
49 assert device_config.device_type == "xpu"
@@ -57,7 +57,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
57 self.parallel_config = parallel_config
58 self.scheduler_config = scheduler_config
59 self.device_config = device_config
60 - self.vision_language_config = vision_language_config
60 + self.multimodal_config = multimodal_config
61
62 placement_group = self.parallel_config.placement_group
63
@@ -199,7 +199,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
199 rank=rank,
200 distributed_init_method=distributed_init_method,
201 lora_config=self.lora_config,
202 - vision_language_config=self.vision_language_config,
202 + multimodal_config=self.multimodal_config,
203 is_driver_worker=rank == 0,
204 ))
205 self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
vllm/executor/tpu_executor.py
+1 -1
@@ -50,7 +50,7 @@ class TPUExecutor(ExecutorBase):
50 local_rank=local_rank,
51 rank=rank,
52 distributed_init_method=distributed_init_method,
53 - vision_language_config=self.vision_language_config,
53 + multimodal_config=self.multimodal_config,
54 is_driver_worker=rank == 0,
55 )
56
vllm/executor/xpu_executor.py
+4 -4
@@ -3,8 +3,8 @@ from typing import List, Optional
3 import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, SchedulerConfig,
7 - SpeculativeConfig, VisionLanguageConfig)
6 + ModelConfig, MultiModalConfig, ParallelConfig,
7 + SchedulerConfig, SpeculativeConfig)
8 from vllm.executor.executor_base import ExecutorAsyncBase
9 from vllm.executor.gpu_executor import GPUExecutor
10 from vllm.logger import init_logger
@@ -26,7 +26,7 @@ class XPUExecutor(GPUExecutor):
26 device_config: DeviceConfig,
27 load_config: LoadConfig,
28 lora_config: Optional[LoRAConfig],
29 - vision_language_config: Optional[VisionLanguageConfig],
29 + multimodal_config: Optional[MultiModalConfig],
30 speculative_config: Optional[SpeculativeConfig],
31 ) -> None:
32 assert device_config.device_type == "xpu"
@@ -42,7 +42,7 @@ class XPUExecutor(GPUExecutor):
42 self.parallel_config = parallel_config
43 self.scheduler_config = scheduler_config
44 self.device_config = device_config
45 - self.vision_language_config = vision_language_config
45 + self.multimodal_config = multimodal_config
46 self.speculative_config = None
47
48 # Instantiate the worker and load the model to GPU.
vllm/inputs/registry.py
+2 -2
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
11 from .data import LLMInputs
12
13 if TYPE_CHECKING:
14 - from vllm.config import ModelConfig, VisionLanguageConfig
14 + from vllm.config import ModelConfig, MultiModalConfig
15 from vllm.multimodal import MultiModalDataDict
16 from vllm.sequence import SequenceData
17
@@ -30,7 +30,7 @@ class InputContext:
30 model_config: "ModelConfig"
31 """The configuration of the model."""
32
33 - def get_multimodal_config(self) -> "VisionLanguageConfig":
33 + def get_multimodal_config(self) -> "MultiModalConfig":
34 """
35 Get the multimodal configuration of the model.
36
vllm/model_executor/model_loader/__init__.py
+4 -4
@@ -3,8 +3,8 @@ from typing import Optional
3 from torch import nn
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, SchedulerConfig,
7 - VisionLanguageConfig)
6 + ModelConfig, MultiModalConfig, ParallelConfig,
7 + SchedulerConfig)
8 from vllm.model_executor.model_loader.loader import (BaseModelLoader,
9 get_model_loader)
10 from vllm.model_executor.model_loader.utils import (
@@ -15,13 +15,13 @@ def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
15 device_config: DeviceConfig, parallel_config: ParallelConfig,
16 scheduler_config: SchedulerConfig,
17 lora_config: Optional[LoRAConfig],
18 - vision_language_config: Optional[VisionLanguageConfig],
18 + multimodal_config: Optional[MultiModalConfig],
19 cache_config: CacheConfig) -> nn.Module:
20 loader = get_model_loader(load_config)
21 return loader.load_model(model_config=model_config,
22 device_config=device_config,
23 lora_config=lora_config,
24 - vision_language_config=vision_language_config,
24 + multimodal_config=multimodal_config,
25 parallel_config=parallel_config,
26 scheduler_config=scheduler_config,
27 cache_config=cache_config)
vllm/model_executor/model_loader/loader.py
+23 -25
@@ -16,8 +16,8 @@ from huggingface_hub import HfApi, hf_hub_download
16 from torch import nn
17
18 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
19 - LoRAConfig, ModelConfig, ParallelConfig,
20 - SchedulerConfig, VisionLanguageConfig)
19 + LoRAConfig, ModelConfig, MultiModalConfig,
20 + ParallelConfig, SchedulerConfig)
21 from vllm.envs import VLLM_USE_MODELSCOPE
22 from vllm.logger import init_logger
23 from vllm.model_executor.layers.quantization.base_config import (
@@ -68,7 +68,7 @@ def _get_quantization_config(
68 def _get_model_initialization_kwargs(
69 model_class: Type[nn.Module],
70 lora_config: Optional[LoRAConfig],
71 - vlm_config: Optional[VisionLanguageConfig],
71 + multimodal_config: Optional[MultiModalConfig],
72 ) -> Dict[str, Any]:
73 """Get extra kwargs for model initialization."""
74 extra_kwargs: Dict[str, Any] = {}
@@ -84,18 +84,18 @@ def _get_model_initialization_kwargs(
84 "please open an issue on github.")
85
86 if supports_vision(model_class):
87 - if vlm_config is None:
87 + if multimodal_config is None:
88 raise ValueError("Provide vision related configurations "
89 "through LLM entrypoint or engine arguments.")
90
91 - extra_kwargs["vlm_config"] = vlm_config
91 + extra_kwargs["multimodal_config"] = multimodal_config
92
93 return extra_kwargs
94
95
96 def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
97 lora_config: Optional[LoRAConfig],
98 - vision_language_config: Optional[VisionLanguageConfig],
98 + multimodal_config: Optional[MultiModalConfig],
99 cache_config: CacheConfig) -> nn.Module:
100 """Initialize a model with the given configurations."""
101 model_class = get_model_architecture(model_config)[0]
@@ -105,7 +105,7 @@ def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
105 cache_config=cache_config,
106 quant_config=quant_config,
107 **_get_model_initialization_kwargs(
108 - model_class, lora_config, vision_language_config))
108 + model_class, lora_config, multimodal_config))
109
110
111 class BaseModelLoader(ABC):
@@ -118,7 +118,7 @@ class BaseModelLoader(ABC):
118 def load_model(self, *, model_config: ModelConfig,
119 device_config: DeviceConfig,
120 lora_config: Optional[LoRAConfig],
121 - vision_language_config: Optional[VisionLanguageConfig],
121 + multimodal_config: Optional[MultiModalConfig],
122 parallel_config: ParallelConfig,
123 scheduler_config: SchedulerConfig,
124 cache_config: CacheConfig) -> nn.Module:
@@ -258,14 +258,14 @@ class DefaultModelLoader(BaseModelLoader):
258 def load_model(self, *, model_config: ModelConfig,
259 device_config: DeviceConfig,
260 lora_config: Optional[LoRAConfig],
261 - vision_language_config: Optional[VisionLanguageConfig],
261 + multimodal_config: Optional[MultiModalConfig],
262 parallel_config: ParallelConfig,
263 scheduler_config: SchedulerConfig,
264 cache_config: CacheConfig) -> nn.Module:
265 with set_default_torch_dtype(model_config.dtype):
266 with torch.device(device_config.device):
267 model = _initialize_model(model_config, self.load_config,
268 - lora_config, vision_language_config,
268 + lora_config, multimodal_config,
269 cache_config)
270 model.load_weights(
271 self._get_weights_iterator(model_config.model,
@@ -298,14 +298,14 @@ class DummyModelLoader(BaseModelLoader):
298 def load_model(self, *, model_config: ModelConfig,
299 device_config: DeviceConfig,
300 lora_config: Optional[LoRAConfig],
301 - vision_language_config: Optional[VisionLanguageConfig],
301 + multimodal_config: Optional[MultiModalConfig],
302 parallel_config: ParallelConfig,
303 scheduler_config: SchedulerConfig,
304 cache_config: CacheConfig) -> nn.Module:
305 with set_default_torch_dtype(model_config.dtype):
306 with torch.device(device_config.device):
307 model = _initialize_model(model_config, self.load_config,
308 - lora_config, vision_language_config,
308 + lora_config, multimodal_config,
309 cache_config)
310 # NOTE(woosuk): For accurate performance evaluation, we assign
311 # random values to the weights.
@@ -339,7 +339,7 @@ class TensorizerLoader(BaseModelLoader):
339 model_config: ModelConfig,
340 device_config: DeviceConfig,
341 lora_config: Optional[LoRAConfig],
342 - vision_language_config: Optional[VisionLanguageConfig],
342 + multimodal_config: Optional[MultiModalConfig],
343 cache_config: CacheConfig,
344 ) -> nn.Module:
345 """Load a serialized model with tensorizer to the CPU.
@@ -352,7 +352,7 @@ class TensorizerLoader(BaseModelLoader):
352 with set_default_torch_dtype(model_config.dtype):
353 with torch.device(device_config.device):
354 model = _initialize_model(model_config, self.load_config,
355 - lora_config, vision_language_config,
355 + lora_config, multimodal_config,
356 cache_config)
357
358 model.load_weights(self._get_weights_iterator())
@@ -363,7 +363,7 @@ class TensorizerLoader(BaseModelLoader):
363 model_config: ModelConfig,
364 device_config: DeviceConfig,
365 lora_config: Optional[LoRAConfig],
366 - vision_language_config: Optional[VisionLanguageConfig],
366 + multimodal_config: Optional[MultiModalConfig],
367 cache_config: CacheConfig,
368 ) -> nn.Module:
369 """Load a serialized model with tensorizer.
@@ -377,7 +377,7 @@ class TensorizerLoader(BaseModelLoader):
377 quant_config = _get_quantization_config(
378 model_config, self.load_config)
379 extra_kwargs = _get_model_initialization_kwargs(
380 - model_class, lora_config, vision_language_config)
380 + model_class, lora_config, multimodal_config)
381 extra_kwargs["quant_config"] = quant_config
382 extra_kwargs["cache_config"] = cache_config
383
@@ -392,7 +392,7 @@ class TensorizerLoader(BaseModelLoader):
392 def load_model(self, *, model_config: ModelConfig,
393 device_config: DeviceConfig,
394 lora_config: Optional[LoRAConfig],
395 - vision_language_config: Optional[VisionLanguageConfig],
395 + multimodal_config: Optional[MultiModalConfig],
396 parallel_config: ParallelConfig,
397 scheduler_config: SchedulerConfig,
398 cache_config: CacheConfig) -> nn.Module:
@@ -406,12 +406,10 @@ class TensorizerLoader(BaseModelLoader):
406
407 if is_vllm_tensorized(self.tensorizer_config):
408 return self._load_model_serialized(model_config, device_config,
409 - lora_config,
410 - vision_language_config,
409 + lora_config, multimodal_config,
410 cache_config)
411 return self._load_model_serialized_cpu(model_config, device_config,
413 - lora_config,
414 - vision_language_config,
412 + lora_config, multimodal_config,
413 cache_config)
414
415 @staticmethod
@@ -494,7 +492,7 @@ class ShardedStateLoader(BaseModelLoader):
492 def load_model(self, *, model_config: ModelConfig,
493 device_config: DeviceConfig,
494 lora_config: Optional[LoRAConfig],
497 - vision_language_config: Optional[VisionLanguageConfig],
495 + multimodal_config: Optional[MultiModalConfig],
496 parallel_config: ParallelConfig,
497 scheduler_config: SchedulerConfig,
498 cache_config: CacheConfig) -> nn.Module:
@@ -508,7 +506,7 @@ class ShardedStateLoader(BaseModelLoader):
506 with set_default_torch_dtype(model_config.dtype):
507 with torch.device(device_config.device):
508 model = _initialize_model(model_config, self.load_config,
511 - lora_config, vision_language_config,
509 + lora_config, multimodal_config,
510 cache_config)
511 rank = get_tensor_model_parallel_rank()
512 pattern = os.path.join(
@@ -804,14 +802,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
802 def load_model(self, *, model_config: ModelConfig,
803 device_config: DeviceConfig,
804 lora_config: Optional[LoRAConfig],
807 - vision_language_config: Optional[VisionLanguageConfig],
805 + multimodal_config: Optional[MultiModalConfig],
806 parallel_config: ParallelConfig,
807 scheduler_config: SchedulerConfig,
808 cache_config: CacheConfig) -> nn.Module:
809 with set_default_torch_dtype(model_config.dtype):
810 with torch.device(device_config.device):
811 model = _initialize_model(model_config, self.load_config,
814 - lora_config, vision_language_config,
812 + lora_config, multimodal_config,
813 cache_config)
814
815 self._load_weights(model_config, model)
vllm/model_executor/models/interfaces.py
+3 -3
@@ -3,7 +3,7 @@ from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
3
4 from typing_extensions import TypeGuard
5
6 -from vllm.config import LoRAConfig, VisionLanguageConfig
6 +from vllm.config import LoRAConfig, MultiModalConfig
7 from vllm.logger import init_logger
8
9 logger = init_logger(__name__)
@@ -22,7 +22,7 @@ class SupportsVision(Protocol):
22 MRO of your model class.
23 """
24
25 - def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
25 + def __init__(self, *, multimodal_config: MultiModalConfig) -> None:
26 ...
27
28
@@ -32,7 +32,7 @@ class SupportsVision(Protocol):
32 class _SupportsVisionType(Protocol):
33 supports_vision: Literal[True]
34
35 - def __call__(self, *, vlm_config: VisionLanguageConfig) -> None:
35 + def __call__(self, *, multimodal_config: MultiModalConfig) -> None:
36 ...
37
38
vllm/model_executor/models/llava.py
+10 -11
@@ -5,7 +5,7 @@ import torch.nn as nn
5 from transformers import CLIPVisionConfig, LlavaConfig
6
7 from vllm.attention import AttentionMetadata
8 -from vllm.config import CacheConfig, VisionLanguageConfig
8 +from vllm.config import CacheConfig, MultiModalConfig
9 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
10 from vllm.model_executor.layers.activation import get_act_fn
11 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -108,13 +108,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
108
109 def __init__(self,
110 config: LlavaConfig,
111 - vlm_config: VisionLanguageConfig,
111 + multimodal_config: MultiModalConfig,
112 cache_config: Optional[CacheConfig] = None,
113 quant_config: Optional[QuantizationConfig] = None) -> None:
114 super().__init__()
115
116 self.config = config
117 - self.vlm_config = vlm_config
117 + self.multimodal_config = multimodal_config
118
119 # TODO: Optionally initializes this for supporting embeddings.
120 self.vision_tower = CLIPVisionModel(config.vision_config)
@@ -138,14 +138,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
138 self.sampler = Sampler()
139
140 def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
141 - if list(data.shape[1:]) != list(self.vlm_config.image_input_shape[1:]):
141 + if list(data.shape)[1:] != [
142 + 3, self.config.vision_config.image_size,
143 + self.config.vision_config.image_size
144 + ]:
145 raise ValueError(
143 - f"The expected image tensor shape is batch dimension plus "
144 - f"{self.vlm_config.image_input_shape[1:]}. "
145 - f"You supplied {data.shape}. "
146 - f"If you are using vLLM's entrypoint, make sure your "
147 - f"supplied image input is consistent with "
148 - f"image_input_shape in engine args.")
146 + "The expected image tensor shape is batch dimension plus "
147 + "channel, height and width.")
148
149 return data
150
@@ -244,7 +243,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
243
244 inputs_embeds = merge_vision_embeddings(
245 input_ids, inputs_embeds, vision_embeddings,
247 - self.vlm_config.image_token_id)
246 + self.config.image_token_index)
247
248 input_ids = None
249 else:
vllm/model_executor/models/llava_next.py
+47 -6
@@ -1,4 +1,4 @@
1 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3 import torch
4 import torch.nn as nn
@@ -9,7 +9,7 @@ from transformers.models.llava_next.modeling_llava_next import (
9 from typing_extensions import NotRequired
10
11 from vllm.attention import AttentionMetadata
12 -from vllm.config import CacheConfig, VisionLanguageConfig
12 +from vllm.config import CacheConfig, MultiModalConfig
13 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14 from vllm.logger import init_logger
15 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -204,13 +204,13 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
204
205 def __init__(self,
206 config: LlavaNextConfig,
207 - vlm_config: VisionLanguageConfig,
207 + multimodal_config: MultiModalConfig,
208 cache_config: Optional[CacheConfig] = None,
209 quant_config: Optional[QuantizationConfig] = None) -> None:
210 super().__init__()
211
212 self.config = config
213 - self.vlm_config = vlm_config
213 + self.multimodal_config = multimodal_config
214
215 # TODO: Optionally initializes this for supporting embeddings.
216 self.vision_tower = CLIPVisionModel(config=config.vision_config)
@@ -244,6 +244,47 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
244
245 return data
246
247 + def _validate_pixel_values(
248 + self, data: Union[torch.Tensor, List[torch.Tensor]]
249 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
250 +
251 + def _validate_shape(data: torch.Tensor):
252 +
253 + dim = data.dim()
254 + height = width = self.config.vision_config.image_size
255 + # All 4d image tensors have the same number of patches,
256 + # so data is a 5d batch of these tensors
257 + if dim == 5:
258 + if list(data.shape)[2:] != [
259 + 3, self.config.vision_config.image_size,
260 + self.config.vision_config.image_size
261 + ]:
262 + raise ValueError(
263 + "Expected pixel value tensor in shape of: (batch size, "
264 + f"patch number, 3, {height}, {width}), got {data.shape}"
265 + )
266 +
267 + # 4d image tensors have different number of patches,
268 + # so data is each individual tensor.
269 + elif dim == 4:
270 + if list(data.shape)[1:] != [
271 + 3, self.config.vision_config.image_size,
272 + self.config.vision_config.image_size
273 + ]:
274 + raise ValueError(
275 + "Expected pixel value tensor in shape of: (patch "
276 + f"number, 3, {height}, {width}), got {data.shape}")
277 + else:
278 + raise ValueError(
279 + f"Invalid pixel value tensor of shape {data.shape}")
280 +
281 + if isinstance(data, torch.Tensor):
282 + _validate_shape(data)
283 + else:
284 + [_validate_shape(d) for d in data]
285 +
286 + return data
287 +
288 def _parse_and_validate_image_input(
289 self, **kwargs: object) -> Optional[LlavaNextImagePixelInputs]:
290 pixel_values = kwargs.pop("pixel_values", None)
@@ -262,7 +303,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
303
304 return LlavaNextImagePixelInputs(
305 type="pixel_values",
265 - data=pixel_values,
306 + data=self._validate_pixel_values(pixel_values),
307 image_sizes=self._validate_image_sizes(image_sizes),
308 )
309
@@ -454,7 +495,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
495
496 inputs_embeds = merge_vision_embeddings(
497 input_ids, inputs_embeds, vision_embeddings,
457 - self.vlm_config.image_token_id)
498 + self.config.image_token_index)
499
500 input_ids = None
501 else:
vllm/model_executor/models/phi3v.py
+45 -17
@@ -15,7 +15,7 @@
15 # limitations under the License.
16 import re
17 from functools import lru_cache
18 -from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
18 +from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
19
20 import numpy as np
21 import torch
@@ -24,7 +24,7 @@ from PIL import Image
24 from transformers import CLIPVisionConfig, PretrainedConfig
25
26 from vllm.attention import AttentionMetadata
27 -from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
27 +from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
28 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
29 from vllm.logger import init_logger
30 from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -50,6 +50,9 @@ _KEYS_TO_MODIFY_MAPPING = {
50 "model.vision_embed_tokens": "vision_embed_tokens",
51 }
52
53 +# Cannot find the following 2 numbers from hf config.
54 +_IMAGE_TOKEN_ID = 32044
55 +
56 CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
57 hidden_act="quick_gelu",
58 hidden_size=1024,
@@ -95,13 +98,10 @@ class Phi3ImageEmbeddingBase(nn.Module):
98 class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
99 """Phi3 Image embedding with HD transform."""
100
98 - def __init__(self,
99 - vision_language_config: VisionLanguageConfig,
100 - config: PretrainedConfig,
101 - wte=None) -> None:
101 + def __init__(self, config: PretrainedConfig, wte=None) -> None:
102 super().__init__(wte)
103
104 - self.image_token_id = vision_language_config.image_token_id
104 + self.image_token_id = _IMAGE_TOKEN_ID
105 # n_embed or hidden_size
106 hidden_size = config.n_embd if hasattr(
107 config, 'n_embd') else config.hidden_size
@@ -333,7 +333,7 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
333 seq_data = dummy_seq_data_for_clip(
334 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335 seq_len,
336 - image_token_id=32044,
336 + image_token_id=_IMAGE_TOKEN_ID,
337 image_feature_size_override=image_feature_size,
338 )
339 mm_data = dummy_image_for_clip(
@@ -370,7 +370,6 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
370 return llm_inputs
371
372 model_config = ctx.model_config
373 - multimodal_config = ctx.get_multimodal_config()
373 hf_config = ctx.get_hf_config(PretrainedConfig)
374
375 image_data = multi_modal_data["image"]
@@ -407,7 +406,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
406 new_token_ids: List[int] = []
407 for i in range(len(prompt_token_ids) - len(image_1_token_ids) + 1):
408 if prompt_token_ids[i:i + len(image_1_token_ids)] == image_1_token_ids:
410 - new_token_ids.append(multimodal_config.image_token_id)
409 + new_token_ids.append(_IMAGE_TOKEN_ID)
410
411 # No need to further scan the list since we only replace once
412 new_token_ids.extend(prompt_token_ids[i + len(image_1_token_ids):])
@@ -424,7 +423,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
423 model_config,
424 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
425 llm_inputs,
427 - image_token_id=multimodal_config.image_token_id,
426 + image_token_id=_IMAGE_TOKEN_ID,
427 image_feature_size_override=image_feature_size,
428 )
429
@@ -436,25 +435,53 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
435
436 def __init__(self,
437 config: PretrainedConfig,
439 - vlm_config: VisionLanguageConfig,
438 + multimodal_config: MultiModalConfig,
439 cache_config: Optional[CacheConfig] = None,
440 quant_config: Optional[QuantizationConfig] = None) -> None:
441 super().__init__()
442
443 self.config = config
445 - self.vlm_config = vlm_config
444 + self.multimodal_config = multimodal_config
445
446 self.model = LlamaModel(config, cache_config, quant_config)
447
448 # TODO: Optionally initializes this for supporting embeddings.
449 self.vision_embed_tokens = Phi3HDImageEmbedding(
451 - vlm_config, config, self.model.embed_tokens)
450 + config, self.model.embed_tokens)
451 self.lm_head = ParallelLMHead(config.vocab_size,
452 config.hidden_size,
453 quant_config=quant_config)
454 self.logits_processor = LogitsProcessor(config.vocab_size)
455 self.sampler = Sampler()
456
457 + def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
458 + if list(data.shape[1:]) != [2]:
459 + raise ValueError(
460 + f"The expected image sizes shape is batch dimension plus "
461 + f"{[2]}. You supplied {data.shape}.")
462 +
463 + return data
464 +
465 + def _validate_pixel_values(
466 + self, data: Union[torch.Tensor, List[torch.Tensor]]
467 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
468 +
469 + def _validate_shape(data: torch.Tensor):
470 + if list(data.shape)[2:] != [
471 + 3, CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size,
472 + CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size
473 + ]:
474 + raise ValueError(
475 + "The expected pixel value tensor shape is batch dimension "
476 + "plus patch number, channel, height and width.")
477 +
478 + if isinstance(data, torch.Tensor):
479 + _validate_shape(data)
480 + else:
481 + [_validate_shape(d) for d in data]
482 +
483 + return data
484 +
485 def _parse_and_validate_image_input(
486 self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
487 pixel_values = kwargs.pop("pixel_values", None)
@@ -471,9 +498,10 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
498 raise ValueError("Incorrect type of image sizes. "
499 f"Got type: {type(image_sizes)}")
500
474 - return Phi3VImagePixelInputs(type="pixel_values",
475 - data=pixel_values,
476 - image_sizes=image_sizes)
501 + return Phi3VImagePixelInputs(
502 + type="pixel_values",
503 + data=self._validate_pixel_values(pixel_values),
504 + image_sizes=self._validate_image_sizes(image_sizes))
505
506 def forward(self,
507 input_ids: torch.Tensor,
vllm/multimodal/registry.py
+7
@@ -120,3 +120,10 @@ class MultiModalRegistry:
120 Create an input mapper (see :meth:`map_input`) for a specific model.
121 """
122 return functools.partial(self.map_input, model_config)
123 +
124 + def get_num_input_tokens(self):
125 + """
126 + Get the number of input tokens for profiling purposes.
127 + """
128 + # TODO: Provide this number on a per model basis.
129 + return 3000
vllm/spec_decode/draft_model_runner.py
+4 -4
@@ -3,8 +3,8 @@ from typing import List, Optional
3 import torch
4
5 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6 - ModelConfig, ParallelConfig, SchedulerConfig,
7 - VisionLanguageConfig)
6 + ModelConfig, MultiModalConfig, ParallelConfig,
7 + SchedulerConfig)
8 from vllm.logger import init_logger
9 from vllm.sequence import (IntermediateTensors, SamplerOutput,
10 SequenceGroupMetadata)
@@ -47,7 +47,7 @@ class TP1DraftModelRunner(ModelRunner):
47 lora_config: Optional[LoRAConfig],
48 kv_cache_dtype: Optional[str] = "auto",
49 is_driver_worker: bool = False,
50 - vision_language_config: Optional[VisionLanguageConfig] = None,
50 + multimodal_config: Optional[MultiModalConfig] = None,
51 return_hidden_states: bool = False,
52 ):
53 if return_hidden_states:
@@ -65,7 +65,7 @@ class TP1DraftModelRunner(ModelRunner):
65 lora_config=lora_config,
66 kv_cache_dtype=kv_cache_dtype,
67 is_driver_worker=is_driver_worker,
68 - vision_language_config=vision_language_config,
68 + multimodal_config=multimodal_config,
69 return_hidden_states=return_hidden_states,
70 )
71
vllm/worker/cpu_model_runner.py
+12 -13
@@ -7,8 +7,8 @@ from torch import nn
7
8 from vllm.attention import AttentionMetadata, get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig)
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader import get_model
@@ -79,7 +79,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
79 cache_config: CacheConfig,
80 load_config: LoadConfig,
81 lora_config: Optional[LoRAConfig],
82 - vision_language_config: Optional[VisionLanguageConfig],
82 + multimodal_config: Optional[MultiModalConfig],
83 kv_cache_dtype: Optional[str] = "auto",
84 is_driver_worker: bool = False,
85 *args,
@@ -93,7 +93,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
93 self.device_config = device_config
94 self.cache_config = cache_config
95 self.lora_config = lora_config
96 - self.vision_language_config = vision_language_config
96 + self.multimodal_config = multimodal_config
97 self.load_config = load_config
98 self.is_driver_worker = is_driver_worker
99
@@ -120,15 +120,14 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
120 self.model: nn.Module # Set after init_Model
121
122 def load_model(self) -> None:
123 - self.model = get_model(
124 - model_config=self.model_config,
125 - load_config=self.load_config,
126 - device_config=self.device_config,
127 - vision_language_config=self.vision_language_config,
128 - lora_config=self.lora_config,
129 - parallel_config=self.parallel_config,
130 - scheduler_config=self.scheduler_config,
131 - cache_config=self.cache_config)
123 + self.model = get_model(model_config=self.model_config,
124 + load_config=self.load_config,
125 + device_config=self.device_config,
126 + multimodal_config=self.multimodal_config,
127 + lora_config=self.lora_config,
128 + parallel_config=self.parallel_config,
129 + scheduler_config=self.scheduler_config,
130 + cache_config=self.cache_config)
131
132 def _prepare_prompt(
133 self,
vllm/worker/cpu_worker.py
+5 -5
@@ -6,8 +6,8 @@ import torch.distributed
6
7 from vllm.attention import get_attn_backend
8 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9 - ModelConfig, ParallelConfig, SchedulerConfig,
10 - VisionLanguageConfig)
9 + ModelConfig, MultiModalConfig, ParallelConfig,
10 + SchedulerConfig)
11 from vllm.distributed import (ensure_model_parallel_initialized,
12 init_distributed_environment)
13 from vllm.logger import init_logger
@@ -131,7 +131,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
131 rank: int,
132 distributed_init_method: str,
133 lora_config: Optional[LoRAConfig] = None,
134 - vision_language_config: Optional[VisionLanguageConfig] = None,
134 + multimodal_config: Optional[MultiModalConfig] = None,
135 kv_cache_dtype: Optional[str] = "auto",
136 is_driver_worker: bool = False,
137 ) -> None:
@@ -145,7 +145,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
145 self.rank = rank
146 self.distributed_init_method = distributed_init_method
147 self.lora_config = lora_config
148 - self.vision_language_config = vision_language_config
148 + self.multimodal_config = multimodal_config
149 self.is_driver_worker = is_driver_worker
150 if self.is_driver_worker:
151 assert self.rank == 0, "The driver worker must have rank 0."
@@ -162,7 +162,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
162 cache_config,
163 load_config=self.load_config,
164 lora_config=self.lora_config,
165 - vision_language_config=self.vision_language_config,
165 + multimodal_config=self.multimodal_config,
166 kv_cache_dtype=kv_cache_dtype,
167 is_driver_worker=is_driver_worker)
168 # Uninitialized cache engine. Will be initialized by
vllm/worker/embedding_model_runner.py
+4 -4
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4 import torch
5
6 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 - ModelConfig, ParallelConfig, SchedulerConfig,
8 - VisionLanguageConfig)
7 + ModelConfig, MultiModalConfig, ParallelConfig,
8 + SchedulerConfig)
9 from vllm.logger import init_logger
10 from vllm.model_executor.pooling_metadata import PoolingMetadata
11 from vllm.pooling_params import PoolingParams
@@ -40,7 +40,7 @@ class EmbeddingModelRunner(
40 lora_config: Optional[LoRAConfig],
41 kv_cache_dtype: Optional[str] = "auto",
42 is_driver_worker: bool = False,
43 - vision_language_config: Optional[VisionLanguageConfig] = None,
43 + multimodal_config: Optional[MultiModalConfig] = None,
44 ):
45 super().__init__(model_config,
46 parallel_config,
@@ -51,7 +51,7 @@ class EmbeddingModelRunner(
51 lora_config=lora_config,
52 kv_cache_dtype=kv_cache_dtype,
53 is_driver_worker=is_driver_worker,
54 - vision_language_config=vision_language_config)
54 + multimodal_config=multimodal_config)
55
56 @torch.inference_mode()
57 def execute_model(
vllm/worker/model_runner.py
+17 -11
@@ -24,8 +24,8 @@ except ImportError:
24
25 from vllm.attention import AttentionMetadata, get_attn_backend
26 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27 - ModelConfig, ParallelConfig, SchedulerConfig,
28 - VisionLanguageConfig)
27 + ModelConfig, MultiModalConfig, ParallelConfig,
28 + SchedulerConfig)
29 from vllm.distributed import get_pp_group
30 from vllm.distributed.parallel_state import graph_capture
31 from vllm.inputs import INPUT_REGISTRY
@@ -36,7 +36,8 @@ from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
36 from vllm.model_executor import SamplingMetadata
37 from vllm.model_executor.model_loader import get_model
38 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
39 -from vllm.model_executor.models.interfaces import supports_lora
39 +from vllm.model_executor.models.interfaces import (supports_lora,
40 + supports_vision)
41 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
42 MultiModalInputs)
43 from vllm.sampling_params import SamplingParams
@@ -171,7 +172,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
172 lora_config: Optional[LoRAConfig],
173 kv_cache_dtype: Optional[str] = "auto",
174 is_driver_worker: bool = False,
174 - vision_language_config: Optional[VisionLanguageConfig] = None,
175 + multimodal_config: Optional[MultiModalConfig] = None,
176 return_hidden_states: bool = False,
177 ):
178 self.model_config = model_config
@@ -182,7 +183,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
183 self.lora_config = lora_config
184 self.load_config = load_config
185 self.is_driver_worker = is_driver_worker
185 - self.vision_language_config = vision_language_config
186 + self.multimodal_config = multimodal_config
187 self.return_hidden_states = return_hidden_states
188
189 self.device = self.device_config.device
@@ -244,7 +245,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
245 device_config=self.device_config,
246 load_config=self.load_config,
247 lora_config=self.lora_config,
247 - vision_language_config=self.vision_language_config,
248 + multimodal_config=self.multimodal_config,
249 parallel_config=self.parallel_config,
250 scheduler_config=self.scheduler_config,
251 cache_config=self.cache_config,
@@ -256,6 +257,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
257
258 if self.lora_config:
259 assert supports_lora(self.model), "Model does not support LoRA"
260 + assert not supports_vision(
261 + self.model
262 + ), "To be tested: vision language model with LoRA settings."
263
264 self.lora_manager = LRUCacheWorkerLoRAManager(
265 self.scheduler_config.max_num_seqs,
@@ -804,12 +808,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
808 # the number of seqs (batch_size) is chosen to maximize the number
809 # of images processed.
810 model_config = self.model_config
807 - vlm_config = self.vision_language_config
811
809 - if vlm_config:
810 - max_num_seqs = min(
811 - max_num_seqs,
812 - int(max_num_batched_tokens / vlm_config.image_feature_size))
812 + if supports_vision(self.model):
813 + max_num_seqs = max(
814 + 1,
815 + min(
816 + max_num_seqs,
817 + int(max_num_batched_tokens /
818 + MULTIMODAL_REGISTRY.get_num_input_tokens())))
819 batch_size = 0
820 for group_id in range(max_num_seqs):
821 seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/openvino_model_runner.py
+4 -4
@@ -7,8 +7,8 @@ from torch import nn
7 from vllm.attention import get_attn_backend
8 from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig)
12 from vllm.logger import init_logger
13 from vllm.model_executor import SamplingMetadata
14 from vllm.model_executor.model_loader.openvino import get_model
@@ -48,7 +48,7 @@ class OpenVINOModelRunner:
48 cache_config: CacheConfig,
49 load_config: LoadConfig,
50 lora_config: Optional[LoRAConfig],
51 - vision_language_config: Optional[VisionLanguageConfig],
51 + multimodal_config: Optional[MultiModalConfig],
52 kv_cache_dtype: Optional[str] = "auto",
53 is_driver_worker: bool = False,
54 *args,
@@ -60,7 +60,7 @@ class OpenVINOModelRunner:
60 self.device_config = device_config
61 self.cache_config = cache_config
62 self.lora_config = lora_config
63 - self.vision_language_config = vision_language_config
63 + self.multimodal_config = multimodal_config
64 self.load_config = load_config
65 self.is_driver_worker = is_driver_worker
66
vllm/worker/openvino_worker.py
+5 -5
@@ -7,8 +7,8 @@ import torch.distributed
7
8 from vllm.attention import get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig)
12 from vllm.distributed import (broadcast_tensor_dict,
13 ensure_model_parallel_initialized,
14 init_distributed_environment)
@@ -148,7 +148,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
148 rank: int,
149 distributed_init_method: str,
150 lora_config: Optional[LoRAConfig] = None,
151 - vision_language_config: Optional[VisionLanguageConfig] = None,
151 + multimodal_config: Optional[MultiModalConfig] = None,
152 kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
153 is_driver_worker: bool = False,
154 ) -> None:
@@ -162,7 +162,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
162 self.rank = rank
163 self.distributed_init_method = distributed_init_method
164 self.lora_config = lora_config
165 - self.vision_language_config = vision_language_config
165 + self.multimodal_config = multimodal_config
166 self.is_driver_worker = is_driver_worker
167 if self.is_driver_worker:
168 assert self.rank == 0, "The driver worker must have rank 0."
@@ -180,7 +180,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
180 cache_config,
181 load_config=self.load_config,
182 lora_config=self.lora_config,
183 - vision_language_config=self.vision_language_config,
183 + multimodal_config=self.multimodal_config,
184 kv_cache_dtype=kv_cache_dtype,
185 is_driver_worker=is_driver_worker,
186 )
vllm/worker/tpu_model_runner.py
+4 -4
@@ -8,7 +8,7 @@ import torch_xla.core.xla_model as xm
8
9 from vllm.attention import AttentionMetadata, get_attn_backend
10 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 + MultiModalConfig, ParallelConfig, SchedulerConfig)
12 from vllm.logger import init_logger
13 from vllm.model_executor.model_loader import get_model
14 from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,7 +39,7 @@ class TPUModelRunner:
39 device_config: DeviceConfig,
40 cache_config: CacheConfig,
41 load_config: LoadConfig,
42 - vision_language_config: Optional[VisionLanguageConfig] = None,
42 + multimodal_config: Optional[MultiModalConfig] = None,
43 is_driver_worker: bool = False,
44 ):
45 self.model_config = model_config
@@ -48,7 +48,7 @@ class TPUModelRunner:
48 self.device_config = device_config
49 self.cache_config = cache_config
50 self.load_config = load_config
51 - self.vision_language_config = vision_language_config
51 + self.multimodal_config = multimodal_config
52 self.is_driver_worker = is_driver_worker
53
54 self.block_size = self.cache_config.block_size
@@ -82,7 +82,7 @@ class TPUModelRunner:
82 parallel_config=self.parallel_config,
83 cache_config=self.cache_config,
84 scheduler_config=self.scheduler_config,
85 - vision_language_config=self.vision_language_config,
85 + multimodal_config=self.multimodal_config,
86 lora_config=None,
87 )
88 xm.wait_device_ops()
vllm/worker/tpu_worker.py
+4 -4
@@ -8,7 +8,7 @@ import torch_xla.runtime as xr
8
9 import vllm.envs as envs
10 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11 + MultiModalConfig, ParallelConfig, SchedulerConfig)
12 from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment)
14 from vllm.logger import init_logger
@@ -31,7 +31,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
31 device_config: DeviceConfig,
32 cache_config: CacheConfig,
33 load_config: LoadConfig,
34 - vision_language_config: Optional[VisionLanguageConfig],
34 + multimodal_config: Optional[MultiModalConfig],
35 local_rank: int,
36 rank: int,
37 distributed_init_method: str,
@@ -43,7 +43,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
43 self.device_config = device_config
44 self.cache_config = cache_config
45 self.load_config = load_config
46 - self.vision_language_config = vision_language_config
46 + self.multimodal_config = multimodal_config
47 self.local_rank = local_rank
48 self.rank = rank
49 self.distributed_init_method = distributed_init_method
@@ -62,7 +62,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
62 device_config,
63 cache_config,
64 load_config,
65 - vision_language_config,
65 + multimodal_config,
66 is_driver_worker=is_driver_worker)
67
68 def init_device(self) -> None:
vllm/worker/worker.py
+5 -8
@@ -7,8 +7,8 @@ import torch
7 import torch.distributed
8
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - SpeculativeConfig, VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig, SpeculativeConfig)
12 from vllm.distributed import (ensure_model_parallel_initialized,
13 init_distributed_environment,
14 set_custom_all_reduce)
@@ -43,7 +43,7 @@ class Worker(LocalOrDistributedWorkerBase):
43 rank: int,
44 distributed_init_method: str,
45 lora_config: Optional[LoRAConfig] = None,
46 - vision_language_config: Optional[VisionLanguageConfig] = None,
46 + multimodal_config: Optional[MultiModalConfig] = None,
47 speculative_config: Optional[SpeculativeConfig] = None,
48 is_driver_worker: bool = False,
49 model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
@@ -66,10 +66,7 @@ class Worker(LocalOrDistributedWorkerBase):
66 # note: lazy import to avoid importing torch before initializing
67 from vllm.utils import init_cached_hf_modules
68 init_cached_hf_modules()
69 - self.vision_language_config = vision_language_config
70 - if self.vision_language_config:
71 - assert not self.lora_config, (
72 - "To be tested: vision language model with LoRA settings.")
69 + self.multimodal_config = multimodal_config
70
71 # Return hidden states from target model if the draft model is an
72 # mlp_speculator
@@ -94,7 +91,7 @@ class Worker(LocalOrDistributedWorkerBase):
91 lora_config=self.lora_config,
92 kv_cache_dtype=self.cache_config.cache_dtype,
93 is_driver_worker=is_driver_worker,
97 - vision_language_config=vision_language_config,
94 + multimodal_config=multimodal_config,
95 **speculative_args,
96 )
97 # Uninitialized cache engine. Will be initialized by
vllm/worker/xpu_model_runner.py
+15 -10
@@ -7,12 +7,13 @@ import torch.nn as nn
7
8 from vllm.attention import get_attn_backend
9 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10 - ModelConfig, ParallelConfig, SchedulerConfig,
11 - VisionLanguageConfig)
10 + ModelConfig, MultiModalConfig, ParallelConfig,
11 + SchedulerConfig)
12 from vllm.distributed import broadcast_tensor_dict
13 from vllm.inputs import INPUT_REGISTRY
14 from vllm.logger import init_logger
15 from vllm.model_executor.model_loader import get_model
16 +from vllm.model_executor.models.interfaces import supports_vision
17 from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
18 MultiModalInputs)
19 from vllm.sampling_params import SamplingParams
@@ -85,7 +86,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
86 cache_config: CacheConfig,
87 load_config: LoadConfig,
88 lora_config: Optional[LoRAConfig],
88 - vision_language_config: Optional[VisionLanguageConfig],
89 + multimodal_config: Optional[MultiModalConfig],
90 kv_cache_dtype: Optional[str] = "auto",
91 is_driver_worker: bool = False,
92 *args,
@@ -97,7 +98,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
98 self.lora_config = lora_config
99 self.load_config = load_config
100 self.cache_config = cache_config
100 - self.vision_language_config = vision_language_config
101 + self.multimodal_config = multimodal_config
102 self.is_driver_worker = is_driver_worker
103
104 self.sliding_window = model_config.get_sliding_window()
@@ -134,7 +135,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
135 device_config=self.device_config,
136 load_config=self.load_config,
137 lora_config=self.lora_config,
137 - vision_language_config=self.vision_language_config,
138 + multimodal_config=self.multimodal_config,
139 parallel_config=self.parallel_config,
140 scheduler_config=self.scheduler_config,
141 cache_config=self.cache_config,
@@ -165,12 +166,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
166 # the number of seqs (batch_size) is chosen to maximize the number
167 # of images processed.
168 model_config = self.model_config
168 - vlm_config = self.vision_language_config
169
170 - if vlm_config:
171 - max_num_seqs = min(
172 - max_num_seqs,
173 - int(max_num_batched_tokens / vlm_config.image_feature_size))
170 + if supports_vision(self.model):
171 + # TODO: properly inject these numbers from MultiModalRegistry.
172 + # Right now, just use an overly conservative number.
173 + max_num_seqs = max(
174 + 1,
175 + min(
176 + max_num_seqs,
177 + int(max_num_batched_tokens /
178 + MULTIMODAL_REGISTRY.get_num_input_tokens())))
179
180 for group_id in range(max_num_seqs):
181 seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/xpu_worker.py
+5 -8
@@ -9,8 +9,8 @@ import torch
9 import torch.distributed
10
11 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12 - ModelConfig, ParallelConfig, SchedulerConfig,
13 - SpeculativeConfig, VisionLanguageConfig)
12 + ModelConfig, MultiModalConfig, ParallelConfig,
13 + SchedulerConfig, SpeculativeConfig)
14 from vllm.distributed import (ensure_model_parallel_initialized,
15 init_distributed_environment)
16 from vllm.logger import init_logger
@@ -45,7 +45,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
45 rank: int,
46 distributed_init_method: str,
47 lora_config: Optional[LoRAConfig] = None,
48 - vision_language_config: Optional[VisionLanguageConfig] = None,
48 + multimodal_config: Optional[MultiModalConfig] = None,
49 speculative_config: Optional[SpeculativeConfig] = None,
50 is_driver_worker: bool = False,
51 ) -> None:
@@ -66,10 +66,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
66 if self.is_driver_worker:
67 assert self.rank == 0, "The driver worker must have rank 0."
68
69 - self.vision_language_config = vision_language_config
70 - if self.vision_language_config:
71 - assert not self.lora_config, (
72 - "To be tested: vision language model with LoRA settings.")
69 + self.multimodal_config = multimodal_config
70
71 self.model_runner = XPUModelRunner( # type: ignore
72 model_config,
@@ -81,7 +78,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
78 lora_config=self.lora_config,
79 kv_cache_dtype=self.cache_config.cache_dtype,
80 is_driver_worker=is_driver_worker,
84 - vision_language_config=vision_language_config,
81 + multimodal_config=multimodal_config,
82 )
83 # Uninitialized cache engine. Will be initialized by
84 # initialize_cache.