A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[vlm] Remove vision language config. (#6089)
Signed-off-by: Xiaowei Jiang <xwjiang2010@gmail.com> Co-authored-by: Roger Wang <ywang@roblox.com>
xwjiang2010 committed
Jul 3, 2024 at 15:14 UTC
d9e98f42e434d1f1a0f8ceed363047060aca6262
43 files changed
+372
-466
docs/source/dev/multimodal/multimodal_index.rst
+5
@@ -10,8 +10,13 @@ vLLM provides experimental support for multi-modal models through the :mod:`vllm
10
:class:`vllm.inputs.PromptStrictInputs` accepts an additional attribute ``multi_modal_data``
11
which allows you to pass in multi-modal input alongside text and token prompts.
12
13
+.. note::
14
+ ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
15
+ :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
16
+
17
By default, vLLM models do not support multi-modal inputs. To enable multi-modal support for a model, please follow :ref:`the guide for adding a new multimodal model. <adding_a_new_multimodal_model>`.
18
19
+
20
# TODO: Add more instructions on how to do that once embeddings is in.
21
22
Guides
docs/source/models/vlm.rst
+39
-39
@@ -8,18 +8,6 @@ vLLM provides experimental support for Vision Language Models (VLMs). This docum
8
.. important::
9
We are actively iterating on VLM support. Expect breaking changes to VLM usage and development in upcoming releases without prior deprecation.
10
11
-Engine Arguments
12
-----------------
13
-
14
-The following :ref:`engine arguments <engine_args>` are specific to VLMs:
15
-
16
-.. argparse::
17
- :module: vllm.engine.arg_utils
18
- :func: _vlm_engine_args_parser
19
- :prog: -m vllm.entrypoints.openai.api_server
20
- :nodefaultconst:
21
-
22
-.. important::
11
Currently, the support for vision language models on vLLM has the following limitations:
12
13
* Only single image input is supported per text prompt.
@@ -33,20 +21,17 @@ To initialize a VLM, the aforementioned arguments must be passed to the ``LLM``
21
22
.. code-block:: python
23
36
- llm = LLM(
37
- model="llava-hf/llava-1.5-7b-hf",
38
- image_token_id=32000,
39
- image_input_shape="1,3,336,336",
40
- image_feature_size=576,
41
- )
24
+ llm = LLM(model="llava-hf/llava-1.5-7b-hf")
25
26
.. important::
44
- Currently, you have to specify ``image_feature_size`` to support memory profiling.
45
- To avoid OOM during runtime, you should set this to the maximum value supported by the model.
46
- The calculation of feature size is specific to the model. For more details, please refer to
47
- the function :code:`get_<model_name>_image_feature_size` inside the corresponding model file.
27
+ We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
28
+ the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
29
+ every model to perform profiling with.
30
49
- We will remove most of the vision-specific arguments in a future release as they can be inferred from the HuggingFace configuration.
31
+ This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
32
+ :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
33
+ for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
34
+ with a more accurate profiling strategy in the future.
35
36
37
To pass an image to the model, note the following in :class:`vllm.inputs.PromptStrictInputs`:
@@ -54,19 +39,15 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
39
* ``prompt``: The prompt should follow the format that is documented on HuggingFace.
40
* ``multi_modal_data``: This is a dictionary that follows the schema defined in :class:`vllm.multimodal.MultiModalDataDict`.
41
57
-.. note::
58
-
59
- ``multi_modal_data`` can accept keys and values beyond the builtin ones, as long as a customized plugin is registered through
60
- :class:`vllm.multimodal.MULTIMODAL_REGISTRY`.
61
-
42
.. code-block:: python
43
44
# Refer to the HuggingFace repo for the correct format to use
45
prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
46
47
# Load the image using PIL.Image
68
- image = ...
69
-
48
+ image = PIL.Image.open(...)
49
+
50
+ # Single prompt inference
51
outputs = llm.generate({
52
"prompt": prompt,
53
"multi_modal_data": {"image": image},
@@ -75,6 +56,26 @@ To pass an image to the model, note the following in :class:`vllm.inputs.PromptS
56
for o in outputs:
57
generated_text = o.outputs[0].text
58
print(generated_text)
59
+
60
+ # Batch inference
61
+ image_1 = PIL.Image.open(...)
62
+ image_2 = PIL.Image.open(...)
63
+ outputs = llm.generate(
64
+ [
65
+ {
66
+ "prompt": "USER: <image>\nWhat is the content of this image?\nASSISTANT:",
67
+ "multi_modal_data": {"image": image_1},
68
+ },
69
+ {
70
+ "prompt": "USER: <image>\nWhat's the color of this image?\nASSISTANT:",
71
+ "multi_modal_data": {"image": image_2},
72
+ }
73
+ ]
74
+ )
75
+
76
+ for o in outputs:
77
+ generated_text = o.outputs[0].text
78
+ print(generated_text)
79
80
A code example can be found in `examples/llava_example.py <https://github.com/vllm-project/vllm/blob/main/examples/llava_example.py>`_.
81
@@ -99,18 +100,17 @@ Below is an example on how to launch the same ``llava-hf/llava-1.5-7b-hf`` with
100
101
python -m vllm.entrypoints.openai.api_server \
102
--model llava-hf/llava-1.5-7b-hf \
102
- --image-token-id 32000 \
103
- --image-input-shape 1,3,336,336 \
104
- --image-feature-size 576 \
103
--chat-template template_llava.jinja
104
105
.. important::
108
- Currently, you have to specify ``image_feature_size`` to support memory profiling.
109
- To avoid OOM during runtime, you should set this to the maximum value supported by the model.
110
- The calculation of feature size is specific to the model. For more details, please refer to
111
- the function :code:`get_<model_name>_image_feature_size` inside the corresponding model file.
112
-
113
- We will remove most of the vision-specific arguments in a future release as they can be inferred from the HuggingFace configuration.
106
+ We have removed all vision language related CLI args in the ``0.5.1`` release. **This is a breaking change**, so please update your code to follow
107
+ the above snippet. Specifically, ``image_feature_size`` is no longer required to be specified, and internally we will construct data structures for
108
+ every model to perform profiling with.
109
+
110
+ This work is still ongoing. In the meantime, we internally hardcode ``image_feature_size = 3000`` through
111
+ :meth:`MULTIMODAL_REGISTRY.get_num_input_tokens <vllm.multimodal.MultiModalRegistry.get_num_input_tokens>`
112
+ for every model to be conservative in terms of GPU memory consumption. This hardcoded value will be replaced
113
+ with a more accurate profiling strategy in the future.
114
115
To consume the server, you can use the OpenAI client like in the example below:
116
examples/llava_example.py
+1
-6
@@ -10,12 +10,7 @@ from vllm import LLM
10
11
12
def run_llava():
13
- llm = LLM(
14
- model="llava-hf/llava-1.5-7b-hf",
15
- image_token_id=32000,
16
- image_input_shape="1,3,336,336",
17
- image_feature_size=576,
18
- )
13
+ llm = LLM(model="llava-hf/llava-1.5-7b-hf")
14
15
prompt = "USER: <image>\nWhat is the content of this image?\nASSISTANT:"
16
examples/llava_next_example.py
+1
-7
@@ -7,13 +7,7 @@ from vllm import LLM, SamplingParams
7
8
9
def run_llava_next():
10
- llm = LLM(
11
- model="llava-hf/llava-v1.6-mistral-7b-hf",
12
- image_token_id=32000,
13
- image_input_shape="1,3,336,336",
14
- # Use the maximum possible value for memory profiling
15
- image_feature_size=2928,
16
- )
10
+ llm = LLM(model="llava-hf/llava-v1.6-mistral-7b-hf", max_model_len=4096)
11
12
prompt = "[INST] <image>\nWhat is shown in this image? [/INST]"
13
url = "https://h2o-release.s3.amazonaws.com/h2ogpt/bigben.jpg"
examples/openai_vision_api_client.py
-3
@@ -3,9 +3,6 @@
3
Launch the vLLM server with the following command:
4
python -m vllm.entrypoints.openai.api_server \
5
--model llava-hf/llava-1.5-7b-hf \
6
- --image-token-id 32000 \
7
- --image-input-shape 1,3,336,336 \
8
- --image-feature-size 576 \
6
--chat-template template_llava.jinja
7
"""
8
import base64
examples/phi3v_example.py
+2
-4
@@ -14,15 +14,13 @@ def run_phi3v():
14
15
# Note: The default setting of max_num_seqs (256) and
16
# max_model_len (128k) for this model may cause OOM.
17
+ # You may lower either to run this example on lower-end GPUs.
18
+
19
# In this example, we override max_num_seqs to 5 while
20
# keeping the original context length of 128k.
21
llm = LLM(
22
model=model_path,
23
trust_remote_code=True,
22
- image_token_id=32044,
23
- image_input_shape="1,3,1008,1344",
24
- # Use the maximum possible value for memory profiling
25
- image_feature_size=2653,
24
max_num_seqs=5,
25
)
26
tests/distributed/test_multimodal_broadcast.py
+3
-3
@@ -20,9 +20,9 @@ from vllm.utils import cuda_device_count_stateless
20
model = os.environ["TEST_DIST_MODEL"]
21
22
if model.startswith("llava-hf/llava"):
23
- from ..models.test_llava import model_and_vl_config, run_test
23
+ from ..models.test_llava import models, run_test
24
elif model.startswith("microsoft/Phi-3-vision"):
25
- from ..models.test_phi3v import model_and_vl_config, run_test
25
+ from ..models.test_phi3v import models, run_test
26
else:
27
raise NotImplementedError(f"Unsupported model: {model}")
28
@@ -44,7 +44,7 @@ def test_models(hf_runner, vllm_runner, image_assets,
44
hf_runner,
45
vllm_runner,
46
image_assets,
47
- model_and_config=model_and_vl_config[0],
47
+ model=models[0],
48
size_factors=[1.0],
49
dtype=dtype,
50
max_tokens=max_tokens,
tests/entrypoints/openai/test_vision.py
-6
@@ -39,12 +39,6 @@ def server(ray_ctx):
39
"--max-model-len",
40
"4096",
41
"--enforce-eager",
42
- "--image-token-id",
43
- "32000",
44
- "--image-input-shape",
45
- "1,3,336,336",
46
- "--image-feature-size",
47
- "576",
42
"--chat-template",
43
str(LLAVA_CHAT_TEMPLATE),
44
])
tests/models/test_llava.py
+17
-43
@@ -3,7 +3,6 @@ from typing import List, Optional, Tuple, Type
3
import pytest
4
from transformers import AutoTokenizer
5
6
-from vllm.config import VisionLanguageConfig
6
from vllm.multimodal.utils import rescale_image_size
7
from vllm.sequence import SampleLogprobs
8
@@ -21,49 +20,27 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
"USER: <image>\nWhat's in this image?\nASSISTANT:",
21
})
22
23
+IMAGE_TOKEN_ID = 32000
24
25
-def iter_llava_configs(model_name: str):
26
- image_hw_to_feature_size = {
27
- (336, 336): 576,
28
- }
29
-
30
- for (h, w), f in image_hw_to_feature_size.items():
31
- input_shape = (1, 3, h, w)
32
- yield (model_name,
33
- VisionLanguageConfig(image_feature_size=f,
34
- image_token_id=32000,
35
- image_input_shape=input_shape))
36
-
37
-
38
-model_and_vl_config = [
39
- *iter_llava_configs("llava-hf/llava-1.5-7b-hf"),
40
-]
25
+models = ["llava-hf/llava-1.5-7b-hf"]
26
27
28
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
29
Optional[SampleLogprobs]],
45
- vlm_config: VisionLanguageConfig, model_id: str):
46
- """Sanitize vllm output to be comparable with hf output.
47
- The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
48
- x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
49
- It also reduces `output_str` from "<image><image>bla" to "bla".
50
- """
30
+ model: str):
31
+ """Sanitize vllm output to be comparable with hf output."""
32
output_ids, output_str, out_logprobs = vllm_output
52
- image_token_id = vlm_config.image_token_id
33
54
- tokenizer = AutoTokenizer.from_pretrained(model_id)
55
- image_token_str = tokenizer.decode(image_token_id)
34
+ tokenizer = AutoTokenizer.from_pretrained(model)
35
eos_token_id = tokenizer.eos_token_id
36
37
hf_output_ids = [
38
token_id for idx, token_id in enumerate(output_ids)
60
- if token_id != image_token_id or output_ids[idx - 1] != image_token_id
39
+ if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
40
]
41
63
- hf_output_str = output_str \
64
- .replace(image_token_str * vlm_config.image_feature_size, "")
65
- assert hf_output_str[0] == " "
66
- hf_output_str = hf_output_str[1:]
42
+ assert output_str[0] == " "
43
+ hf_output_str = output_str[1:]
44
if hf_output_ids[-1] == eos_token_id:
45
hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
46
@@ -74,7 +51,7 @@ def run_test(
51
hf_runner: Type[HfRunner],
52
vllm_runner: Type[VllmRunner],
53
image_assets: _ImageAssets,
77
- model_and_config: Tuple[str, VisionLanguageConfig],
54
+ model: str,
55
*,
56
size_factors: List[float],
57
dtype: str,
@@ -92,7 +69,6 @@ def run_test(
69
Note, the text input is also adjusted to abide by vllm contract.
70
The text output is sanitized to be able to compare with hf.
71
"""
95
- model_id, vlm_config = model_and_config
72
images = [asset.pil_image for asset in image_assets]
73
74
inputs_per_image = [(
@@ -106,12 +82,11 @@ def run_test(
82
# will hurt multiprocessing backend with fork method (the default method).
83
84
# max_model_len should be greater than image_feature_size
109
- with vllm_runner(model_id,
85
+ with vllm_runner(model,
86
dtype=dtype,
87
tensor_parallel_size=tensor_parallel_size,
88
distributed_executor_backend=distributed_executor_backend,
113
- enforce_eager=True,
114
- **vlm_config.as_cli_args_dict()) as vllm_model:
89
+ enforce_eager=True) as vllm_model:
90
vllm_outputs_per_image = [
91
vllm_model.generate_greedy_logprobs(prompts,
92
max_tokens,
@@ -120,7 +95,7 @@ def run_test(
95
for prompts, images in inputs_per_image
96
]
97
123
- with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
98
+ with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
99
hf_outputs_per_image = [
100
hf_model.generate_greedy_logprobs_limit(prompts,
101
max_tokens,
@@ -136,7 +111,7 @@ def run_test(
111
check_logprobs_close(
112
outputs_0_lst=hf_outputs,
113
outputs_1_lst=[
139
- vllm_to_hf_output(vllm_output, vlm_config, model_id)
114
+ vllm_to_hf_output(vllm_output, model)
115
for vllm_output in vllm_outputs
116
],
117
name_0="hf",
@@ -144,7 +119,7 @@ def run_test(
119
)
120
121
147
-@pytest.mark.parametrize("model_and_config", model_and_vl_config)
122
+@pytest.mark.parametrize("model", models)
123
@pytest.mark.parametrize(
124
"size_factors",
125
[
@@ -161,14 +136,13 @@ def run_test(
136
@pytest.mark.parametrize("dtype", ["half"])
137
@pytest.mark.parametrize("max_tokens", [128])
138
@pytest.mark.parametrize("num_logprobs", [5])
164
-def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
165
- size_factors, dtype: str, max_tokens: int,
166
- num_logprobs: int) -> None:
139
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
140
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
141
run_test(
142
hf_runner,
143
vllm_runner,
144
image_assets,
171
- model_and_config,
145
+ model,
146
size_factors=size_factors,
147
dtype=dtype,
148
max_tokens=max_tokens,
tests/models/test_llava_next.py
+13
-41
@@ -4,7 +4,6 @@ from typing import List, Optional, Tuple
4
import pytest
5
from transformers import AutoTokenizer
6
7
-from vllm.config import VisionLanguageConfig
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
@@ -27,46 +26,22 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
26
f"{_PREFACE} USER: <image>\nWhat's in this image? ASSISTANT:",
27
})
28
30
-
31
-def iter_llava_next_configs(model_name: str):
32
- # Need to use the max possible feature size for profile_run
33
- image_hw_to_feature_size = {
34
- (336, 336): 2928,
35
- }
36
-
37
- for (h, w), f in image_hw_to_feature_size.items():
38
- input_shape = (1, 3, h, w)
39
- yield (model_name,
40
- VisionLanguageConfig(
41
- image_feature_size=f,
42
- image_token_id=32000,
43
- image_input_shape=input_shape,
44
- ))
45
-
46
-
47
-model_and_vl_config = [
48
- *iter_llava_next_configs("llava-hf/llava-v1.6-vicuna-7b-hf"),
49
-]
29
+IMAGE_TOKEN_ID = 32000
30
31
32
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
33
Optional[SampleLogprobs]],
54
- vlm_config: VisionLanguageConfig, model_id: str):
55
- """Sanitize vllm output to be comparable with hf output.
56
- The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
57
- x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
58
- It also reduces `output_str` from "<image><image>bla" to "bla".
59
- """
34
+ model: str):
35
+ """Sanitize vllm output to be comparable with hf output."""
36
output_ids, output_str, out_logprobs = vllm_output
61
- image_token_id = vlm_config.image_token_id
37
63
- tokenizer = AutoTokenizer.from_pretrained(model_id)
64
- image_token_str = tokenizer.decode(image_token_id)
38
+ tokenizer = AutoTokenizer.from_pretrained(model)
39
+ image_token_str = tokenizer.decode(IMAGE_TOKEN_ID)
40
eos_token_id = tokenizer.eos_token_id
41
42
hf_output_ids = [
43
token_id for idx, token_id in enumerate(output_ids)
69
- if token_id != image_token_id or output_ids[idx - 1] != image_token_id
44
+ if token_id != IMAGE_TOKEN_ID or output_ids[idx - 1] != IMAGE_TOKEN_ID
45
]
46
47
hf_output_str = re.sub(fr"({image_token_str})+", "", output_str)
@@ -78,7 +53,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
53
return hf_output_ids, hf_output_str, out_logprobs
54
55
81
-@pytest.mark.parametrize("model_and_config", model_and_vl_config)
56
+@pytest.mark.parametrize("model", ["llava-hf/llava-v1.6-vicuna-7b-hf"])
57
@pytest.mark.parametrize(
58
"size_factors",
59
[
@@ -95,9 +70,8 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
70
@pytest.mark.parametrize("dtype", ["half"])
71
@pytest.mark.parametrize("max_tokens", [128])
72
@pytest.mark.parametrize("num_logprobs", [5])
98
-def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
99
- size_factors, dtype: str, max_tokens: int,
100
- num_logprobs: int) -> None:
73
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
74
+ dtype, max_tokens, num_logprobs) -> None:
75
"""Inference result should be the same between hf and vllm.
76
77
All the image fixtures for the test is under tests/images.
@@ -107,7 +81,6 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
81
Note, the text input is also adjusted to abide by vllm contract.
82
The text output is sanitized to be able to compare with hf.
83
"""
110
- model_id, vlm_config = model_and_config
84
images = [asset.pil_image for asset in image_assets]
85
86
inputs_per_image = [(
@@ -116,11 +89,10 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
89
) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
90
91
# max_model_len should be greater than image_feature_size
119
- with vllm_runner(model_id,
92
+ with vllm_runner(model,
93
dtype=dtype,
94
max_model_len=4096,
122
- enforce_eager=True,
123
- **vlm_config.as_cli_args_dict()) as vllm_model:
95
+ enforce_eager=True) as vllm_model:
96
vllm_outputs_per_image = [
97
vllm_model.generate_greedy_logprobs(prompts,
98
max_tokens,
@@ -129,7 +101,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
101
for prompts, images in inputs_per_image
102
]
103
132
- with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
104
+ with hf_runner(model, dtype=dtype, is_vision_model=True) as hf_model:
105
hf_outputs_per_image = [
106
hf_model.generate_greedy_logprobs_limit(prompts,
107
max_tokens,
@@ -145,7 +117,7 @@ def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
117
check_logprobs_close(
118
outputs_0_lst=hf_outputs,
119
outputs_1_lst=[
148
- vllm_to_hf_output(vllm_output, vlm_config, model_id)
120
+ vllm_to_hf_output(vllm_output, model)
121
for vllm_output in vllm_outputs
122
],
123
name_0="hf",
tests/models/test_phi3v.py
+15
-39
@@ -4,7 +4,6 @@ from typing import List, Optional, Tuple, Type
4
import pytest
5
from transformers import AutoTokenizer
6
7
-from vllm.config import VisionLanguageConfig
7
from vllm.multimodal.utils import rescale_image_size
8
from vllm.sequence import SampleLogprobs
9
from vllm.utils import is_cpu
@@ -23,35 +22,14 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
22
"<|user|>\n<|image_1|>\nWhat's in this image?<|end|>\n<|assistant|>\n",
23
})
24
26
-
27
-def iter_phi3v_configs(model_name: str):
28
- # Need to use the max possible feature size for profile_run
29
- image_hw_to_feature_size = {
30
- (1008, 1344): 2653,
31
- }
32
-
33
- for (h, w), f in image_hw_to_feature_size.items():
34
- input_shape = (1, 3, h, w)
35
- yield (model_name,
36
- VisionLanguageConfig(image_feature_size=f,
37
- image_token_id=32044,
38
- image_input_shape=input_shape))
39
-
40
-
41
-model_and_vl_config = [
42
- *iter_phi3v_configs("microsoft/Phi-3-vision-128k-instruct"),
43
-]
25
+models = ["microsoft/Phi-3-vision-128k-instruct"]
26
27
28
def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
29
Optional[SampleLogprobs]],
48
- vlm_config: VisionLanguageConfig, model_id: str):
49
- """Sanitize vllm output to be comparable with hf output.
50
- The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
51
- x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
52
- It also reduces `output_str` from "<image><image>bla" to "bla".
53
- """
54
- output_ids, output_str, out_logprobs = vllm_output
30
+ model: str):
31
+ """Sanitize vllm output to be comparable with hf output."""
32
+ _, output_str, out_logprobs = vllm_output
33
34
output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
35
assert output_str_without_image[0] == " "
@@ -60,7 +38,7 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
38
hf_output_str = output_str_without_image.replace("<|user|>", "") \
39
.replace("<|end|>\n<|assistant|>", " ")
40
63
- tokenizer = AutoTokenizer.from_pretrained(model_id)
41
+ tokenizer = AutoTokenizer.from_pretrained(model)
42
hf_output_ids = tokenizer.encode(output_str_without_image)
43
assert hf_output_ids[0] == 1
44
hf_output_ids = hf_output_ids[1:]
@@ -77,7 +55,7 @@ def run_test(
55
hf_runner: Type[HfRunner],
56
vllm_runner: Type[VllmRunner],
57
image_assets: _ImageAssets,
80
- model_and_config: Tuple[str, VisionLanguageConfig],
58
+ model: str,
59
*,
60
size_factors: List[float],
61
dtype: str,
@@ -95,7 +73,6 @@ def run_test(
73
Note, the text input is also adjusted to abide by vllm contract.
74
The text output is sanitized to be able to compare with hf.
75
"""
98
- model_id, vlm_config = model_and_config
76
images = [asset.pil_image for asset in image_assets]
77
78
inputs_per_image = [(
@@ -109,13 +86,13 @@ def run_test(
86
# will hurt multiprocessing backend with fork method (the default method).
87
88
# max_model_len should be greater than image_feature_size
112
- with vllm_runner(model_id,
89
+ with vllm_runner(model,
90
max_model_len=4096,
91
+ max_num_seqs=1,
92
dtype=dtype,
93
tensor_parallel_size=tensor_parallel_size,
94
distributed_executor_backend=distributed_executor_backend,
117
- enforce_eager=True,
118
- **vlm_config.as_cli_args_dict()) as vllm_model:
95
+ enforce_eager=True) as vllm_model:
96
vllm_outputs_per_image = [
97
vllm_model.generate_greedy_logprobs(prompts,
98
max_tokens,
@@ -126,7 +103,7 @@ def run_test(
103
104
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
105
hf_model_kwargs = {"_attn_implementation": "eager"}
129
- with hf_runner(model_id, dtype=dtype,
106
+ with hf_runner(model, dtype=dtype,
107
model_kwargs=hf_model_kwargs) as hf_model:
108
eos_token_id = hf_model.processor.tokenizer.eos_token_id
109
hf_outputs_per_image = [
@@ -143,7 +120,7 @@ def run_test(
120
check_logprobs_close(
121
outputs_0_lst=hf_outputs,
122
outputs_1_lst=[
146
- vllm_to_hf_output(vllm_output, vlm_config, model_id)
123
+ vllm_to_hf_output(vllm_output, model)
124
for vllm_output in vllm_outputs
125
],
126
name_0="hf",
@@ -153,7 +130,7 @@ def run_test(
130
131
# Since we use _attn_implementation="eager" for hf_runner, there is more
132
# significant numerical difference. The basic `logprobs=5` fails to pass.
156
-@pytest.mark.parametrize("model_and_config", model_and_vl_config)
133
+@pytest.mark.parametrize("model", models)
134
@pytest.mark.parametrize(
135
"size_factors",
136
[
@@ -170,14 +147,13 @@ def run_test(
147
@pytest.mark.parametrize("dtype", [target_dtype])
148
@pytest.mark.parametrize("max_tokens", [128])
149
@pytest.mark.parametrize("num_logprobs", [10])
173
-def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
174
- size_factors, dtype: str, max_tokens: int,
175
- num_logprobs: int) -> None:
150
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
151
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
152
run_test(
153
hf_runner,
154
vllm_runner,
155
image_assets,
180
- model_and_config,
156
+ model,
157
size_factors=size_factors,
158
dtype=dtype,
159
max_tokens=max_tokens,
vllm/config.py
+7
-31
@@ -1,8 +1,7 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
5
- Union)
4
+from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
5
6
import torch
7
from transformers import PretrainedConfig
@@ -120,7 +119,7 @@ class ModelConfig:
119
disable_sliding_window: bool = False,
120
skip_tokenizer_init: bool = False,
121
served_model_name: Optional[Union[str, List[str]]] = None,
123
- multimodal_config: Optional["VisionLanguageConfig"] = None,
122
+ multimodal_config: Optional["MultiModalConfig"] = None,
123
) -> None:
124
self.model = model
125
self.tokenizer = tokenizer
@@ -1289,35 +1288,12 @@ class LoRAConfig:
1288
raise ValueError("LoRA is not supported with chunked prefill yet.")
1289
1290
1292
-# TODO: To be replaced by MultiModalConfig.
1291
@dataclass
1294
-class VisionLanguageConfig:
1292
+class MultiModalConfig:
1293
"""Configs the input data format and how models should run for
1296
- vision language models."""
1297
- # The input id corresponding to image token.
1298
- image_token_id: int
1299
- # Used for running `run_prefill_max_token`.
1300
- # For models that support varying resolution, this corresponds to
1301
- # worst case scenario (biggest supported resolution).
1302
- image_input_shape: tuple
1303
- image_feature_size: int
1304
-
1305
- def as_cli_args_dict(self) -> Dict[str, Any]:
1306
- """Flatten vision language config to pure args.
1307
-
1308
- Compatible with what llm entrypoint expects.
1309
- """
1310
- result: Dict[str, Any] = {}
1311
- for f in fields(self):
1312
- value = getattr(self, f.name)
1313
- if isinstance(value, enum.Enum):
1314
- result[f.name] = value.name.lower()
1315
- elif isinstance(value, tuple):
1316
- result[f.name] = ",".join([str(item) for item in value])
1317
- else:
1318
- result[f.name] = value
1319
-
1320
- return result
1294
+ multimodal models."""
1295
+ # TODO: Add configs to init vision tower or not.
1296
+ pass
1297
1298
1299
_STR_DTYPE_TO_TORCH_DTYPE = {
@@ -1541,7 +1517,7 @@ class EngineConfig:
1517
device_config: DeviceConfig
1518
load_config: LoadConfig
1519
lora_config: Optional[LoRAConfig]
1544
- vision_language_config: Optional[VisionLanguageConfig]
1520
+ multimodal_config: Optional[MultiModalConfig]
1521
speculative_config: Optional[SpeculativeConfig]
1522
decoding_config: Optional[DecodingConfig]
1523
observability_config: Optional[ObservabilityConfig]
vllm/engine/arg_utils.py
+7
-52
@@ -6,11 +6,11 @@ from typing import List, Optional, Tuple, Union
6
7
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
8
EngineConfig, LoadConfig, LoRAConfig, ModelConfig,
9
- ObservabilityConfig, ParallelConfig, SchedulerConfig,
10
- SpeculativeConfig, TokenizerPoolConfig,
11
- VisionLanguageConfig)
9
+ MultiModalConfig, ObservabilityConfig, ParallelConfig,
10
+ SchedulerConfig, SpeculativeConfig,
11
+ TokenizerPoolConfig)
12
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
13
-from vllm.utils import FlexibleArgumentParser, str_to_int_tuple
13
+from vllm.utils import FlexibleArgumentParser
14
15
16
def nullable_str(val: str):
@@ -78,11 +78,6 @@ class EngineArgs:
78
model_loader_extra_config: Optional[dict] = None
79
preemption_mode: Optional[str] = None
80
81
- # Related to Vision-language models such as llava
82
- image_token_id: Optional[int] = None
83
- image_input_shape: Optional[str] = None
84
- image_feature_size: Optional[int] = None
85
-
81
scheduler_delay_factor: float = 0.0
82
enable_chunked_prefill: bool = False
83
@@ -106,27 +101,6 @@ class EngineArgs:
101
if self.tokenizer is None:
102
self.tokenizer = self.model
103
109
- @staticmethod
110
- def add_cli_args_for_vlm(
111
- parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
112
- parser.add_argument('--image-token-id',
113
- type=int,
114
- default=None,
115
- help=('Input id for image token.'))
116
- parser.add_argument(
117
- '--image-input-shape',
118
- type=nullable_str,
119
- default=None,
120
- help=('The biggest image input shape (worst for memory footprint) '
121
- 'given an input type. Only used for vLLM\'s profile_run.'))
122
- parser.add_argument(
123
- '--image-feature-size',
124
- type=int,
125
- default=None,
126
- help=('The image feature size along the context dimension.'))
127
-
128
- return parser
129
-
104
@staticmethod
105
def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
106
"""Shared CLI arguments for vLLM engine."""
@@ -484,9 +458,6 @@ class EngineArgs:
458
],
459
help='Device type for vLLM execution.')
460
487
- # Related to Vision-language models such as llava
488
- parser = EngineArgs.add_cli_args_for_vlm(parser)
489
-
461
parser.add_argument(
462
'--scheduler-delay-factor',
463
type=float,
@@ -648,19 +619,7 @@ class EngineArgs:
619
raise ValueError(
620
"BitsAndBytes load format and QLoRA adapter only support "
621
f"'bitsandbytes' quantization, but got {self.quantization}")
651
- if self.image_token_id is not None:
652
- if (not self.image_input_shape or not self.image_feature_size):
653
- raise ValueError(
654
- 'Specify `image_input_shape` and '
655
- '`image_feature_size` together with `image_token_id`.')
656
-
657
- vision_language_config = VisionLanguageConfig(
658
- image_token_id=self.image_token_id,
659
- image_input_shape=str_to_int_tuple(self.image_input_shape),
660
- image_feature_size=self.image_feature_size,
661
- )
662
- else:
663
- vision_language_config = None
622
+ multimodal_config = MultiModalConfig()
623
624
device_config = DeviceConfig(device=self.device)
625
model_config = ModelConfig(
@@ -685,7 +644,7 @@ class EngineArgs:
644
disable_sliding_window=self.disable_sliding_window,
645
skip_tokenizer_init=self.skip_tokenizer_init,
646
served_model_name=self.served_model_name,
688
- multimodal_config=vision_language_config)
647
+ multimodal_config=multimodal_config)
648
cache_config = CacheConfig(
649
block_size=self.block_size,
650
gpu_memory_utilization=self.gpu_memory_utilization,
@@ -787,7 +746,7 @@ class EngineArgs:
746
scheduler_config=scheduler_config,
747
device_config=device_config,
748
lora_config=lora_config,
790
- vision_language_config=vision_language_config,
749
+ multimodal_config=multimodal_config,
750
speculative_config=speculative_config,
751
load_config=load_config,
752
decoding_config=decoding_config,
@@ -831,7 +790,3 @@ def _engine_args_parser():
790
def _async_engine_args_parser():
791
return AsyncEngineArgs.add_cli_args(FlexibleArgumentParser(),
792
async_args_only=True)
834
-
835
-
836
-def _vlm_engine_args_parser():
837
- return EngineArgs.add_cli_args_for_vlm(FlexibleArgumentParser())
vllm/engine/llm_engine.py
+8
-8
@@ -7,9 +7,9 @@ from typing import Set, Type, TypeVar, Union
7
from transformers import PreTrainedTokenizer
8
9
from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig, LoadConfig,
10
- LoRAConfig, ModelConfig, ObservabilityConfig,
11
- ParallelConfig, SchedulerConfig, SpeculativeConfig,
12
- VisionLanguageConfig)
10
+ LoRAConfig, ModelConfig, MultiModalConfig,
11
+ ObservabilityConfig, ParallelConfig, SchedulerConfig,
12
+ SpeculativeConfig)
13
from vllm.core.scheduler import (ScheduledSequenceGroup, Scheduler,
14
SchedulerOutputs)
15
from vllm.engine.arg_utils import EngineArgs
@@ -87,8 +87,8 @@ class LLMEngine:
87
scheduler_config: The configuration related to the request scheduler.
88
device_config: The configuration related to the device.
89
lora_config (Optional): The configuration related to serving multi-LoRA.
90
- vision_language_config (Optional): The configuration related to vision
91
- language models.
90
+ multimodal_config (Optional): The configuration related to multimodal
91
+ models.
92
speculative_config (Optional): The configuration related to speculative
93
decoding.
94
executor_class: The model executor class for managing distributed
@@ -157,7 +157,7 @@ class LLMEngine:
157
device_config: DeviceConfig,
158
load_config: LoadConfig,
159
lora_config: Optional[LoRAConfig],
160
- vision_language_config: Optional[VisionLanguageConfig],
160
+ multimodal_config: Optional[MultiModalConfig],
161
speculative_config: Optional[SpeculativeConfig],
162
decoding_config: Optional[DecodingConfig],
163
observability_config: Optional[ObservabilityConfig],
@@ -215,7 +215,7 @@ class LLMEngine:
215
self.model_config = model_config
216
self.cache_config = cache_config
217
self.lora_config = lora_config
218
- self.vision_language_config = vision_language_config
218
+ self.multimodal_config = multimodal_config
219
self.parallel_config = parallel_config
220
self.scheduler_config = scheduler_config
221
self.device_config = device_config
@@ -247,7 +247,7 @@ class LLMEngine:
247
scheduler_config=scheduler_config,
248
device_config=device_config,
249
lora_config=lora_config,
250
- vision_language_config=vision_language_config,
250
+ multimodal_config=multimodal_config,
251
speculative_config=speculative_config,
252
load_config=load_config,
253
)
vllm/entrypoints/llm.py
+5
@@ -121,6 +121,11 @@ class LLM:
121
) -> None:
122
if "disable_log_stats" not in kwargs:
123
kwargs["disable_log_stats"] = True
124
+ removed_vision_keys = ("image_token_id", "image_feature_size",
125
+ "image_input_shape", "image_input_type")
126
+ if any(k in kwargs for k in removed_vision_keys):
127
+ raise TypeError(
128
+ "There is no need to pass vision-related arguments anymore.")
129
engine_args = EngineArgs(
130
model=model,
131
tokenizer=tokenizer,
vllm/entrypoints/openai/serving_chat.py
+5
-16
@@ -109,23 +109,12 @@ class OpenAIServingChat(OpenAIServing):
109
"paligemma"):
110
# These models do not use image tokens in the prompt
111
return None
112
+ if model_type.startswith("llava"):
113
+ return self.tokenizer.decode(
114
+ self.model_config.hf_config.image_token_index)
115
113
- # The default behaviour assumes that the image token is
114
- # available to the tokenizer.
115
- # (Suitable for LLaVA, Idefics2, DeepSeek-VL)
116
- vlm_config = self.model_config.multimodal_config
117
- if vlm_config is None:
118
- raise ValueError(
119
- "'image_url' input is not supported as the loaded "
120
- "model is not multimodal.")
121
-
122
- image_token_id = vlm_config.image_token_id
123
- if vlm_config.image_token_id is None:
124
- raise ValueError(
125
- "'image_url' input is not supported as the loaded "
126
- "model does not specify an image token.")
127
-
128
- return self.tokenizer.decode(image_token_id)
116
+ else:
117
+ raise TypeError("Unknown model type: {model_type}")
118
119
# TODO: Let user specify how to insert image tokens into prompt
120
# (similar to chat template)
vllm/executor/cpu_executor.py
+1
-1
@@ -46,7 +46,7 @@ class CPUExecutor(ExecutorBase):
46
rank=0,
47
distributed_init_method=distributed_init_method,
48
lora_config=self.lora_config,
49
- vision_language_config=self.vision_language_config,
49
+ multimodal_config=self.multimodal_config,
50
kv_cache_dtype=self.cache_config.cache_dtype,
51
is_driver_worker=True,
52
)
vllm/executor/executor_base.py
+6
-7
@@ -3,8 +3,8 @@ from abc import ABC, abstractmethod
3
from typing import List, Optional, Set, Tuple
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, SchedulerConfig,
7
- SpeculativeConfig, VisionLanguageConfig)
6
+ ModelConfig, MultiModalConfig, ParallelConfig,
7
+ SchedulerConfig, SpeculativeConfig)
8
from vllm.lora.request import LoRARequest
9
from vllm.sequence import ExecuteModelRequest, SamplerOutput
10
@@ -26,7 +26,7 @@ class ExecutorBase(ABC):
26
device_config: DeviceConfig,
27
load_config: LoadConfig,
28
lora_config: Optional[LoRAConfig],
29
- vision_language_config: Optional[VisionLanguageConfig],
29
+ multimodal_config: Optional[MultiModalConfig],
30
speculative_config: Optional[SpeculativeConfig],
31
) -> None:
32
self.model_config = model_config
@@ -36,7 +36,7 @@ class ExecutorBase(ABC):
36
self.parallel_config = parallel_config
37
self.scheduler_config = scheduler_config
38
self.device_config = device_config
39
- self.vision_language_config = vision_language_config
39
+ self.multimodal_config = multimodal_config
40
self.speculative_config = speculative_config
41
42
self._init_executor()
@@ -120,7 +120,7 @@ class ExecutorAsyncBase(ExecutorBase):
120
device_config: DeviceConfig,
121
load_config: LoadConfig,
122
lora_config: Optional[LoRAConfig],
123
- vision_language_config: Optional[VisionLanguageConfig],
123
+ multimodal_config: Optional[MultiModalConfig],
124
speculative_config: Optional[SpeculativeConfig],
125
) -> None:
126
# This locks each pipeline parallel stage so multiple virtual engines
@@ -132,8 +132,7 @@ class ExecutorAsyncBase(ExecutorBase):
132
133
super().__init__(model_config, cache_config, parallel_config,
134
scheduler_config, device_config, load_config,
135
- lora_config, vision_language_config,
136
- speculative_config)
135
+ lora_config, multimodal_config, speculative_config)
136
137
@abstractmethod
138
async def execute_model_async(
vllm/executor/gpu_executor.py
+1
-1
@@ -43,7 +43,7 @@ class GPUExecutor(ExecutorBase):
43
rank=rank,
44
distributed_init_method=distributed_init_method,
45
lora_config=self.lora_config,
46
- vision_language_config=self.vision_language_config,
46
+ multimodal_config=self.multimodal_config,
47
speculative_config=self.speculative_config,
48
is_driver_worker=(not self.parallel_config)
49
or (rank % self.parallel_config.tensor_parallel_size == 0),
vllm/executor/openvino_executor.py
+1
-1
@@ -47,7 +47,7 @@ class OpenVINOExecutor(ExecutorBase):
47
rank=0,
48
distributed_init_method=distributed_init_method,
49
lora_config=self.lora_config,
50
- vision_language_config=self.vision_language_config,
50
+ multimodal_config=self.multimodal_config,
51
kv_cache_dtype=self.cache_config.cache_dtype,
52
is_driver_worker=True,
53
)
vllm/executor/ray_xpu_executor.py
+5
-5
@@ -7,8 +7,8 @@ from typing import (TYPE_CHECKING, Any, Awaitable, Dict, List, Optional, Set,
7
Tuple, Union)
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- SpeculativeConfig, VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig, SpeculativeConfig)
12
from vllm.executor.distributed_gpu_executor import ( # yapf: disable
13
DistributedGPUExecutor, DistributedGPUExecutorAsync)
14
from vllm.executor.ray_utils import RayWorkerWrapper, ray
@@ -43,7 +43,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
43
device_config: DeviceConfig,
44
load_config: LoadConfig,
45
lora_config: Optional[LoRAConfig],
46
- vision_language_config: Optional[VisionLanguageConfig],
46
+ multimodal_config: Optional[MultiModalConfig],
47
speculative_config: Optional[SpeculativeConfig],
48
) -> None:
49
assert device_config.device_type == "xpu"
@@ -57,7 +57,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
57
self.parallel_config = parallel_config
58
self.scheduler_config = scheduler_config
59
self.device_config = device_config
60
- self.vision_language_config = vision_language_config
60
+ self.multimodal_config = multimodal_config
61
62
placement_group = self.parallel_config.placement_group
63
@@ -199,7 +199,7 @@ class RayXPUExecutor(DistributedGPUExecutor):
199
rank=rank,
200
distributed_init_method=distributed_init_method,
201
lora_config=self.lora_config,
202
- vision_language_config=self.vision_language_config,
202
+ multimodal_config=self.multimodal_config,
203
is_driver_worker=rank == 0,
204
))
205
self._run_workers("init_worker", all_kwargs=init_worker_all_kwargs)
vllm/executor/tpu_executor.py
+1
-1
@@ -50,7 +50,7 @@ class TPUExecutor(ExecutorBase):
50
local_rank=local_rank,
51
rank=rank,
52
distributed_init_method=distributed_init_method,
53
- vision_language_config=self.vision_language_config,
53
+ multimodal_config=self.multimodal_config,
54
is_driver_worker=rank == 0,
55
)
56
vllm/executor/xpu_executor.py
+4
-4
@@ -3,8 +3,8 @@ from typing import List, Optional
3
import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, SchedulerConfig,
7
- SpeculativeConfig, VisionLanguageConfig)
6
+ ModelConfig, MultiModalConfig, ParallelConfig,
7
+ SchedulerConfig, SpeculativeConfig)
8
from vllm.executor.executor_base import ExecutorAsyncBase
9
from vllm.executor.gpu_executor import GPUExecutor
10
from vllm.logger import init_logger
@@ -26,7 +26,7 @@ class XPUExecutor(GPUExecutor):
26
device_config: DeviceConfig,
27
load_config: LoadConfig,
28
lora_config: Optional[LoRAConfig],
29
- vision_language_config: Optional[VisionLanguageConfig],
29
+ multimodal_config: Optional[MultiModalConfig],
30
speculative_config: Optional[SpeculativeConfig],
31
) -> None:
32
assert device_config.device_type == "xpu"
@@ -42,7 +42,7 @@ class XPUExecutor(GPUExecutor):
42
self.parallel_config = parallel_config
43
self.scheduler_config = scheduler_config
44
self.device_config = device_config
45
- self.vision_language_config = vision_language_config
45
+ self.multimodal_config = multimodal_config
46
self.speculative_config = None
47
48
# Instantiate the worker and load the model to GPU.
vllm/inputs/registry.py
+2
-2
@@ -11,7 +11,7 @@ from vllm.logger import init_logger
11
from .data import LLMInputs
12
13
if TYPE_CHECKING:
14
- from vllm.config import ModelConfig, VisionLanguageConfig
14
+ from vllm.config import ModelConfig, MultiModalConfig
15
from vllm.multimodal import MultiModalDataDict
16
from vllm.sequence import SequenceData
17
@@ -30,7 +30,7 @@ class InputContext:
30
model_config: "ModelConfig"
31
"""The configuration of the model."""
32
33
- def get_multimodal_config(self) -> "VisionLanguageConfig":
33
+ def get_multimodal_config(self) -> "MultiModalConfig":
34
"""
35
Get the multimodal configuration of the model.
36
vllm/model_executor/model_loader/__init__.py
+4
-4
@@ -3,8 +3,8 @@ from typing import Optional
3
from torch import nn
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, SchedulerConfig,
7
- VisionLanguageConfig)
6
+ ModelConfig, MultiModalConfig, ParallelConfig,
7
+ SchedulerConfig)
8
from vllm.model_executor.model_loader.loader import (BaseModelLoader,
9
get_model_loader)
10
from vllm.model_executor.model_loader.utils import (
@@ -15,13 +15,13 @@ def get_model(*, model_config: ModelConfig, load_config: LoadConfig,
15
device_config: DeviceConfig, parallel_config: ParallelConfig,
16
scheduler_config: SchedulerConfig,
17
lora_config: Optional[LoRAConfig],
18
- vision_language_config: Optional[VisionLanguageConfig],
18
+ multimodal_config: Optional[MultiModalConfig],
19
cache_config: CacheConfig) -> nn.Module:
20
loader = get_model_loader(load_config)
21
return loader.load_model(model_config=model_config,
22
device_config=device_config,
23
lora_config=lora_config,
24
- vision_language_config=vision_language_config,
24
+ multimodal_config=multimodal_config,
25
parallel_config=parallel_config,
26
scheduler_config=scheduler_config,
27
cache_config=cache_config)
vllm/model_executor/model_loader/loader.py
+23
-25
@@ -16,8 +16,8 @@ from huggingface_hub import HfApi, hf_hub_download
16
from torch import nn
17
18
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoadFormat,
19
- LoRAConfig, ModelConfig, ParallelConfig,
20
- SchedulerConfig, VisionLanguageConfig)
19
+ LoRAConfig, ModelConfig, MultiModalConfig,
20
+ ParallelConfig, SchedulerConfig)
21
from vllm.envs import VLLM_USE_MODELSCOPE
22
from vllm.logger import init_logger
23
from vllm.model_executor.layers.quantization.base_config import (
@@ -68,7 +68,7 @@ def _get_quantization_config(
68
def _get_model_initialization_kwargs(
69
model_class: Type[nn.Module],
70
lora_config: Optional[LoRAConfig],
71
- vlm_config: Optional[VisionLanguageConfig],
71
+ multimodal_config: Optional[MultiModalConfig],
72
) -> Dict[str, Any]:
73
"""Get extra kwargs for model initialization."""
74
extra_kwargs: Dict[str, Any] = {}
@@ -84,18 +84,18 @@ def _get_model_initialization_kwargs(
84
"please open an issue on github.")
85
86
if supports_vision(model_class):
87
- if vlm_config is None:
87
+ if multimodal_config is None:
88
raise ValueError("Provide vision related configurations "
89
"through LLM entrypoint or engine arguments.")
90
91
- extra_kwargs["vlm_config"] = vlm_config
91
+ extra_kwargs["multimodal_config"] = multimodal_config
92
93
return extra_kwargs
94
95
96
def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
97
lora_config: Optional[LoRAConfig],
98
- vision_language_config: Optional[VisionLanguageConfig],
98
+ multimodal_config: Optional[MultiModalConfig],
99
cache_config: CacheConfig) -> nn.Module:
100
"""Initialize a model with the given configurations."""
101
model_class = get_model_architecture(model_config)[0]
@@ -105,7 +105,7 @@ def _initialize_model(model_config: ModelConfig, load_config: LoadConfig,
105
cache_config=cache_config,
106
quant_config=quant_config,
107
**_get_model_initialization_kwargs(
108
- model_class, lora_config, vision_language_config))
108
+ model_class, lora_config, multimodal_config))
109
110
111
class BaseModelLoader(ABC):
@@ -118,7 +118,7 @@ class BaseModelLoader(ABC):
118
def load_model(self, *, model_config: ModelConfig,
119
device_config: DeviceConfig,
120
lora_config: Optional[LoRAConfig],
121
- vision_language_config: Optional[VisionLanguageConfig],
121
+ multimodal_config: Optional[MultiModalConfig],
122
parallel_config: ParallelConfig,
123
scheduler_config: SchedulerConfig,
124
cache_config: CacheConfig) -> nn.Module:
@@ -258,14 +258,14 @@ class DefaultModelLoader(BaseModelLoader):
258
def load_model(self, *, model_config: ModelConfig,
259
device_config: DeviceConfig,
260
lora_config: Optional[LoRAConfig],
261
- vision_language_config: Optional[VisionLanguageConfig],
261
+ multimodal_config: Optional[MultiModalConfig],
262
parallel_config: ParallelConfig,
263
scheduler_config: SchedulerConfig,
264
cache_config: CacheConfig) -> nn.Module:
265
with set_default_torch_dtype(model_config.dtype):
266
with torch.device(device_config.device):
267
model = _initialize_model(model_config, self.load_config,
268
- lora_config, vision_language_config,
268
+ lora_config, multimodal_config,
269
cache_config)
270
model.load_weights(
271
self._get_weights_iterator(model_config.model,
@@ -298,14 +298,14 @@ class DummyModelLoader(BaseModelLoader):
298
def load_model(self, *, model_config: ModelConfig,
299
device_config: DeviceConfig,
300
lora_config: Optional[LoRAConfig],
301
- vision_language_config: Optional[VisionLanguageConfig],
301
+ multimodal_config: Optional[MultiModalConfig],
302
parallel_config: ParallelConfig,
303
scheduler_config: SchedulerConfig,
304
cache_config: CacheConfig) -> nn.Module:
305
with set_default_torch_dtype(model_config.dtype):
306
with torch.device(device_config.device):
307
model = _initialize_model(model_config, self.load_config,
308
- lora_config, vision_language_config,
308
+ lora_config, multimodal_config,
309
cache_config)
310
# NOTE(woosuk): For accurate performance evaluation, we assign
311
# random values to the weights.
@@ -339,7 +339,7 @@ class TensorizerLoader(BaseModelLoader):
339
model_config: ModelConfig,
340
device_config: DeviceConfig,
341
lora_config: Optional[LoRAConfig],
342
- vision_language_config: Optional[VisionLanguageConfig],
342
+ multimodal_config: Optional[MultiModalConfig],
343
cache_config: CacheConfig,
344
) -> nn.Module:
345
"""Load a serialized model with tensorizer to the CPU.
@@ -352,7 +352,7 @@ class TensorizerLoader(BaseModelLoader):
352
with set_default_torch_dtype(model_config.dtype):
353
with torch.device(device_config.device):
354
model = _initialize_model(model_config, self.load_config,
355
- lora_config, vision_language_config,
355
+ lora_config, multimodal_config,
356
cache_config)
357
358
model.load_weights(self._get_weights_iterator())
@@ -363,7 +363,7 @@ class TensorizerLoader(BaseModelLoader):
363
model_config: ModelConfig,
364
device_config: DeviceConfig,
365
lora_config: Optional[LoRAConfig],
366
- vision_language_config: Optional[VisionLanguageConfig],
366
+ multimodal_config: Optional[MultiModalConfig],
367
cache_config: CacheConfig,
368
) -> nn.Module:
369
"""Load a serialized model with tensorizer.
@@ -377,7 +377,7 @@ class TensorizerLoader(BaseModelLoader):
377
quant_config = _get_quantization_config(
378
model_config, self.load_config)
379
extra_kwargs = _get_model_initialization_kwargs(
380
- model_class, lora_config, vision_language_config)
380
+ model_class, lora_config, multimodal_config)
381
extra_kwargs["quant_config"] = quant_config
382
extra_kwargs["cache_config"] = cache_config
383
@@ -392,7 +392,7 @@ class TensorizerLoader(BaseModelLoader):
392
def load_model(self, *, model_config: ModelConfig,
393
device_config: DeviceConfig,
394
lora_config: Optional[LoRAConfig],
395
- vision_language_config: Optional[VisionLanguageConfig],
395
+ multimodal_config: Optional[MultiModalConfig],
396
parallel_config: ParallelConfig,
397
scheduler_config: SchedulerConfig,
398
cache_config: CacheConfig) -> nn.Module:
@@ -406,12 +406,10 @@ class TensorizerLoader(BaseModelLoader):
406
407
if is_vllm_tensorized(self.tensorizer_config):
408
return self._load_model_serialized(model_config, device_config,
409
- lora_config,
410
- vision_language_config,
409
+ lora_config, multimodal_config,
410
cache_config)
411
return self._load_model_serialized_cpu(model_config, device_config,
413
- lora_config,
414
- vision_language_config,
412
+ lora_config, multimodal_config,
413
cache_config)
414
415
@staticmethod
@@ -494,7 +492,7 @@ class ShardedStateLoader(BaseModelLoader):
492
def load_model(self, *, model_config: ModelConfig,
493
device_config: DeviceConfig,
494
lora_config: Optional[LoRAConfig],
497
- vision_language_config: Optional[VisionLanguageConfig],
495
+ multimodal_config: Optional[MultiModalConfig],
496
parallel_config: ParallelConfig,
497
scheduler_config: SchedulerConfig,
498
cache_config: CacheConfig) -> nn.Module:
@@ -508,7 +506,7 @@ class ShardedStateLoader(BaseModelLoader):
506
with set_default_torch_dtype(model_config.dtype):
507
with torch.device(device_config.device):
508
model = _initialize_model(model_config, self.load_config,
511
- lora_config, vision_language_config,
509
+ lora_config, multimodal_config,
510
cache_config)
511
rank = get_tensor_model_parallel_rank()
512
pattern = os.path.join(
@@ -804,14 +802,14 @@ class BitsAndBytesModelLoader(BaseModelLoader):
802
def load_model(self, *, model_config: ModelConfig,
803
device_config: DeviceConfig,
804
lora_config: Optional[LoRAConfig],
807
- vision_language_config: Optional[VisionLanguageConfig],
805
+ multimodal_config: Optional[MultiModalConfig],
806
parallel_config: ParallelConfig,
807
scheduler_config: SchedulerConfig,
808
cache_config: CacheConfig) -> nn.Module:
809
with set_default_torch_dtype(model_config.dtype):
810
with torch.device(device_config.device):
811
model = _initialize_model(model_config, self.load_config,
814
- lora_config, vision_language_config,
812
+ lora_config, multimodal_config,
813
cache_config)
814
815
self._load_weights(model_config, model)
vllm/model_executor/models/interfaces.py
+3
-3
@@ -3,7 +3,7 @@ from typing import (ClassVar, Dict, List, Literal, Optional, Protocol, Type,
3
4
from typing_extensions import TypeGuard
5
6
-from vllm.config import LoRAConfig, VisionLanguageConfig
6
+from vllm.config import LoRAConfig, MultiModalConfig
7
from vllm.logger import init_logger
8
9
logger = init_logger(__name__)
@@ -22,7 +22,7 @@ class SupportsVision(Protocol):
22
MRO of your model class.
23
"""
24
25
- def __init__(self, *, vlm_config: VisionLanguageConfig) -> None:
25
+ def __init__(self, *, multimodal_config: MultiModalConfig) -> None:
26
...
27
28
@@ -32,7 +32,7 @@ class SupportsVision(Protocol):
32
class _SupportsVisionType(Protocol):
33
supports_vision: Literal[True]
34
35
- def __call__(self, *, vlm_config: VisionLanguageConfig) -> None:
35
+ def __call__(self, *, multimodal_config: MultiModalConfig) -> None:
36
...
37
38
vllm/model_executor/models/llava.py
+10
-11
@@ -5,7 +5,7 @@ import torch.nn as nn
5
from transformers import CLIPVisionConfig, LlavaConfig
6
7
from vllm.attention import AttentionMetadata
8
-from vllm.config import CacheConfig, VisionLanguageConfig
8
+from vllm.config import CacheConfig, MultiModalConfig
9
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
10
from vllm.model_executor.layers.activation import get_act_fn
11
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -108,13 +108,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
108
109
def __init__(self,
110
config: LlavaConfig,
111
- vlm_config: VisionLanguageConfig,
111
+ multimodal_config: MultiModalConfig,
112
cache_config: Optional[CacheConfig] = None,
113
quant_config: Optional[QuantizationConfig] = None) -> None:
114
super().__init__()
115
116
self.config = config
117
- self.vlm_config = vlm_config
117
+ self.multimodal_config = multimodal_config
118
119
# TODO: Optionally initializes this for supporting embeddings.
120
self.vision_tower = CLIPVisionModel(config.vision_config)
@@ -138,14 +138,13 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
138
self.sampler = Sampler()
139
140
def _validate_image_data(self, data: torch.Tensor) -> torch.Tensor:
141
- if list(data.shape[1:]) != list(self.vlm_config.image_input_shape[1:]):
141
+ if list(data.shape)[1:] != [
142
+ 3, self.config.vision_config.image_size,
143
+ self.config.vision_config.image_size
144
+ ]:
145
raise ValueError(
143
- f"The expected image tensor shape is batch dimension plus "
144
- f"{self.vlm_config.image_input_shape[1:]}. "
145
- f"You supplied {data.shape}. "
146
- f"If you are using vLLM's entrypoint, make sure your "
147
- f"supplied image input is consistent with "
148
- f"image_input_shape in engine args.")
146
+ "The expected image tensor shape is batch dimension plus "
147
+ "channel, height and width.")
148
149
return data
150
@@ -244,7 +243,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsVision):
243
244
inputs_embeds = merge_vision_embeddings(
245
input_ids, inputs_embeds, vision_embeddings,
247
- self.vlm_config.image_token_id)
246
+ self.config.image_token_index)
247
248
input_ids = None
249
else:
vllm/model_executor/models/llava_next.py
+47
-6
@@ -1,4 +1,4 @@
1
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
1
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
2
3
import torch
4
import torch.nn as nn
@@ -9,7 +9,7 @@ from transformers.models.llava_next.modeling_llava_next import (
9
from typing_extensions import NotRequired
10
11
from vllm.attention import AttentionMetadata
12
-from vllm.config import CacheConfig, VisionLanguageConfig
12
+from vllm.config import CacheConfig, MultiModalConfig
13
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
14
from vllm.logger import init_logger
15
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -204,13 +204,13 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
204
205
def __init__(self,
206
config: LlavaNextConfig,
207
- vlm_config: VisionLanguageConfig,
207
+ multimodal_config: MultiModalConfig,
208
cache_config: Optional[CacheConfig] = None,
209
quant_config: Optional[QuantizationConfig] = None) -> None:
210
super().__init__()
211
212
self.config = config
213
- self.vlm_config = vlm_config
213
+ self.multimodal_config = multimodal_config
214
215
# TODO: Optionally initializes this for supporting embeddings.
216
self.vision_tower = CLIPVisionModel(config=config.vision_config)
@@ -244,6 +244,47 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
244
245
return data
246
247
+ def _validate_pixel_values(
248
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
249
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
250
+
251
+ def _validate_shape(data: torch.Tensor):
252
+
253
+ dim = data.dim()
254
+ height = width = self.config.vision_config.image_size
255
+ # All 4d image tensors have the same number of patches,
256
+ # so data is a 5d batch of these tensors
257
+ if dim == 5:
258
+ if list(data.shape)[2:] != [
259
+ 3, self.config.vision_config.image_size,
260
+ self.config.vision_config.image_size
261
+ ]:
262
+ raise ValueError(
263
+ "Expected pixel value tensor in shape of: (batch size, "
264
+ f"patch number, 3, {height}, {width}), got {data.shape}"
265
+ )
266
+
267
+ # 4d image tensors have different number of patches,
268
+ # so data is each individual tensor.
269
+ elif dim == 4:
270
+ if list(data.shape)[1:] != [
271
+ 3, self.config.vision_config.image_size,
272
+ self.config.vision_config.image_size
273
+ ]:
274
+ raise ValueError(
275
+ "Expected pixel value tensor in shape of: (patch "
276
+ f"number, 3, {height}, {width}), got {data.shape}")
277
+ else:
278
+ raise ValueError(
279
+ f"Invalid pixel value tensor of shape {data.shape}")
280
+
281
+ if isinstance(data, torch.Tensor):
282
+ _validate_shape(data)
283
+ else:
284
+ [_validate_shape(d) for d in data]
285
+
286
+ return data
287
+
288
def _parse_and_validate_image_input(
289
self, **kwargs: object) -> Optional[LlavaNextImagePixelInputs]:
290
pixel_values = kwargs.pop("pixel_values", None)
@@ -262,7 +303,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
303
304
return LlavaNextImagePixelInputs(
305
type="pixel_values",
265
- data=pixel_values,
306
+ data=self._validate_pixel_values(pixel_values),
307
image_sizes=self._validate_image_sizes(image_sizes),
308
)
309
@@ -454,7 +495,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsVision):
495
496
inputs_embeds = merge_vision_embeddings(
497
input_ids, inputs_embeds, vision_embeddings,
457
- self.vlm_config.image_token_id)
498
+ self.config.image_token_index)
499
500
input_ids = None
501
else:
vllm/model_executor/models/phi3v.py
+45
-17
@@ -15,7 +15,7 @@
15
# limitations under the License.
16
import re
17
from functools import lru_cache
18
-from typing import Iterable, List, Literal, Optional, Tuple, TypedDict
18
+from typing import Iterable, List, Literal, Optional, Tuple, TypedDict, Union
19
20
import numpy as np
21
import torch
@@ -24,7 +24,7 @@ from PIL import Image
24
from transformers import CLIPVisionConfig, PretrainedConfig
25
26
from vllm.attention import AttentionMetadata
27
-from vllm.config import CacheConfig, ModelConfig, VisionLanguageConfig
27
+from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
28
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
29
from vllm.logger import init_logger
30
from vllm.model_executor.layers.logits_processor import LogitsProcessor
@@ -50,6 +50,9 @@ _KEYS_TO_MODIFY_MAPPING = {
50
"model.vision_embed_tokens": "vision_embed_tokens",
51
}
52
53
+# Cannot find the following 2 numbers from hf config.
54
+_IMAGE_TOKEN_ID = 32044
55
+
56
CLIP_VIT_LARGE_PATCH14_336_CONFIG = CLIPVisionConfig(dropout=0.0,
57
hidden_act="quick_gelu",
58
hidden_size=1024,
@@ -95,13 +98,10 @@ class Phi3ImageEmbeddingBase(nn.Module):
98
class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
99
"""Phi3 Image embedding with HD transform."""
100
98
- def __init__(self,
99
- vision_language_config: VisionLanguageConfig,
100
- config: PretrainedConfig,
101
- wte=None) -> None:
101
+ def __init__(self, config: PretrainedConfig, wte=None) -> None:
102
super().__init__(wte)
103
104
- self.image_token_id = vision_language_config.image_token_id
104
+ self.image_token_id = _IMAGE_TOKEN_ID
105
# n_embed or hidden_size
106
hidden_size = config.n_embd if hasattr(
107
config, 'n_embd') else config.hidden_size
@@ -333,7 +333,7 @@ def dummy_data_for_phi3v(ctx: InputContext, seq_len: int):
333
seq_data = dummy_seq_data_for_clip(
334
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335
seq_len,
336
- image_token_id=32044,
336
+ image_token_id=_IMAGE_TOKEN_ID,
337
image_feature_size_override=image_feature_size,
338
)
339
mm_data = dummy_image_for_clip(
@@ -370,7 +370,6 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
370
return llm_inputs
371
372
model_config = ctx.model_config
373
- multimodal_config = ctx.get_multimodal_config()
373
hf_config = ctx.get_hf_config(PretrainedConfig)
374
375
image_data = multi_modal_data["image"]
@@ -407,7 +406,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
406
new_token_ids: List[int] = []
407
for i in range(len(prompt_token_ids) - len(image_1_token_ids) + 1):
408
if prompt_token_ids[i:i + len(image_1_token_ids)] == image_1_token_ids:
410
- new_token_ids.append(multimodal_config.image_token_id)
409
+ new_token_ids.append(_IMAGE_TOKEN_ID)
410
411
# No need to further scan the list since we only replace once
412
new_token_ids.extend(prompt_token_ids[i + len(image_1_token_ids):])
@@ -424,7 +423,7 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
423
model_config,
424
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
425
llm_inputs,
427
- image_token_id=multimodal_config.image_token_id,
426
+ image_token_id=_IMAGE_TOKEN_ID,
427
image_feature_size_override=image_feature_size,
428
)
429
@@ -436,25 +435,53 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
435
436
def __init__(self,
437
config: PretrainedConfig,
439
- vlm_config: VisionLanguageConfig,
438
+ multimodal_config: MultiModalConfig,
439
cache_config: Optional[CacheConfig] = None,
440
quant_config: Optional[QuantizationConfig] = None) -> None:
441
super().__init__()
442
443
self.config = config
445
- self.vlm_config = vlm_config
444
+ self.multimodal_config = multimodal_config
445
446
self.model = LlamaModel(config, cache_config, quant_config)
447
448
# TODO: Optionally initializes this for supporting embeddings.
449
self.vision_embed_tokens = Phi3HDImageEmbedding(
451
- vlm_config, config, self.model.embed_tokens)
450
+ config, self.model.embed_tokens)
451
self.lm_head = ParallelLMHead(config.vocab_size,
452
config.hidden_size,
453
quant_config=quant_config)
454
self.logits_processor = LogitsProcessor(config.vocab_size)
455
self.sampler = Sampler()
456
457
+ def _validate_image_sizes(self, data: torch.Tensor) -> torch.Tensor:
458
+ if list(data.shape[1:]) != [2]:
459
+ raise ValueError(
460
+ f"The expected image sizes shape is batch dimension plus "
461
+ f"{[2]}. You supplied {data.shape}.")
462
+
463
+ return data
464
+
465
+ def _validate_pixel_values(
466
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
467
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
468
+
469
+ def _validate_shape(data: torch.Tensor):
470
+ if list(data.shape)[2:] != [
471
+ 3, CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size,
472
+ CLIP_VIT_LARGE_PATCH14_336_CONFIG.image_size
473
+ ]:
474
+ raise ValueError(
475
+ "The expected pixel value tensor shape is batch dimension "
476
+ "plus patch number, channel, height and width.")
477
+
478
+ if isinstance(data, torch.Tensor):
479
+ _validate_shape(data)
480
+ else:
481
+ [_validate_shape(d) for d in data]
482
+
483
+ return data
484
+
485
def _parse_and_validate_image_input(
486
self, **kwargs: object) -> Optional[Phi3VImagePixelInputs]:
487
pixel_values = kwargs.pop("pixel_values", None)
@@ -471,9 +498,10 @@ class Phi3VForCausalLM(nn.Module, SupportsVision):
498
raise ValueError("Incorrect type of image sizes. "
499
f"Got type: {type(image_sizes)}")
500
474
- return Phi3VImagePixelInputs(type="pixel_values",
475
- data=pixel_values,
476
- image_sizes=image_sizes)
501
+ return Phi3VImagePixelInputs(
502
+ type="pixel_values",
503
+ data=self._validate_pixel_values(pixel_values),
504
+ image_sizes=self._validate_image_sizes(image_sizes))
505
506
def forward(self,
507
input_ids: torch.Tensor,
vllm/multimodal/registry.py
+7
@@ -120,3 +120,10 @@ class MultiModalRegistry:
120
Create an input mapper (see :meth:`map_input`) for a specific model.
121
"""
122
return functools.partial(self.map_input, model_config)
123
+
124
+ def get_num_input_tokens(self):
125
+ """
126
+ Get the number of input tokens for profiling purposes.
127
+ """
128
+ # TODO: Provide this number on a per model basis.
129
+ return 3000
vllm/spec_decode/draft_model_runner.py
+4
-4
@@ -3,8 +3,8 @@ from typing import List, Optional
3
import torch
4
5
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
6
- ModelConfig, ParallelConfig, SchedulerConfig,
7
- VisionLanguageConfig)
6
+ ModelConfig, MultiModalConfig, ParallelConfig,
7
+ SchedulerConfig)
8
from vllm.logger import init_logger
9
from vllm.sequence import (IntermediateTensors, SamplerOutput,
10
SequenceGroupMetadata)
@@ -47,7 +47,7 @@ class TP1DraftModelRunner(ModelRunner):
47
lora_config: Optional[LoRAConfig],
48
kv_cache_dtype: Optional[str] = "auto",
49
is_driver_worker: bool = False,
50
- vision_language_config: Optional[VisionLanguageConfig] = None,
50
+ multimodal_config: Optional[MultiModalConfig] = None,
51
return_hidden_states: bool = False,
52
):
53
if return_hidden_states:
@@ -65,7 +65,7 @@ class TP1DraftModelRunner(ModelRunner):
65
lora_config=lora_config,
66
kv_cache_dtype=kv_cache_dtype,
67
is_driver_worker=is_driver_worker,
68
- vision_language_config=vision_language_config,
68
+ multimodal_config=multimodal_config,
69
return_hidden_states=return_hidden_states,
70
)
71
vllm/worker/cpu_model_runner.py
+12
-13
@@ -7,8 +7,8 @@ from torch import nn
7
8
from vllm.attention import AttentionMetadata, get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig)
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
@@ -79,7 +79,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
79
cache_config: CacheConfig,
80
load_config: LoadConfig,
81
lora_config: Optional[LoRAConfig],
82
- vision_language_config: Optional[VisionLanguageConfig],
82
+ multimodal_config: Optional[MultiModalConfig],
83
kv_cache_dtype: Optional[str] = "auto",
84
is_driver_worker: bool = False,
85
*args,
@@ -93,7 +93,7 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
93
self.device_config = device_config
94
self.cache_config = cache_config
95
self.lora_config = lora_config
96
- self.vision_language_config = vision_language_config
96
+ self.multimodal_config = multimodal_config
97
self.load_config = load_config
98
self.is_driver_worker = is_driver_worker
99
@@ -120,15 +120,14 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
120
self.model: nn.Module # Set after init_Model
121
122
def load_model(self) -> None:
123
- self.model = get_model(
124
- model_config=self.model_config,
125
- load_config=self.load_config,
126
- device_config=self.device_config,
127
- vision_language_config=self.vision_language_config,
128
- lora_config=self.lora_config,
129
- parallel_config=self.parallel_config,
130
- scheduler_config=self.scheduler_config,
131
- cache_config=self.cache_config)
123
+ self.model = get_model(model_config=self.model_config,
124
+ load_config=self.load_config,
125
+ device_config=self.device_config,
126
+ multimodal_config=self.multimodal_config,
127
+ lora_config=self.lora_config,
128
+ parallel_config=self.parallel_config,
129
+ scheduler_config=self.scheduler_config,
130
+ cache_config=self.cache_config)
131
132
def _prepare_prompt(
133
self,
vllm/worker/cpu_worker.py
+5
-5
@@ -6,8 +6,8 @@ import torch.distributed
6
7
from vllm.attention import get_attn_backend
8
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
9
- ModelConfig, ParallelConfig, SchedulerConfig,
10
- VisionLanguageConfig)
9
+ ModelConfig, MultiModalConfig, ParallelConfig,
10
+ SchedulerConfig)
11
from vllm.distributed import (ensure_model_parallel_initialized,
12
init_distributed_environment)
13
from vllm.logger import init_logger
@@ -131,7 +131,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
131
rank: int,
132
distributed_init_method: str,
133
lora_config: Optional[LoRAConfig] = None,
134
- vision_language_config: Optional[VisionLanguageConfig] = None,
134
+ multimodal_config: Optional[MultiModalConfig] = None,
135
kv_cache_dtype: Optional[str] = "auto",
136
is_driver_worker: bool = False,
137
) -> None:
@@ -145,7 +145,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
145
self.rank = rank
146
self.distributed_init_method = distributed_init_method
147
self.lora_config = lora_config
148
- self.vision_language_config = vision_language_config
148
+ self.multimodal_config = multimodal_config
149
self.is_driver_worker = is_driver_worker
150
if self.is_driver_worker:
151
assert self.rank == 0, "The driver worker must have rank 0."
@@ -162,7 +162,7 @@ class CPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
162
cache_config,
163
load_config=self.load_config,
164
lora_config=self.lora_config,
165
- vision_language_config=self.vision_language_config,
165
+ multimodal_config=self.multimodal_config,
166
kv_cache_dtype=kv_cache_dtype,
167
is_driver_worker=is_driver_worker)
168
# Uninitialized cache engine. Will be initialized by
vllm/worker/embedding_model_runner.py
+4
-4
@@ -4,8 +4,8 @@ from typing import Any, Dict, List, Optional, Tuple, Type
4
import torch
5
6
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
- ModelConfig, ParallelConfig, SchedulerConfig,
8
- VisionLanguageConfig)
7
+ ModelConfig, MultiModalConfig, ParallelConfig,
8
+ SchedulerConfig)
9
from vllm.logger import init_logger
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.pooling_params import PoolingParams
@@ -40,7 +40,7 @@ class EmbeddingModelRunner(
40
lora_config: Optional[LoRAConfig],
41
kv_cache_dtype: Optional[str] = "auto",
42
is_driver_worker: bool = False,
43
- vision_language_config: Optional[VisionLanguageConfig] = None,
43
+ multimodal_config: Optional[MultiModalConfig] = None,
44
):
45
super().__init__(model_config,
46
parallel_config,
@@ -51,7 +51,7 @@ class EmbeddingModelRunner(
51
lora_config=lora_config,
52
kv_cache_dtype=kv_cache_dtype,
53
is_driver_worker=is_driver_worker,
54
- vision_language_config=vision_language_config)
54
+ multimodal_config=multimodal_config)
55
56
@torch.inference_mode()
57
def execute_model(
vllm/worker/model_runner.py
+17
-11
@@ -24,8 +24,8 @@ except ImportError:
24
25
from vllm.attention import AttentionMetadata, get_attn_backend
26
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
27
- ModelConfig, ParallelConfig, SchedulerConfig,
28
- VisionLanguageConfig)
27
+ ModelConfig, MultiModalConfig, ParallelConfig,
28
+ SchedulerConfig)
29
from vllm.distributed import get_pp_group
30
from vllm.distributed.parallel_state import graph_capture
31
from vllm.inputs import INPUT_REGISTRY
@@ -36,7 +36,8 @@ from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
36
from vllm.model_executor import SamplingMetadata
37
from vllm.model_executor.model_loader import get_model
38
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
39
-from vllm.model_executor.models.interfaces import supports_lora
39
+from vllm.model_executor.models.interfaces import (supports_lora,
40
+ supports_vision)
41
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
42
MultiModalInputs)
43
from vllm.sampling_params import SamplingParams
@@ -171,7 +172,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
172
lora_config: Optional[LoRAConfig],
173
kv_cache_dtype: Optional[str] = "auto",
174
is_driver_worker: bool = False,
174
- vision_language_config: Optional[VisionLanguageConfig] = None,
175
+ multimodal_config: Optional[MultiModalConfig] = None,
176
return_hidden_states: bool = False,
177
):
178
self.model_config = model_config
@@ -182,7 +183,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
183
self.lora_config = lora_config
184
self.load_config = load_config
185
self.is_driver_worker = is_driver_worker
185
- self.vision_language_config = vision_language_config
186
+ self.multimodal_config = multimodal_config
187
self.return_hidden_states = return_hidden_states
188
189
self.device = self.device_config.device
@@ -244,7 +245,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
245
device_config=self.device_config,
246
load_config=self.load_config,
247
lora_config=self.lora_config,
247
- vision_language_config=self.vision_language_config,
248
+ multimodal_config=self.multimodal_config,
249
parallel_config=self.parallel_config,
250
scheduler_config=self.scheduler_config,
251
cache_config=self.cache_config,
@@ -256,6 +257,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
257
258
if self.lora_config:
259
assert supports_lora(self.model), "Model does not support LoRA"
260
+ assert not supports_vision(
261
+ self.model
262
+ ), "To be tested: vision language model with LoRA settings."
263
264
self.lora_manager = LRUCacheWorkerLoRAManager(
265
self.scheduler_config.max_num_seqs,
@@ -804,12 +808,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
808
# the number of seqs (batch_size) is chosen to maximize the number
809
# of images processed.
810
model_config = self.model_config
807
- vlm_config = self.vision_language_config
811
809
- if vlm_config:
810
- max_num_seqs = min(
811
- max_num_seqs,
812
- int(max_num_batched_tokens / vlm_config.image_feature_size))
812
+ if supports_vision(self.model):
813
+ max_num_seqs = max(
814
+ 1,
815
+ min(
816
+ max_num_seqs,
817
+ int(max_num_batched_tokens /
818
+ MULTIMODAL_REGISTRY.get_num_input_tokens())))
819
batch_size = 0
820
for group_id in range(max_num_seqs):
821
seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/openvino_model_runner.py
+4
-4
@@ -7,8 +7,8 @@ from torch import nn
7
from vllm.attention import get_attn_backend
8
from vllm.attention.backends.openvino import OpenVINOAttentionMetadata
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig)
12
from vllm.logger import init_logger
13
from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader.openvino import get_model
@@ -48,7 +48,7 @@ class OpenVINOModelRunner:
48
cache_config: CacheConfig,
49
load_config: LoadConfig,
50
lora_config: Optional[LoRAConfig],
51
- vision_language_config: Optional[VisionLanguageConfig],
51
+ multimodal_config: Optional[MultiModalConfig],
52
kv_cache_dtype: Optional[str] = "auto",
53
is_driver_worker: bool = False,
54
*args,
@@ -60,7 +60,7 @@ class OpenVINOModelRunner:
60
self.device_config = device_config
61
self.cache_config = cache_config
62
self.lora_config = lora_config
63
- self.vision_language_config = vision_language_config
63
+ self.multimodal_config = multimodal_config
64
self.load_config = load_config
65
self.is_driver_worker = is_driver_worker
66
vllm/worker/openvino_worker.py
+5
-5
@@ -7,8 +7,8 @@ import torch.distributed
7
8
from vllm.attention import get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig)
12
from vllm.distributed import (broadcast_tensor_dict,
13
ensure_model_parallel_initialized,
14
init_distributed_environment)
@@ -148,7 +148,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
148
rank: int,
149
distributed_init_method: str,
150
lora_config: Optional[LoRAConfig] = None,
151
- vision_language_config: Optional[VisionLanguageConfig] = None,
151
+ multimodal_config: Optional[MultiModalConfig] = None,
152
kv_cache_dtype: Optional[ov.Type] = ov.Type.undefined,
153
is_driver_worker: bool = False,
154
) -> None:
@@ -162,7 +162,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
162
self.rank = rank
163
self.distributed_init_method = distributed_init_method
164
self.lora_config = lora_config
165
- self.vision_language_config = vision_language_config
165
+ self.multimodal_config = multimodal_config
166
self.is_driver_worker = is_driver_worker
167
if self.is_driver_worker:
168
assert self.rank == 0, "The driver worker must have rank 0."
@@ -180,7 +180,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
180
cache_config,
181
load_config=self.load_config,
182
lora_config=self.lora_config,
183
- vision_language_config=self.vision_language_config,
183
+ multimodal_config=self.multimodal_config,
184
kv_cache_dtype=kv_cache_dtype,
185
is_driver_worker=is_driver_worker,
186
)
vllm/worker/tpu_model_runner.py
+4
-4
@@ -8,7 +8,7 @@ import torch_xla.core.xla_model as xm
8
9
from vllm.attention import AttentionMetadata, get_attn_backend
10
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
+ MultiModalConfig, ParallelConfig, SchedulerConfig)
12
from vllm.logger import init_logger
13
from vllm.model_executor.model_loader import get_model
14
from vllm.model_executor.sampling_metadata import SamplingMetadata
@@ -39,7 +39,7 @@ class TPUModelRunner:
39
device_config: DeviceConfig,
40
cache_config: CacheConfig,
41
load_config: LoadConfig,
42
- vision_language_config: Optional[VisionLanguageConfig] = None,
42
+ multimodal_config: Optional[MultiModalConfig] = None,
43
is_driver_worker: bool = False,
44
):
45
self.model_config = model_config
@@ -48,7 +48,7 @@ class TPUModelRunner:
48
self.device_config = device_config
49
self.cache_config = cache_config
50
self.load_config = load_config
51
- self.vision_language_config = vision_language_config
51
+ self.multimodal_config = multimodal_config
52
self.is_driver_worker = is_driver_worker
53
54
self.block_size = self.cache_config.block_size
@@ -82,7 +82,7 @@ class TPUModelRunner:
82
parallel_config=self.parallel_config,
83
cache_config=self.cache_config,
84
scheduler_config=self.scheduler_config,
85
- vision_language_config=self.vision_language_config,
85
+ multimodal_config=self.multimodal_config,
86
lora_config=None,
87
)
88
xm.wait_device_ops()
vllm/worker/tpu_worker.py
+4
-4
@@ -8,7 +8,7 @@ import torch_xla.runtime as xr
8
9
import vllm.envs as envs
10
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, ModelConfig,
11
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
11
+ MultiModalConfig, ParallelConfig, SchedulerConfig)
12
from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment)
14
from vllm.logger import init_logger
@@ -31,7 +31,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
31
device_config: DeviceConfig,
32
cache_config: CacheConfig,
33
load_config: LoadConfig,
34
- vision_language_config: Optional[VisionLanguageConfig],
34
+ multimodal_config: Optional[MultiModalConfig],
35
local_rank: int,
36
rank: int,
37
distributed_init_method: str,
@@ -43,7 +43,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
43
self.device_config = device_config
44
self.cache_config = cache_config
45
self.load_config = load_config
46
- self.vision_language_config = vision_language_config
46
+ self.multimodal_config = multimodal_config
47
self.local_rank = local_rank
48
self.rank = rank
49
self.distributed_init_method = distributed_init_method
@@ -62,7 +62,7 @@ class TPUWorker(LoraNotSupportedWorkerBase):
62
device_config,
63
cache_config,
64
load_config,
65
- vision_language_config,
65
+ multimodal_config,
66
is_driver_worker=is_driver_worker)
67
68
def init_device(self) -> None:
vllm/worker/worker.py
+5
-8
@@ -7,8 +7,8 @@ import torch
7
import torch.distributed
8
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- SpeculativeConfig, VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig, SpeculativeConfig)
12
from vllm.distributed import (ensure_model_parallel_initialized,
13
init_distributed_environment,
14
set_custom_all_reduce)
@@ -43,7 +43,7 @@ class Worker(LocalOrDistributedWorkerBase):
43
rank: int,
44
distributed_init_method: str,
45
lora_config: Optional[LoRAConfig] = None,
46
- vision_language_config: Optional[VisionLanguageConfig] = None,
46
+ multimodal_config: Optional[MultiModalConfig] = None,
47
speculative_config: Optional[SpeculativeConfig] = None,
48
is_driver_worker: bool = False,
49
model_runner_cls: Optional[Type[GPUModelRunnerBase]] = None,
@@ -66,10 +66,7 @@ class Worker(LocalOrDistributedWorkerBase):
66
# note: lazy import to avoid importing torch before initializing
67
from vllm.utils import init_cached_hf_modules
68
init_cached_hf_modules()
69
- self.vision_language_config = vision_language_config
70
- if self.vision_language_config:
71
- assert not self.lora_config, (
72
- "To be tested: vision language model with LoRA settings.")
69
+ self.multimodal_config = multimodal_config
70
71
# Return hidden states from target model if the draft model is an
72
# mlp_speculator
@@ -94,7 +91,7 @@ class Worker(LocalOrDistributedWorkerBase):
91
lora_config=self.lora_config,
92
kv_cache_dtype=self.cache_config.cache_dtype,
93
is_driver_worker=is_driver_worker,
97
- vision_language_config=vision_language_config,
94
+ multimodal_config=multimodal_config,
95
**speculative_args,
96
)
97
# Uninitialized cache engine. Will be initialized by
vllm/worker/xpu_model_runner.py
+15
-10
@@ -7,12 +7,13 @@ import torch.nn as nn
7
8
from vllm.attention import get_attn_backend
9
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
10
- ModelConfig, ParallelConfig, SchedulerConfig,
11
- VisionLanguageConfig)
10
+ ModelConfig, MultiModalConfig, ParallelConfig,
11
+ SchedulerConfig)
12
from vllm.distributed import broadcast_tensor_dict
13
from vllm.inputs import INPUT_REGISTRY
14
from vllm.logger import init_logger
15
from vllm.model_executor.model_loader import get_model
16
+from vllm.model_executor.models.interfaces import supports_vision
17
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
18
MultiModalInputs)
19
from vllm.sampling_params import SamplingParams
@@ -85,7 +86,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
86
cache_config: CacheConfig,
87
load_config: LoadConfig,
88
lora_config: Optional[LoRAConfig],
88
- vision_language_config: Optional[VisionLanguageConfig],
89
+ multimodal_config: Optional[MultiModalConfig],
90
kv_cache_dtype: Optional[str] = "auto",
91
is_driver_worker: bool = False,
92
*args,
@@ -97,7 +98,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
98
self.lora_config = lora_config
99
self.load_config = load_config
100
self.cache_config = cache_config
100
- self.vision_language_config = vision_language_config
101
+ self.multimodal_config = multimodal_config
102
self.is_driver_worker = is_driver_worker
103
104
self.sliding_window = model_config.get_sliding_window()
@@ -134,7 +135,7 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
135
device_config=self.device_config,
136
load_config=self.load_config,
137
lora_config=self.lora_config,
137
- vision_language_config=self.vision_language_config,
138
+ multimodal_config=self.multimodal_config,
139
parallel_config=self.parallel_config,
140
scheduler_config=self.scheduler_config,
141
cache_config=self.cache_config,
@@ -165,12 +166,16 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
166
# the number of seqs (batch_size) is chosen to maximize the number
167
# of images processed.
168
model_config = self.model_config
168
- vlm_config = self.vision_language_config
169
170
- if vlm_config:
171
- max_num_seqs = min(
172
- max_num_seqs,
173
- int(max_num_batched_tokens / vlm_config.image_feature_size))
170
+ if supports_vision(self.model):
171
+ # TODO: properly inject these numbers from MultiModalRegistry.
172
+ # Right now, just use an overly conservative number.
173
+ max_num_seqs = max(
174
+ 1,
175
+ min(
176
+ max_num_seqs,
177
+ int(max_num_batched_tokens /
178
+ MULTIMODAL_REGISTRY.get_num_input_tokens())))
179
180
for group_id in range(max_num_seqs):
181
seq_len = (max_num_batched_tokens // max_num_seqs +
vllm/worker/xpu_worker.py
+5
-8
@@ -9,8 +9,8 @@ import torch
9
import torch.distributed
10
11
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
12
- ModelConfig, ParallelConfig, SchedulerConfig,
13
- SpeculativeConfig, VisionLanguageConfig)
12
+ ModelConfig, MultiModalConfig, ParallelConfig,
13
+ SchedulerConfig, SpeculativeConfig)
14
from vllm.distributed import (ensure_model_parallel_initialized,
15
init_distributed_environment)
16
from vllm.logger import init_logger
@@ -45,7 +45,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
45
rank: int,
46
distributed_init_method: str,
47
lora_config: Optional[LoRAConfig] = None,
48
- vision_language_config: Optional[VisionLanguageConfig] = None,
48
+ multimodal_config: Optional[MultiModalConfig] = None,
49
speculative_config: Optional[SpeculativeConfig] = None,
50
is_driver_worker: bool = False,
51
) -> None:
@@ -66,10 +66,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
66
if self.is_driver_worker:
67
assert self.rank == 0, "The driver worker must have rank 0."
68
69
- self.vision_language_config = vision_language_config
70
- if self.vision_language_config:
71
- assert not self.lora_config, (
72
- "To be tested: vision language model with LoRA settings.")
69
+ self.multimodal_config = multimodal_config
70
71
self.model_runner = XPUModelRunner( # type: ignore
72
model_config,
@@ -81,7 +78,7 @@ class XPUWorker(LoraNotSupportedWorkerBase, Worker):
78
lora_config=self.lora_config,
79
kv_cache_dtype=self.cache_config.cache_dtype,
80
is_driver_worker=is_driver_worker,
84
- vision_language_config=vision_language_config,
81
+ multimodal_config=multimodal_config,
82
)
83
# Uninitialized cache engine. Will be initialized by
84
# initialize_cache.