A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] merged input processor for Phi-3-Vision models (#10977)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Dec 10, 2024 at 04:55 UTC
a811dd660856a5c222a1447fe1d93deccbc162fd
7 files changed
+235
-409
tests/entrypoints/openai/test_vision.py
+2
-2
@@ -89,7 +89,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
89
choice = chat_completion.choices[0]
90
assert choice.finish_reason == "length"
91
assert chat_completion.usage == openai.types.CompletionUsage(
92
- completion_tokens=10, prompt_tokens=772, total_tokens=782)
92
+ completion_tokens=10, prompt_tokens=775, total_tokens=785)
93
94
message = choice.message
95
message = chat_completion.choices[0].message
@@ -181,7 +181,7 @@ async def test_single_chat_session_image_base64encoded(
181
choice = chat_completion.choices[0]
182
assert choice.finish_reason == "length"
183
assert chat_completion.usage == openai.types.CompletionUsage(
184
- completion_tokens=10, prompt_tokens=772, total_tokens=782)
184
+ completion_tokens=10, prompt_tokens=775, total_tokens=785)
185
186
message = choice.message
187
message = chat_completion.choices[0].message
tests/entrypoints/openai/test_vision_embedding.py
+2
-2
@@ -95,5 +95,5 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
95
assert len(embeddings["data"]) == 1
96
assert len(embeddings["data"][0]["embedding"]) == 3072
97
assert embeddings["usage"]["completion_tokens"] == 0
98
- assert embeddings["usage"]["prompt_tokens"] == 762
99
- assert embeddings["usage"]["total_tokens"] == 762
98
+ assert embeddings["usage"]["prompt_tokens"] == 765
99
+ assert embeddings["usage"]["total_tokens"] == 765
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+27
-109
@@ -2,12 +2,10 @@
2
from typing import Optional
3
4
import pytest
5
-import torch
6
-from transformers import AutoImageProcessor, AutoTokenizer
5
+from transformers import AutoTokenizer
6
8
-from vllm.inputs import InputContext, token_inputs
7
+from vllm.inputs import InputContext, InputProcessingContext
8
from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
10
-from vllm.multimodal import MultiModalRegistry
9
10
from .....conftest import _ImageAssets
11
from ....utils import build_model_context
@@ -17,15 +15,9 @@ models = ["microsoft/Phi-3.5-vision-instruct"]
15
16
# Wrap lazy imports to avoid initializing CUDA during test collection
17
@pytest.fixture()
20
-def input_processor_for_phi3v():
21
- from vllm.model_executor.models.phi3v import input_processor_for_phi3v
22
- return input_processor_for_phi3v
23
-
24
-
25
-@pytest.fixture()
26
-def dummy_data_for_phi3v():
27
- from vllm.model_executor.models.phi3v import dummy_data_for_phi3v
28
- return dummy_data_for_phi3v
18
+def processor_for_phi3v():
19
+ from vllm.model_executor.models.phi3v import Phi3VProcessor
20
+ return Phi3VProcessor
21
22
23
@pytest.fixture()
@@ -34,53 +26,6 @@ def get_max_phi3v_image_tokens():
26
return get_max_phi3v_image_tokens
27
28
37
-@pytest.mark.parametrize("model", models)
38
-@pytest.mark.parametrize("num_crops", [4, 16, None])
39
-def test_input_mapper_override(model: str, image_assets: _ImageAssets,
40
- num_crops: Optional[int]):
41
- """Ensure that the [default] input mapper handles num_crops properly."""
42
- # We pass the processor kwargs here since for this model, we fall back to
43
- # the default mapper; this will fall back to the HF mapper and forward
44
- # mm_processor_kwargs to it.
45
- mm_processor_kwargs = {
46
- "num_crops": num_crops
47
- } if num_crops is not None else {}
48
- ctx = build_model_context(
49
- model_name=model,
50
- tokenizer_name=model,
51
- trust_remote_code=True,
52
- mm_processor_kwargs=mm_processor_kwargs,
53
- )
54
-
55
- hf_processor = AutoImageProcessor.from_pretrained(model,
56
- trust_remote_code=True,
57
- **mm_processor_kwargs)
58
-
59
- mm_registry = MultiModalRegistry()
60
- mm_registry.init_mm_limits_per_prompt(ctx.model_config)
61
-
62
- image = image_assets[0].pil_image
63
- hf_result = hf_processor.preprocess(
64
- image,
65
- return_tensors="pt",
66
- )
67
-
68
- vllm_result = mm_registry.map_input(
69
- ctx.model_config,
70
- {"image": image},
71
- )
72
-
73
- assert torch.all(hf_result["image_sizes"] == vllm_result["image_sizes"])
74
- assert torch.all(
75
- hf_result["num_img_tokens"] == vllm_result["num_img_tokens"])
76
-
77
- # For pixel values, the second axis should be the num_crops + 1
78
- # for the rescaled original image. The default value in VLLM falls
79
- # back to the HF config, which is why we compare to the processor num_crops
80
- assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
81
- assert vllm_result["pixel_values"].shape[1] == hf_processor.num_crops + 1
82
-
83
-
29
@pytest.mark.parametrize("model", models)
30
@pytest.mark.parametrize("num_crops,expected_max_tokens", [
31
(4, 781),
@@ -112,48 +57,20 @@ def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
57
58
59
@pytest.mark.parametrize("model", models)
115
-@pytest.mark.parametrize("num_crops,toks_per_img,num_imgs", [
116
- (4, 781, 1),
117
- (4, 781, 2),
118
- (16, 2653, 1),
119
- (16, 2653, 2),
120
-])
121
-def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
122
- toks_per_img: int, num_imgs: int):
123
- """Ensure dummy_data_for_phi3v handles num_crops properly."""
124
- # Same as the previous test - don't initialize mm_processor_kwargs
125
- # in this test and assume that the kwargs will be correctly expanded by
126
- # the partial when calling the dummy data func.
127
- ctx = build_model_context(
128
- model_name=model,
129
- tokenizer_name=model,
130
- trust_remote_code=True,
131
- mm_processor_kwargs=None,
132
- )
133
-
134
- dummy_data = dummy_data_for_phi3v(
135
- ctx=ctx,
136
- seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137
- mm_counts={"image": num_imgs},
138
- num_crops=num_crops,
139
- )
140
- sequence_data = dummy_data.seq_data
141
- # Ensure we have the right number of placeholders per num_crops size
142
- img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
143
- assert img_tok_count == toks_per_img * num_imgs
144
-
145
-
146
-@pytest.mark.parametrize("model", models)
147
-@pytest.mark.parametrize("num_crops,expected_toks_per_img,num_imgs", [
148
- (4, 757, 1),
149
- (4, 757, 2),
150
- (16, 1921, 1),
151
- (16, 1921, 2),
152
-])
153
-def test_input_processor_override(input_processor_for_phi3v,
154
- image_assets: _ImageAssets, model: str,
155
- num_crops: int, expected_toks_per_img: int,
156
- num_imgs: int):
60
+@pytest.mark.parametrize(
61
+ "num_crops,expected_toks_per_img,num_imgs",
62
+ [
63
+ (4, 757, 1),
64
+ (4, 757, 2),
65
+ (16, 1921, 1),
66
+ (16, 1921, 2),
67
+ # the default num_crops of phi-3.5-vision is 4
68
+ (None, 757, 2),
69
+ (None, 757, 2),
70
+ ])
71
+def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
72
+ model: str, num_crops: Optional[int],
73
+ expected_toks_per_img: int, num_imgs: int):
74
"""Ensure input_processor_for_phi3v handles num_crops properly."""
75
# Same as the previous test - don't initialize mm_processor_kwargs
76
# in this test and assume that the kwargs will be correctly expanded by
@@ -163,19 +80,20 @@ def test_input_processor_override(input_processor_for_phi3v,
80
tokenizer_name=model,
81
trust_remote_code=True,
82
)
166
- tokenizer = AutoTokenizer.from_pretrained(model)
83
+ tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
84
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
85
# Build the image str / prompt based on the number of images we pass
86
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
87
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
88
images = [image_assets[0].pil_image] * num_imgs
89
172
- inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
173
- prompt=prompt,
174
- multi_modal_data={"image": images})
90
+ mm_data = {"image": images}
91
+ mm_processor_kwargs = {}
92
+ if num_crops is not None:
93
+ mm_processor_kwargs = {"num_crops": num_crops}
94
176
- processed_inputs = input_processor_for_phi3v(ctx,
177
- inputs,
178
- num_crops=num_crops)
95
+ processor = processor_for_phi3v(ctx)
96
+ processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
97
98
# Ensure we have the right number of placeholders per num_crops size
99
img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
tests/multimodal/test_processor_kwargs.py
+93
-76
@@ -15,13 +15,13 @@ from ..models.utils import build_model_context
15
# Used for fast tests where the model doesn't matter
16
DUMMY_MODEL_ID = "facebook/opt-125m"
17
# Used for tests that need a multimodal model
18
-MULTIMODAL_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
18
+MULTIMODAL_MODEL_ID = "OpenGVLab/InternVL2-2B"
19
20
# For mm_processor_kwargs - we test overrides by defining mocks for each place
21
# it is used, and ensuring that we can pass processor kwargs an override value
22
# to receive the intended result for things like sequence length etc.
23
-DEFAULT_NUM_CROPS = 4
24
-NUM_CROPS_OVERRIDE = 16
23
+DEFAULT_MAX_DYNAMIC_PATCH = 6
24
+MAX_DYNAMIC_PATCH_OVERRIDE = 4
25
26
27
# Mocks for all of the places that we use the mm_processor_kwargs
@@ -33,10 +33,11 @@ def use_processor_mock():
33
def custom_processor(ctx: InputContext,
34
inputs: DecoderOnlyInputs,
35
*,
36
- num_crops=DEFAULT_NUM_CROPS):
36
+ max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH):
37
# For testing purposes, we don't worry about the prompt
38
- return token_inputs(prompt_token_ids=[],
39
- mm_processor_kwargs={"num_crops": num_crops})
38
+ return token_inputs(
39
+ prompt_token_ids=[],
40
+ mm_processor_kwargs={"max_dynamic_patch": max_dynamic_patch})
41
42
with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
43
return_value=custom_processor):
@@ -52,9 +53,9 @@ def use_dummy_data_mock():
53
seq_len: int,
54
mm_counts: Mapping[str, int],
55
*,
55
- num_crops=DEFAULT_NUM_CROPS):
56
+ max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH):
57
seq_data = SequenceData(
57
- array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
58
+ array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * max_dynamic_patch))
59
return DummyData(seq_data, None)
60
61
with patch(
@@ -65,15 +66,15 @@ def use_dummy_data_mock():
66
67
# Lazy import to avoid CUDA reinitialization error
68
def mm_model_cls():
68
- from vllm.model_executor.models.phi3v import Phi3VForCausalLM
69
+ from vllm.model_executor.models.internvl import InternVLChatModel
70
70
- return Phi3VForCausalLM
71
+ return InternVLChatModel
72
73
74
# lambda whose signature matches max token calcs extra & mapper + extra kwargs
74
-get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
75
-custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
76
- "pixel_values": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
75
+get_max_dynamic_patch = lambda ctx, *, max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH: max_dynamic_patch # noqa: E501
76
+custom_mapper = lambda ctx, data, *, max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH: { # noqa: E501
77
+ "pixel_values": torch.zeros(size=(1, max_dynamic_patch + 1, 3, 448, 448))
78
}
79
80
@@ -88,27 +89,28 @@ def test_default_processor_is_a_noop():
89
assert proc_inputs is proc_outputs
90
91
91
-def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
92
- """Get the init / inference kwargs and expected num_crops for this test."""
93
- # If we have a value for num_crops, pass the override value and make
92
+def _get_max_dynamic_patch_info(init_max_dynamic_patch: int,
93
+ inference_max_dynamic_patch: int):
94
+ """Get the init / inference kwargs and expected max_dynamic_patch."""
95
+ # If we have a value for max_dynamic_patch, pass the override value and make
96
# sure we get that value as a return-value from out mock processor,
97
# otherwise fall back to the default value
96
- init_kwargs = None if init_num_crops is None else {
97
- "num_crops": init_num_crops
98
+ init_kwargs = None if init_max_dynamic_patch is None else {
99
+ "max_dynamic_patch": init_max_dynamic_patch
100
}
99
- inference_kwargs = None if inference_num_crops is None else {
100
- "num_crops": inference_num_crops
101
+ inference_kwargs = None if inference_max_dynamic_patch is None else {
102
+ "max_dynamic_patch": inference_max_dynamic_patch
103
}
102
- if inference_num_crops is not None:
103
- expected_seq_count = inference_num_crops
104
- elif init_num_crops is not None:
105
- expected_seq_count = init_num_crops
104
+ if inference_max_dynamic_patch is not None:
105
+ expected_seq_count = inference_max_dynamic_patch
106
+ elif init_max_dynamic_patch is not None:
107
+ expected_seq_count = init_max_dynamic_patch
108
else:
107
- expected_seq_count = DEFAULT_NUM_CROPS
109
+ expected_seq_count = DEFAULT_MAX_DYNAMIC_PATCH
110
return init_kwargs, inference_kwargs, expected_seq_count
111
112
111
-def _get_processed_num_crops(
113
+def _get_processed_max_dynamic_patch(
114
processor: Callable[[ProcessorInputs], ProcessorInputs],
115
inference_kwargs: Optional[Dict[str, int]],
116
) -> int:
@@ -120,27 +122,30 @@ def _get_processed_num_crops(
122
assert "type" in processed_inputs
123
assert processed_inputs["type"] == "token"
124
assert "mm_processor_kwargs" in processed_inputs
123
- return processed_inputs["mm_processor_kwargs"]["num_crops"]
125
+ return processed_inputs["mm_processor_kwargs"]["max_dynamic_patch"]
126
127
126
-@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
127
- (None, None),
128
- (NUM_CROPS_OVERRIDE, None),
129
- (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
130
-])
131
-def test_input_processor_kwargs(use_processor_mock, init_num_crops,
132
- inference_num_crops):
128
+@pytest.mark.parametrize(
129
+ "init_max_dynamic_patch,inference_max_dynamic_patch", [
130
+ (None, None),
131
+ (MAX_DYNAMIC_PATCH_OVERRIDE, None),
132
+ (DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE),
133
+ ])
134
+def test_input_processor_kwargs(use_processor_mock, init_max_dynamic_patch,
135
+ inference_max_dynamic_patch):
136
"""Ensure input processors can use processor kwargs."""
137
dummy_registry = InputRegistry()
138
136
- init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
137
- init_num_crops, inference_num_crops)
139
+ (init_kwargs, inference_kwargs,
140
+ expected_seq_count) = _get_max_dynamic_patch_info(
141
+ init_max_dynamic_patch, inference_max_dynamic_patch)
142
143
ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
144
processor = dummy_registry.create_input_processor(ctx.model_config)
141
- num_crops_val = _get_processed_num_crops(processor, inference_kwargs)
145
+ max_dynamic_patch_val = _get_processed_max_dynamic_patch(
146
+ processor, inference_kwargs)
147
143
- assert num_crops_val == expected_seq_count
148
+ assert max_dynamic_patch_val == expected_seq_count
149
150
151
@pytest.mark.parametrize(
@@ -165,18 +170,21 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
170
171
processor = dummy_registry.create_input_processor(ctx.model_config)
172
# Should filter out the inference time kwargs
168
- num_crops_val = _get_processed_num_crops(processor, mm_processor_kwargs)
169
- assert num_crops_val == DEFAULT_NUM_CROPS
173
+ max_dynamic_patch_val = _get_processed_max_dynamic_patch(
174
+ processor, mm_processor_kwargs)
175
+ assert max_dynamic_patch_val == DEFAULT_MAX_DYNAMIC_PATCH
176
177
178
### Test overrides for the dummy data
173
-@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
174
-def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
179
+@pytest.mark.parametrize("max_dynamic_patch",
180
+ [None, MAX_DYNAMIC_PATCH_OVERRIDE])
181
+def test_dummy_data_kwarg_overrides(use_dummy_data_mock, max_dynamic_patch):
182
"""Ensure dummy data factories can use processor kwargs."""
176
- mm_processor_kwargs = None if num_crops is None else {
177
- "num_crops": num_crops
183
+ mm_processor_kwargs = None if max_dynamic_patch is None else {
184
+ "max_dynamic_patch": max_dynamic_patch
185
}
179
- expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
186
+ expected_seq_count = (DEFAULT_MAX_DYNAMIC_PATCH
187
+ if max_dynamic_patch is None else max_dynamic_patch)
188
dummy_registry = InputRegistry()
189
ctx = build_model_context(DUMMY_MODEL_ID,
190
mm_processor_kwargs=mm_processor_kwargs)
@@ -217,17 +225,20 @@ def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
225
# len is solely dependent on the value of the mm_processor_kwargs.
226
dummy_data = dummy_registry.dummy_data_for_profiling(
227
ctx.model_config, seq_len=-1, mm_registry=mm_registry)
220
- assert len(dummy_data.seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
228
+ assert len(
229
+ dummy_data.seq_data.prompt_token_ids) == DEFAULT_MAX_DYNAMIC_PATCH
230
231
232
### Test overrides for the max token count per multimodal instance
224
-@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
225
-def test_max_tokens_kwarg_overrides(num_crops):
233
+@pytest.mark.parametrize("max_dynamic_patch",
234
+ [None, MAX_DYNAMIC_PATCH_OVERRIDE])
235
+def test_max_tokens_kwarg_overrides(max_dynamic_patch):
236
"""Ensure max token calcs can use processor kwargs."""
227
- mm_processor_kwargs = None if num_crops is None else {
228
- "num_crops": num_crops
237
+ mm_processor_kwargs = None if max_dynamic_patch is None else {
238
+ "max_dynamic_patch": max_dynamic_patch
239
}
230
- expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
240
+ expected_seq_count = (DEFAULT_MAX_DYNAMIC_PATCH
241
+ if max_dynamic_patch is None else max_dynamic_patch)
242
243
ctx = build_model_context(MULTIMODAL_MODEL_ID,
244
task="generate",
@@ -239,11 +250,11 @@ def test_max_tokens_kwarg_overrides(num_crops):
250
mm_registry.init_mm_limits_per_prompt(ctx.model_config)
251
# Patch the image registry for phi3v with our lambda that is compatible
252
# with overrides, then ensure that calling the method correctly echos
242
- # our num_crops value back from the mm_processor_kwargs.
253
+ # our max_dynamic_patch value back from the mm_processor_kwargs.
254
with patch.object(
255
mm_registry._get_plugin("image"),
256
"_max_mm_tokens",
246
- {mm_model_cls(): get_num_crops},
257
+ {mm_model_cls(): get_max_dynamic_patch},
258
):
259
max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
260
ctx.model_config)
@@ -279,26 +290,29 @@ def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
290
with patch.object(
291
mm_registry._get_plugin("image"),
292
"_max_mm_tokens",
282
- {mm_model_cls(): get_num_crops},
293
+ {mm_model_cls(): get_max_dynamic_patch},
294
):
295
max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
296
ctx.model_config)
297
287
- assert max_multimodal_tokens == DEFAULT_NUM_CROPS
298
+ assert max_multimodal_tokens == DEFAULT_MAX_DYNAMIC_PATCH
299
300
301
### Test overrides for the mapper
291
-@pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
292
-def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
302
+@pytest.mark.parametrize(
303
+ "max_dynamic_patch",
304
+ [DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE])
305
+def test_default_mapper_with_processor_kwargs(image_assets, max_dynamic_patch):
306
"""Ensure that the mapper processor kwargs can fall back to HF models."""
307
# NOTE - we don't validate bad inputs for the default mapper, because it's
308
# through the automodel interface in transformers, so we can't easily
309
# inspect what kwargs are or are not allowed.
297
- ctx = build_model_context(MULTIMODAL_MODEL_ID,
298
- task="generate",
299
- trust_remote_code=True,
300
- mm_processor_kwargs={"num_crops": num_crops},
301
- limit_mm_per_prompt={"image": 1})
310
+ ctx = build_model_context(
311
+ MULTIMODAL_MODEL_ID,
312
+ task="generate",
313
+ trust_remote_code=True,
314
+ mm_processor_kwargs={"max_dynamic_patch": max_dynamic_patch},
315
+ limit_mm_per_prompt={"image": 1})
316
317
mm_registry = MultiModalRegistry()
318
mm_registry.init_mm_limits_per_prompt(ctx.model_config)
@@ -307,20 +321,22 @@ def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
321
mm_inputs = {"image": image}
322
323
mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
310
- # Phi3v pixel vals should have shape: [batch, num_crops+1, 3, 336, 336]
311
- assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
324
+ # pixel vals should have shape: [batch, max_dynamic_patch+1, ...]
325
+ assert mapped_inputs["pixel_values"].shape[1] == max_dynamic_patch + 1
326
327
314
-@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
315
- (None, None),
316
- (NUM_CROPS_OVERRIDE, None),
317
- (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
318
-])
319
-def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
320
- inference_num_crops):
328
+@pytest.mark.parametrize(
329
+ "init_max_dynamic_patch,inference_max_dynamic_patch", [
330
+ (None, None),
331
+ (MAX_DYNAMIC_PATCH_OVERRIDE, None),
332
+ (DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE),
333
+ ])
334
+def test_custom_mapper_kwarg_overrides(image_assets, init_max_dynamic_patch,
335
+ inference_max_dynamic_patch):
336
"""Ensure custom mappers can use processor kwargs."""
322
- init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
323
- init_num_crops, inference_num_crops)
337
+ (init_kwargs, inference_kwargs,
338
+ expected_seq_count) = _get_max_dynamic_patch_info(
339
+ init_max_dynamic_patch, inference_max_dynamic_patch)
340
341
ctx = build_model_context(MULTIMODAL_MODEL_ID,
342
task="generate",
@@ -335,7 +351,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
351
352
# Patch the image registry for phi3v with our lambda that is compatible
353
# with overrides, then ensure that calling the method correctly echos
338
- # our num_crops value back from the mm_processor_kwargs.
354
+ # our max_dynamic_patch value back from the mm_processor_kwargs.
355
mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
356
mm_model_cls())
357
mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs,
@@ -373,11 +389,12 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
389
390
# Patch the image registry for phi3v with our lambda that is compatible
391
# with overrides, then ensure that calling the method correctly echos
376
- # our num_crops value back from the mm_processor_kwargs.
392
+ # our max_dynamic_patch value back from the mm_processor_kwargs.
393
mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
394
mm_model_cls())
395
# Should filter out the inference time kwargs
396
mapped_inputs = mm_registry.map_input(
397
ctx.model_config, mm_inputs, mm_processor_kwargs=mm_processor_kwargs)
398
383
- assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
399
+ assert mapped_inputs["pixel_values"].shape[1] == (
400
+ DEFAULT_MAX_DYNAMIC_PATCH + 1)
vllm/inputs/registry.py
+2
-2
@@ -69,12 +69,12 @@ class InputProcessingContext(InputContext):
69
tokenizer: AnyTokenizer
70
"""The tokenizer used to tokenize the inputs."""
71
72
- def get_hf_processor(self) -> ProcessorMixin:
72
+ def get_hf_processor(self, **kwargs) -> ProcessorMixin:
73
return cached_get_processor(
74
self.model_config.tokenizer,
75
tokenizer=self.tokenizer, # Override the tokenizer with ours
76
trust_remote_code=self.model_config.trust_remote_code,
77
- )
77
+ **kwargs)
78
79
80
N = TypeVar("N", bound=Type[nn.Module])
vllm/model_executor/models/phi3v.py
+84
-214
@@ -12,22 +12,18 @@
12
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
# See the License for the specific language governing permissions and
14
# limitations under the License.
15
-import itertools
16
-import re
17
-from functools import cached_property, lru_cache
18
-from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
19
- Tuple, TypedDict, Union)
15
+from functools import cached_property
16
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17
+ TypedDict, Union)
18
21
-import numpy as np
19
import torch
20
import torch.nn as nn
24
-from PIL import Image
25
-from transformers import CLIPVisionConfig, PretrainedConfig
21
+from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,
22
+ ProcessorMixin)
23
24
from vllm.attention import AttentionMetadata
28
-from vllm.config import ModelConfig, VllmConfig
29
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30
- InputContext, token_inputs)
25
+from vllm.config import VllmConfig
26
+from vllm.inputs import InputContext
27
from vllm.logger import init_logger
28
from vllm.model_executor.layers.quantization import QuantizationConfig
29
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -36,12 +32,18 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32
from vllm.model_executor.models.clip import CLIPVisionModel
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.multimodal import MULTIMODAL_REGISTRY
39
-from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
40
-from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
35
+from vllm.multimodal.image import cached_get_image_processor
36
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
37
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
38
+ InputProcessingContext,
39
+ ModalityProcessingMetadata,
40
+ MultiModalDataDict,
41
+ MultiModalProcessingMetadata,
42
+ PromptReplacement)
43
from vllm.sequence import IntermediateTensors
44
from vllm.utils import is_list_of
45
44
-from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46
+from .clip import dummy_image_for_clip
47
from .interfaces import SupportsMultiModal, SupportsPP
48
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
49
init_vllm_registered_model, maybe_prefix,
@@ -303,231 +305,99 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
305
return image_features_hd_newline
306
307
306
-# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
307
-def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336):
308
- target_height = int(np.ceil(height / padding_unit) * padding_unit)
309
- top_padding = int((target_height - height) / 2)
310
- bottom_padding = target_height - height - top_padding
311
- padded_width = width
312
- padded_height = height + top_padding + bottom_padding
313
- return padded_width, padded_height
314
-
315
-
316
-# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L90
317
-def _calc_hd_transform_size(*, width: int, height: int, hd_num: int):
318
- transposed = False
319
- if width < height:
320
- width, height = height, width
321
- transposed = True
322
-
323
- ratio = width / height
324
- scale = 1
325
- while scale * np.ceil(scale / ratio) <= hd_num:
326
- scale += 1
327
- scale -= 1
328
-
329
- new_width = int(scale * 336)
330
- new_height = int(new_width / ratio)
331
-
332
- padded_width, padded_height = _calc_padded_size(width=new_width,
333
- height=new_height)
334
-
335
- if transposed:
336
- padded_width, padded_height = padded_height, padded_width
337
-
338
- return padded_width, padded_height
339
-
340
-
341
-# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L181
342
-def get_phi3v_image_feature_size(
343
- hf_config: Dict[str, Any],
344
- *,
345
- input_height: int,
346
- input_width: int,
347
- num_crops: int,
348
-) -> int:
349
- if num_crops is None:
350
- num_crops = hf_config.get("num_crops", 16)
351
- new_width, new_height = _calc_hd_transform_size(width=input_width,
352
- height=input_height,
353
- hd_num=num_crops)
354
-
355
- return (new_height // 336 * new_width // 336 + 1) * 144 + 1 \
356
- + (new_height // 336 + 1) * 12
357
-
358
-
308
def get_max_phi3v_image_tokens(ctx: InputContext,
309
*,
310
num_crops: Optional[int] = None):
311
+ mm_processor_kwargs = {}
312
+ if num_crops is not None:
313
+ mm_processor_kwargs["num_crops"] = num_crops
314
363
- return get_phi3v_image_feature_size(
364
- ctx.get_hf_image_processor_config(),
365
- input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
366
- input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
367
- num_crops=num_crops,
315
+ model_config = ctx.model_config
316
+ image_processor = cached_get_image_processor(
317
+ model_config.model,
318
+ trust_remote_code=model_config.trust_remote_code,
319
+ **mm_processor_kwargs,
320
+ )
321
+
322
+ num_tokens = image_processor.calc_num_image_tokens_from_image_size(
323
+ width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
324
+ height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
325
)
326
+ return num_tokens
327
328
371
-def dummy_data_for_phi3v(ctx: InputContext,
372
- seq_len: int,
373
- mm_counts: Mapping[str, int],
374
- *,
375
- num_crops: Optional[int] = None):
329
+def dummy_mm_kwargs_for_phi3v(ctx: InputProcessingContext,
330
+ mm_counts: Mapping[str, int]):
331
num_images = mm_counts["image"]
332
378
- image_feature_size = get_max_phi3v_image_tokens(ctx, num_crops=num_crops)
379
-
380
- seq_data, ranges = dummy_seq_data_for_clip(
381
- CLIP_VIT_LARGE_PATCH14_336_CONFIG,
382
- seq_len,
383
- num_images,
384
- image_token_id=_IMAGE_TOKEN_ID,
385
- image_feature_size_override=image_feature_size,
386
- )
387
- mm_data = dummy_image_for_clip(
333
+ data = dummy_image_for_clip(
334
CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335
num_images,
336
image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
337
image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
338
)
339
394
- return DummyData(seq_data, mm_data, ranges)
395
-
340
+ hf_processor = ctx.get_hf_processor()
341
+ image_processor = hf_processor.image_processor # type: ignore
342
+ hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
343
397
-@lru_cache
398
-def _get_image_placeholder_token_id_candidates(
399
- model_config: ModelConfig,
400
- idx: int,
401
-) -> List[List[int]]:
402
- assert idx > 0
344
+ return MultiModalKwargs(**hf_inputs)
345
404
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
346
406
- # This is used when the image token is at the start of the string
407
- start_candidate = tokenizer.encode(f"<|image_{idx}|>",
408
- add_special_tokens=False)
347
+def create_metadata_for_phi3v(
348
+ ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
349
+ return {
350
+ "image":
351
+ ModalityProcessingMetadata(prompt_repls=[
352
+ PromptReplacement(target=[_IMAGE_TOKEN_ID],
353
+ repl_unit=[_IMAGE_TOKEN_ID],
354
+ repl_count=get_max_phi3v_image_tokens(ctx)),
355
+ ]),
356
+ }
357
410
- # This is used when the image token is in the middle of the string
411
- # We need to get the token for "<", not "▁<"
412
- # https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/raw/main/tokenizer.json
413
- a_token_id, = tokenizer.encode("a", add_special_tokens=False)
414
- a_token_id_, *middle_candidate = tokenizer.encode(f"a<|image_{idx}|>",
415
- add_special_tokens=False)
416
- assert a_token_id == a_token_id_
358
418
- return [start_candidate, middle_candidate]
359
+class Phi3VProcessor(BaseMultiModalProcessor):
360
361
+ def __init__(self, ctx: InputProcessingContext) -> None:
362
+ super().__init__(
363
+ ctx=ctx,
364
+ metadata=create_metadata_for_phi3v(ctx),
365
+ )
366
421
-def input_processor_for_phi3v(ctx: InputContext,
422
- inputs: DecoderOnlyInputs,
423
- *,
424
- num_crops: Optional[int] = None):
425
- multi_modal_data = inputs.get("multi_modal_data")
426
- if multi_modal_data is None or "image" not in multi_modal_data:
427
- return inputs
428
-
429
- model_config = ctx.model_config
430
- hf_config = ctx.get_hf_image_processor_config()
431
-
432
- image_data = multi_modal_data["image"]
433
- if isinstance(image_data, Image.Image):
434
- w, h = image_data.size
435
- image_feature_size = [
436
- get_phi3v_image_feature_size(hf_config,
437
- input_width=w,
438
- input_height=h,
439
- num_crops=num_crops)
440
- ]
441
- image_data = [image_data]
442
- elif is_list_of(image_data, Image.Image):
443
- image_feature_size = []
444
- for image in image_data:
445
- w, h = image.size
446
- image_feature_size.append(
447
- get_phi3v_image_feature_size(hf_config,
448
- input_width=w,
449
- input_height=h,
450
- num_crops=num_crops))
451
- elif isinstance(image_data, torch.Tensor):
452
- image_feature_size = [image_data.shape[0]]
453
- image_data = [image_data]
454
- elif is_list_of(image_data, torch.Tensor):
455
- image_feature_size = [item.shape[0] for item in image_data]
456
- else:
457
- raise TypeError(f"Invalid image type: {type(image_data)}")
458
-
459
- prompt = inputs.get("prompt")
460
- if prompt is None:
461
- # for async server request, we assume prompt and its token_ids is always
462
- # in correct format. And num_image_tags == len(image_data) always True.
463
- image_idx = range(1, len(image_data) + 1)
464
- new_prompt = None
465
- else:
466
- image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))
467
- if prompt.count("<|image|>") > 0:
468
- logger.warning("Please follow the prompt format that is "
469
- "documented on HuggingFace which does not involve "
470
- "repeating <|image|> tokens.")
471
- elif (num_image_tags := len(image_idx)) > 1:
472
- assert num_image_tags == len(
473
- image_data), "The count of image_placeholder not match image's"
474
- new_prompt = prompt
475
-
476
- prompt_token_ids = inputs["prompt_token_ids"].copy()
477
-
478
- # masked placeholder with image token id
479
- for idx in image_idx:
480
- candidates = _get_image_placeholder_token_id_candidates(model_config,
481
- idx=idx)
482
-
483
- for candidate in candidates:
484
- for i in range(len(prompt_token_ids) - len(candidate) + 1):
485
- if prompt_token_ids[i:i + len(candidate)] == candidate:
486
- prompt_token_ids[i:i +
487
- len(candidate)] = ([_IMAGE_TOKEN_ID] *
488
- len(candidate))
489
- break
490
-
491
- # merge consecutive tag ids
492
- merged_token_ids: List[int] = []
493
- for is_placeholder, token_ids in itertools.groupby(
494
- prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID):
495
- if is_placeholder:
496
- merged_token_ids.append(_IMAGE_TOKEN_ID)
497
- else:
498
- merged_token_ids.extend(list(token_ids))
499
-
500
- # TODO: Move this to utils or integrate with clip.
501
- new_token_ids: List[int] = []
502
- placeholder_ranges: List[PlaceholderRange] = []
503
- placeholder_idx = 0
504
- while merged_token_ids:
505
- token_id = merged_token_ids.pop(0)
506
- if token_id == _IMAGE_TOKEN_ID:
507
- replacement_ids = repeat_and_pad_token(
508
- _IMAGE_TOKEN_ID,
509
- repeat_count=image_feature_size[placeholder_idx],
510
- )
511
- placeholder_ranges.append({
512
- "offset": len(new_token_ids),
513
- "length": len(replacement_ids)
514
- })
515
- new_token_ids.extend(replacement_ids)
516
- placeholder_idx += 1
517
- else:
518
- new_token_ids.append(token_id)
519
-
520
- # NOTE: Create a defensive copy of the original inputs
521
- return token_inputs(prompt_token_ids=new_token_ids,
522
- prompt=new_prompt,
523
- multi_modal_data=multi_modal_data,
524
- multi_modal_placeholders={"image": placeholder_ranges})
367
+ def _get_hf_processor(
368
+ self,
369
+ *,
370
+ num_crops: Optional[int] = None,
371
+ ) -> ProcessorMixin:
372
+ if num_crops is not None:
373
+ return self.ctx.get_hf_processor(num_crops=num_crops)
374
+ return self.ctx.get_hf_processor()
375
+
376
+ def _apply_hf_processor(
377
+ self,
378
+ prompt: str,
379
+ mm_data: MultiModalDataDict,
380
+ mm_processor_kwargs: Mapping[str, object],
381
+ ) -> BatchFeature:
382
+ processed_outputs = super()._apply_hf_processor(
383
+ prompt, mm_data, mm_processor_kwargs)
384
+ # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
385
+ # which will cause OverflowError when decoding the prompt_ids.
386
+ # Therefore, we need to do an early replacement here
387
+ token_ids = processed_outputs['input_ids']
388
+ token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
389
+ processed_outputs['input_ids'] = token_ids
390
+ return processed_outputs
391
+
392
+ def _get_dummy_mm_kwargs(
393
+ self,
394
+ mm_counts: Mapping[str, int],
395
+ ) -> MultiModalKwargs:
396
+ return dummy_mm_kwargs_for_phi3v(self.ctx, mm_counts)
397
398
527
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
399
@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
529
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
530
-@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
400
+@MULTIMODAL_REGISTRY.register_processor(Phi3VProcessor)
401
class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
402
403
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/multimodal/processing.py
+25
-4
@@ -3,7 +3,8 @@ from abc import ABC, abstractmethod
3
from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4
from dataclasses import dataclass
5
from functools import lru_cache
6
-from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
6
+from typing import (Any, Dict, Generic, NamedTuple, Optional, Protocol,
7
+ TypeVar, Union, cast)
8
9
import torch
10
from transformers import BatchFeature, ProcessorMixin
@@ -11,7 +12,8 @@ from typing_extensions import TypeAlias, TypedDict
12
13
from vllm.inputs import DummyData, InputProcessingContext
14
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
14
-from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
15
+from vllm.utils import (flatten_2d_lists, full_groupby, is_list_of,
16
+ resolve_mm_processor_kwargs)
17
18
from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
19
MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
@@ -543,8 +545,14 @@ class BaseMultiModalProcessor(ABC):
545
546
self.ctx = ctx
547
self.metadata = metadata
548
+ self.init_mm_processor_kwargs = (ctx.model_config.mm_processor_kwargs
549
+ or {})
550
547
- def _get_hf_processor(self) -> ProcessorMixin:
551
+ def _get_hf_processor(
552
+ self,
553
+ **mm_processor_kwargs: Mapping[str, object],
554
+ ) -> ProcessorMixin:
555
+ # by default, we won't pass any kwargs to the processor initialization
556
return self.ctx.get_hf_processor()
557
558
def _get_tokenizer(self) -> AnyTokenizer:
@@ -581,7 +589,13 @@ class BaseMultiModalProcessor(ABC):
589
mm_data: MultiModalDataDict,
590
mm_processor_kwargs: Mapping[str, object],
591
) -> BatchFeature:
584
- hf_processor = self._get_hf_processor()
592
+ # some mm_processor_kwargs may be used in processor initialization
593
+ # instead of processor call
594
+ processor_init_kwargs = {
595
+ **self.init_mm_processor_kwargs,
596
+ **mm_processor_kwargs,
597
+ }
598
+ hf_processor = self._get_hf_processor(**processor_init_kwargs)
599
600
processor_data = dict[str, Any]()
601
passthrough_data = dict[str, Any]()
@@ -601,6 +615,13 @@ class BaseMultiModalProcessor(ABC):
615
else:
616
processor_data[k] = v
617
618
+ # filter mm_processor_kwargs used in processor call
619
+ mm_processor_kwargs = resolve_mm_processor_kwargs(
620
+ self.init_mm_processor_kwargs,
621
+ cast(Dict[str, Any], mm_processor_kwargs),
622
+ hf_processor,
623
+ )
624
+
625
try:
626
hf_inputs = hf_processor(
627
text=prompt, # type: ignore