@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a811dd6608
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] merged input processor for Phi-3-Vision models (#10977)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Dec 10, 2024 at 04:55 UTC a811dd660856a5c222a1447fe1d93deccbc162fd
7 files changed +235 -409
tests/entrypoints/openai/test_vision.py
+2 -2
@@ -89,7 +89,7 @@ async def test_single_chat_session_image(client: openai.AsyncOpenAI,
89 choice = chat_completion.choices[0]
90 assert choice.finish_reason == "length"
91 assert chat_completion.usage == openai.types.CompletionUsage(
92 - completion_tokens=10, prompt_tokens=772, total_tokens=782)
92 + completion_tokens=10, prompt_tokens=775, total_tokens=785)
93
94 message = choice.message
95 message = chat_completion.choices[0].message
@@ -181,7 +181,7 @@ async def test_single_chat_session_image_base64encoded(
181 choice = chat_completion.choices[0]
182 assert choice.finish_reason == "length"
183 assert chat_completion.usage == openai.types.CompletionUsage(
184 - completion_tokens=10, prompt_tokens=772, total_tokens=782)
184 + completion_tokens=10, prompt_tokens=775, total_tokens=785)
185
186 message = choice.message
187 message = chat_completion.choices[0].message
tests/entrypoints/openai/test_vision_embedding.py
+2 -2
@@ -95,5 +95,5 @@ async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
95 assert len(embeddings["data"]) == 1
96 assert len(embeddings["data"][0]["embedding"]) == 3072
97 assert embeddings["usage"]["completion_tokens"] == 0
98 - assert embeddings["usage"]["prompt_tokens"] == 762
99 - assert embeddings["usage"]["total_tokens"] == 762
98 + assert embeddings["usage"]["prompt_tokens"] == 765
99 + assert embeddings["usage"]["total_tokens"] == 765
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py
+27 -109
@@ -2,12 +2,10 @@
2 from typing import Optional
3
4 import pytest
5 -import torch
6 -from transformers import AutoImageProcessor, AutoTokenizer
5 +from transformers import AutoTokenizer
6
8 -from vllm.inputs import InputContext, token_inputs
7 +from vllm.inputs import InputContext, InputProcessingContext
8 from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
10 -from vllm.multimodal import MultiModalRegistry
9
10 from .....conftest import _ImageAssets
11 from ....utils import build_model_context
@@ -17,15 +15,9 @@ models = ["microsoft/Phi-3.5-vision-instruct"]
15
16 # Wrap lazy imports to avoid initializing CUDA during test collection
17 @pytest.fixture()
20 -def input_processor_for_phi3v():
21 - from vllm.model_executor.models.phi3v import input_processor_for_phi3v
22 - return input_processor_for_phi3v
23 -
24 -
25 -@pytest.fixture()
26 -def dummy_data_for_phi3v():
27 - from vllm.model_executor.models.phi3v import dummy_data_for_phi3v
28 - return dummy_data_for_phi3v
18 +def processor_for_phi3v():
19 + from vllm.model_executor.models.phi3v import Phi3VProcessor
20 + return Phi3VProcessor
21
22
23 @pytest.fixture()
@@ -34,53 +26,6 @@ def get_max_phi3v_image_tokens():
26 return get_max_phi3v_image_tokens
27
28
37 -@pytest.mark.parametrize("model", models)
38 -@pytest.mark.parametrize("num_crops", [4, 16, None])
39 -def test_input_mapper_override(model: str, image_assets: _ImageAssets,
40 - num_crops: Optional[int]):
41 - """Ensure that the [default] input mapper handles num_crops properly."""
42 - # We pass the processor kwargs here since for this model, we fall back to
43 - # the default mapper; this will fall back to the HF mapper and forward
44 - # mm_processor_kwargs to it.
45 - mm_processor_kwargs = {
46 - "num_crops": num_crops
47 - } if num_crops is not None else {}
48 - ctx = build_model_context(
49 - model_name=model,
50 - tokenizer_name=model,
51 - trust_remote_code=True,
52 - mm_processor_kwargs=mm_processor_kwargs,
53 - )
54 -
55 - hf_processor = AutoImageProcessor.from_pretrained(model,
56 - trust_remote_code=True,
57 - **mm_processor_kwargs)
58 -
59 - mm_registry = MultiModalRegistry()
60 - mm_registry.init_mm_limits_per_prompt(ctx.model_config)
61 -
62 - image = image_assets[0].pil_image
63 - hf_result = hf_processor.preprocess(
64 - image,
65 - return_tensors="pt",
66 - )
67 -
68 - vllm_result = mm_registry.map_input(
69 - ctx.model_config,
70 - {"image": image},
71 - )
72 -
73 - assert torch.all(hf_result["image_sizes"] == vllm_result["image_sizes"])
74 - assert torch.all(
75 - hf_result["num_img_tokens"] == vllm_result["num_img_tokens"])
76 -
77 - # For pixel values, the second axis should be the num_crops + 1
78 - # for the rescaled original image. The default value in VLLM falls
79 - # back to the HF config, which is why we compare to the processor num_crops
80 - assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
81 - assert vllm_result["pixel_values"].shape[1] == hf_processor.num_crops + 1
82 -
83 -
29 @pytest.mark.parametrize("model", models)
30 @pytest.mark.parametrize("num_crops,expected_max_tokens", [
31 (4, 781),
@@ -112,48 +57,20 @@ def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
57
58
59 @pytest.mark.parametrize("model", models)
115 -@pytest.mark.parametrize("num_crops,toks_per_img,num_imgs", [
116 - (4, 781, 1),
117 - (4, 781, 2),
118 - (16, 2653, 1),
119 - (16, 2653, 2),
120 -])
121 -def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
122 - toks_per_img: int, num_imgs: int):
123 - """Ensure dummy_data_for_phi3v handles num_crops properly."""
124 - # Same as the previous test - don't initialize mm_processor_kwargs
125 - # in this test and assume that the kwargs will be correctly expanded by
126 - # the partial when calling the dummy data func.
127 - ctx = build_model_context(
128 - model_name=model,
129 - tokenizer_name=model,
130 - trust_remote_code=True,
131 - mm_processor_kwargs=None,
132 - )
133 -
134 - dummy_data = dummy_data_for_phi3v(
135 - ctx=ctx,
136 - seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137 - mm_counts={"image": num_imgs},
138 - num_crops=num_crops,
139 - )
140 - sequence_data = dummy_data.seq_data
141 - # Ensure we have the right number of placeholders per num_crops size
142 - img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
143 - assert img_tok_count == toks_per_img * num_imgs
144 -
145 -
146 -@pytest.mark.parametrize("model", models)
147 -@pytest.mark.parametrize("num_crops,expected_toks_per_img,num_imgs", [
148 - (4, 757, 1),
149 - (4, 757, 2),
150 - (16, 1921, 1),
151 - (16, 1921, 2),
152 -])
153 -def test_input_processor_override(input_processor_for_phi3v,
154 - image_assets: _ImageAssets, model: str,
155 - num_crops: int, expected_toks_per_img: int,
156 - num_imgs: int):
60 +@pytest.mark.parametrize(
61 + "num_crops,expected_toks_per_img,num_imgs",
62 + [
63 + (4, 757, 1),
64 + (4, 757, 2),
65 + (16, 1921, 1),
66 + (16, 1921, 2),
67 + # the default num_crops of phi-3.5-vision is 4
68 + (None, 757, 2),
69 + (None, 757, 2),
70 + ])
71 +def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
72 + model: str, num_crops: Optional[int],
73 + expected_toks_per_img: int, num_imgs: int):
74 """Ensure input_processor_for_phi3v handles num_crops properly."""
75 # Same as the previous test - don't initialize mm_processor_kwargs
76 # in this test and assume that the kwargs will be correctly expanded by
@@ -163,19 +80,20 @@ def test_input_processor_override(input_processor_for_phi3v,
80 tokenizer_name=model,
81 trust_remote_code=True,
82 )
166 - tokenizer = AutoTokenizer.from_pretrained(model)
83 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
84 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
85 # Build the image str / prompt based on the number of images we pass
86 img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
87 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
88 images = [image_assets[0].pil_image] * num_imgs
89
172 - inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
173 - prompt=prompt,
174 - multi_modal_data={"image": images})
90 + mm_data = {"image": images}
91 + mm_processor_kwargs = {}
92 + if num_crops is not None:
93 + mm_processor_kwargs = {"num_crops": num_crops}
94
176 - processed_inputs = input_processor_for_phi3v(ctx,
177 - inputs,
178 - num_crops=num_crops)
95 + processor = processor_for_phi3v(ctx)
96 + processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
97
98 # Ensure we have the right number of placeholders per num_crops size
99 img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
tests/multimodal/test_processor_kwargs.py
+93 -76
@@ -15,13 +15,13 @@ from ..models.utils import build_model_context
15 # Used for fast tests where the model doesn't matter
16 DUMMY_MODEL_ID = "facebook/opt-125m"
17 # Used for tests that need a multimodal model
18 -MULTIMODAL_MODEL_ID = "microsoft/Phi-3.5-vision-instruct"
18 +MULTIMODAL_MODEL_ID = "OpenGVLab/InternVL2-2B"
19
20 # For mm_processor_kwargs - we test overrides by defining mocks for each place
21 # it is used, and ensuring that we can pass processor kwargs an override value
22 # to receive the intended result for things like sequence length etc.
23 -DEFAULT_NUM_CROPS = 4
24 -NUM_CROPS_OVERRIDE = 16
23 +DEFAULT_MAX_DYNAMIC_PATCH = 6
24 +MAX_DYNAMIC_PATCH_OVERRIDE = 4
25
26
27 # Mocks for all of the places that we use the mm_processor_kwargs
@@ -33,10 +33,11 @@ def use_processor_mock():
33 def custom_processor(ctx: InputContext,
34 inputs: DecoderOnlyInputs,
35 *,
36 - num_crops=DEFAULT_NUM_CROPS):
36 + max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH):
37 # For testing purposes, we don't worry about the prompt
38 - return token_inputs(prompt_token_ids=[],
39 - mm_processor_kwargs={"num_crops": num_crops})
38 + return token_inputs(
39 + prompt_token_ids=[],
40 + mm_processor_kwargs={"max_dynamic_patch": max_dynamic_patch})
41
42 with patch("vllm.inputs.registry.InputRegistry._get_model_input_processor",
43 return_value=custom_processor):
@@ -52,9 +53,9 @@ def use_dummy_data_mock():
53 seq_len: int,
54 mm_counts: Mapping[str, int],
55 *,
55 - num_crops=DEFAULT_NUM_CROPS):
56 + max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH):
57 seq_data = SequenceData(
57 - array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * num_crops))
58 + array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * max_dynamic_patch))
59 return DummyData(seq_data, None)
60
61 with patch(
@@ -65,15 +66,15 @@ def use_dummy_data_mock():
66
67 # Lazy import to avoid CUDA reinitialization error
68 def mm_model_cls():
68 - from vllm.model_executor.models.phi3v import Phi3VForCausalLM
69 + from vllm.model_executor.models.internvl import InternVLChatModel
70
70 - return Phi3VForCausalLM
71 + return InternVLChatModel
72
73
74 # lambda whose signature matches max token calcs extra & mapper + extra kwargs
74 -get_num_crops = lambda ctx, *, num_crops=DEFAULT_NUM_CROPS: num_crops
75 -custom_mapper = lambda ctx, data, *, num_crops=DEFAULT_NUM_CROPS: {
76 - "pixel_values": torch.zeros(size=(1, num_crops + 1, 3, 336, 336))
75 +get_max_dynamic_patch = lambda ctx, *, max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH: max_dynamic_patch # noqa: E501
76 +custom_mapper = lambda ctx, data, *, max_dynamic_patch=DEFAULT_MAX_DYNAMIC_PATCH: { # noqa: E501
77 + "pixel_values": torch.zeros(size=(1, max_dynamic_patch + 1, 3, 448, 448))
78 }
79
80
@@ -88,27 +89,28 @@ def test_default_processor_is_a_noop():
89 assert proc_inputs is proc_outputs
90
91
91 -def _get_num_crops_info(init_num_crops: int, inference_num_crops: int):
92 - """Get the init / inference kwargs and expected num_crops for this test."""
93 - # If we have a value for num_crops, pass the override value and make
92 +def _get_max_dynamic_patch_info(init_max_dynamic_patch: int,
93 + inference_max_dynamic_patch: int):
94 + """Get the init / inference kwargs and expected max_dynamic_patch."""
95 + # If we have a value for max_dynamic_patch, pass the override value and make
96 # sure we get that value as a return-value from out mock processor,
97 # otherwise fall back to the default value
96 - init_kwargs = None if init_num_crops is None else {
97 - "num_crops": init_num_crops
98 + init_kwargs = None if init_max_dynamic_patch is None else {
99 + "max_dynamic_patch": init_max_dynamic_patch
100 }
99 - inference_kwargs = None if inference_num_crops is None else {
100 - "num_crops": inference_num_crops
101 + inference_kwargs = None if inference_max_dynamic_patch is None else {
102 + "max_dynamic_patch": inference_max_dynamic_patch
103 }
102 - if inference_num_crops is not None:
103 - expected_seq_count = inference_num_crops
104 - elif init_num_crops is not None:
105 - expected_seq_count = init_num_crops
104 + if inference_max_dynamic_patch is not None:
105 + expected_seq_count = inference_max_dynamic_patch
106 + elif init_max_dynamic_patch is not None:
107 + expected_seq_count = init_max_dynamic_patch
108 else:
107 - expected_seq_count = DEFAULT_NUM_CROPS
109 + expected_seq_count = DEFAULT_MAX_DYNAMIC_PATCH
110 return init_kwargs, inference_kwargs, expected_seq_count
111
112
111 -def _get_processed_num_crops(
113 +def _get_processed_max_dynamic_patch(
114 processor: Callable[[ProcessorInputs], ProcessorInputs],
115 inference_kwargs: Optional[Dict[str, int]],
116 ) -> int:
@@ -120,27 +122,30 @@ def _get_processed_num_crops(
122 assert "type" in processed_inputs
123 assert processed_inputs["type"] == "token"
124 assert "mm_processor_kwargs" in processed_inputs
123 - return processed_inputs["mm_processor_kwargs"]["num_crops"]
125 + return processed_inputs["mm_processor_kwargs"]["max_dynamic_patch"]
126
127
126 -@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
127 - (None, None),
128 - (NUM_CROPS_OVERRIDE, None),
129 - (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
130 -])
131 -def test_input_processor_kwargs(use_processor_mock, init_num_crops,
132 - inference_num_crops):
128 +@pytest.mark.parametrize(
129 + "init_max_dynamic_patch,inference_max_dynamic_patch", [
130 + (None, None),
131 + (MAX_DYNAMIC_PATCH_OVERRIDE, None),
132 + (DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE),
133 + ])
134 +def test_input_processor_kwargs(use_processor_mock, init_max_dynamic_patch,
135 + inference_max_dynamic_patch):
136 """Ensure input processors can use processor kwargs."""
137 dummy_registry = InputRegistry()
138
136 - init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
137 - init_num_crops, inference_num_crops)
139 + (init_kwargs, inference_kwargs,
140 + expected_seq_count) = _get_max_dynamic_patch_info(
141 + init_max_dynamic_patch, inference_max_dynamic_patch)
142
143 ctx = build_model_context(DUMMY_MODEL_ID, mm_processor_kwargs=init_kwargs)
144 processor = dummy_registry.create_input_processor(ctx.model_config)
141 - num_crops_val = _get_processed_num_crops(processor, inference_kwargs)
145 + max_dynamic_patch_val = _get_processed_max_dynamic_patch(
146 + processor, inference_kwargs)
147
143 - assert num_crops_val == expected_seq_count
148 + assert max_dynamic_patch_val == expected_seq_count
149
150
151 @pytest.mark.parametrize(
@@ -165,18 +170,21 @@ def test_processor_with_sad_kwarg_overrides(use_processor_mock,
170
171 processor = dummy_registry.create_input_processor(ctx.model_config)
172 # Should filter out the inference time kwargs
168 - num_crops_val = _get_processed_num_crops(processor, mm_processor_kwargs)
169 - assert num_crops_val == DEFAULT_NUM_CROPS
173 + max_dynamic_patch_val = _get_processed_max_dynamic_patch(
174 + processor, mm_processor_kwargs)
175 + assert max_dynamic_patch_val == DEFAULT_MAX_DYNAMIC_PATCH
176
177
178 ### Test overrides for the dummy data
173 -@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
174 -def test_dummy_data_kwarg_overrides(use_dummy_data_mock, num_crops):
179 +@pytest.mark.parametrize("max_dynamic_patch",
180 + [None, MAX_DYNAMIC_PATCH_OVERRIDE])
181 +def test_dummy_data_kwarg_overrides(use_dummy_data_mock, max_dynamic_patch):
182 """Ensure dummy data factories can use processor kwargs."""
176 - mm_processor_kwargs = None if num_crops is None else {
177 - "num_crops": num_crops
183 + mm_processor_kwargs = None if max_dynamic_patch is None else {
184 + "max_dynamic_patch": max_dynamic_patch
185 }
179 - expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
186 + expected_seq_count = (DEFAULT_MAX_DYNAMIC_PATCH
187 + if max_dynamic_patch is None else max_dynamic_patch)
188 dummy_registry = InputRegistry()
189 ctx = build_model_context(DUMMY_MODEL_ID,
190 mm_processor_kwargs=mm_processor_kwargs)
@@ -217,17 +225,20 @@ def test_dummy_data_with_sad_kwarg_overrides(use_dummy_data_mock,
225 # len is solely dependent on the value of the mm_processor_kwargs.
226 dummy_data = dummy_registry.dummy_data_for_profiling(
227 ctx.model_config, seq_len=-1, mm_registry=mm_registry)
220 - assert len(dummy_data.seq_data.prompt_token_ids) == DEFAULT_NUM_CROPS
228 + assert len(
229 + dummy_data.seq_data.prompt_token_ids) == DEFAULT_MAX_DYNAMIC_PATCH
230
231
232 ### Test overrides for the max token count per multimodal instance
224 -@pytest.mark.parametrize("num_crops", [None, NUM_CROPS_OVERRIDE])
225 -def test_max_tokens_kwarg_overrides(num_crops):
233 +@pytest.mark.parametrize("max_dynamic_patch",
234 + [None, MAX_DYNAMIC_PATCH_OVERRIDE])
235 +def test_max_tokens_kwarg_overrides(max_dynamic_patch):
236 """Ensure max token calcs can use processor kwargs."""
227 - mm_processor_kwargs = None if num_crops is None else {
228 - "num_crops": num_crops
237 + mm_processor_kwargs = None if max_dynamic_patch is None else {
238 + "max_dynamic_patch": max_dynamic_patch
239 }
230 - expected_seq_count = DEFAULT_NUM_CROPS if num_crops is None else num_crops
240 + expected_seq_count = (DEFAULT_MAX_DYNAMIC_PATCH
241 + if max_dynamic_patch is None else max_dynamic_patch)
242
243 ctx = build_model_context(MULTIMODAL_MODEL_ID,
244 task="generate",
@@ -239,11 +250,11 @@ def test_max_tokens_kwarg_overrides(num_crops):
250 mm_registry.init_mm_limits_per_prompt(ctx.model_config)
251 # Patch the image registry for phi3v with our lambda that is compatible
252 # with overrides, then ensure that calling the method correctly echos
242 - # our num_crops value back from the mm_processor_kwargs.
253 + # our max_dynamic_patch value back from the mm_processor_kwargs.
254 with patch.object(
255 mm_registry._get_plugin("image"),
256 "_max_mm_tokens",
246 - {mm_model_cls(): get_num_crops},
257 + {mm_model_cls(): get_max_dynamic_patch},
258 ):
259 max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
260 ctx.model_config)
@@ -279,26 +290,29 @@ def test_max_tokens_with_sad_kwarg_overrides(mm_processor_kwargs):
290 with patch.object(
291 mm_registry._get_plugin("image"),
292 "_max_mm_tokens",
282 - {mm_model_cls(): get_num_crops},
293 + {mm_model_cls(): get_max_dynamic_patch},
294 ):
295 max_multimodal_tokens = mm_registry.get_max_multimodal_tokens(
296 ctx.model_config)
297
287 - assert max_multimodal_tokens == DEFAULT_NUM_CROPS
298 + assert max_multimodal_tokens == DEFAULT_MAX_DYNAMIC_PATCH
299
300
301 ### Test overrides for the mapper
291 -@pytest.mark.parametrize("num_crops", [DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE])
292 -def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
302 +@pytest.mark.parametrize(
303 + "max_dynamic_patch",
304 + [DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE])
305 +def test_default_mapper_with_processor_kwargs(image_assets, max_dynamic_patch):
306 """Ensure that the mapper processor kwargs can fall back to HF models."""
307 # NOTE - we don't validate bad inputs for the default mapper, because it's
308 # through the automodel interface in transformers, so we can't easily
309 # inspect what kwargs are or are not allowed.
297 - ctx = build_model_context(MULTIMODAL_MODEL_ID,
298 - task="generate",
299 - trust_remote_code=True,
300 - mm_processor_kwargs={"num_crops": num_crops},
301 - limit_mm_per_prompt={"image": 1})
310 + ctx = build_model_context(
311 + MULTIMODAL_MODEL_ID,
312 + task="generate",
313 + trust_remote_code=True,
314 + mm_processor_kwargs={"max_dynamic_patch": max_dynamic_patch},
315 + limit_mm_per_prompt={"image": 1})
316
317 mm_registry = MultiModalRegistry()
318 mm_registry.init_mm_limits_per_prompt(ctx.model_config)
@@ -307,20 +321,22 @@ def test_default_mapper_with_processor_kwargs(image_assets, num_crops):
321 mm_inputs = {"image": image}
322
323 mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs)
310 - # Phi3v pixel vals should have shape: [batch, num_crops+1, 3, 336, 336]
311 - assert mapped_inputs["pixel_values"].shape[1] == num_crops + 1
324 + # pixel vals should have shape: [batch, max_dynamic_patch+1, ...]
325 + assert mapped_inputs["pixel_values"].shape[1] == max_dynamic_patch + 1
326
327
314 -@pytest.mark.parametrize("init_num_crops,inference_num_crops", [
315 - (None, None),
316 - (NUM_CROPS_OVERRIDE, None),
317 - (DEFAULT_NUM_CROPS, NUM_CROPS_OVERRIDE),
318 -])
319 -def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
320 - inference_num_crops):
328 +@pytest.mark.parametrize(
329 + "init_max_dynamic_patch,inference_max_dynamic_patch", [
330 + (None, None),
331 + (MAX_DYNAMIC_PATCH_OVERRIDE, None),
332 + (DEFAULT_MAX_DYNAMIC_PATCH, MAX_DYNAMIC_PATCH_OVERRIDE),
333 + ])
334 +def test_custom_mapper_kwarg_overrides(image_assets, init_max_dynamic_patch,
335 + inference_max_dynamic_patch):
336 """Ensure custom mappers can use processor kwargs."""
322 - init_kwargs, inference_kwargs, expected_seq_count = _get_num_crops_info(
323 - init_num_crops, inference_num_crops)
337 + (init_kwargs, inference_kwargs,
338 + expected_seq_count) = _get_max_dynamic_patch_info(
339 + init_max_dynamic_patch, inference_max_dynamic_patch)
340
341 ctx = build_model_context(MULTIMODAL_MODEL_ID,
342 task="generate",
@@ -335,7 +351,7 @@ def test_custom_mapper_kwarg_overrides(image_assets, init_num_crops,
351
352 # Patch the image registry for phi3v with our lambda that is compatible
353 # with overrides, then ensure that calling the method correctly echos
338 - # our num_crops value back from the mm_processor_kwargs.
354 + # our max_dynamic_patch value back from the mm_processor_kwargs.
355 mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
356 mm_model_cls())
357 mapped_inputs = mm_registry.map_input(ctx.model_config, mm_inputs,
@@ -373,11 +389,12 @@ def test_custom_mapper_with_sad_kwarg_overrides(image_assets,
389
390 # Patch the image registry for phi3v with our lambda that is compatible
391 # with overrides, then ensure that calling the method correctly echos
376 - # our num_crops value back from the mm_processor_kwargs.
392 + # our max_dynamic_patch value back from the mm_processor_kwargs.
393 mm_registry._get_plugin("image").register_input_mapper(custom_mapper)(
394 mm_model_cls())
395 # Should filter out the inference time kwargs
396 mapped_inputs = mm_registry.map_input(
397 ctx.model_config, mm_inputs, mm_processor_kwargs=mm_processor_kwargs)
398
383 - assert mapped_inputs["pixel_values"].shape[1] == DEFAULT_NUM_CROPS + 1
399 + assert mapped_inputs["pixel_values"].shape[1] == (
400 + DEFAULT_MAX_DYNAMIC_PATCH + 1)
vllm/inputs/registry.py
+2 -2
@@ -69,12 +69,12 @@ class InputProcessingContext(InputContext):
69 tokenizer: AnyTokenizer
70 """The tokenizer used to tokenize the inputs."""
71
72 - def get_hf_processor(self) -> ProcessorMixin:
72 + def get_hf_processor(self, **kwargs) -> ProcessorMixin:
73 return cached_get_processor(
74 self.model_config.tokenizer,
75 tokenizer=self.tokenizer, # Override the tokenizer with ours
76 trust_remote_code=self.model_config.trust_remote_code,
77 - )
77 + **kwargs)
78
79
80 N = TypeVar("N", bound=Type[nn.Module])
vllm/model_executor/models/phi3v.py
+84 -214
@@ -12,22 +12,18 @@
12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 # See the License for the specific language governing permissions and
14 # limitations under the License.
15 -import itertools
16 -import re
17 -from functools import cached_property, lru_cache
18 -from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional, Set,
19 - Tuple, TypedDict, Union)
15 +from functools import cached_property
16 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
17 + TypedDict, Union)
18
21 -import numpy as np
19 import torch
20 import torch.nn as nn
24 -from PIL import Image
25 -from transformers import CLIPVisionConfig, PretrainedConfig
21 +from transformers import (BatchFeature, CLIPVisionConfig, PretrainedConfig,
22 + ProcessorMixin)
23
24 from vllm.attention import AttentionMetadata
28 -from vllm.config import ModelConfig, VllmConfig
29 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
30 - InputContext, token_inputs)
25 +from vllm.config import VllmConfig
26 +from vllm.inputs import InputContext
27 from vllm.logger import init_logger
28 from vllm.model_executor.layers.quantization import QuantizationConfig
29 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
@@ -36,12 +32,18 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
32 from vllm.model_executor.models.clip import CLIPVisionModel
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.multimodal import MULTIMODAL_REGISTRY
39 -from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
40 -from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
35 +from vllm.multimodal.image import cached_get_image_processor
36 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
37 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
38 + InputProcessingContext,
39 + ModalityProcessingMetadata,
40 + MultiModalDataDict,
41 + MultiModalProcessingMetadata,
42 + PromptReplacement)
43 from vllm.sequence import IntermediateTensors
44 from vllm.utils import is_list_of
45
44 -from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46 +from .clip import dummy_image_for_clip
47 from .interfaces import SupportsMultiModal, SupportsPP
48 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
49 init_vllm_registered_model, maybe_prefix,
@@ -303,231 +305,99 @@ class Phi3HDImageEmbedding(Phi3ImageEmbeddingBase):
305 return image_features_hd_newline
306
307
306 -# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L57
307 -def _calc_padded_size(*, width: int, height: int, padding_unit: int = 336):
308 - target_height = int(np.ceil(height / padding_unit) * padding_unit)
309 - top_padding = int((target_height - height) / 2)
310 - bottom_padding = target_height - height - top_padding
311 - padded_width = width
312 - padded_height = height + top_padding + bottom_padding
313 - return padded_width, padded_height
314 -
315 -
316 -# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L90
317 -def _calc_hd_transform_size(*, width: int, height: int, hd_num: int):
318 - transposed = False
319 - if width < height:
320 - width, height = height, width
321 - transposed = True
322 -
323 - ratio = width / height
324 - scale = 1
325 - while scale * np.ceil(scale / ratio) <= hd_num:
326 - scale += 1
327 - scale -= 1
328 -
329 - new_width = int(scale * 336)
330 - new_height = int(new_width / ratio)
331 -
332 - padded_width, padded_height = _calc_padded_size(width=new_width,
333 - height=new_height)
334 -
335 - if transposed:
336 - padded_width, padded_height = padded_height, padded_width
337 -
338 - return padded_width, padded_height
339 -
340 -
341 -# Based on https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/blob/main/image_processing_phi3_v.py#L181
342 -def get_phi3v_image_feature_size(
343 - hf_config: Dict[str, Any],
344 - *,
345 - input_height: int,
346 - input_width: int,
347 - num_crops: int,
348 -) -> int:
349 - if num_crops is None:
350 - num_crops = hf_config.get("num_crops", 16)
351 - new_width, new_height = _calc_hd_transform_size(width=input_width,
352 - height=input_height,
353 - hd_num=num_crops)
354 -
355 - return (new_height // 336 * new_width // 336 + 1) * 144 + 1 \
356 - + (new_height // 336 + 1) * 12
357 -
358 -
308 def get_max_phi3v_image_tokens(ctx: InputContext,
309 *,
310 num_crops: Optional[int] = None):
311 + mm_processor_kwargs = {}
312 + if num_crops is not None:
313 + mm_processor_kwargs["num_crops"] = num_crops
314
363 - return get_phi3v_image_feature_size(
364 - ctx.get_hf_image_processor_config(),
365 - input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
366 - input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
367 - num_crops=num_crops,
315 + model_config = ctx.model_config
316 + image_processor = cached_get_image_processor(
317 + model_config.model,
318 + trust_remote_code=model_config.trust_remote_code,
319 + **mm_processor_kwargs,
320 + )
321 +
322 + num_tokens = image_processor.calc_num_image_tokens_from_image_size(
323 + width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
324 + height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
325 )
326 + return num_tokens
327
328
371 -def dummy_data_for_phi3v(ctx: InputContext,
372 - seq_len: int,
373 - mm_counts: Mapping[str, int],
374 - *,
375 - num_crops: Optional[int] = None):
329 +def dummy_mm_kwargs_for_phi3v(ctx: InputProcessingContext,
330 + mm_counts: Mapping[str, int]):
331 num_images = mm_counts["image"]
332
378 - image_feature_size = get_max_phi3v_image_tokens(ctx, num_crops=num_crops)
379 -
380 - seq_data, ranges = dummy_seq_data_for_clip(
381 - CLIP_VIT_LARGE_PATCH14_336_CONFIG,
382 - seq_len,
383 - num_images,
384 - image_token_id=_IMAGE_TOKEN_ID,
385 - image_feature_size_override=image_feature_size,
386 - )
387 - mm_data = dummy_image_for_clip(
333 + data = dummy_image_for_clip(
334 CLIP_VIT_LARGE_PATCH14_336_CONFIG,
335 num_images,
336 image_width_override=MAX_IMAGE_FEATURE_SIZE_WIDTH,
337 image_height_override=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
338 )
339
394 - return DummyData(seq_data, mm_data, ranges)
395 -
340 + hf_processor = ctx.get_hf_processor()
341 + image_processor = hf_processor.image_processor # type: ignore
342 + hf_inputs = image_processor.preprocess(data['image'], return_tensors="pt")
343
397 -@lru_cache
398 -def _get_image_placeholder_token_id_candidates(
399 - model_config: ModelConfig,
400 - idx: int,
401 -) -> List[List[int]]:
402 - assert idx > 0
344 + return MultiModalKwargs(**hf_inputs)
345
404 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
346
406 - # This is used when the image token is at the start of the string
407 - start_candidate = tokenizer.encode(f"<|image_{idx}|>",
408 - add_special_tokens=False)
347 +def create_metadata_for_phi3v(
348 + ctx: InputProcessingContext) -> MultiModalProcessingMetadata:
349 + return {
350 + "image":
351 + ModalityProcessingMetadata(prompt_repls=[
352 + PromptReplacement(target=[_IMAGE_TOKEN_ID],
353 + repl_unit=[_IMAGE_TOKEN_ID],
354 + repl_count=get_max_phi3v_image_tokens(ctx)),
355 + ]),
356 + }
357
410 - # This is used when the image token is in the middle of the string
411 - # We need to get the token for "<", not "▁<"
412 - # https://huggingface.co/microsoft/Phi-3-vision-128k-instruct/raw/main/tokenizer.json
413 - a_token_id, = tokenizer.encode("a", add_special_tokens=False)
414 - a_token_id_, *middle_candidate = tokenizer.encode(f"a<|image_{idx}|>",
415 - add_special_tokens=False)
416 - assert a_token_id == a_token_id_
358
418 - return [start_candidate, middle_candidate]
359 +class Phi3VProcessor(BaseMultiModalProcessor):
360
361 + def __init__(self, ctx: InputProcessingContext) -> None:
362 + super().__init__(
363 + ctx=ctx,
364 + metadata=create_metadata_for_phi3v(ctx),
365 + )
366
421 -def input_processor_for_phi3v(ctx: InputContext,
422 - inputs: DecoderOnlyInputs,
423 - *,
424 - num_crops: Optional[int] = None):
425 - multi_modal_data = inputs.get("multi_modal_data")
426 - if multi_modal_data is None or "image" not in multi_modal_data:
427 - return inputs
428 -
429 - model_config = ctx.model_config
430 - hf_config = ctx.get_hf_image_processor_config()
431 -
432 - image_data = multi_modal_data["image"]
433 - if isinstance(image_data, Image.Image):
434 - w, h = image_data.size
435 - image_feature_size = [
436 - get_phi3v_image_feature_size(hf_config,
437 - input_width=w,
438 - input_height=h,
439 - num_crops=num_crops)
440 - ]
441 - image_data = [image_data]
442 - elif is_list_of(image_data, Image.Image):
443 - image_feature_size = []
444 - for image in image_data:
445 - w, h = image.size
446 - image_feature_size.append(
447 - get_phi3v_image_feature_size(hf_config,
448 - input_width=w,
449 - input_height=h,
450 - num_crops=num_crops))
451 - elif isinstance(image_data, torch.Tensor):
452 - image_feature_size = [image_data.shape[0]]
453 - image_data = [image_data]
454 - elif is_list_of(image_data, torch.Tensor):
455 - image_feature_size = [item.shape[0] for item in image_data]
456 - else:
457 - raise TypeError(f"Invalid image type: {type(image_data)}")
458 -
459 - prompt = inputs.get("prompt")
460 - if prompt is None:
461 - # for async server request, we assume prompt and its token_ids is always
462 - # in correct format. And num_image_tags == len(image_data) always True.
463 - image_idx = range(1, len(image_data) + 1)
464 - new_prompt = None
465 - else:
466 - image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))
467 - if prompt.count("<|image|>") > 0:
468 - logger.warning("Please follow the prompt format that is "
469 - "documented on HuggingFace which does not involve "
470 - "repeating <|image|> tokens.")
471 - elif (num_image_tags := len(image_idx)) > 1:
472 - assert num_image_tags == len(
473 - image_data), "The count of image_placeholder not match image's"
474 - new_prompt = prompt
475 -
476 - prompt_token_ids = inputs["prompt_token_ids"].copy()
477 -
478 - # masked placeholder with image token id
479 - for idx in image_idx:
480 - candidates = _get_image_placeholder_token_id_candidates(model_config,
481 - idx=idx)
482 -
483 - for candidate in candidates:
484 - for i in range(len(prompt_token_ids) - len(candidate) + 1):
485 - if prompt_token_ids[i:i + len(candidate)] == candidate:
486 - prompt_token_ids[i:i +
487 - len(candidate)] = ([_IMAGE_TOKEN_ID] *
488 - len(candidate))
489 - break
490 -
491 - # merge consecutive tag ids
492 - merged_token_ids: List[int] = []
493 - for is_placeholder, token_ids in itertools.groupby(
494 - prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID):
495 - if is_placeholder:
496 - merged_token_ids.append(_IMAGE_TOKEN_ID)
497 - else:
498 - merged_token_ids.extend(list(token_ids))
499 -
500 - # TODO: Move this to utils or integrate with clip.
501 - new_token_ids: List[int] = []
502 - placeholder_ranges: List[PlaceholderRange] = []
503 - placeholder_idx = 0
504 - while merged_token_ids:
505 - token_id = merged_token_ids.pop(0)
506 - if token_id == _IMAGE_TOKEN_ID:
507 - replacement_ids = repeat_and_pad_token(
508 - _IMAGE_TOKEN_ID,
509 - repeat_count=image_feature_size[placeholder_idx],
510 - )
511 - placeholder_ranges.append({
512 - "offset": len(new_token_ids),
513 - "length": len(replacement_ids)
514 - })
515 - new_token_ids.extend(replacement_ids)
516 - placeholder_idx += 1
517 - else:
518 - new_token_ids.append(token_id)
519 -
520 - # NOTE: Create a defensive copy of the original inputs
521 - return token_inputs(prompt_token_ids=new_token_ids,
522 - prompt=new_prompt,
523 - multi_modal_data=multi_modal_data,
524 - multi_modal_placeholders={"image": placeholder_ranges})
367 + def _get_hf_processor(
368 + self,
369 + *,
370 + num_crops: Optional[int] = None,
371 + ) -> ProcessorMixin:
372 + if num_crops is not None:
373 + return self.ctx.get_hf_processor(num_crops=num_crops)
374 + return self.ctx.get_hf_processor()
375 +
376 + def _apply_hf_processor(
377 + self,
378 + prompt: str,
379 + mm_data: MultiModalDataDict,
380 + mm_processor_kwargs: Mapping[str, object],
381 + ) -> BatchFeature:
382 + processed_outputs = super()._apply_hf_processor(
383 + prompt, mm_data, mm_processor_kwargs)
384 + # Phi3v processor has inserted -1, -2 etc as placeholder in prompt_ids,
385 + # which will cause OverflowError when decoding the prompt_ids.
386 + # Therefore, we need to do an early replacement here
387 + token_ids = processed_outputs['input_ids']
388 + token_ids[token_ids < 0] = _IMAGE_TOKEN_ID
389 + processed_outputs['input_ids'] = token_ids
390 + return processed_outputs
391 +
392 + def _get_dummy_mm_kwargs(
393 + self,
394 + mm_counts: Mapping[str, int],
395 + ) -> MultiModalKwargs:
396 + return dummy_mm_kwargs_for_phi3v(self.ctx, mm_counts)
397
398
527 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
399 @MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_phi3v_image_tokens)
529 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_phi3v)
530 -@INPUT_REGISTRY.register_input_processor(input_processor_for_phi3v)
400 +@MULTIMODAL_REGISTRY.register_processor(Phi3VProcessor)
401 class Phi3VForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
402
403 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/multimodal/processing.py
+25 -4
@@ -3,7 +3,8 @@ from abc import ABC, abstractmethod
3 from collections.abc import Callable, ItemsView, Iterable, Mapping, Sequence
4 from dataclasses import dataclass
5 from functools import lru_cache
6 -from typing import Any, Generic, NamedTuple, Optional, Protocol, TypeVar, Union
6 +from typing import (Any, Dict, Generic, NamedTuple, Optional, Protocol,
7 + TypeVar, Union, cast)
8
9 import torch
10 from transformers import BatchFeature, ProcessorMixin
@@ -11,7 +12,8 @@ from typing_extensions import TypeAlias, TypedDict
12
13 from vllm.inputs import DummyData, InputProcessingContext
14 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
14 -from vllm.utils import flatten_2d_lists, full_groupby, is_list_of
15 +from vllm.utils import (flatten_2d_lists, full_groupby, is_list_of,
16 + resolve_mm_processor_kwargs)
17
18 from .inputs import (AudioItem, ImageItem, MultiModalDataDict,
19 MultiModalInputsV2, MultiModalKwargs, PlaceholderRange,
@@ -543,8 +545,14 @@ class BaseMultiModalProcessor(ABC):
545
546 self.ctx = ctx
547 self.metadata = metadata
548 + self.init_mm_processor_kwargs = (ctx.model_config.mm_processor_kwargs
549 + or {})
550
547 - def _get_hf_processor(self) -> ProcessorMixin:
551 + def _get_hf_processor(
552 + self,
553 + **mm_processor_kwargs: Mapping[str, object],
554 + ) -> ProcessorMixin:
555 + # by default, we won't pass any kwargs to the processor initialization
556 return self.ctx.get_hf_processor()
557
558 def _get_tokenizer(self) -> AnyTokenizer:
@@ -581,7 +589,13 @@ class BaseMultiModalProcessor(ABC):
589 mm_data: MultiModalDataDict,
590 mm_processor_kwargs: Mapping[str, object],
591 ) -> BatchFeature:
584 - hf_processor = self._get_hf_processor()
592 + # some mm_processor_kwargs may be used in processor initialization
593 + # instead of processor call
594 + processor_init_kwargs = {
595 + **self.init_mm_processor_kwargs,
596 + **mm_processor_kwargs,
597 + }
598 + hf_processor = self._get_hf_processor(**processor_init_kwargs)
599
600 processor_data = dict[str, Any]()
601 passthrough_data = dict[str, Any]()
@@ -601,6 +615,13 @@ class BaseMultiModalProcessor(ABC):
615 else:
616 processor_data[k] = v
617
618 + # filter mm_processor_kwargs used in processor call
619 + mm_processor_kwargs = resolve_mm_processor_kwargs(
620 + self.init_mm_processor_kwargs,
621 + cast(Dict[str, Any], mm_processor_kwargs),
622 + hf_processor,
623 + )
624 +
625 try:
626 hf_inputs = hf_processor(
627 text=prompt, # type: ignore