A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model] Expose size to Idefics3 as mm_processor_kwargs (#10146)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 8, 2024 at 17:56 UTC
1ff4aed5bddd995c5a2847993e2fb5be88763872
4 files changed
+270
-23
examples/offline_inference_vision_language.py
+14
-5
@@ -382,10 +382,19 @@ def run_idefics3(question: str, modality: str):
382
assert modality == "image"
383
model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
384
385
- llm = LLM(model=model_name,
386
- max_model_len=8192,
387
- max_num_seqs=2,
388
- enforce_eager=True)
385
+ llm = LLM(
386
+ model=model_name,
387
+ max_model_len=8192,
388
+ max_num_seqs=2,
389
+ enforce_eager=True,
390
+ # if you are running out of memory, you can reduce the "longest_edge".
391
+ # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
392
+ mm_processor_kwargs={
393
+ "size": {
394
+ "longest_edge": 3 * 364
395
+ },
396
+ },
397
+ )
398
prompt = (
399
f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
400
)
@@ -518,4 +527,4 @@ if __name__ == "__main__":
527
default=16,
528
help='Number of frames to extract from the video.')
529
args = parser.parse_args()
521
- main(args)
\ No newline at end of file
530
+ main(args)
examples/offline_inference_vision_language_multi_image.py
+7
@@ -300,6 +300,13 @@ def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
300
max_num_seqs=16,
301
enforce_eager=True,
302
limit_mm_per_prompt={"image": len(image_urls)},
303
+ # if you are running out of memory, you can reduce the "longest_edge".
304
+ # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
305
+ mm_processor_kwargs={
306
+ "size": {
307
+ "longest_edge": 2 * 364
308
+ },
309
+ },
310
)
311
312
placeholders = "\n".join(f"Image-{i}: <image>\n"
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_idefics3.py
new
+187
@@ -0,0 +1,187 @@
1
+"""Tests for Idefics3's multimodal preprocessing kwargs."""
2
+from typing import Optional
3
+
4
+import pytest
5
+import torch
6
+import transformers
7
+from transformers import AutoImageProcessor, AutoTokenizer
8
+
9
+from vllm.inputs import InputContext, token_inputs
10
+from vllm.multimodal import MultiModalRegistry
11
+
12
+from .....conftest import _ImageAssets
13
+from ....utils import build_model_context
14
+
15
+models = ["HuggingFaceM4/Idefics3-8B-Llama3"]
16
+
17
+
18
+# Wrap lazy imports to avoid initializing CUDA during test collection
19
+@pytest.fixture()
20
+def input_processor_for_idefics3():
21
+ from vllm.model_executor.models.idefics3 import (
22
+ input_processor_for_idefics3)
23
+ return input_processor_for_idefics3
24
+
25
+
26
+@pytest.fixture()
27
+def dummy_data_for_idefics3():
28
+ from vllm.model_executor.models.idefics3 import dummy_data_for_idefics3
29
+ return dummy_data_for_idefics3
30
+
31
+
32
+@pytest.fixture()
33
+def get_max_idefics3_image_tokens():
34
+ from vllm.model_executor.models.idefics3 import (
35
+ get_max_idefics3_image_tokens)
36
+ return get_max_idefics3_image_tokens
37
+
38
+
39
+@pytest.mark.skipif(transformers.__version__ < "4.46.0",
40
+ reason="Model introduced in HF >= 4.46.0")
41
+@pytest.mark.parametrize("model", models)
42
+@pytest.mark.parametrize("longest_edge", [None, 168, 336, 400, 2 * 336])
43
+def test_input_mapper_override(model: str, image_assets: _ImageAssets,
44
+ longest_edge: Optional[int]):
45
+ """Ensure that the [default] input mapper handles size properly."""
46
+
47
+ mm_processor_kwargs = {
48
+ "size": {
49
+ "longest_edge": longest_edge
50
+ }
51
+ } if longest_edge is not None else {}
52
+ ctx = build_model_context(
53
+ model_name=model,
54
+ tokenizer_name=model,
55
+ trust_remote_code=True,
56
+ mm_processor_kwargs=mm_processor_kwargs,
57
+ )
58
+
59
+ hf_processor = AutoImageProcessor.from_pretrained(model,
60
+ trust_remote_code=True,
61
+ **mm_processor_kwargs)
62
+
63
+ mm_registry = MultiModalRegistry()
64
+ mm_registry.init_mm_limits_per_prompt(ctx.model_config)
65
+
66
+ image = image_assets[0].pil_image
67
+ hf_result = hf_processor.preprocess(
68
+ image,
69
+ return_tensors="pt",
70
+ )
71
+
72
+ vllm_result = mm_registry.map_input(
73
+ ctx.model_config,
74
+ {"image": image},
75
+ )
76
+
77
+ assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
78
+
79
+
80
+@pytest.mark.skipif(transformers.__version__ < "4.46.0",
81
+ reason="Model introduced in HF >= 4.46.0")
82
+@pytest.mark.parametrize("model", models)
83
+@pytest.mark.parametrize("longest_edge, expected_max_tokens", [
84
+ (None, 2873),
85
+ (168, 169),
86
+ (336, 169),
87
+ (400, 338),
88
+ (672, 338),
89
+])
90
+def test_max_tokens_override(get_max_idefics3_image_tokens, model: str,
91
+ longest_edge: Optional[int],
92
+ expected_max_tokens: int):
93
+ """Ensure get_max_idefics3_image_tokens handles mm_processor_kwargs."""
94
+ size = {"longest_edge": longest_edge} if longest_edge is not None else None
95
+ ctx = build_model_context(
96
+ model_name=model,
97
+ tokenizer_name=model,
98
+ trust_remote_code=True,
99
+ mm_processor_kwargs=None,
100
+ )
101
+
102
+ actual_max_tokens = get_max_idefics3_image_tokens(
103
+ ctx=InputContext(ctx.model_config),
104
+ size=size,
105
+ )
106
+
107
+ assert expected_max_tokens == actual_max_tokens
108
+
109
+
110
+@pytest.mark.skipif(transformers.__version__ < "4.46.0",
111
+ reason="Model introduced in HF >= 4.46.0")
112
+@pytest.mark.parametrize("model", models)
113
+@pytest.mark.parametrize("longest_edge, toks_per_img, num_imgs", [
114
+ (168, 169, 1),
115
+ (168, 169, 2),
116
+ (400, 338, 1),
117
+ (400, 338, 2),
118
+])
119
+def test_dummy_data_override(dummy_data_for_idefics3, model: str,
120
+ longest_edge: int, toks_per_img: int,
121
+ num_imgs: int):
122
+ """Ensure dummy_data_for_idefics3 handles num_crops properly."""
123
+ # Same as the previous test - don't initialize mm_processor_kwargs
124
+ # in this test and assume that the kwargs will be correctly expanded by
125
+ # the partial when calling the dummy data func.
126
+ size = {"longest_edge": longest_edge} if longest_edge is not None else None
127
+ ctx = build_model_context(
128
+ model_name=model,
129
+ tokenizer_name=model,
130
+ trust_remote_code=True,
131
+ mm_processor_kwargs=None,
132
+ )
133
+
134
+ dummy_data = dummy_data_for_idefics3(
135
+ ctx=ctx,
136
+ seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137
+ mm_counts={"image": num_imgs},
138
+ size=size)
139
+ sequence_data = dummy_data.seq_data
140
+ # Ensure we have the right number of placeholders per size
141
+ image_token_id = ctx.get_hf_config().image_token_id
142
+ img_tok_count = sequence_data.get_token_ids().count(image_token_id)
143
+ assert img_tok_count == toks_per_img * num_imgs
144
+
145
+
146
+@pytest.mark.skipif(transformers.__version__ < "4.46.0",
147
+ reason="Model introduced in HF >= 4.46.0")
148
+@pytest.mark.parametrize("model", models)
149
+@pytest.mark.parametrize("longest_edge,expected_toks_per_img,num_imgs", [
150
+ (336, 169 * (1**2 + 1), 1),
151
+ (336, 169 * (1**2 + 1), 2),
152
+ (400, 169 * (2**2 + 1), 1),
153
+ (400, 169 * (2**2 + 1), 2),
154
+])
155
+def test_input_processor_override(input_processor_for_idefics3,
156
+ image_assets: _ImageAssets, model: str,
157
+ longest_edge: int,
158
+ expected_toks_per_img: int, num_imgs: int):
159
+ """Ensure input_processor_for_idefics3 handles num_crops properly."""
160
+ # Same as the previous test - don't initialize mm_processor_kwargs
161
+ # in this test and assume that the kwargs will be correctly expanded by
162
+ # the partial when calling the custom input processor.
163
+ size = {"longest_edge": longest_edge} if longest_edge is not None else None
164
+ ctx = build_model_context(
165
+ model_name=model,
166
+ tokenizer_name=model,
167
+ trust_remote_code=True,
168
+ mm_processor_kwargs=None,
169
+ )
170
+
171
+ # Build the image str / prompt based on the number of images we pass
172
+ tokenizer = AutoTokenizer.from_pretrained(model)
173
+ placeholders = "<image>" if num_imgs == 1 else "\n".join(
174
+ f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
175
+ prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
176
+ images = [image_assets[0].pil_image.resize((336 * 4, 336 * 4))] * num_imgs
177
+
178
+ inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
179
+ prompt=prompt,
180
+ multi_modal_data={"image": images})
181
+
182
+ processed_inputs = input_processor_for_idefics3(ctx, inputs, size=size)
183
+
184
+ # Ensure we have the right number of placeholders per num_crops size
185
+ image_token_id = ctx.get_hf_config().image_token_id
186
+ img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
187
+ assert img_tok_count == expected_toks_per_img * num_imgs
vllm/model_executor/models/idefics3.py
+62
-18
@@ -14,8 +14,8 @@
14
"""Inference-only Idefics3 model compatible with HuggingFace weights."""
15
16
import math
17
-from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18
- TypedDict, Union)
17
+from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
18
+ Optional, Tuple, TypedDict, Union)
19
20
import torch
21
import torch.utils.checkpoint
@@ -23,6 +23,7 @@ from PIL import Image
23
from torch import nn
24
# Temporary solution for transformers below 4.46.0.
25
from transformers import PretrainedConfig as Idefics3Config
26
+from transformers import ProcessorMixin as Idefics3ImageProcessor
27
28
from vllm.attention import AttentionMetadata
29
from vllm.config import CacheConfig, MultiModalConfig
@@ -72,16 +73,41 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
73
"""
74
75
76
+class Idefics3ProcessorSize(NamedTuple):
77
+ """Hashable wrapper for unhashable `size` dict of Idefics3Processor."""
78
+ # NOTE: cached_get_processor/cached_get_image_processor uses lru_cache,
79
+ # we need to use NamedTuple instead of TypedDict to avoid hashing issues.
80
+ longest_edge: int
81
+
82
+ def __contains__(self, key: str) -> bool:
83
+ return key in self._asdict() and getattr(self, key) is not None
84
+
85
+ def __getitem__(self, key: str) -> int:
86
+ return getattr(self, key)
87
+
88
+
89
ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
90
91
92
+def get_mm_processor_kwargs(size: Optional[Dict[str, int]] = None) -> Dict:
93
+ mm_processor_kwargs = {}
94
+ if size:
95
+ mm_processor_kwargs["size"] = Idefics3ProcessorSize(**size)
96
+ return mm_processor_kwargs
97
+
98
+
99
def input_mapper_for_idefics3(
100
ctx: InputContext,
101
data: object,
102
+ *,
103
+ size: Optional[Dict[str, int]] = None,
104
):
105
model_config = ctx.model_config
106
+ mm_processor_kwargs = get_mm_processor_kwargs(size)
107
image_processor = cached_get_image_processor(
84
- model_config.model, trust_remote_code=model_config.trust_remote_code)
108
+ model_config.model,
109
+ trust_remote_code=model_config.trust_remote_code,
110
+ **mm_processor_kwargs)
111
if image_processor is None:
112
raise RuntimeError("No HuggingFace processor is available "
113
"to process the image object")
@@ -201,13 +227,17 @@ def _get_image_prompt_string(image_rows: int, image_cols: int,
227
global_img_token)
228
229
204
-def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
230
+def input_processor_for_idefics3(ctx: InputContext,
231
+ inputs: DecoderOnlyInputs,
232
+ *,
233
+ size: Optional[Dict[str, int]] = None):
234
multi_modal_data = inputs.get("multi_modal_data")
235
if multi_modal_data is None or "image" not in multi_modal_data:
236
return inputs
237
238
model_config = ctx.model_config
210
- processor = cached_get_processor(model_config.model)
239
+ mm_processor_kwargs = get_mm_processor_kwargs(size)
240
+ processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
241
image_processor = processor.image_processor
242
tokenizer = processor.tokenizer
243
size = image_processor.size['longest_edge']
@@ -286,32 +316,46 @@ def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
316
)
317
318
289
-def get_max_idefics3_image_tokens(ctx: InputContext,
290
- *,
291
- num_crops: Optional[int] = None):
292
- model_config = ctx.model_config
293
- processor = cached_get_processor(model_config.model)
294
- image_seq_len = processor.image_seq_len
295
- image_processor = processor.image_processor
296
-
319
+def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int:
320
size = image_processor.size['longest_edge']
321
max_image_size = image_processor.max_image_size['longest_edge']
322
resized_height, resized_width = size, size
323
324
grid_h = resized_height // max_image_size
325
grid_w = resized_width // max_image_size
326
+ return (grid_h * grid_w + 1)
327
+
328
+
329
+def get_max_idefics3_image_tokens(ctx: InputContext,
330
+ *,
331
+ size: Optional[Dict[str,
332
+ int]] = None) -> int:
333
+ model_config = ctx.model_config
334
+ mm_processor_kwargs = get_mm_processor_kwargs(size)
335
+ processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
336
+ image_seq_len = processor.image_seq_len
337
+ image_processor = processor.image_processor
338
+
339
+ max_num_image_patches = _get_max_num_image_patch(image_processor)
340
304
- return (grid_h * grid_w + 1) * image_seq_len
341
+ return max_num_image_patches * image_seq_len
342
343
307
-def dummy_data_for_idefics3(ctx: InputContext, seq_len: int,
308
- mm_counts: Mapping[str, int]) -> DummyData:
344
+def dummy_data_for_idefics3(
345
+ ctx: InputContext,
346
+ seq_len: int,
347
+ mm_counts: Mapping[str, int],
348
+ *,
349
+ size: Optional[Dict[str, int]] = None) -> DummyData:
350
hf_config = ctx.get_hf_config()
351
num_images = mm_counts["image"]
352
312
- processor = cached_get_processor(ctx.model_config.model)
353
+ mm_processor_kwargs = get_mm_processor_kwargs(size)
354
+ processor = cached_get_processor(ctx.model_config.model,
355
+ **mm_processor_kwargs)
356
+ max_num_image_patches = _get_max_num_image_patch(processor.image_processor)
357
image_seq_len = processor.image_seq_len
314
- max_llm_image_tokens = 17 * image_seq_len * num_images
358
+ max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
359
360
seq_data = SequenceData.from_prompt_token_counts(
361
(hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))