@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1ff4aed5bd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model] Expose size to Idefics3 as mm_processor_kwargs (#10146)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 8, 2024 at 17:56 UTC 1ff4aed5bddd995c5a2847993e2fb5be88763872
4 files changed +270 -23
examples/offline_inference_vision_language.py
+14 -5
@@ -382,10 +382,19 @@ def run_idefics3(question: str, modality: str):
382 assert modality == "image"
383 model_name = "HuggingFaceM4/Idefics3-8B-Llama3"
384
385 - llm = LLM(model=model_name,
386 - max_model_len=8192,
387 - max_num_seqs=2,
388 - enforce_eager=True)
385 + llm = LLM(
386 + model=model_name,
387 + max_model_len=8192,
388 + max_num_seqs=2,
389 + enforce_eager=True,
390 + # if you are running out of memory, you can reduce the "longest_edge".
391 + # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
392 + mm_processor_kwargs={
393 + "size": {
394 + "longest_edge": 3 * 364
395 + },
396 + },
397 + )
398 prompt = (
399 f"<|begin_of_text|>User:<image>{question}<end_of_utterance>\nAssistant:"
400 )
@@ -518,4 +527,4 @@ if __name__ == "__main__":
527 default=16,
528 help='Number of frames to extract from the video.')
529 args = parser.parse_args()
521 - main(args)
\ No newline at end of file
530 + main(args)
examples/offline_inference_vision_language_multi_image.py
+7
@@ -300,6 +300,13 @@ def load_idefics3(question, image_urls: List[str]) -> ModelRequestData:
300 max_num_seqs=16,
301 enforce_eager=True,
302 limit_mm_per_prompt={"image": len(image_urls)},
303 + # if you are running out of memory, you can reduce the "longest_edge".
304 + # see: https://huggingface.co/HuggingFaceM4/Idefics3-8B-Llama3#model-optimizations
305 + mm_processor_kwargs={
306 + "size": {
307 + "longest_edge": 2 * 364
308 + },
309 + },
310 )
311
312 placeholders = "\n".join(f"Image-{i}: <image>\n"
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_idefics3.py new
+187
@@ -0,0 +1,187 @@
1 +"""Tests for Idefics3's multimodal preprocessing kwargs."""
2 +from typing import Optional
3 +
4 +import pytest
5 +import torch
6 +import transformers
7 +from transformers import AutoImageProcessor, AutoTokenizer
8 +
9 +from vllm.inputs import InputContext, token_inputs
10 +from vllm.multimodal import MultiModalRegistry
11 +
12 +from .....conftest import _ImageAssets
13 +from ....utils import build_model_context
14 +
15 +models = ["HuggingFaceM4/Idefics3-8B-Llama3"]
16 +
17 +
18 +# Wrap lazy imports to avoid initializing CUDA during test collection
19 +@pytest.fixture()
20 +def input_processor_for_idefics3():
21 + from vllm.model_executor.models.idefics3 import (
22 + input_processor_for_idefics3)
23 + return input_processor_for_idefics3
24 +
25 +
26 +@pytest.fixture()
27 +def dummy_data_for_idefics3():
28 + from vllm.model_executor.models.idefics3 import dummy_data_for_idefics3
29 + return dummy_data_for_idefics3
30 +
31 +
32 +@pytest.fixture()
33 +def get_max_idefics3_image_tokens():
34 + from vllm.model_executor.models.idefics3 import (
35 + get_max_idefics3_image_tokens)
36 + return get_max_idefics3_image_tokens
37 +
38 +
39 +@pytest.mark.skipif(transformers.__version__ < "4.46.0",
40 + reason="Model introduced in HF >= 4.46.0")
41 +@pytest.mark.parametrize("model", models)
42 +@pytest.mark.parametrize("longest_edge", [None, 168, 336, 400, 2 * 336])
43 +def test_input_mapper_override(model: str, image_assets: _ImageAssets,
44 + longest_edge: Optional[int]):
45 + """Ensure that the [default] input mapper handles size properly."""
46 +
47 + mm_processor_kwargs = {
48 + "size": {
49 + "longest_edge": longest_edge
50 + }
51 + } if longest_edge is not None else {}
52 + ctx = build_model_context(
53 + model_name=model,
54 + tokenizer_name=model,
55 + trust_remote_code=True,
56 + mm_processor_kwargs=mm_processor_kwargs,
57 + )
58 +
59 + hf_processor = AutoImageProcessor.from_pretrained(model,
60 + trust_remote_code=True,
61 + **mm_processor_kwargs)
62 +
63 + mm_registry = MultiModalRegistry()
64 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
65 +
66 + image = image_assets[0].pil_image
67 + hf_result = hf_processor.preprocess(
68 + image,
69 + return_tensors="pt",
70 + )
71 +
72 + vllm_result = mm_registry.map_input(
73 + ctx.model_config,
74 + {"image": image},
75 + )
76 +
77 + assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
78 +
79 +
80 +@pytest.mark.skipif(transformers.__version__ < "4.46.0",
81 + reason="Model introduced in HF >= 4.46.0")
82 +@pytest.mark.parametrize("model", models)
83 +@pytest.mark.parametrize("longest_edge, expected_max_tokens", [
84 + (None, 2873),
85 + (168, 169),
86 + (336, 169),
87 + (400, 338),
88 + (672, 338),
89 +])
90 +def test_max_tokens_override(get_max_idefics3_image_tokens, model: str,
91 + longest_edge: Optional[int],
92 + expected_max_tokens: int):
93 + """Ensure get_max_idefics3_image_tokens handles mm_processor_kwargs."""
94 + size = {"longest_edge": longest_edge} if longest_edge is not None else None
95 + ctx = build_model_context(
96 + model_name=model,
97 + tokenizer_name=model,
98 + trust_remote_code=True,
99 + mm_processor_kwargs=None,
100 + )
101 +
102 + actual_max_tokens = get_max_idefics3_image_tokens(
103 + ctx=InputContext(ctx.model_config),
104 + size=size,
105 + )
106 +
107 + assert expected_max_tokens == actual_max_tokens
108 +
109 +
110 +@pytest.mark.skipif(transformers.__version__ < "4.46.0",
111 + reason="Model introduced in HF >= 4.46.0")
112 +@pytest.mark.parametrize("model", models)
113 +@pytest.mark.parametrize("longest_edge, toks_per_img, num_imgs", [
114 + (168, 169, 1),
115 + (168, 169, 2),
116 + (400, 338, 1),
117 + (400, 338, 2),
118 +])
119 +def test_dummy_data_override(dummy_data_for_idefics3, model: str,
120 + longest_edge: int, toks_per_img: int,
121 + num_imgs: int):
122 + """Ensure dummy_data_for_idefics3 handles num_crops properly."""
123 + # Same as the previous test - don't initialize mm_processor_kwargs
124 + # in this test and assume that the kwargs will be correctly expanded by
125 + # the partial when calling the dummy data func.
126 + size = {"longest_edge": longest_edge} if longest_edge is not None else None
127 + ctx = build_model_context(
128 + model_name=model,
129 + tokenizer_name=model,
130 + trust_remote_code=True,
131 + mm_processor_kwargs=None,
132 + )
133 +
134 + dummy_data = dummy_data_for_idefics3(
135 + ctx=ctx,
136 + seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137 + mm_counts={"image": num_imgs},
138 + size=size)
139 + sequence_data = dummy_data.seq_data
140 + # Ensure we have the right number of placeholders per size
141 + image_token_id = ctx.get_hf_config().image_token_id
142 + img_tok_count = sequence_data.get_token_ids().count(image_token_id)
143 + assert img_tok_count == toks_per_img * num_imgs
144 +
145 +
146 +@pytest.mark.skipif(transformers.__version__ < "4.46.0",
147 + reason="Model introduced in HF >= 4.46.0")
148 +@pytest.mark.parametrize("model", models)
149 +@pytest.mark.parametrize("longest_edge,expected_toks_per_img,num_imgs", [
150 + (336, 169 * (1**2 + 1), 1),
151 + (336, 169 * (1**2 + 1), 2),
152 + (400, 169 * (2**2 + 1), 1),
153 + (400, 169 * (2**2 + 1), 2),
154 +])
155 +def test_input_processor_override(input_processor_for_idefics3,
156 + image_assets: _ImageAssets, model: str,
157 + longest_edge: int,
158 + expected_toks_per_img: int, num_imgs: int):
159 + """Ensure input_processor_for_idefics3 handles num_crops properly."""
160 + # Same as the previous test - don't initialize mm_processor_kwargs
161 + # in this test and assume that the kwargs will be correctly expanded by
162 + # the partial when calling the custom input processor.
163 + size = {"longest_edge": longest_edge} if longest_edge is not None else None
164 + ctx = build_model_context(
165 + model_name=model,
166 + tokenizer_name=model,
167 + trust_remote_code=True,
168 + mm_processor_kwargs=None,
169 + )
170 +
171 + # Build the image str / prompt based on the number of images we pass
172 + tokenizer = AutoTokenizer.from_pretrained(model)
173 + placeholders = "<image>" if num_imgs == 1 else "\n".join(
174 + f"Image-{i}: <image>\n" for i in range(1, num_imgs + 1))
175 + prompt = f"<|begin_of_text|>User:{placeholders}\n<end_of_utterance>\nAssistant:" # noqa: E501
176 + images = [image_assets[0].pil_image.resize((336 * 4, 336 * 4))] * num_imgs
177 +
178 + inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
179 + prompt=prompt,
180 + multi_modal_data={"image": images})
181 +
182 + processed_inputs = input_processor_for_idefics3(ctx, inputs, size=size)
183 +
184 + # Ensure we have the right number of placeholders per num_crops size
185 + image_token_id = ctx.get_hf_config().image_token_id
186 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
187 + assert img_tok_count == expected_toks_per_img * num_imgs
vllm/model_executor/models/idefics3.py
+62 -18
@@ -14,8 +14,8 @@
14 """Inference-only Idefics3 model compatible with HuggingFace weights."""
15
16 import math
17 -from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
18 - TypedDict, Union)
17 +from typing import (Dict, Iterable, List, Literal, Mapping, NamedTuple,
18 + Optional, Tuple, TypedDict, Union)
19
20 import torch
21 import torch.utils.checkpoint
@@ -23,6 +23,7 @@ from PIL import Image
23 from torch import nn
24 # Temporary solution for transformers below 4.46.0.
25 from transformers import PretrainedConfig as Idefics3Config
26 +from transformers import ProcessorMixin as Idefics3ImageProcessor
27
28 from vllm.attention import AttentionMetadata
29 from vllm.config import CacheConfig, MultiModalConfig
@@ -72,16 +73,41 @@ class Idefics3ImageEmbeddingInputs(TypedDict):
73 """
74
75
76 +class Idefics3ProcessorSize(NamedTuple):
77 + """Hashable wrapper for unhashable `size` dict of Idefics3Processor."""
78 + # NOTE: cached_get_processor/cached_get_image_processor uses lru_cache,
79 + # we need to use NamedTuple instead of TypedDict to avoid hashing issues.
80 + longest_edge: int
81 +
82 + def __contains__(self, key: str) -> bool:
83 + return key in self._asdict() and getattr(self, key) is not None
84 +
85 + def __getitem__(self, key: str) -> int:
86 + return getattr(self, key)
87 +
88 +
89 ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
90
91
92 +def get_mm_processor_kwargs(size: Optional[Dict[str, int]] = None) -> Dict:
93 + mm_processor_kwargs = {}
94 + if size:
95 + mm_processor_kwargs["size"] = Idefics3ProcessorSize(**size)
96 + return mm_processor_kwargs
97 +
98 +
99 def input_mapper_for_idefics3(
100 ctx: InputContext,
101 data: object,
102 + *,
103 + size: Optional[Dict[str, int]] = None,
104 ):
105 model_config = ctx.model_config
106 + mm_processor_kwargs = get_mm_processor_kwargs(size)
107 image_processor = cached_get_image_processor(
84 - model_config.model, trust_remote_code=model_config.trust_remote_code)
108 + model_config.model,
109 + trust_remote_code=model_config.trust_remote_code,
110 + **mm_processor_kwargs)
111 if image_processor is None:
112 raise RuntimeError("No HuggingFace processor is available "
113 "to process the image object")
@@ -201,13 +227,17 @@ def _get_image_prompt_string(image_rows: int, image_cols: int,
227 global_img_token)
228
229
204 -def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
230 +def input_processor_for_idefics3(ctx: InputContext,
231 + inputs: DecoderOnlyInputs,
232 + *,
233 + size: Optional[Dict[str, int]] = None):
234 multi_modal_data = inputs.get("multi_modal_data")
235 if multi_modal_data is None or "image" not in multi_modal_data:
236 return inputs
237
238 model_config = ctx.model_config
210 - processor = cached_get_processor(model_config.model)
239 + mm_processor_kwargs = get_mm_processor_kwargs(size)
240 + processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
241 image_processor = processor.image_processor
242 tokenizer = processor.tokenizer
243 size = image_processor.size['longest_edge']
@@ -286,32 +316,46 @@ def input_processor_for_idefics3(ctx: InputContext, inputs: DecoderOnlyInputs):
316 )
317
318
289 -def get_max_idefics3_image_tokens(ctx: InputContext,
290 - *,
291 - num_crops: Optional[int] = None):
292 - model_config = ctx.model_config
293 - processor = cached_get_processor(model_config.model)
294 - image_seq_len = processor.image_seq_len
295 - image_processor = processor.image_processor
296 -
319 +def _get_max_num_image_patch(image_processor: Idefics3ImageProcessor) -> int:
320 size = image_processor.size['longest_edge']
321 max_image_size = image_processor.max_image_size['longest_edge']
322 resized_height, resized_width = size, size
323
324 grid_h = resized_height // max_image_size
325 grid_w = resized_width // max_image_size
326 + return (grid_h * grid_w + 1)
327 +
328 +
329 +def get_max_idefics3_image_tokens(ctx: InputContext,
330 + *,
331 + size: Optional[Dict[str,
332 + int]] = None) -> int:
333 + model_config = ctx.model_config
334 + mm_processor_kwargs = get_mm_processor_kwargs(size)
335 + processor = cached_get_processor(model_config.model, **mm_processor_kwargs)
336 + image_seq_len = processor.image_seq_len
337 + image_processor = processor.image_processor
338 +
339 + max_num_image_patches = _get_max_num_image_patch(image_processor)
340
304 - return (grid_h * grid_w + 1) * image_seq_len
341 + return max_num_image_patches * image_seq_len
342
343
307 -def dummy_data_for_idefics3(ctx: InputContext, seq_len: int,
308 - mm_counts: Mapping[str, int]) -> DummyData:
344 +def dummy_data_for_idefics3(
345 + ctx: InputContext,
346 + seq_len: int,
347 + mm_counts: Mapping[str, int],
348 + *,
349 + size: Optional[Dict[str, int]] = None) -> DummyData:
350 hf_config = ctx.get_hf_config()
351 num_images = mm_counts["image"]
352
312 - processor = cached_get_processor(ctx.model_config.model)
353 + mm_processor_kwargs = get_mm_processor_kwargs(size)
354 + processor = cached_get_processor(ctx.model_config.model,
355 + **mm_processor_kwargs)
356 + max_num_image_patches = _get_max_num_image_patch(processor.image_processor)
357 image_seq_len = processor.image_seq_len
314 - max_llm_image_tokens = 17 * image_seq_len * num_images
358 + max_llm_image_tokens = max_num_image_patches * image_seq_len * num_images
359
360 seq_data = SequenceData.from_prompt_token_counts(
361 (hf_config.image_token_id, max_llm_image_tokens), (0, seq_len))