@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 377d10bd14
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 22:15 UTC

[VLM][Bugfix] Pass processor kwargs properly on init (#13516)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 19, 2025 at 21:13 UTC 377d10bd1456268350b20a37e27a25257d0de0b3
44 files changed +675 -453
examples/offline_inference/vision_language_multi_image.py
+1
@@ -85,6 +85,7 @@ def load_h2ovl(question: str, image_urls: List[str]) -> ModelRequestData:
85 trust_remote_code=True,
86 max_model_len=8192,
87 limit_mm_per_prompt={"image": len(image_urls)},
88 + mm_processor_kwargs={"max_dynamic_patch": 4},
89 )
90
91 placeholders = "\n".join(f"Image-{i}: <image>\n"
tests/models/multimodal/processing/test_common.py
+2 -5
@@ -10,7 +10,7 @@ from vllm.config import ModelConfig
10 from vllm.inputs import InputProcessingContext
11 from vllm.multimodal import MULTIMODAL_REGISTRY
12 from vllm.multimodal.processing import ProcessingCache
13 -from vllm.multimodal.utils import cached_get_tokenizer
13 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
14
15 from ....multimodal.utils import random_audio, random_image, random_video
16 from ...registry import HF_EXAMPLE_MODELS
@@ -42,10 +42,7 @@ def _test_processing_correctness(
42 factories = MULTIMODAL_REGISTRY._processor_factories[model_cls]
43 ctx = InputProcessingContext(
44 model_config,
45 - tokenizer=cached_get_tokenizer(
46 - model_config.tokenizer,
47 - trust_remote_code=model_info.trust_remote_code,
48 - ),
45 + tokenizer=cached_tokenizer_from_config(model_config),
46 )
47 # Ensure that it can fit all of the data
48 cache = ProcessingCache(capacity=1 << 30)
tests/models/multimodal/processing/test_h2ovl.py
+131 -94
@@ -1,17 +1,118 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """Tests for H2OVL's multimodal preprocessing kwargs."""
3 -from typing import Optional
3 +from typing import Mapping, Optional
4
5 import pytest
6 +from PIL import Image
7 +from transformers import PretrainedConfig
8
9 from vllm.multimodal import MULTIMODAL_REGISTRY
10 from vllm.multimodal.image import rescale_image_size
9 -from vllm.multimodal.utils import cached_get_tokenizer
11 +from vllm.multimodal.processing import BaseMultiModalProcessor
12 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
13
14 from ....conftest import _ImageAssets
15 from ...utils import build_model_context
16
17
18 +def _get_expected_num_patches(
19 + config: PretrainedConfig,
20 + image: Image.Image,
21 + num_imgs: int,
22 + min_num: int,
23 + max_num: int,
24 +):
25 + from vllm.model_executor.models.h2ovl import (calculate_h2ovl_targets,
26 + get_h2ovl_target_ratios)
27 +
28 + width, height = image.size
29 +
30 + # Calculate the expected number of blocks
31 + if num_imgs == 1 and config.use_msac:
32 + # First pass
33 + blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
34 + orig_width=width,
35 + orig_height=height,
36 + target_ratios=get_h2ovl_target_ratios(
37 + min_num=1,
38 + max_num=max_num,
39 + prior_aspect_ratio=None,
40 + ),
41 + image_size=config.vision_config.image_size,
42 + use_thumbnail=False, # Thumbnail is handled separately
43 + )
44 +
45 + # Second pass
46 + blocks2, _, _, _ = calculate_h2ovl_targets(
47 + orig_width=width,
48 + orig_height=height,
49 + target_ratios=get_h2ovl_target_ratios(
50 + min_num=3,
51 + max_num=max_num,
52 + prior_aspect_ratio=aspect_ratio,
53 + ),
54 + image_size=config.vision_config.image_size,
55 + use_thumbnail=False,
56 + )
57 +
58 + # Add thumbnail if use_thumbnail is True and total_blocks > 1
59 + if config.use_thumbnail:
60 + blocks1 += 1 if blocks1 > 1 else 0
61 + blocks2 += 1 if blocks2 > 1 else 0
62 +
63 + # Total blocks is the sum of blocks from both passes minus
64 + # overlapping
65 + total_blocks = blocks1 + blocks2 - 1
66 +
67 + return total_blocks
68 +
69 + blocks, _, _, _ = calculate_h2ovl_targets(
70 + orig_width=width,
71 + orig_height=height,
72 + target_ratios=get_h2ovl_target_ratios(
73 + min_num,
74 + max_num,
75 + prior_aspect_ratio=None,
76 + ),
77 + image_size=config.vision_config.image_size,
78 + use_thumbnail=False,
79 + )
80 + expected_num_patches = blocks
81 +
82 + if config.use_thumbnail and expected_num_patches > 1:
83 + expected_num_patches += 1
84 +
85 + return expected_num_patches
86 +
87 +
88 +def _run_check(
89 + processor: BaseMultiModalProcessor,
90 + images: list[Image.Image],
91 + min_num: int,
92 + max_num: int,
93 + mm_processor_kwargs: Mapping[str, object],
94 +):
95 + tokenizer = processor.info.get_tokenizer()
96 + config = processor.info.get_hf_config()
97 +
98 + mm_data = {"image": images}
99 +
100 + total_expected_num_patches = sum(
101 + _get_expected_num_patches(config, image, len(images), min_num, max_num)
102 + for image in images)
103 +
104 + processed_inputs = processor.apply("<image>" * len(images), mm_data,
105 + mm_processor_kwargs)
106 +
107 + # Ensure we have the right number of placeholders per num_crops size
108 + image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
109 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
110 + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
111 +
112 + assert img_tok_count == 256 * total_expected_num_patches
113 + assert pixel_shape[0] == total_expected_num_patches
114 +
115 +
116 @pytest.mark.parametrize("model_id", [
117 "h2oai/h2ovl-mississippi-800m",
118 "h2oai/h2ovl-mississippi-2b",
@@ -25,118 +126,54 @@ from ...utils import build_model_context
126 [1.0, 1.0, 1.0],
127 # Multi-scale
128 [0.25, 0.5, 1.0],
129 + [4.0, 2.0, 1.0],
130 ],
131 )
30 -@pytest.mark.parametrize("max_dynamic_patch", [1, 2, 4, 8])
132 +@pytest.mark.parametrize(
133 + ("min_dynamic_patch", "max_dynamic_patch"),
134 + [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
135 +)
136 @pytest.mark.parametrize("dynamic_image_size", [True, False])
32 -@pytest.mark.parametrize("num_imgs", [1, 2])
137 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
138 def test_processor_override(
139 model_id: str,
140 image_assets: _ImageAssets,
141 size_factors: list[int],
142 + min_dynamic_patch: int,
143 max_dynamic_patch: int,
144 dynamic_image_size: Optional[bool],
39 - num_imgs: int,
145 + kwargs_on_init: bool,
146 ):
41 - from vllm.model_executor.models.h2ovl import (calculate_h2ovl_targets,
42 - get_h2ovl_target_ratios)
147 + mm_processor_kwargs = {
148 + "min_dynamic_patch": min_dynamic_patch,
149 + "max_dynamic_patch": max_dynamic_patch,
150 + "dynamic_image_size": dynamic_image_size,
151 + }
152
153 ctx = build_model_context(
154 model_name=model_id,
155 tokenizer_name=model_id,
156 trust_remote_code=True,
48 - mm_processor_kwargs=None,
49 - limit_mm_per_prompt={"image": num_imgs},
50 - )
51 - tokenizer = cached_get_tokenizer(
52 - ctx.model_config.tokenizer,
53 - trust_remote_code=ctx.model_config.trust_remote_code,
157 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
158 + limit_mm_per_prompt={"image": len(size_factors)},
159 )
160 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
161 processor = MULTIMODAL_REGISTRY.create_processor(
162 ctx.model_config,
163 tokenizer=tokenizer,
164 )
165 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
166
60 - config = processor.info.get_hf_config()
61 - use_msac = config.use_msac
62 -
63 - mm_processor_kwargs = {
64 - "max_dynamic_patch": max_dynamic_patch,
65 - }
66 - if dynamic_image_size is not None:
67 - mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
68 -
69 - min_num = config.min_dynamic_patch
167 + min_num = min_dynamic_patch if dynamic_image_size else 1
168 max_num = max_dynamic_patch if dynamic_image_size else 1
169
72 - # Build the image str / prompt based on the number of images we pass
73 - prompt = "<image>" * num_imgs
74 -
75 - for asset in image_assets:
76 - for factor in size_factors:
77 - image = rescale_image_size(asset.pil_image, factor)
78 - mm_data = {"image": [image] * num_imgs}
79 -
80 - width, height = image.size
81 -
82 - # Calculate the expected number of blocks
83 - if num_imgs == 1 and use_msac:
84 - # First pass
85 - blocks1, _, _, aspect_ratio = calculate_h2ovl_targets(
86 - orig_width=width,
87 - orig_height=height,
88 - target_ratios=get_h2ovl_target_ratios(
89 - min_num,
90 - max_num,
91 - prior_aspect_ratio=None,
92 - ),
93 - image_size=config.vision_config.image_size,
94 - use_thumbnail=False, # Thumbnail is handled separately
95 - )
96 -
97 - # Second pass
98 - blocks2, _, _, _ = calculate_h2ovl_targets(
99 - orig_width=width,
100 - orig_height=height,
101 - target_ratios=get_h2ovl_target_ratios(
102 - min_num,
103 - max_num,
104 - prior_aspect_ratio=aspect_ratio,
105 - ),
106 - image_size=config.vision_config.image_size,
107 - use_thumbnail=False,
108 - )
109 -
110 - # Add thumbnail if use_thumbnail is True and total_blocks > 1
111 - if config.use_thumbnail:
112 - blocks1 += 1 if blocks1 > 1 else 0
113 - blocks2 += 1 if blocks2 > 1 else 0
114 -
115 - # Total blocks is the sum of blocks from both passes minus
116 - # overlapping
117 - total_blocks = blocks1 + blocks2 - 1
118 -
119 - expected_num_patches = total_blocks
120 - else:
121 - blocks, _, _, _ = calculate_h2ovl_targets(
122 - orig_width=width,
123 - orig_height=height,
124 - target_ratios=get_h2ovl_target_ratios(
125 - min_num,
126 - max_num,
127 - prior_aspect_ratio=None,
128 - ),
129 - image_size=config.vision_config.image_size,
130 - use_thumbnail=False,
131 - )
132 - expected_num_patches = blocks
133 -
134 - if config.use_thumbnail and expected_num_patches != 1:
135 - expected_num_patches += 1
136 -
137 - processed_inputs = processor.apply(prompt, mm_data,
138 - mm_processor_kwargs)
139 - pixel_shape = (
140 - processed_inputs["mm_kwargs"]["pixel_values_flat"].shape)
141 -
142 - assert pixel_shape[0] == expected_num_patches * num_imgs
170 + _run_check(
171 + processor,
172 + [
173 + rescale_image_size(image_assets[0].pil_image, f)
174 + for f in size_factors
175 + ],
176 + min_num,
177 + max_num,
178 + hf_processor_mm_kwargs,
179 + )
tests/models/multimodal/processing/test_idefics3.py
+16 -8
@@ -4,7 +4,7 @@ import pytest
4 from transformers import Idefics3Config
5
6 from vllm.multimodal import MULTIMODAL_REGISTRY
7 -from vllm.multimodal.utils import cached_get_tokenizer
7 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
8
9 from ....conftest import _ImageAssets
10 from ...utils import build_model_context
@@ -22,9 +22,15 @@ models = ["HuggingFaceM4/Idefics3-8B-Llama3"]
22 ])
23 # yapf: enable
24 @pytest.mark.parametrize("num_imgs", [1, 2])
25 -def test_processor_override(image_assets: _ImageAssets, model: str,
26 - mm_processor_kwargs: dict[str, object],
27 - expected_toks_per_img: int, num_imgs: int):
25 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
26 +def test_processor_override(
27 + image_assets: _ImageAssets,
28 + model: str,
29 + mm_processor_kwargs: dict[str, object],
30 + expected_toks_per_img: int,
31 + num_imgs: int,
32 + kwargs_on_init: bool,
33 +):
34 """Ensure input_processor_for_idefics3 handles num_crops properly."""
35 # Same as the previous test - don't initialize mm_processor_kwargs
36 # in this test and assume that the kwargs will be correctly expanded by
@@ -33,15 +39,15 @@ def test_processor_override(image_assets: _ImageAssets, model: str,
39 model_name=model,
40 tokenizer_name=model,
41 trust_remote_code=True,
36 - mm_processor_kwargs=None,
42 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
43 limit_mm_per_prompt={"image": num_imgs},
44 )
39 - tokenizer = cached_get_tokenizer(ctx.model_config.tokenizer)
45 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
46 processor = MULTIMODAL_REGISTRY.create_processor(
47 ctx.model_config,
48 tokenizer=tokenizer,
49 )
44 - hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
50 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
51
52 # Build the image str / prompt based on the number of images we pass
53 placeholders = "<image>" if num_imgs == 1 else "\n".join(
@@ -54,8 +60,10 @@ def test_processor_override(image_assets: _ImageAssets, model: str,
60 dummy_image = image_assets[0].pil_image.resize(dummy_image_size)
61 mm_data = {"image": [dummy_image] * num_imgs}
62
57 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
63 + processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
64 +
65 # Ensure the placeholders format are correct
66 + hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
67 hf_processed_inputs = hf_processor(text=prompt, images=mm_data["image"])
68 assert processed_inputs["prompt_token_ids"] == hf_processed_inputs[
69 "input_ids"][0]
tests/models/multimodal/processing/test_internvl.py
+107 -35
@@ -1,64 +1,136 @@
1 # SPDX-License-Identifier: Apache-2.0
2 """Tests for InternVL's multimodal preprocessing kwargs."""
3 -from typing import Optional
3 +from typing import Mapping, Optional
4
5 import pytest
6 +from PIL import Image
7 +from transformers import PretrainedConfig
8
9 from vllm.multimodal import MULTIMODAL_REGISTRY
8 -from vllm.multimodal.utils import cached_get_tokenizer
10 +from vllm.multimodal.image import rescale_image_size
11 +from vllm.multimodal.processing import BaseMultiModalProcessor
12 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
13
14 from ....conftest import _ImageAssets
15 from ...utils import build_model_context
16
17
18 +def _get_expected_num_patches(
19 + config: PretrainedConfig,
20 + image: Image.Image,
21 + num_imgs: int,
22 + min_num: int,
23 + max_num: int,
24 +):
25 + from vllm.model_executor.models.internvl import (
26 + calculate_internvl_targets, get_internvl_target_ratios)
27 +
28 + width, height = image.size
29 +
30 + blocks, _, _ = calculate_internvl_targets(
31 + orig_width=width,
32 + orig_height=height,
33 + target_ratios=get_internvl_target_ratios(
34 + min_num,
35 + max_num,
36 + ),
37 + image_size=config.vision_config.image_size,
38 + use_thumbnail=False,
39 + )
40 + expected_num_patches = blocks
41 +
42 + if config.use_thumbnail and expected_num_patches > 1:
43 + expected_num_patches += 1
44 +
45 + return expected_num_patches
46 +
47 +
48 +def _run_check(
49 + processor: BaseMultiModalProcessor,
50 + images: list[Image.Image],
51 + min_num: int,
52 + max_num: int,
53 + mm_processor_kwargs: Mapping[str, object],
54 +):
55 + tokenizer = processor.info.get_tokenizer()
56 + config = processor.info.get_hf_config()
57 +
58 + mm_data = {"image": images}
59 +
60 + total_expected_num_patches = sum(
61 + _get_expected_num_patches(config, image, len(images), min_num, max_num)
62 + for image in images)
63 +
64 + processed_inputs = processor.apply("<image>" * len(images), mm_data,
65 + mm_processor_kwargs)
66 +
67 + # Ensure we have the right number of placeholders per num_crops size
68 + image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
69 + img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
70 + pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
71 +
72 + assert img_tok_count == 256 * total_expected_num_patches
73 + assert pixel_shape[0] == total_expected_num_patches
74 +
75 +
76 @pytest.mark.parametrize("model_id", ["OpenGVLab/InternVL2-2B"])
15 -@pytest.mark.parametrize("max_dynamic_patch", [1, 4])
16 -@pytest.mark.parametrize("dynamic_image_size", [True, False, None])
17 -@pytest.mark.parametrize("num_imgs", [1, 2])
77 +@pytest.mark.parametrize(
78 + "size_factors",
79 + [
80 + # Single-scale
81 + [1.0],
82 + # Single-scale, batched
83 + [1.0, 1.0, 1.0],
84 + # Multi-scale
85 + [0.25, 0.5, 1.0],
86 + [4.0, 2.0, 1.0],
87 + ],
88 +)
89 +@pytest.mark.parametrize(
90 + ("min_dynamic_patch", "max_dynamic_patch"),
91 + [(1, 1), (1, 2), (1, 4), (1, 8), (2, 4), (4, 8)],
92 +)
93 +@pytest.mark.parametrize("dynamic_image_size", [True, False])
94 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
95 def test_processor_override(
96 model_id: str,
97 image_assets: _ImageAssets,
98 + size_factors: list[int],
99 + min_dynamic_patch: int,
100 max_dynamic_patch: int,
101 dynamic_image_size: Optional[bool],
23 - num_imgs: int,
102 + kwargs_on_init: bool,
103 ):
104 + mm_processor_kwargs = {
105 + "min_dynamic_patch": min_dynamic_patch,
106 + "max_dynamic_patch": max_dynamic_patch,
107 + "dynamic_image_size": dynamic_image_size,
108 + }
109 +
110 ctx = build_model_context(
111 model_name=model_id,
112 tokenizer_name=model_id,
113 trust_remote_code=True,
29 - mm_processor_kwargs=None,
30 - limit_mm_per_prompt={"image": num_imgs},
31 - )
32 - tokenizer = cached_get_tokenizer(
33 - ctx.model_config.tokenizer,
34 - trust_remote_code=ctx.model_config.trust_remote_code,
114 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
115 + limit_mm_per_prompt={"image": len(size_factors)},
116 )
117 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
118 processor = MULTIMODAL_REGISTRY.create_processor(
119 ctx.model_config,
120 tokenizer=tokenizer,
121 )
122 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
123
41 - mm_processor_kwargs = {
42 - "max_dynamic_patch": max_dynamic_patch,
43 - }
44 - if dynamic_image_size is not None:
45 - mm_processor_kwargs["dynamic_image_size"] = dynamic_image_size
124 + min_num = min_dynamic_patch if dynamic_image_size else 1
125 + max_num = max_dynamic_patch if dynamic_image_size else 1
126
47 - # Build the image str / prompt based on the number of images we pass
48 - prompt = "<image>" * num_imgs
49 - image = image_assets[0].pil_image.resize((448 * 2, 448 * 2))
50 - mm_data = {"image": [image] * num_imgs}
51 -
52 - expected_num_patches = max_dynamic_patch + 1 if max_dynamic_patch > 1 else 1
53 - if dynamic_image_size is False:
54 - expected_num_patches = 1
55 -
56 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
57 -
58 - # Ensure we have the right number of placeholders per num_crops size
59 - image_token_id = tokenizer.convert_tokens_to_ids("<IMG_CONTEXT>")
60 - img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
61 - pixel_shape = processed_inputs["mm_kwargs"]["pixel_values_flat"].shape
62 -
63 - assert img_tok_count == 256 * expected_num_patches * num_imgs
64 - assert pixel_shape[0] == expected_num_patches * num_imgs
127 + _run_check(
128 + processor,
129 + [
130 + rescale_image_size(image_assets[0].pil_image, f)
131 + for f in size_factors
132 + ],
133 + min_num,
134 + max_num,
135 + hf_processor_mm_kwargs,
136 + )
tests/models/multimodal/processing/test_llava_next.py
+4 -13
@@ -10,7 +10,7 @@ from pqdm.threads import pqdm
10 from vllm.multimodal import MULTIMODAL_REGISTRY
11 from vllm.multimodal.parse import ImageSize
12 from vllm.multimodal.processing import BaseMultiModalProcessor
13 -from vllm.multimodal.utils import cached_get_tokenizer
13 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
14
15 from ...utils import build_model_context
16
@@ -43,10 +43,7 @@ def test_processor_max_tokens(model_id):
43 )
44 processor = MULTIMODAL_REGISTRY.create_processor(
45 ctx.model_config,
46 - tokenizer=cached_get_tokenizer(
47 - ctx.model_config.tokenizer,
48 - trust_remote_code=ctx.model_config.trust_remote_code,
49 - ),
46 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
47 )
48 info = processor.info
49
@@ -146,10 +143,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
143 )
144 processor = MULTIMODAL_REGISTRY.create_processor(
145 ctx.model_config,
149 - tokenizer=cached_get_tokenizer(
150 - ctx.model_config.tokenizer,
151 - trust_remote_code=ctx.model_config.trust_remote_code,
152 - ),
146 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
147 )
148
149 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -179,10 +173,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
173 )
174 processor = MULTIMODAL_REGISTRY.create_processor(
175 ctx.model_config,
182 - tokenizer=cached_get_tokenizer(
183 - ctx.model_config.tokenizer,
184 - trust_remote_code=ctx.model_config.trust_remote_code,
185 - ),
176 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
177 )
178
179 seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_llava_onevision.py
+4 -13
@@ -10,7 +10,7 @@ from pqdm.threads import pqdm
10 from vllm.multimodal import MULTIMODAL_REGISTRY
11 from vllm.multimodal.parse import ImageSize
12 from vllm.multimodal.processing import BaseMultiModalProcessor
13 -from vllm.multimodal.utils import cached_get_tokenizer
13 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
14
15 from ...utils import build_model_context
16
@@ -44,10 +44,7 @@ def test_processor_max_tokens(model_id):
44 )
45 processor = MULTIMODAL_REGISTRY.create_processor(
46 ctx.model_config,
47 - tokenizer=cached_get_tokenizer(
48 - ctx.model_config.tokenizer,
49 - trust_remote_code=ctx.model_config.trust_remote_code,
50 - ),
47 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
48 )
49 info = processor.info
50
@@ -146,10 +143,7 @@ def test_processor_prompt_replacements_regression(model_id, num_imgs):
143 )
144 processor = MULTIMODAL_REGISTRY.create_processor(
145 ctx.model_config,
149 - tokenizer=cached_get_tokenizer(
150 - ctx.model_config.tokenizer,
151 - trust_remote_code=ctx.model_config.trust_remote_code,
152 - ),
146 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
147 )
148
149 image_ratios = [(171, 152), (184, 161), (198, 176), (333, 296), (369, 328),
@@ -180,10 +174,7 @@ def test_processor_prompt_replacements_all(model_id, num_imgs):
174 )
175 processor = MULTIMODAL_REGISTRY.create_processor(
176 ctx.model_config,
183 - tokenizer=cached_get_tokenizer(
184 - ctx.model_config.tokenizer,
185 - trust_remote_code=ctx.model_config.trust_remote_code,
186 - ),
177 + tokenizer=cached_tokenizer_from_config(ctx.model_config),
178 )
179
180 seen_aspect_ratios = set[float]()
tests/models/multimodal/processing/test_phi3v.py
+7 -6
@@ -3,7 +3,7 @@
3 import pytest
4
5 from vllm.multimodal import MULTIMODAL_REGISTRY
6 -from vllm.multimodal.utils import cached_get_tokenizer
6 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
7
8 from ....conftest import _ImageAssets
9 from ...utils import build_model_context
@@ -21,12 +21,14 @@ from ...utils import build_model_context
21 ])
22 # yapf: enable
23 @pytest.mark.parametrize("num_imgs", [1, 2])
24 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
25 def test_processor_override(
26 image_assets: _ImageAssets,
27 model_id: str,
28 mm_processor_kwargs: dict[str, int],
29 expected_toks_per_img: int,
30 num_imgs: int,
31 + kwargs_on_init: bool,
32 ):
33 """Ensure input_processor_for_phi3v handles num_crops properly."""
34 # Avoid initializing CUDA early
@@ -36,23 +38,22 @@ def test_processor_override(
38 model_name=model_id,
39 tokenizer_name=model_id,
40 trust_remote_code=True,
41 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
42 limit_mm_per_prompt={"image": num_imgs},
43 )
41 - tokenizer = cached_get_tokenizer(
42 - ctx.model_config.tokenizer,
43 - trust_remote_code=ctx.model_config.trust_remote_code,
44 - )
44 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
45 processor = MULTIMODAL_REGISTRY.create_processor(
46 ctx.model_config,
47 tokenizer=tokenizer,
48 )
49 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
50
51 # Build the image str / prompt based on the number of images we pass
52 img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
53 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
54 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
55
55 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
56 + processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
57
58 # Ensure we have the right number of placeholders per num_crops size
59 img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
tests/models/multimodal/processing/test_qwen2_vl.py
+8 -8
@@ -3,7 +3,7 @@
3 import pytest
4
5 from vllm.multimodal import MULTIMODAL_REGISTRY
6 -from vllm.multimodal.utils import cached_get_tokenizer
6 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
7
8 from ....conftest import _ImageAssets
9 from ...utils import build_model_context
@@ -18,6 +18,7 @@ from ...utils import build_model_context
18 ])
19 # yapf: enable
20 @pytest.mark.parametrize("num_imgs", [1, 2])
21 +@pytest.mark.parametrize("kwargs_on_init", [True, False])
22 def test_processor_override(
23 image_assets: _ImageAssets,
24 model_id: str,
@@ -25,31 +26,30 @@ def test_processor_override(
26 expected_toks_per_img: int,
27 expected_pixels_shape: tuple[int, int],
28 num_imgs: int,
29 + kwargs_on_init: bool,
30 ):
31 """Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
32 ctx = build_model_context(
33 model_name=model_id,
34 tokenizer_name=model_id,
33 - mm_processor_kwargs=None,
35 + mm_processor_kwargs=mm_processor_kwargs if kwargs_on_init else None,
36 limit_mm_per_prompt={"image": num_imgs},
37 )
36 - tokenizer = cached_get_tokenizer(
37 - ctx.model_config.tokenizer,
38 - trust_remote_code=ctx.model_config.trust_remote_code,
39 - )
38 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
39 processor = MULTIMODAL_REGISTRY.create_processor(
40 ctx.model_config,
41 tokenizer=tokenizer,
42 )
43 + hf_processor_mm_kwargs = {} if kwargs_on_init else mm_processor_kwargs
44
45 # Build the image str / prompt based on the number of images we pass
46 prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
47 mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
48
49 - processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
49 + processed_inputs = processor.apply(prompt, mm_data, hf_processor_mm_kwargs)
50
51 # Ensure we have the right number of placeholders per num_crops size
52 - hf_processor = processor.info.get_hf_processor(**mm_processor_kwargs)
52 + hf_processor = processor.info.get_hf_processor(**hf_processor_mm_kwargs)
53 image_token_id = tokenizer.convert_tokens_to_ids(hf_processor.image_token)
54 img_tok_count = processed_inputs["prompt_token_ids"].count(image_token_id)
55 pixel_shape = processed_inputs["mm_kwargs"]["pixel_values"].shape
tests/models/utils.py
+11 -7
@@ -248,13 +248,16 @@ def check_logprobs_close(
248 warnings.warn(fail_msg, stacklevel=2)
249
250
251 -def build_model_context(model_name: str,
252 - task: TaskOption = "auto",
253 - tokenizer_name: Optional[str] = None,
254 - trust_remote_code: bool = False,
255 - dtype: Optional[Union[str, torch.dtype]] = None,
256 - mm_processor_kwargs: Optional[Dict] = None,
257 - limit_mm_per_prompt: Optional[Dict] = None):
251 +def build_model_context(
252 + model_name: str,
253 + task: TaskOption = "auto",
254 + tokenizer_name: Optional[str] = None,
255 + trust_remote_code: bool = False,
256 + dtype: Optional[Union[str, torch.dtype]] = None,
257 + mm_processor_kwargs: Optional[Dict] = None,
258 + limit_mm_per_prompt: Optional[Dict] = None,
259 + disable_mm_preprocessor_cache: bool = True,
260 +):
261 """Creates an InputContext for a given model.
262
263 Args:
@@ -283,5 +286,6 @@ def build_model_context(model_name: str,
286 seed=0,
287 mm_processor_kwargs=mm_processor_kwargs,
288 limit_mm_per_prompt=limit_mm_per_prompt,
289 + disable_mm_preprocessor_cache=disable_mm_preprocessor_cache,
290 )
291 return InputContext(model_config)
tests/multimodal/test_processing.py
+5 -5
@@ -22,8 +22,8 @@ from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
22 replace_token_matches)
23 # yapf: enable
24 from vllm.multimodal.profiling import MultiModalProfiler
25 -from vllm.multimodal.utils import cached_get_tokenizer
26 -from vllm.transformers_utils.tokenizer import AnyTokenizer
25 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,
26 + cached_tokenizer_from_config)
27 from vllm.utils import full_groupby
28
29 from .utils import random_image
@@ -577,7 +577,7 @@ def test_limit_mm_per_prompt_dummy(model_id, limit, num_supported, is_valid):
577
578 processor = MULTIMODAL_REGISTRY.create_processor(
579 model_config,
580 - tokenizer=cached_get_tokenizer(model_config.tokenizer),
580 + tokenizer=cached_tokenizer_from_config(model_config),
581 )
582 profiler = MultiModalProfiler(processor)
583
@@ -617,7 +617,7 @@ def test_limit_mm_per_prompt_apply(model_id, num_images, limit, is_valid):
617
618 processor = MULTIMODAL_REGISTRY.create_processor(
619 model_config,
620 - tokenizer=cached_get_tokenizer(model_config.tokenizer),
620 + tokenizer=cached_tokenizer_from_config(model_config),
621 )
622
623 rng = np.random.RandomState(0)
@@ -689,7 +689,7 @@ def test_hf_processor_kwargs(model_id, call_kwargs, expected_kwargs):
689
690 processor = MULTIMODAL_REGISTRY.create_processor(
691 model_config,
692 - tokenizer=cached_get_tokenizer(model_config.tokenizer),
692 + tokenizer=cached_tokenizer_from_config(model_config),
693 )
694 orig_get_hf_processor = processor.info.get_hf_processor
695
vllm/inputs/registry.py
+30 -47
@@ -11,8 +11,9 @@ from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
11 from typing_extensions import TypeVar, assert_never
12
13 from vllm.logger import init_logger
14 -from vllm.transformers_utils.processor import cached_get_processor
15 -from vllm.transformers_utils.tokenizer import AnyTokenizer
14 +from vllm.transformers_utils.processor import cached_processor_from_config
15 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,
16 + cached_tokenizer_from_config)
17 from vllm.utils import (ClassRegistry, get_allowed_kwarg_only_overrides,
18 resolve_mm_processor_kwargs)
19
@@ -27,19 +28,9 @@ if TYPE_CHECKING:
28
29 logger = init_logger(__name__)
30
30 -C = TypeVar("C", bound=PretrainedConfig, default=PretrainedConfig)
31 -P = TypeVar("P", bound=ProcessorMixin, default=ProcessorMixin)
32 -
33 -
34 -class HashableDict(dict):
35 - """
36 - A dictionary that can be hashed by lru_cache.
37 - """
38 -
39 - # NOTE: pythonic dict is not hashable,
40 - # we override on it directly for simplicity
41 - def __hash__(self) -> int: # type: ignore[override]
42 - return hash(frozenset(self.items()))
31 +_T = TypeVar("_T")
32 +_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
33 +_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
34
35
36 @dataclass(frozen=True)
@@ -54,9 +45,9 @@ class InputContext:
45
46 def get_hf_config(
47 self,
57 - typ: Union[type[C], tuple[type[C], ...]] = PretrainedConfig,
48 + typ: Union[type[_C], tuple[type[_C], ...]] = PretrainedConfig,
49 /,
59 - ) -> C:
50 + ) -> _C:
51 """
52 Get the HuggingFace configuration
53 (:class:`transformers.PretrainedConfig`) of the model,
@@ -94,10 +85,10 @@ class InputContext:
85
86 def get_hf_processor(
87 self,
97 - typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
88 + typ: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
89 /,
90 **kwargs: object,
100 - ) -> P:
91 + ) -> _P:
92 """
93 Get the HuggingFace processor
94 (:class:`transformers.ProcessorMixin`) of the model,
@@ -106,33 +97,29 @@ class InputContext:
97 Raises:
98 TypeError: If the processor is not of the specified type.
99 """
100 + return cached_processor_from_config(
101 + self.model_config,
102 + processor_cls=typ,
103 + **kwargs,
104 + )
105 +
106 + def init_processor(
107 + self,
108 + typ: type[_T],
109 + /,
110 + **kwargs: object,
111 + ) -> _T:
112 + """
113 + Initialize a HuggingFace-like processor class, merging the
114 + keyword arguments with those in the model's configuration.
115 + """
116 base_kwargs = self.model_config.mm_processor_kwargs
117 if base_kwargs is None:
118 base_kwargs = {}
119
120 merged_kwargs = {**base_kwargs, **kwargs}
121
115 - if isinstance(typ, type):
116 - merged_kwargs["processor_cls"] = typ
117 -
118 - # NOTE: Pythonic dict is not hashable and will raise unhashable type
119 - # error when calling `cached_get_processor`, therefore we need to
120 - # wrap it to a hashable dict.
121 - for key, value in merged_kwargs.items():
122 - if isinstance(value, dict):
123 - merged_kwargs[key] = HashableDict(value)
124 -
125 - hf_processor = cached_get_processor(
126 - self.model_config.model,
127 - trust_remote_code=self.model_config.trust_remote_code,
128 - **merged_kwargs,
129 - )
130 - if not isinstance(hf_processor, typ):
131 - raise TypeError("Invalid type of HuggingFace processor. "
132 - f"Expected type: {typ}, but "
133 - f"found type: {type(hf_processor)}")
134 -
135 - return hf_processor
122 + return typ(**merged_kwargs)
123
124
125 @dataclass(frozen=True)
@@ -142,10 +129,10 @@ class InputProcessingContext(InputContext):
129
130 def get_hf_processor(
131 self,
145 - typ: Union[type[P], tuple[type[P], ...]] = ProcessorMixin,
132 + typ: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
133 /,
134 **kwargs: object,
148 - ) -> P:
135 + ) -> _P:
136 return super().get_hf_processor(
137 typ,
138 tokenizer=self.tokenizer,
@@ -341,13 +328,9 @@ class InputRegistry:
328 from vllm.model_executor.model_loader import get_model_architecture
329 from vllm.multimodal import MultiModalKwargs
330 from vllm.multimodal.profiling import MultiModalProfiler
344 - from vllm.multimodal.utils import cached_get_tokenizer
331
332 if mm_registry.has_processor(model_config):
347 - tokenizer = cached_get_tokenizer(
348 - model_config.tokenizer,
349 - trust_remote_code=model_config.trust_remote_code,
350 - )
333 + tokenizer = cached_tokenizer_from_config(model_config)
334 processor = mm_registry.create_processor(model_config, tokenizer)
335 profiler = MultiModalProfiler(processor)
336 dummy_data = profiler.get_dummy_data(
vllm/model_executor/models/aria.py
+2 -2
@@ -400,8 +400,8 @@ class AriaProcessingInfo(BaseProcessingInfo):
400 def get_vision_config(self):
401 return self.get_hf_config().vision_config
402
403 - def get_hf_processor(self):
404 - return self.ctx.get_hf_processor(AriaProcessor)
403 + def get_hf_processor(self, **kwargs: object):
404 + return self.ctx.get_hf_processor(AriaProcessor, **kwargs)
405
406 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
407 return {"image": None}
vllm/model_executor/models/chameleon.py
+2 -2
@@ -58,8 +58,8 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
58 def get_hf_config(self):
59 return self.ctx.get_hf_config(ChameleonConfig)
60
61 - def get_hf_processor(self):
62 - return self.ctx.get_hf_processor(ChameleonProcessor)
61 + def get_hf_processor(self, **kwargs: object):
62 + return self.ctx.get_hf_processor(ChameleonProcessor, **kwargs)
63
64 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
65 return {"image": 1}
vllm/model_executor/models/deepseek_vl2.py
+5 -10
@@ -28,13 +28,13 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
28 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 BaseProcessingInfo, PromptReplacement)
30 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 -from vllm.multimodal.utils import cached_get_tokenizer
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
33 MlpProjectorConfig,
34 VisionEncoderConfig)
35 from vllm.transformers_utils.processors.deepseek_vl2 import (
36 DeepseekVLV2Processor)
37 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
38 from vllm.utils import is_list_of
39
40 from .interfaces import SupportsMultiModal, SupportsPP
@@ -133,8 +133,8 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
133 def get_hf_config(self):
134 return self.ctx.get_hf_config(DeepseekVLV2Config)
135
136 - def get_hf_processor(self) -> DeepseekVLV2Processor:
137 - return self.ctx.get_hf_processor(DeepseekVLV2Processor)
136 + def get_hf_processor(self, **kwargs: object):
137 + return self.ctx.get_hf_processor(DeepseekVLV2Processor, **kwargs)
138
139 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
140 return {"image": None}
@@ -308,13 +308,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
308 self.text_config = config.text_config
309
310 model_config = vllm_config.model_config
311 - tokenizer = cached_get_tokenizer(
312 - model_config.tokenizer,
313 - tokenizer_mode=model_config.tokenizer_mode,
314 - tokenizer_revision=model_config.tokenizer_revision,
315 - trust_remote_code=model_config.trust_remote_code,
316 - )
317 - self.image_token_id = tokenizer.vocab.get(_IMAGE_TOKEN)
311 + tokenizer = cached_tokenizer_from_config(model_config)
312 + self.image_token_id = tokenizer.vocab[_IMAGE_TOKEN]
313
314 self.vision = self._init_vision_module(self.vision_config,
315 quant_config,
vllm/model_executor/models/fuyu.py
+2 -2
@@ -71,8 +71,8 @@ class FuyuProcessingInfo(BaseProcessingInfo):
71 def get_hf_config(self):
72 return self.ctx.get_hf_config(FuyuConfig)
73
74 - def get_hf_processor(self):
75 - return self.ctx.get_hf_processor(FuyuProcessor)
74 + def get_hf_processor(self, **kwargs: object):
75 + return self.ctx.get_hf_processor(FuyuProcessor, **kwargs)
76
77 def get_image_processor(self) -> FuyuImageProcessor:
78 return self.get_hf_processor().image_processor
vllm/model_executor/models/glm4v.py
+6 -9
@@ -416,18 +416,15 @@ class GLM4VProcessor:
416
417 class GLM4VProcessingInfo(BaseProcessingInfo):
418
419 - def get_tokenizer(self):
420 - tokenizer = self.ctx.tokenizer
421 - assert isinstance(tokenizer, PreTrainedTokenizer)
422 - return tokenizer
423 -
419 def get_hf_config(self):
420 return self.ctx.get_hf_config(ChatGLMConfig)
421
427 - def get_hf_processor(self) -> GLM4VProcessor:
428 - return GLM4VProcessor(
429 - self.get_hf_config(),
430 - self.get_tokenizer(),
422 + def get_hf_processor(self, **kwargs: object) -> GLM4VProcessor:
423 + return self.ctx.init_processor(
424 + GLM4VProcessor,
425 + config=self.get_hf_config(),
426 + tokenizer=self.get_tokenizer(),
427 + **kwargs,
428 )
429
430 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
vllm/model_executor/models/gritlm.py
+2 -7
@@ -15,9 +15,9 @@ from vllm.model_executor.layers.pooler import PoolerHead
15 from vllm.model_executor.models.llama import LlamaForCausalLM
16 from vllm.model_executor.pooling_metadata import (PoolingMetadata,
17 PoolingTensors)
18 -from vllm.multimodal.utils import cached_get_tokenizer
18 from vllm.sequence import (IntermediateTensors, PoolerOutput,
19 PoolingSequenceGroupOutput)
20 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
21
22 logger = init_logger(__name__)
23
@@ -29,12 +29,7 @@ class GritLMPooler(nn.Module):
29
30 self.model_config = model_config
31
32 - tokenizer = cached_get_tokenizer(
33 - self.model_config.tokenizer,
34 - tokenizer_mode=self.model_config.tokenizer_mode,
35 - tokenizer_revision=self.model_config.tokenizer_revision,
36 - trust_remote_code=self.model_config.trust_remote_code,
37 - )
32 + tokenizer = cached_tokenizer_from_config(self.model_config)
33
34 # Collect the tokens needed for pattern matching.
35 # "▁<" is different from "_<". The former uses "▁" to indicate that
vllm/model_executor/models/h2ovl.py
+31 -10
@@ -41,6 +41,7 @@ def resolve_h2ovl_min_max_num(
41 dynamic_image_size: bool,
42 use_thumbnail: bool,
43 ) -> tuple[int, int]:
44 + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
45 max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
46
47 if use_thumbnail and max_dynamic_patch != 1:
@@ -190,7 +191,7 @@ def image_to_pixel_values_h2ovl(
191 pixel_values1, aspect_ratio1 = _preprocess_image(
192 image,
193 input_size=input_size,
193 - min_num=min_num,
194 + min_num=1,
195 max_num=max_num,
196 use_thumbnail=True,
197 prior_aspect_ratio=None,
@@ -199,7 +200,7 @@ def image_to_pixel_values_h2ovl(
200 pixel_values2, _ = _preprocess_image(
201 image,
202 input_size=input_size,
202 - min_num=3, # Hardcoded value
203 + min_num=3,
204 max_num=max_num,
205 use_thumbnail=True,
206 prior_aspect_ratio=aspect_ratio1,
@@ -228,6 +229,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
229 config: PretrainedConfig,
230 tokenizer: AnyTokenizer,
231 *,
232 + min_dynamic_patch: Optional[int] = None,
233 max_dynamic_patch: Optional[int] = None,
234 dynamic_image_size: Optional[bool] = None,
235 use_msac: Optional[bool] = None,
@@ -235,6 +237,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
237 super().__init__(
238 config,
239 tokenizer,
240 + min_dynamic_patch=min_dynamic_patch,
241 max_dynamic_patch=max_dynamic_patch,
242 dynamic_image_size=dynamic_image_size,
243 )
@@ -267,11 +270,13 @@ class H2OVLProcessor(BaseInternVLProcessor):
270 def resolve_min_max_num(
271 self,
272 *,
273 + min_dynamic_patch: Optional[int] = None,
274 max_dynamic_patch: Optional[int] = None,
275 dynamic_image_size: Optional[bool] = None,
276 use_thumbnail: Optional[bool] = None,
277 ) -> tuple[int, int]:
274 - min_dynamic_patch = self.min_dynamic_patch
278 + min_dynamic_patch = (self.min_dynamic_patch if min_dynamic_patch
279 + is None else min_dynamic_patch)
280 max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
281 is None else max_dynamic_patch)
282 dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
@@ -289,18 +294,21 @@ class H2OVLProcessor(BaseInternVLProcessor):
294 def resolve_target_ratios(
295 self,
296 *,
297 + min_dynamic_patch: Optional[int] = None,
298 max_dynamic_patch: Optional[int] = None,
299 dynamic_image_size: Optional[bool] = None,
300 use_thumbnail: Optional[bool] = None,
301 prior_aspect_ratio: Optional[tuple[int, int]] = None,
302 + override_min_num: Optional[int] = None,
303 ) -> list[tuple[int, int]]:
304 min_num, max_num = self.resolve_min_max_num(
305 + min_dynamic_patch=min_dynamic_patch,
306 max_dynamic_patch=max_dynamic_patch,
307 dynamic_image_size=dynamic_image_size,
308 use_thumbnail=use_thumbnail,
309 )
302 - if prior_aspect_ratio: # hardcoded value for second pass of use_msac
303 - min_num = 3
310 + if override_min_num is not None:
311 + min_num = override_min_num
312
313 return get_h2ovl_target_ratios(
314 min_num,
@@ -322,6 +330,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
330 if use_msac:
331 target_ratios_1 = self.resolve_target_ratios(
332 use_thumbnail=False, # Applied in calculate_targets
333 + override_min_num=1,
334 )
335 num_patches_1, _, _, aspect_ratio_1 = calculate_h2ovl_targets(
336 orig_width=image_width,
@@ -334,6 +343,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
343 target_ratios_2 = self.resolve_target_ratios(
344 use_thumbnail=False, # Applied in calculate_targets
345 prior_aspect_ratio=aspect_ratio_1,
346 + override_min_num=3,
347 )
348 num_patches_2, _, _, _ = calculate_h2ovl_targets(
349 orig_width=image_width,
@@ -361,12 +371,14 @@ class H2OVLProcessor(BaseInternVLProcessor):
371 def _images_to_pixel_values_lst(
372 self,
373 images: list[Image.Image],
374 + min_dynamic_patch: Optional[int] = None,
375 max_dynamic_patch: Optional[int] = None,
376 dynamic_image_size: Optional[bool] = None,
377 ) -> list[torch.Tensor]:
378 use_msac = self.use_msac if len(images) == 1 else False
379
380 min_num, max_num = self.resolve_min_max_num(
381 + min_dynamic_patch=min_dynamic_patch,
382 max_dynamic_patch=max_dynamic_patch,
383 dynamic_image_size=dynamic_image_size,
384 use_thumbnail=False, # Applied in image_to_pixel_values
@@ -389,14 +401,23 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
401 def get_hf_processor(
402 self,
403 *,
404 + min_dynamic_patch: Optional[int] = None,
405 max_dynamic_patch: Optional[int] = None,
406 dynamic_image_size: Optional[bool] = None,
407 + **kwargs: object,
408 ) -> H2OVLProcessor:
395 - return H2OVLProcessor(
396 - self.get_hf_config(),
397 - self.get_tokenizer(),
398 - max_dynamic_patch=max_dynamic_patch,
399 - dynamic_image_size=dynamic_image_size,
409 + if min_dynamic_patch is not None:
410 + kwargs["min_dynamic_patch"] = min_dynamic_patch
411 + if max_dynamic_patch is not None:
412 + kwargs["max_dynamic_patch"] = max_dynamic_patch
413 + if dynamic_image_size is not None:
414 + kwargs["dynamic_image_size"] = dynamic_image_size
415 +
416 + return self.ctx.init_processor(
417 + H2OVLProcessor,
418 + config=self.get_hf_config(),
419 + tokenizer=self.get_tokenizer(),
420 + **kwargs,
421 )
422
423 def get_mm_max_tokens_per_item(
vllm/model_executor/models/idefics3.py
+7 -5
@@ -83,13 +83,15 @@ ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
83 class Idefics3ProcessingInfo(BaseProcessingInfo):
84
85 def get_hf_processor(
86 - self,
87 - *,
88 - size: Optional[Dict[str, int]] = None) -> Idefics3Processor:
86 + self,
87 + *,
88 + size: Optional[Dict[str, int]] = None,
89 + **kwargs: object,
90 + ) -> Idefics3Processor:
91 if size is not None:
90 - return self.ctx.get_hf_processor(Idefics3Processor, size=size)
92 + kwargs["size"] = size
93
92 - return self.ctx.get_hf_processor(Idefics3Processor)
94 + return self.ctx.get_hf_processor(Idefics3Processor, **kwargs)
95
96 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
97 return {"image": None}
vllm/model_executor/models/internvl.py
+35 -10
@@ -120,6 +120,7 @@ def resolve_internvl_min_max_num(
120 dynamic_image_size: bool,
121 use_thumbnail: bool,
122 ) -> tuple[int, int]:
123 + min_dynamic_patch = min_dynamic_patch if dynamic_image_size else 1
124 max_dynamic_patch = max_dynamic_patch if dynamic_image_size else 1
125
126 if use_thumbnail and max_dynamic_patch != 1:
@@ -247,6 +248,7 @@ class BaseInternVLProcessor(ABC):
248 config: PretrainedConfig,
249 tokenizer: AnyTokenizer,
250 *,
251 + min_dynamic_patch: Optional[int] = None,
252 max_dynamic_patch: Optional[int] = None,
253 dynamic_image_size: Optional[bool] = None,
254 ) -> None:
@@ -258,18 +260,22 @@ class BaseInternVLProcessor(ABC):
260 image_size: int = config.vision_config.image_size
261 patch_size: int = config.vision_config.patch_size
262
261 - if dynamic_image_size is None:
262 - dynamic_image_size = config.dynamic_image_size
263 - assert isinstance(dynamic_image_size, bool)
263 + if min_dynamic_patch is None:
264 + min_dynamic_patch = config.min_dynamic_patch
265 + assert isinstance(min_dynamic_patch, int)
266
267 if max_dynamic_patch is None:
268 max_dynamic_patch = config.max_dynamic_patch
269 assert isinstance(max_dynamic_patch, int)
270
271 + if dynamic_image_size is None:
272 + dynamic_image_size = config.dynamic_image_size
273 + assert isinstance(dynamic_image_size, bool)
274 +
275 self.num_image_token = int(
276 (image_size // patch_size)**2 * (config.downsample_ratio**2))
277 self.image_size = image_size
272 - self.min_dynamic_patch: int = config.min_dynamic_patch
278 + self.min_dynamic_patch = min_dynamic_patch
279 self.max_dynamic_patch = max_dynamic_patch
280 self.dynamic_image_size = dynamic_image_size
281 self.use_thumbnail: bool = config.use_thumbnail
@@ -298,11 +304,13 @@ class BaseInternVLProcessor(ABC):
304 def resolve_min_max_num(
305 self,
306 *,
307 + min_dynamic_patch: Optional[int] = None,
308 max_dynamic_patch: Optional[int] = None,
309 dynamic_image_size: Optional[bool] = None,
310 use_thumbnail: Optional[bool] = None,
311 ) -> tuple[int, int]:
305 - min_dynamic_patch = self.min_dynamic_patch
312 + min_dynamic_patch = (self.min_dynamic_patch if min_dynamic_patch
313 + is None else min_dynamic_patch)
314 max_dynamic_patch = (self.max_dynamic_patch if max_dynamic_patch
315 is None else max_dynamic_patch)
316 dynamic_image_size = (self.dynamic_image_size if dynamic_image_size
@@ -320,11 +328,13 @@ class BaseInternVLProcessor(ABC):
328 def resolve_target_ratios(
329 self,
330 *,
331 + min_dynamic_patch: Optional[int] = None,
332 max_dynamic_patch: Optional[int] = None,
333 dynamic_image_size: Optional[bool] = None,
334 use_thumbnail: Optional[bool] = None,
335 ) -> list[tuple[int, int]]:
336 min_num, max_num = self.resolve_min_max_num(
337 + min_dynamic_patch=min_dynamic_patch,
338 max_dynamic_patch=max_dynamic_patch,
339 dynamic_image_size=dynamic_image_size,
340 use_thumbnail=use_thumbnail,
@@ -355,10 +365,12 @@ class BaseInternVLProcessor(ABC):
365 def _images_to_pixel_values_lst(
366 self,
367 images: list[Image.Image],
368 + min_dynamic_patch: Optional[int] = None,
369 max_dynamic_patch: Optional[int] = None,
370 dynamic_image_size: Optional[bool] = None,
371 ) -> list[torch.Tensor]:
372 min_num, max_num = self.resolve_min_max_num(
373 + min_dynamic_patch=min_dynamic_patch,
374 max_dynamic_patch=max_dynamic_patch,
375 dynamic_image_size=dynamic_image_size,
376 use_thumbnail=False, # Applied in image_to_pixel_values
@@ -378,6 +390,7 @@ class BaseInternVLProcessor(ABC):
390 self,
391 text: Optional[Union[str, list[str]]] = None,
392 images: Optional[Union[Image.Image, list[Image.Image]]] = None,
393 + min_dynamic_patch: Optional[int] = None,
394 max_dynamic_patch: Optional[int] = None,
395 dynamic_image_size: Optional[bool] = None,
396 return_tensors: Optional[Union[str, TensorType]] = None,
@@ -396,6 +409,7 @@ class BaseInternVLProcessor(ABC):
409 else:
410 pixel_values_lst = self._images_to_pixel_values_lst(
411 images,
412 + min_dynamic_patch=min_dynamic_patch,
413 max_dynamic_patch=max_dynamic_patch,
414 dynamic_image_size=dynamic_image_size,
415 )
@@ -451,8 +465,10 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
465 def get_hf_processor(
466 self,
467 *,
468 + min_dynamic_patch: Optional[int] = None,
469 max_dynamic_patch: Optional[int] = None,
470 dynamic_image_size: Optional[bool] = None,
471 + **kwargs: object,
472 ) -> BaseInternVLProcessor:
473 raise NotImplementedError
474
@@ -642,14 +658,23 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
658 def get_hf_processor(
659 self,
660 *,
661 + min_dynamic_patch: Optional[int] = None,
662 max_dynamic_patch: Optional[int] = None,
663 dynamic_image_size: Optional[bool] = None,
664 + **kwargs: object,
665 ) -> InternVLProcessor:
648 - return InternVLProcessor(
649 - self.get_hf_config(),
650 - self.get_tokenizer(),
651 - max_dynamic_patch=max_dynamic_patch,
652 - dynamic_image_size=dynamic_image_size,
666 + if min_dynamic_patch is not None:
667 + kwargs["min_dynamic_patch"] = min_dynamic_patch
668 + if max_dynamic_patch is not None:
669 + kwargs["max_dynamic_patch"] = max_dynamic_patch
670 + if dynamic_image_size is not None:
671 + kwargs["dynamic_image_size"] = dynamic_image_size
672 +
673 + return self.ctx.init_processor(
674 + InternVLProcessor,
675 + config=self.get_hf_config(),
676 + tokenizer=self.get_tokenizer(),
677 + **kwargs,
678 )
679
680
vllm/model_executor/models/llava.py
+14 -13
@@ -119,7 +119,7 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
119 return get_vision_encoder_info(self.get_hf_config())
120
121 @abstractmethod
122 - def get_hf_processor(self) -> LlavaLikeProcessor:
122 + def get_hf_processor(self, **kwargs: object) -> LlavaLikeProcessor:
123 raise NotImplementedError
124
125 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
@@ -208,8 +208,8 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
208
209 class LlavaProcessingInfo(BaseLlavaProcessingInfo):
210
211 - def get_hf_processor(self):
212 - return self.ctx.get_hf_processor(LlavaProcessor)
211 + def get_hf_processor(self, **kwargs: object):
212 + return self.ctx.get_hf_processor(LlavaProcessor, **kwargs)
213
214
215 class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
@@ -272,8 +272,8 @@ class LlavaMultiModalProcessor(
272
273 class PixtralHFProcessingInfo(BaseLlavaProcessingInfo):
274
275 - def get_hf_processor(self):
276 - return self.ctx.get_hf_processor(PixtralProcessor)
275 + def get_hf_processor(self, **kwargs: object):
276 + return self.ctx.get_hf_processor(PixtralProcessor, **kwargs)
277
278
279 class PixtralHFMultiModalProcessor(
@@ -742,23 +742,24 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
742
743 class MantisProcessingInfo(LlavaProcessingInfo):
744
745 - def get_hf_processor(self):
745 + def get_hf_processor(self, **kwargs: object):
746 hf_config = self.get_hf_config()
747 vision_info = self.get_vision_encoder_info()
748
749 + kwargs.setdefault("patch_size", vision_info.get_patch_size())
750 +
751 if Version(TRANSFORMERS_VERSION) < Version("4.48"):
752 # BUG: num_additional_image_tokens = 0 but treated as 1,
753 # so we set vision_feature_select_strategy to None to offset this
752 - vision_feature_select_strategy = None
754 + kwargs.setdefault("vision_feature_select_strategy", None)
755 else:
756 # FIXED: https://github.com/huggingface/transformers/pull/33424/files#diff-6a37acc21efcadaae622b079b2712a131131448ff64262bd219aa346aeec38faL150
755 - vision_feature_select_strategy = hf_config.vision_feature_select_strategy # noqa: E501
757 + kwargs.setdefault(
758 + "vision_feature_select_strategy",
759 + hf_config.vision_feature_select_strategy,
760 + )
761
757 - return self.ctx.get_hf_processor(
758 - LlavaProcessor,
759 - patch_size=vision_info.get_patch_size(),
760 - vision_feature_select_strategy=vision_feature_select_strategy,
761 - )
762 + return self.ctx.get_hf_processor(LlavaProcessor, **kwargs)
763
764
765 class MantisMultiModalProcessor(LlavaMultiModalProcessor):
vllm/model_executor/models/llava_next.py
+2 -2
@@ -72,8 +72,8 @@ class LlavaNextProcessingInfo(BaseLlavaProcessingInfo):
72 def get_hf_config(self) -> LlavaNextLikeConfig:
73 return self.ctx.get_hf_config(LlavaNextConfig)
74
75 - def get_hf_processor(self):
76 - hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor)
75 + def get_hf_processor(self, **kwargs: object):
76 + hf_processor = self.ctx.get_hf_processor(LlavaNextProcessor, **kwargs)
77
78 # In case patch_size is omitted from `processor_config.json`
79 # e.g. for E5-V: https://huggingface.co/royokong/e5-v
vllm/model_executor/models/llava_next_video.py
+2 -2
@@ -56,8 +56,8 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
56 def get_vision_encoder_info(self):
57 return get_vision_encoder_info(self.get_hf_config())
58
59 - def get_hf_processor(self):
60 - return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
59 + def get_hf_processor(self, **kwargs: object):
60 + return self.ctx.get_hf_processor(LlavaNextVideoProcessor, **kwargs)
61
62 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
63 return {"video": 1}
vllm/model_executor/models/llava_onevision.py
+2 -2
@@ -97,8 +97,8 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
97 def get_hf_config(self) -> LlavaOnevisionLikeConfig:
98 return self.ctx.get_hf_config(LlavaOnevisionConfig)
99
100 - def get_hf_processor(self):
101 - return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
100 + def get_hf_processor(self, **kwargs: object):
101 + return self.ctx.get_hf_processor(LlavaOnevisionProcessor, **kwargs)
102
103 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
104 return {"image": None, "video": None}
vllm/model_executor/models/minicpmv.py
+2 -5
@@ -331,11 +331,8 @@ class MiniCPMVProcessingInfo(BaseProcessingInfo):
331 def get_hf_config(self):
332 return self.ctx.get_hf_config()
333
334 - def get_hf_processor(
335 - self,
336 - **kwargs: object,
337 - ):
338 - hf_processor = self.ctx.get_hf_processor()
334 + def get_hf_processor(self, **kwargs: object):
335 + hf_processor = self.ctx.get_hf_processor(**kwargs)
336
337 # NumPy arrays are considered as Iterable but not Sequence in
338 # https://github.com/huggingface/transformers/blob/main/src/transformers/image_transforms.py#L428
vllm/model_executor/models/mllama.py
+2 -2
@@ -94,8 +94,8 @@ class MllamaProcessingInfo(BaseProcessingInfo):
94 def get_hf_config(self) -> MllamaConfig:
95 return self.ctx.get_hf_config(MllamaConfig)
96
97 - def get_hf_processor(self) -> MllamaProcessor:
98 - return self.ctx.get_hf_processor(MllamaProcessor)
97 + def get_hf_processor(self, **kwargs: object) -> MllamaProcessor:
98 + return self.ctx.get_hf_processor(MllamaProcessor, **kwargs)
99
100 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
101 return {"image": None}
vllm/model_executor/models/molmo.py
+2 -2
@@ -1200,8 +1200,8 @@ class MolmoProcessorWrapper:
1200
1201 class MolmoProcessingInfo(BaseProcessingInfo):
1202
1203 - def get_hf_processor(self) -> MolmoProcessorWrapper:
1204 - processor = self.ctx.get_hf_processor()
1203 + def get_hf_processor(self, **kwargs: object) -> MolmoProcessorWrapper:
1204 + processor = self.ctx.get_hf_processor(**kwargs)
1205 return MolmoProcessorWrapper(processor)
1206
1207 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
vllm/model_executor/models/nvlm_d.py
+14 -5
@@ -69,14 +69,23 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):
69 def get_hf_processor(
70 self,
71 *,
72 + min_dynamic_patch: Optional[int] = None,
73 max_dynamic_patch: Optional[int] = None,
74 dynamic_image_size: Optional[bool] = None,
75 + **kwargs: object,
76 ) -> NVLMProcessor:
75 - return NVLMProcessor(
76 - self.get_hf_config(),
77 - self.get_tokenizer(),
78 - max_dynamic_patch=max_dynamic_patch,
79 - dynamic_image_size=dynamic_image_size,
77 + if min_dynamic_patch is not None:
78 + kwargs["min_dynamic_patch"] = min_dynamic_patch
79 + if max_dynamic_patch is not None:
80 + kwargs["max_dynamic_patch"] = max_dynamic_patch
81 + if dynamic_image_size is not None:
82 + kwargs["dynamic_image_size"] = dynamic_image_size
83 +
84 + return self.ctx.init_processor(
85 + NVLMProcessor,
86 + config=self.get_hf_config(),
87 + tokenizer=self.get_tokenizer(),
88 + **kwargs,
89 )
90
91 def get_max_image_tokens(self) -> int:
vllm/model_executor/models/paligemma.py
+2 -2
@@ -16,8 +16,8 @@ from vllm.model_executor.layers.sampler import SamplerOutput
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import NestedTensors
19 -from vllm.multimodal.utils import cached_get_tokenizer
19 from vllm.sequence import IntermediateTensors
20 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
21
22 from .interfaces import SupportsMultiModal, SupportsPP
23 from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
@@ -88,7 +88,7 @@ def input_processor_for_paligemma(ctx: InputContext,
88 model_config = ctx.model_config
89 hf_config = ctx.get_hf_config(PaliGemmaConfig)
90
91 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
91 + tokenizer = cached_tokenizer_from_config(model_config)
92 image_feature_size = hf_config.text_config.num_image_tokens
93 image_token_str = tokenizer.decode(hf_config.image_token_index)
94 bos_token = tokenizer.decode(hf_config.bos_token_id)
vllm/model_executor/models/phi3v.py
+3 -2
@@ -313,11 +313,12 @@ class Phi3VProcessingInfo(BaseProcessingInfo):
313 self,
314 *,
315 num_crops: Optional[int] = None,
316 + **kwargs: object,
317 ) -> ProcessorMixin:
318 if num_crops is not None:
318 - return self.ctx.get_hf_processor(num_crops=num_crops)
319 + kwargs["num_crops"] = num_crops
320
320 - return self.ctx.get_hf_processor()
321 + return self.ctx.get_hf_processor(**kwargs)
322
323 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
324 return {"image": None}
vllm/model_executor/models/pixtral.py
+6 -14
@@ -32,9 +32,9 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
32 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
34 from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
35 -from vllm.multimodal.utils import (cached_get_tokenizer,
36 - consecutive_placeholder_ranges)
35 +from vllm.multimodal.utils import consecutive_placeholder_ranges
36 from vllm.sequence import IntermediateTensors, SequenceData
37 +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config
38
39 from .interfaces import SupportsMultiModal, SupportsPP
40 from .utils import (init_vllm_registered_model, maybe_prefix,
@@ -49,9 +49,7 @@ except ImportError:
49
50
51 def get_max_pixtral_image_tokens(ctx: InputContext):
52 - tokenizer = cached_get_tokenizer(
53 - ctx.model_config.tokenizer,
54 - tokenizer_mode=ctx.model_config.tokenizer_mode)
52 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
53 mm_encoder = tokenizer.instruct.mm_encoder
54
55 image_config = mm_encoder.mm_config if hasattr(
@@ -65,9 +63,7 @@ def get_max_pixtral_image_tokens(ctx: InputContext):
63
64 def dummy_data_for_pixtral(ctx: InputContext, seq_len: int,
65 mm_counts: Mapping[str, int]):
68 - tokenizer = cached_get_tokenizer(
69 - ctx.model_config.tokenizer,
70 - tokenizer_mode=ctx.model_config.tokenizer_mode)
66 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
67
68 mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
69 image_token_id = mm_encoder.special_ids.img
@@ -109,9 +105,7 @@ def input_mapper_for_pixtral(ctx: InputContext,
105 MultiModalKwargs containing the stacked normalized images tensor or
106 image embeddings.
107 """
112 - model_config = ctx.model_config
113 - tokenizer = cached_get_tokenizer(
114 - model_config.tokenizer, tokenizer_mode=model_config.tokenizer_mode)
108 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
109
110 data_list = data if isinstance(data, list) else [data]
111
@@ -138,9 +132,7 @@ def input_processor_for_pixtral(ctx: InputContext, inputs: DecoderOnlyInputs):
132
133 prompt_token_ids = inputs.get("prompt_token_ids")
134 prompt = inputs.get("prompt")
141 - tokenizer = cached_get_tokenizer(
142 - ctx.model_config.tokenizer,
143 - tokenizer_mode=ctx.model_config.tokenizer_mode)
135 + tokenizer = cached_tokenizer_from_config(ctx.model_config)
136
137 mm_encoder = tokenizer.mistral.instruct_tokenizer.mm_encoder
138 image_token_id = mm_encoder.special_ids.img
vllm/model_executor/models/qwen2_5_vl.py
+12 -35
@@ -36,8 +36,6 @@ from transformers import BatchFeature
36 from transformers.models.qwen2_5_vl import Qwen2_5_VLProcessor
37 from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
38 Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig)
39 -from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
40 - Qwen2VLImageProcessorFast)
39
40 from vllm.attention import AttentionMetadata
41 from vllm.config import VllmConfig
@@ -690,41 +688,20 @@ class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
688 *,
689 min_pixels: Optional[int] = None,
690 max_pixels: Optional[int] = None,
693 - fps: Optional[float] = 2.0,
691 + size: Optional[dict[str, int]] = None,
692 + fps: Optional[float] = None,
693 + **kwargs: object,
694 ) -> Qwen2_5_VLProcessor:
695 - hf_processor = self.ctx.get_hf_processor(Qwen2_5_VLProcessor)
696 - image_processor = hf_processor.image_processor # type: ignore
697 - assert isinstance(image_processor,
698 - (Qwen2VLImageProcessor, Qwen2VLImageProcessorFast))
699 -
700 - if min_pixels:
701 - image_processor.min_pixels = min_pixels
702 - if max_pixels:
703 - image_processor.max_pixels = max_pixels
704 - if max_pixels or min_pixels:
705 - image_processor.size = {
706 - "min_pixels": image_processor.min_pixels,
707 - "max_pixels": image_processor.max_pixels,
708 - }
709 -
710 - return hf_processor
711 -
712 - def get_image_processor(
713 - self,
714 - *,
715 - min_pixels: Optional[int] = None,
716 - max_pixels: Optional[int] = None,
717 - fps: Optional[float] = 2.0,
718 - ) -> Union[Qwen2VLImageProcessor, Qwen2VLImageProcessorFast]:
719 - hf_processor = self.get_hf_processor(
720 - min_pixels=min_pixels,
721 - max_pixels=max_pixels,
722 - fps=fps,
695 + if fps is not None:
696 + kwargs["fps"] = fps
697 +
698 + return self.ctx.get_hf_processor(
699 + Qwen2_5_VLProcessor,
700 + image_processor=self.get_image_processor(min_pixels=min_pixels,
701 + max_pixels=max_pixels,
702 + size=size),
703 + **kwargs,
704 )
724 - image_processor = hf_processor.image_processor # type: ignore
725 - assert isinstance(image_processor,
726 - (Qwen2VLImageProcessor, Qwen2VLImageProcessorFast))
727 - return image_processor
705
706
707 class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
vllm/model_executor/models/qwen2_audio.py
+2 -1
@@ -93,8 +93,9 @@ class Qwen2AudioProcessingInfo(BaseProcessingInfo):
93 *,
94 # Ignored in initialization
95 sampling_rate: Optional[int] = None,
96 + **kwargs: object,
97 ) -> Qwen2AudioProcessor:
97 - return self.ctx.get_hf_processor(Qwen2AudioProcessor)
98 + return self.ctx.get_hf_processor(Qwen2AudioProcessor, **kwargs)
99
100 def get_feature_extractor(
101 self,
vllm/model_executor/models/qwen2_vl.py
+64 -30
@@ -31,9 +31,7 @@ import torch
31 import torch.nn as nn
32 import torch.nn.functional as F
33 from einops import rearrange, repeat
34 -from packaging.version import Version
34 from transformers import BatchFeature
36 -from transformers import __version__ as TRANSFORMERS_VERSION
35 from transformers.models.qwen2_vl import (Qwen2VLImageProcessor,
36 Qwen2VLProcessor)
37 from transformers.models.qwen2_vl.configuration_qwen2_vl import (
@@ -69,6 +67,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
67 from vllm.platforms import _Backend
68 from vllm.sequence import IntermediateTensors
69 from vllm.transformers_utils.config import uses_mrope
70 +from vllm.transformers_utils.processor import (
71 + cached_image_processor_from_config)
72
73 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
74 from .utils import (AutoWeightsLoader, WeightsMapper,
@@ -722,40 +722,64 @@ class Qwen2VLProcessingInfo(BaseProcessingInfo):
722 *,
723 min_pixels: Optional[int] = None,
724 max_pixels: Optional[int] = None,
725 + size: Optional[dict[str, int]] = None,
726 + **kwargs: object,
727 ) -> Qwen2VLProcessor:
726 - hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
727 - image_processor = hf_processor.image_processor # type: ignore
728 - assert isinstance(image_processor, Qwen2VLImageProcessor)
729 -
730 - if min_pixels:
731 - image_processor.min_pixels = min_pixels
732 - if max_pixels:
733 - image_processor.max_pixels = max_pixels
734 - if max_pixels or min_pixels:
735 - image_processor.size = {
736 - "min_pixels": image_processor.min_pixels,
737 - "max_pixels": image_processor.max_pixels,
738 - }
739 -
740 - return hf_processor
728 + return self.ctx.get_hf_processor(
729 + Qwen2VLProcessor,
730 + image_processor=self.get_image_processor(min_pixels=min_pixels,
731 + max_pixels=max_pixels,
732 + size=size),
733 + **kwargs,
734 + )
735 +
736 + def _get_image_processor_kwargs(
737 + self,
738 + *,
739 + min_pixels: Optional[int] = None,
740 + max_pixels: Optional[int] = None,
741 + size: Optional[dict[str, int]] = None,
742 + **kwargs: object,
743 + ):
744 + if self.ctx.model_config.mm_processor_kwargs:
745 + kwargs.update(self.ctx.model_config.mm_processor_kwargs)
746 +
747 + if min_pixels is not None:
748 + kwargs["min_pixels"] = min_pixels
749 +
750 + if size is None:
751 + size = {"shortest_edge": min_pixels}
752 + else:
753 + size["shortest_edge"] = min_pixels
754 +
755 + if max_pixels is not None:
756 + kwargs["max_pixels"] = max_pixels
757 +
758 + if size is None:
759 + size = {"longest_edge": max_pixels}
760 + else:
761 + size["longest_edge"] = max_pixels
762 +
763 + if size is not None:
764 + kwargs["size"] = size
765 +
766 + return kwargs
767
768 def get_image_processor(
769 self,
770 *,
771 min_pixels: Optional[int] = None,
772 max_pixels: Optional[int] = None,
773 + size: Optional[dict[str, int]] = None,
774 + **kwargs: object,
775 ):
748 - hf_processor = self.get_hf_processor(min_pixels=min_pixels,
749 - max_pixels=max_pixels)
750 - image_processor = hf_processor.image_processor # type: ignore
751 - if Version(TRANSFORMERS_VERSION) >= Version("4.49"):
752 - from transformers.models.qwen2_vl import Qwen2VLImageProcessorFast
753 - assert isinstance(
754 - image_processor,
755 - (Qwen2VLImageProcessor, Qwen2VLImageProcessorFast))
756 - else:
757 - assert isinstance(image_processor, Qwen2VLImageProcessor)
758 - return image_processor
776 + return cached_image_processor_from_config(
777 + self.ctx.model_config,
778 + **self._get_image_processor_kwargs(min_pixels=min_pixels,
779 + max_pixels=max_pixels,
780 + size=size,
781 + **kwargs),
782 + )
783
784 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
785 return {"image": None, "video": None}
@@ -952,6 +976,18 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
976 def _get_data_parser(self) -> MultiModalDataParser:
977 return Qwen2VLMultiModalDataParser()
978
979 + def _call_hf_processor(
980 + self,
981 + prompt: str,
982 + mm_data: Mapping[str, object],
983 + mm_kwargs: Mapping[str, object],
984 + ) -> BatchFeature:
985 + return self.info.ctx.call_hf_processor(
986 + self.info.get_hf_processor(**mm_kwargs),
987 + dict(text=prompt, **mm_data),
988 + self.info._get_image_processor_kwargs(**mm_kwargs),
989 + )
990 +
991 def _get_prompt_replacements(
992 self,
993 mm_items: MultiModalDataItems,
@@ -964,8 +1000,6 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
1000 tokenizer = self.info.get_tokenizer()
1001 vocab = tokenizer.get_vocab()
1002
967 - # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
968 - # image_token and video_token registered
1003 placeholder = {
1004 "image": vocab[hf_processor.image_token],
1005 "video": vocab[hf_processor.video_token],
vllm/model_executor/models/qwen_vl.py
+7 -2
@@ -519,8 +519,13 @@ class QwenVLProcessingInfo(BaseProcessingInfo):
519
520 return _get_tokenizer_without_image_pad(tokenizer)
521
522 - def get_hf_processor(self) -> QwenVLProcessor:
523 - return QwenVLProcessor(self.get_hf_config(), self.get_tokenizer())
522 + def get_hf_processor(self, **kwargs: object) -> QwenVLProcessor:
523 + return self.ctx.init_processor(
524 + QwenVLProcessor,
525 + config=self.get_hf_config(),
526 + tokenizer=self.get_tokenizer(),
527 + **kwargs,
528 + )
529
530 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
531 return {"image": None}
vllm/model_executor/models/ultravox.py
+2 -1
@@ -68,8 +68,9 @@ class UltravoxProcessingInfo(BaseProcessingInfo):
68 *,
69 # Ignored in initialization
70 sampling_rate: Optional[int] = None,
71 + **kwargs: object,
72 ) -> ProcessorMixin:
72 - hf_processor = self.ctx.get_hf_processor()
73 + hf_processor = self.ctx.get_hf_processor(**kwargs)
74
75 # NOTE: Ultravox processing definition uses '<|eot_id|>' as the
76 # placeholder that will cause confusion with the actual end of turn
vllm/model_executor/models/whisper.py
+4 -4
@@ -29,7 +29,7 @@ from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalKwargs,
29 NestedTensors)
30 from vllm.multimodal.audio import resample_audio
31 from vllm.sequence import SequenceData
32 -from vllm.transformers_utils.processor import cached_get_processor
32 +from vllm.transformers_utils.processor import cached_processor_from_config
33
34 from .interfaces import SupportsMultiModal, SupportsTranscription
35 from .utils import AutoWeightsLoader, WeightsMapper, make_layers
@@ -579,7 +579,7 @@ def dummy_encoder_data_for_whisper(ctx: InputContext, seq_len: int,
579 mm_counts: Mapping[str, int]):
580 assert mm_counts["audio"] == 1
581 num_tokens = get_max_whisper_audio_tokens(ctx)
582 - processor = cached_get_processor(ctx.model_config.model)
582 + processor = cached_processor_from_config(ctx.model_config)
583 chunk_length = processor.feature_extractor.chunk_length
584 sampling_rate = processor.feature_extractor.sampling_rate
585 num_samples = chunk_length * sampling_rate
@@ -596,7 +596,7 @@ def input_processor_for_whisper(ctx: InputContext, inputs):
596 multi_modal_data["audio"] = multi_modal_data["audio"][0]
597 # Resample and process audio
598 audio, orig_sr = multi_modal_data["audio"]
599 - processor = cached_get_processor(ctx.model_config.model)
599 + processor = cached_processor_from_config(ctx.model_config)
600 target_sr = processor.feature_extractor.sampling_rate
601 audio = resample_audio(audio, orig_sr=orig_sr, target_sr=target_sr)
602 multi_modal_data["audio"] = (audio, target_sr)
@@ -618,7 +618,7 @@ def input_mapper_for_whisper(
618 if len(multi_modal_data) == 0:
619 return MultiModalKwargs()
620
621 - processor = cached_get_processor(ctx.model_config.model)
621 + processor = cached_processor_from_config(ctx.model_config)
622 sampling_rate = processor.feature_extractor.sampling_rate
623
624 audios = [audio for audio, _ in multi_modal_data]
vllm/multimodal/image.py
+1 -4
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import base64
4 -from functools import lru_cache
4 from io import BytesIO
5 from pathlib import Path
6 from typing import TYPE_CHECKING, Any, Dict, Optional
@@ -11,7 +10,7 @@ from PIL import Image
10
11 from vllm.inputs.registry import InputContext
12 from vllm.logger import init_logger
14 -from vllm.transformers_utils.processor import get_image_processor
13 +from vllm.transformers_utils.processor import cached_get_image_processor
14 from vllm.utils import is_list_of
15
16 from .base import MediaIO, MultiModalPlugin
@@ -22,8 +21,6 @@ if TYPE_CHECKING:
21
22 logger = init_logger(__name__)
23
25 -cached_get_image_processor = lru_cache(get_image_processor)
26 -
24
25 class ImagePlugin(MultiModalPlugin):
26 """Plugin for image data."""
vllm/multimodal/registry.py
+4 -10
@@ -11,7 +11,8 @@ import torch.nn as nn
11 from vllm.envs import VLLM_MM_INPUT_CACHE_SIZE
12 from vllm.inputs import InputProcessingContext
13 from vllm.logger import init_logger
14 -from vllm.transformers_utils.tokenizer import AnyTokenizer
14 +from vllm.transformers_utils.tokenizer import (AnyTokenizer,
15 + cached_tokenizer_from_config)
16 from vllm.utils import ClassRegistry
17
18 from .audio import AudioPlugin
@@ -21,7 +22,6 @@ from .inputs import MultiModalDataDict, MultiModalKwargs, NestedTensors
22 from .processing import (BaseMultiModalProcessor, BaseProcessingInfo,
23 ProcessingCache)
24 from .profiling import BaseDummyInputsBuilder, MultiModalProfiler
24 -from .utils import cached_get_tokenizer
25 from .video import VideoPlugin
26
27 if TYPE_CHECKING:
@@ -256,10 +256,7 @@ class MultiModalRegistry:
256 on underlying model configuration.
257 """
258 if self.has_processor(model_config):
259 - tokenizer = cached_get_tokenizer(
260 - model_config.tokenizer,
261 - trust_remote_code=model_config.trust_remote_code,
262 - )
259 + tokenizer = cached_tokenizer_from_config(model_config)
260 processor = self.create_processor(model_config, tokenizer)
261 seq_len = model_config.max_model_len
262 mm_limits = self.get_mm_limits_per_prompt(model_config)
@@ -374,10 +371,7 @@ class MultiModalRegistry:
371 This should be called after :meth:`init_mm_limits_per_prompt`.
372 """
373 if self.has_processor(model_config):
377 - tokenizer = cached_get_tokenizer(
378 - model_config.tokenizer,
379 - trust_remote_code=model_config.trust_remote_code,
380 - )
374 + tokenizer = cached_tokenizer_from_config(model_config)
375 processor = self.create_processor(model_config, tokenizer)
376 profiler = MultiModalProfiler(processor)
377 return profiler.get_mm_limits()
vllm/multimodal/utils.py
+1 -4
@@ -1,6 +1,5 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 -from functools import lru_cache
3 from itertools import groupby
4 from pathlib import Path
5 from typing import TYPE_CHECKING, Optional, TypeVar, Union
@@ -13,7 +12,7 @@ from PIL import Image
12 import vllm.envs as envs
13 from vllm.connections import HTTPConnection, global_http_connection
14 from vllm.logger import init_logger
16 -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer
15 +from vllm.transformers_utils.tokenizer import AnyTokenizer
16
17 from .audio import AudioMediaIO
18 from .base import MediaIO
@@ -23,8 +22,6 @@ from .video import VideoMediaIO
22
23 logger = init_logger(__name__)
24
26 -cached_get_tokenizer = lru_cache(get_tokenizer)
27 -
25 _M = TypeVar("_M")
26
27 if TYPE_CHECKING:
vllm/multimodal/video.py
+2 -6
@@ -1,7 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import base64
4 -from functools import lru_cache, partial
4 +from functools import partial
5 from io import BytesIO
6 from pathlib import Path
7 from typing import TYPE_CHECKING, Any, Dict, Optional
@@ -12,8 +12,7 @@ from PIL import Image
12
13 from vllm.inputs.registry import InputContext
14 from vllm.logger import init_logger
15 -from vllm.transformers_utils.processor import get_video_processor
16 -from vllm.transformers_utils.tokenizer import get_tokenizer
15 +from vllm.transformers_utils.processor import cached_get_video_processor
16 from vllm.utils import PlaceholderModule, is_list_of
17
18 from .base import MediaIO, ModalityData
@@ -30,9 +29,6 @@ except ImportError:
29
30 logger = init_logger(__name__)
31
33 -cached_get_video_processor = lru_cache(get_video_processor)
34 -cached_get_tokenizer = lru_cache(get_tokenizer)
35 -
32
33 class VideoPlugin(ImagePlugin):
34 """Plugin for video data."""
vllm/transformers_utils/processor.py
+86 -6
@@ -1,25 +1,59 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from functools import lru_cache
4 -from typing import Any, cast
4 +from typing import TYPE_CHECKING, Any, Union, cast
5
6 from transformers.processing_utils import ProcessorMixin
7 +from typing_extensions import TypeVar
8 +
9 +if TYPE_CHECKING:
10 + from vllm.config import ModelConfig
11 +
12 +_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin)
13 +
14 +
15 +class HashableDict(dict):
16 + """
17 + A dictionary that can be hashed by lru_cache.
18 + """
19 +
20 + # NOTE: pythonic dict is not hashable,
21 + # we override on it directly for simplicity
22 + def __hash__(self) -> int: # type: ignore[override]
23 + return hash(frozenset(self.items()))
24 +
25 +
26 +def _merge_mm_kwargs(model_config: "ModelConfig", **kwargs):
27 + base_kwargs = model_config.mm_processor_kwargs
28 + if base_kwargs is None:
29 + base_kwargs = {}
30 +
31 + merged_kwargs = {**base_kwargs, **kwargs}
32 +
33 + # NOTE: Pythonic dict is not hashable and will raise unhashable type
34 + # error when calling `cached_get_processor`, therefore we need to
35 + # wrap it to a hashable dict.
36 + for key, value in merged_kwargs.items():
37 + if isinstance(value, dict):
38 + merged_kwargs[key] = HashableDict(value)
39 +
40 + return merged_kwargs
41
42
43 def get_processor(
44 processor_name: str,
45 *args: Any,
46 trust_remote_code: bool = False,
13 - processor_cls: type[ProcessorMixin] = ProcessorMixin,
47 + processor_cls: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
48 **kwargs: Any,
15 -):
49 +) -> _P:
50 """Load a processor for the given model name via HuggingFace."""
51 # don't put this import at the top level
52 # it will call torch.cuda.device_count()
53 from transformers import AutoProcessor
54
21 - processor_factory = (AutoProcessor
22 - if processor_cls == ProcessorMixin else processor_cls)
55 + processor_factory = (AutoProcessor if processor_cls == ProcessorMixin or
56 + isinstance(processor_cls, tuple) else processor_cls)
57
58 try:
59 processor = processor_factory.from_pretrained(
@@ -43,12 +77,30 @@ def get_processor(
77 else:
78 raise e
79
46 - return cast(ProcessorMixin, processor)
80 + if not isinstance(processor, processor_cls):
81 + raise TypeError("Invalid type of HuggingFace processor. "
82 + f"Expected type: {processor_cls}, but "
83 + f"found type: {type(processor)}")
84 +
85 + return processor
86
87
88 cached_get_processor = lru_cache(get_processor)
89
90
91 +def cached_processor_from_config(
92 + model_config: "ModelConfig",
93 + processor_cls: Union[type[_P], tuple[type[_P], ...]] = ProcessorMixin,
94 + **kwargs: Any,
95 +) -> _P:
96 + return cached_get_processor(
97 + model_config.model,
98 + trust_remote_code=model_config.trust_remote_code,
99 + processor_cls=processor_cls, # type: ignore[arg-type]
100 + **_merge_mm_kwargs(model_config, **kwargs),
101 + )
102 +
103 +
104 def get_image_processor(
105 processor_name: str,
106 *args: Any,
@@ -85,6 +137,20 @@ def get_image_processor(
137 return cast(BaseImageProcessor, processor)
138
139
140 +cached_get_image_processor = lru_cache(get_image_processor)
141 +
142 +
143 +def cached_image_processor_from_config(
144 + model_config: "ModelConfig",
145 + **kwargs: Any,
146 +):
147 + return cached_get_image_processor(
148 + model_config.model,
149 + trust_remote_code=model_config.trust_remote_code,
150 + **_merge_mm_kwargs(model_config, **kwargs),
151 + )
152 +
153 +
154 def get_video_processor(
155 processor_name: str,
156 *args: Any,
@@ -104,3 +170,17 @@ def get_video_processor(
170 )
171
172 return cast(BaseImageProcessor, processor.video_processor)
173 +
174 +
175 +cached_get_video_processor = lru_cache(get_video_processor)
176 +
177 +
178 +def cached_video_processor_from_config(
179 + model_config: "ModelConfig",
180 + **kwargs: Any,
181 +):
182 + return cached_get_video_processor(
183 + model_config.model,
184 + trust_remote_code=model_config.trust_remote_code,
185 + **_merge_mm_kwargs(model_config, **kwargs),
186 + )
vllm/transformers_utils/tokenizer.py
+21 -1
@@ -3,9 +3,10 @@
3 import contextlib
4 import os
5 import warnings
6 +from functools import lru_cache
7 from pathlib import Path
8 from types import MethodType
8 -from typing import Optional, Union
9 +from typing import TYPE_CHECKING, Any, Optional, Union
10
11 import huggingface_hub
12 from transformers import (AutoTokenizer, PreTrainedTokenizer,
@@ -20,6 +21,9 @@ from vllm.transformers_utils.tokenizers import MistralTokenizer
21 from vllm.transformers_utils.utils import check_gguf_file
22 from vllm.utils import make_async
23
24 +if TYPE_CHECKING:
25 + from vllm.config import ModelConfig
26 +
27 logger = init_logger(__name__)
28
29 AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
@@ -232,6 +236,22 @@ def get_tokenizer(
236 return tokenizer
237
238
239 +cached_get_tokenizer = lru_cache(get_tokenizer)
240 +
241 +
242 +def cached_tokenizer_from_config(
243 + model_config: "ModelConfig",
244 + **kwargs: Any,
245 +):
246 + return cached_get_tokenizer(
247 + model_config.tokenizer,
248 + tokenizer_mode=model_config.tokenizer_mode,
249 + tokenizer_revision=model_config.tokenizer_revision,
250 + trust_remote_code=model_config.trust_remote_code,
251 + **kwargs,
252 + )
253 +
254 +
255 def get_lora_tokenizer(lora_request: LoRARequest, *args,
256 **kwargs) -> Optional[AnyTokenizer]:
257 if lora_request is None: