A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model] Add support for H2OVL-Mississippi models (#9747)
Signed-off-by: Shanshan Wang <shanshan.wang@h2o.ai> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>
shanshan wang committed
Nov 3, 2024 at 18:15 UTC
54597724f4c6b52d50152f3cc46e86c101d9c820
12 files changed
+698
-4
docs/source/models/supported_models.rst
+6
@@ -440,6 +440,12 @@ Text Generation
440
- :code:`THUDM/glm-4v-9b` etc.
441
-
442
- ✅︎
443
+ * - :code:`H2OVLChatModel`
444
+ - H2OVL
445
+ - T + I\ :sup:`E+`
446
+ - :code:`h2oai/h2ovl-mississippi-800m`, :code:`h2oai/h2ovl-mississippi-2b`, etc.
447
+ -
448
+ - ✅︎
449
* - :code:`InternVLChatModel`
450
- InternVL2
451
- T + I\ :sup:`E+`
examples/offline_inference_vision_language.py
+27
-1
@@ -176,6 +176,31 @@ def run_minicpmv(question: str, modality: str):
176
return llm, prompt, stop_token_ids
177
178
179
+# H2OVL-Mississippi
180
+def run_h2ovl(question: str, modality: str):
181
+ assert modality == "image"
182
+
183
+ model_name = "h2oai/h2ovl-mississippi-2b"
184
+
185
+ llm = LLM(
186
+ model=model_name,
187
+ trust_remote_code=True,
188
+ max_model_len=8192,
189
+ )
190
+
191
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
192
+ trust_remote_code=True)
193
+ messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
194
+ prompt = tokenizer.apply_chat_template(messages,
195
+ tokenize=False,
196
+ add_generation_prompt=True)
197
+
198
+ # Stop tokens for H2OVL-Mississippi
199
+ # https://huggingface.co/h2oai/h2ovl-mississippi-2b
200
+ stop_token_ids = [tokenizer.eos_token_id]
201
+ return llm, prompt, stop_token_ids
202
+
203
+
204
# InternVL
205
def run_internvl(question: str, modality: str):
206
assert modality == "image"
@@ -363,6 +388,7 @@ model_example_map = {
388
"chameleon": run_chameleon,
389
"minicpmv": run_minicpmv,
390
"blip-2": run_blip2,
391
+ "h2ovl_chat": run_h2ovl,
392
"internvl_chat": run_internvl,
393
"NVLM_D": run_nvlm_d,
394
"qwen_vl": run_qwen_vl,
@@ -475,4 +501,4 @@ if __name__ == "__main__":
501
default=16,
502
help='Number of frames to extract from the video.')
503
args = parser.parse_args()
478
- main(args)
504
+ main(args)
\ No newline at end of file
examples/offline_inference_vision_language_multi_image.py
+35
@@ -107,6 +107,40 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
107
)
108
109
110
+def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
111
+ model_name = "h2oai/h2ovl-mississippi-2b"
112
+
113
+ llm = LLM(
114
+ model=model_name,
115
+ trust_remote_code=True,
116
+ max_model_len=8192,
117
+ limit_mm_per_prompt={"image": len(image_urls)},
118
+ mm_processor_kwargs={"max_dynamic_patch": 4},
119
+ )
120
+
121
+ placeholders = "\n".join(f"Image-{i}: <image>\n"
122
+ for i, _ in enumerate(image_urls, start=1))
123
+ messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
124
+
125
+ tokenizer = AutoTokenizer.from_pretrained(model_name,
126
+ trust_remote_code=True)
127
+ prompt = tokenizer.apply_chat_template(messages,
128
+ tokenize=False,
129
+ add_generation_prompt=True)
130
+
131
+ # Stop tokens for H2OVL-Mississippi
132
+ # https://huggingface.co/h2oai/h2ovl-mississippi-2b
133
+ stop_token_ids = [tokenizer.eos_token_id]
134
+
135
+ return ModelRequestData(
136
+ llm=llm,
137
+ prompt=prompt,
138
+ stop_token_ids=stop_token_ids,
139
+ image_data=[fetch_image(url) for url in image_urls],
140
+ chat_template=None,
141
+ )
142
+
143
+
144
def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
145
model_name = "OpenGVLab/InternVL2-2B"
146
@@ -258,6 +292,7 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
292
293
model_example_map = {
294
"phi3_v": load_phi3v,
295
+ "h2ovl_chat": load_h2onvl,
296
"internvl_chat": load_internvl,
297
"NVLM_D": load_nvlm_d,
298
"qwen2_vl": load_qwen2_vl,
tests/models/decoder_only/vision_language/test_h2ovl.py
new
+130
@@ -0,0 +1,130 @@
1
+from typing import Optional, Tuple
2
+
3
+import pytest
4
+import torch
5
+from PIL.Image import Image
6
+from transformers import AutoConfig
7
+
8
+# Import the functions to test
9
+from vllm.model_executor.models.h2ovl import (calculate_num_blocks,
10
+ image_to_pixel_values_wrapper)
11
+from vllm.multimodal.utils import rescale_image_size
12
+
13
+models = [
14
+ "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
15
+ "h2oai/h2ovl-mississippi-2b",
16
+]
17
+target_dtype = "bfloat16"
18
+
19
+
20
+def run_preprocessing_test(
21
+ image: Image,
22
+ config,
23
+ max_dynamic_patch: Optional[int] = None,
24
+) -> Tuple[torch.Tensor, int]:
25
+ """Test the image preprocessing and calculate expected blocks."""
26
+
27
+ if max_dynamic_patch is None:
28
+ max_dynamic_patch = config.max_dynamic_patch
29
+
30
+ width, height = image.size
31
+ use_MSAC = config.use_msac
32
+
33
+ # Create the mapper function with the provided configuration
34
+ mapper = image_to_pixel_values_wrapper(config, max_dynamic_patch, use_MSAC)
35
+ pixel_values = mapper(image)
36
+
37
+ # Calculate the expected number of blocks
38
+ if use_MSAC:
39
+ # First pass
40
+ blocks1, _, _, aspect_ratio = calculate_num_blocks(
41
+ width,
42
+ height,
43
+ config.min_dynamic_patch,
44
+ max_dynamic_patch,
45
+ config.vision_config.image_size,
46
+ use_thumbnail=False, # Thumbnail is handled separately
47
+ prior_aspect_ratio=None,
48
+ )
49
+
50
+ # Second pass
51
+ blocks2, _, _, _ = calculate_num_blocks(
52
+ width,
53
+ height,
54
+ config.min_dynamic_patch,
55
+ max_dynamic_patch,
56
+ config.vision_config.image_size,
57
+ use_thumbnail=False,
58
+ prior_aspect_ratio=aspect_ratio,
59
+ )
60
+
61
+ # Add thumbnail if use_thumbnail is True and total_blocks > 1
62
+ if config.use_thumbnail:
63
+ blocks1 += 1 if blocks1 > 1 else 0
64
+ blocks2 += 1 if blocks2 > 1 else 0
65
+
66
+ # Total blocks is the sum of blocks from both passes minus overlapping
67
+ total_blocks = blocks1 + blocks2 - 1
68
+
69
+ expected_blocks = total_blocks
70
+
71
+ else:
72
+ blocks, _, _, _ = calculate_num_blocks(
73
+ width,
74
+ height,
75
+ config.min_dynamic_patch,
76
+ max_dynamic_patch,
77
+ config.vision_config.image_size,
78
+ use_thumbnail=False,
79
+ prior_aspect_ratio=None,
80
+ )
81
+ expected_blocks = blocks
82
+
83
+ if config.use_thumbnail and expected_blocks > 1:
84
+ expected_blocks += 1
85
+
86
+ return pixel_values, expected_blocks
87
+
88
+
89
+@pytest.mark.parametrize("model_name", models)
90
+@pytest.mark.parametrize(
91
+ "size_factors",
92
+ [
93
+ # Single-scale
94
+ [1.0],
95
+ # Single-scale, batched
96
+ [1.0, 1.0, 1.0],
97
+ # Multi-scale
98
+ [0.25, 0.5, 1.0],
99
+ ],
100
+)
101
+@pytest.mark.parametrize("max_dynamic_patch", [None, 2, 4, 8])
102
+def test_image_preprocessing(image_assets, model_name, size_factors,
103
+ max_dynamic_patch):
104
+ """Test image preprocessing pipeline with different configurations."""
105
+ # Load the configuration from the model
106
+ config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
107
+
108
+ for asset in image_assets:
109
+ image = asset.pil_image
110
+ for factor in size_factors:
111
+ scaled_image = rescale_image_size(image, factor)
112
+
113
+ # Test preprocessing and get expected number of blocks
114
+ pixel_values, expected_blocks = run_preprocessing_test(
115
+ scaled_image, config, max_dynamic_patch)
116
+
117
+ # Verify output shapes and properties
118
+ actual_blocks = pixel_values.shape[0]
119
+ assert actual_blocks == expected_blocks, (
120
+ f"Expected {expected_blocks} blocks, got {actual_blocks}")
121
+
122
+ # Check image dimensions
123
+ expected_size = (
124
+ 3, # Number of channels (C, H, W)
125
+ config.vision_config.image_size,
126
+ config.vision_config.image_size,
127
+ )
128
+ for img in pixel_values:
129
+ assert img.shape == expected_size, (
130
+ f"Expected image size {expected_size}, got {img.shape}")
tests/models/decoder_only/vision_language/test_models.py
+17
@@ -187,6 +187,23 @@ VLM_TEST_SETTINGS = {
187
marks=[large_gpu_mark(min_gb=48)],
188
patch_hf_runner=model_utils.glm_patch_hf_runner,
189
),
190
+ "h2ovl": VLMTestInfo(
191
+ models = [
192
+ "h2oai/h2ovl-mississippi-800m",
193
+ "h2oai/h2ovl-mississippi-2b",
194
+ ],
195
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
196
+ prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
197
+ single_image_prompts=IMAGE_ASSETS.prompts({
198
+ "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
199
+ "cherry_blossom": "<image>\nWhat is the season?",
200
+ }),
201
+ multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
202
+ max_model_len=8192,
203
+ dtype="bfloat16",
204
+ use_tokenizer_eos=True,
205
+ patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
206
+ ),
207
"intern_vl": VLMTestInfo(
208
models=[
209
"OpenGVLab/InternVL2-1B",
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+60
@@ -259,6 +259,66 @@ def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
259
return hf_model
260
261
262
+def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
263
+ """Patches and returns an instance of the HfRunner to use for H2OVL."""
264
+
265
+ class H2OVLProcessor:
266
+ """A simple processor for H2OVL models."""
267
+
268
+ def __init__(self, hf_runner: HfRunner):
269
+ self.num_image_token = hf_runner.model.num_image_token
270
+ self.tokenizer = hf_runner.tokenizer
271
+ self.dtype = hf_runner.model.dtype
272
+
273
+ self.config = AutoConfig.from_pretrained(hf_runner.model_name,
274
+ trust_remote_code=True)
275
+ self.vision_config = self.config.vision_config
276
+ self.use_thumbnail = self.config.use_thumbnail
277
+ self.min_num = self.config.min_dynamic_patch
278
+ self.max_num = self.config.max_dynamic_patch
279
+ self.image_size = self.vision_config.image_size
280
+
281
+ def __call__(self, text: str, images: Union[Image, List[Image]],
282
+ **kwargs):
283
+ # yapf: disable
284
+ from vllm.model_executor.models.h2ovl import (
285
+ IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
286
+
287
+ # yapf: enable
288
+ images = [images] if isinstance(images, Image) else images
289
+ pixel_values = [
290
+ image_to_pixel_values(image,
291
+ self.image_size,
292
+ self.min_num,
293
+ self.max_num,
294
+ self.use_thumbnail,
295
+ use_MSAC=self.config.use_msac).to(
296
+ self.dtype) for image in images
297
+ ]
298
+ num_patches_list = [
299
+ pixel_value.shape[0] for pixel_value in pixel_values
300
+ ]
301
+ pixel_values = torch.cat(pixel_values, dim=0)
302
+ for num_patches in num_patches_list:
303
+ context_tokens = IMG_CONTEXT * self.num_image_token \
304
+ * num_patches
305
+ image_tokens = IMG_START + context_tokens + IMG_END
306
+ text = text.replace('<image>', image_tokens, 1)
307
+ prompt = self.tokenizer(text, return_tensors="pt")
308
+ prompt.update({"pixel_values": pixel_values})
309
+ return prompt
310
+
311
+ img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
312
+ "<IMG_CONTEXT>")
313
+ hf_model.model.img_context_token_id = img_context_token_id
314
+ hf_model.processor = H2OVLProcessor(hf_model)
315
+ hf_model.model.get_output_embeddings = lambda: \
316
+ hf_model.model.language_model.get_output_embeddings()
317
+ hf_model.model.generate = types.MethodType(_internvl_generate,
318
+ hf_model.model)
319
+ return hf_model
320
+
321
+
322
def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
323
"""Patches and returns an instance of the HfRunner to use for InternVL."""
324
vllm/entrypoints/chat_utils.py
+2
-1
@@ -187,7 +187,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
187
if model_type.startswith("llava"):
188
return self._cached_token_str(self._tokenizer,
189
hf_config.image_token_index)
190
- if model_type in ("chameleon", "internvl_chat", "NVLM_D"):
190
+ if model_type in ("chameleon", "internvl_chat", "NVLM_D",
191
+ "h2ovl_chat"):
192
return "<image>"
193
if model_type == "mllama":
194
return "<|image|>"
vllm/model_executor/models/h2ovl.py
new
+401
@@ -0,0 +1,401 @@
1
+# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py
2
+# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py
3
+# --------------------------------------------------------
4
+# H2OVL-Mississippi
5
+# Copyright (c) 2024 H2O.AI
6
+# Licensed under Apache 2.0 License [see LICENSE for details]
7
+# --------------------------------------------------------
8
+from functools import partial
9
+from typing import List, Optional, Tuple
10
+
11
+import torch
12
+from PIL import Image
13
+from transformers import PretrainedConfig
14
+
15
+from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
16
+ token_inputs)
17
+from vllm.model_executor.layers.quantization import QuantizationConfig
18
+from vllm.multimodal import MULTIMODAL_REGISTRY
19
+from vllm.multimodal.base import MultiModalInputs
20
+from vllm.multimodal.utils import cached_get_tokenizer
21
+from vllm.utils import is_list_of
22
+
23
+from .intern_vit import InternVisionModel
24
+from .internvl import (IMG_CONTEXT, IMG_END, IMG_START, InternVLChatModel,
25
+ InternVLInputPipeline, build_transform,
26
+ find_closest_aspect_ratio, get_internvl_num_patches)
27
+
28
+
29
+# modified to include blocks generated in second pass
30
+def calculate_num_blocks(
31
+ orig_width: int,
32
+ orig_height: int,
33
+ min_num: int,
34
+ max_num: int,
35
+ image_size: int,
36
+ use_thumbnail: bool,
37
+ prior_aspect_ratio=None,
38
+) -> Tuple[int, int, int, Tuple[int, int]]:
39
+ aspect_ratio = orig_width / orig_height
40
+
41
+ # calculate the existing image aspect ratio
42
+ target_ratios = set((i, j) for n in range(min_num, max_num + 1)
43
+ for i in range(1, n + 1) for j in range(1, n + 1)
44
+ if i * j <= max_num and i * j >= min_num)
45
+ target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
46
+
47
+ # if prior_aspect_ratio is provided, filter the target ratios
48
+ if prior_aspect_ratio is not None:
49
+ target_ratios = [
50
+ ratio for ratio in target_ratios if prior_aspect_ratio[0] %
51
+ ratio[0] != 0 and prior_aspect_ratio[1] % ratio[1] != 0
52
+ ]
53
+
54
+ # find the closest aspect ratio to the target
55
+ target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
56
+ target_ratios, orig_width,
57
+ orig_height, image_size)
58
+
59
+ # calculate the target width and height
60
+ target_width = image_size * target_aspect_ratio[0]
61
+ target_height = image_size * target_aspect_ratio[1]
62
+ blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
63
+ # add thumbnail image if num_blocks > 1
64
+ if use_thumbnail and blocks > 1:
65
+ blocks += 1
66
+ return blocks, target_width, target_height, target_aspect_ratio
67
+
68
+
69
+# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
70
+# refactored to handle prior_aspect_ratio as optional
71
+def dynamic_preprocess(
72
+ image: Image.Image,
73
+ min_num: int,
74
+ max_num: int,
75
+ image_size: int,
76
+ use_thumbnail: bool,
77
+ prior_aspect_ratio: Optional[Tuple[int, int]] = None,
78
+) -> Tuple[List[Image.Image], Tuple[int, int]]:
79
+ orig_width, orig_height = image.size
80
+
81
+ # calculate the number of blocks based on prior aspect ratio if available
82
+ blocks, target_width, target_height, target_aspect_ratio = (
83
+ calculate_num_blocks(
84
+ orig_width,
85
+ orig_height,
86
+ min_num,
87
+ max_num,
88
+ image_size,
89
+ use_thumbnail=False,
90
+ prior_aspect_ratio=prior_aspect_ratio,
91
+ ))
92
+ # resize the image
93
+ resized_img = image.resize((target_width, target_height))
94
+ processed_images = []
95
+ for i in range(blocks):
96
+ box = (
97
+ (i % (target_width // image_size)) * image_size,
98
+ (i // (target_width // image_size)) * image_size,
99
+ ((i % (target_width // image_size)) + 1) * image_size,
100
+ ((i // (target_width // image_size)) + 1) * image_size,
101
+ )
102
+ # split the image
103
+ split_img = resized_img.crop(box)
104
+ processed_images.append(split_img)
105
+ assert len(processed_images) == blocks
106
+ if use_thumbnail and len(processed_images) != 1:
107
+ thumbnail_img = image.resize((image_size, image_size))
108
+ processed_images.append(thumbnail_img)
109
+ return processed_images, target_aspect_ratio
110
+
111
+
112
+def load_image(
113
+ image: Image.Image,
114
+ input_size=448,
115
+ min_num=1,
116
+ max_num=6,
117
+ use_thumbnail=True,
118
+ prior_aspect_ratio: Optional[Tuple[int, int]] = None,
119
+) -> Tuple[torch.Tensor, Tuple[int, int]]:
120
+ transform = build_transform(input_size=input_size)
121
+ images, target_aspect_ratio = dynamic_preprocess(
122
+ image,
123
+ image_size=input_size,
124
+ use_thumbnail=use_thumbnail,
125
+ min_num=min_num,
126
+ max_num=max_num,
127
+ prior_aspect_ratio=prior_aspect_ratio,
128
+ )
129
+ pixel_values = [transform(image) for image in images]
130
+ pixel_values = torch.stack(pixel_values)
131
+ return pixel_values, target_aspect_ratio
132
+
133
+
134
+# refactored to use the combined load_image function
135
+def image_to_pixel_values(
136
+ image: Image.Image,
137
+ input_size: int,
138
+ min_num: int,
139
+ max_num: int,
140
+ use_thumbnail: bool,
141
+ use_MSAC: bool,
142
+) -> torch.Tensor:
143
+ # when MSAC is turned on, we need to process the image twice
144
+ if use_MSAC:
145
+ # first pass
146
+ pixel_values, target_aspect_ratio = load_image(
147
+ image,
148
+ input_size=input_size,
149
+ min_num=min_num,
150
+ max_num=max_num,
151
+ use_thumbnail=True,
152
+ )
153
+ # second pass
154
+ pixel_values2, _ = load_image(
155
+ image,
156
+ input_size=input_size,
157
+ min_num=min_num,
158
+ max_num=max_num,
159
+ prior_aspect_ratio=target_aspect_ratio,
160
+ )
161
+ # combine pixel values
162
+ pixel_values = torch.cat(
163
+ [pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0)
164
+
165
+ else:
166
+ pixel_values, _ = load_image(
167
+ image,
168
+ input_size=input_size,
169
+ min_num=min_num,
170
+ max_num=max_num,
171
+ use_thumbnail=use_thumbnail,
172
+ )
173
+
174
+ return pixel_values
175
+
176
+
177
+def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
178
+ max_dynamic_patch: Optional[int] = None,
179
+ use_MSAC: Optional[bool] = None):
180
+ image_size = hf_config.vision_config.image_size
181
+ min_num = hf_config.min_dynamic_patch
182
+ if max_dynamic_patch is None:
183
+ max_dynamic_patch = hf_config.max_dynamic_patch
184
+ if use_MSAC is None:
185
+ use_MSAC = hf_config.use_msac
186
+ use_thumbnail = hf_config.use_thumbnail
187
+ return partial(
188
+ image_to_pixel_values,
189
+ input_size=image_size,
190
+ min_num=min_num,
191
+ max_num=max_dynamic_patch,
192
+ use_thumbnail=use_thumbnail,
193
+ use_MSAC=use_MSAC,
194
+ )
195
+
196
+
197
+def get_max_internvl_image_tokens(ctx: InputContext,
198
+ *,
199
+ max_dynamic_patch: Optional[int] = None):
200
+ """
201
+ Calculate the maximum number of tokens with/without MSAC and thumbnail
202
+ """
203
+ hf_config = ctx.get_hf_config()
204
+ use_thumbnail = hf_config.use_thumbnail
205
+ use_MSAC = hf_config.use_msac
206
+
207
+ if max_dynamic_patch is None:
208
+ max_dynamic_patch = hf_config.max_dynamic_patch
209
+
210
+ num_patches = get_internvl_num_patches(hf_config)
211
+
212
+ coefficient = 2 if use_MSAC else 1
213
+ num_blocks = coefficient * max_dynamic_patch + (1 if use_thumbnail else 0)
214
+
215
+ return num_blocks * num_patches
216
+
217
+
218
+class H2OVLInputPipeline(InternVLInputPipeline):
219
+ """
220
+ Input pipeline for processing image and text data for the H2OVL model.
221
+ """
222
+
223
+ def input_processor(
224
+ self,
225
+ ctx: InputContext,
226
+ inputs: DecoderOnlyInputs,
227
+ *,
228
+ max_dynamic_patch: Optional[int] = None,
229
+ ) -> DecoderOnlyInputs:
230
+ # get multi_modal_data
231
+ multi_modal_data = inputs.get("multi_modal_data")
232
+ if multi_modal_data is None or "image" not in multi_modal_data:
233
+ return inputs
234
+
235
+ model_config = ctx.model_config
236
+ hf_config = ctx.get_hf_config()
237
+ use_MSAC = hf_config.use_msac
238
+
239
+ image_data = multi_modal_data["image"]
240
+ num_patches = get_internvl_num_patches(hf_config)
241
+
242
+ image_pixel_values_mapper = image_to_pixel_values_wrapper(
243
+ hf_config, max_dynamic_patch=max_dynamic_patch)
244
+
245
+ # single image
246
+ if isinstance(image_data, Image.Image):
247
+ pixel_values = image_pixel_values_mapper(image_data,
248
+ use_MSAC=use_MSAC)
249
+ num_blocks = pixel_values.shape[0]
250
+ image_feature_sizes = [num_blocks * num_patches]
251
+ pixel_values = pixel_values.unsqueeze(0)
252
+
253
+ # multi images
254
+ elif is_list_of(image_data, Image.Image):
255
+ # Do not use MSAC for multi images
256
+ image_feature_sizes = []
257
+ pixel_values = [
258
+ image_pixel_values_mapper(image, use_MSAC=False)
259
+ for image in image_data
260
+ ]
261
+ for pixel_value in pixel_values:
262
+ num_blocks = pixel_value.shape[0]
263
+ image_feature_sizes.append(num_blocks * num_patches)
264
+
265
+ # image embeddings as input
266
+ elif isinstance(image_data, torch.Tensor):
267
+ _, image_feature_size, _ = image_data.shape
268
+ image_feature_sizes = [image_feature_size]
269
+ pixel_values = None
270
+
271
+ # multi-image image embeddings
272
+ elif is_list_of(image_data, torch.Tensor):
273
+
274
+ image_feature_sizes = []
275
+ for image_embed in image_data:
276
+ _, image_feature_size, _ = image_embed.shape
277
+ image_feature_sizes.append(image_feature_size)
278
+ pixel_values = None
279
+
280
+ else:
281
+ raise TypeError(f"Invalid image type: {type(image_data)}")
282
+
283
+ tokenizer = cached_get_tokenizer(
284
+ model_config.tokenizer,
285
+ trust_remote_code=model_config.trust_remote_code,
286
+ )
287
+
288
+ prompt = inputs.get("prompt")
289
+ prompt_token_ids = inputs["prompt_token_ids"]
290
+ if prompt is None:
291
+ prompt = tokenizer.decode(prompt_token_ids)
292
+
293
+ new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
294
+ num_patches)
295
+ new_prompt_token_ids = tokenizer.encode(new_prompt)
296
+
297
+ # Wrap image processing in input_processor to avoid duplication
298
+ image_token_id = tokenizer.encode(
299
+ self.img_context_token,
300
+ add_special_tokens=False,
301
+ return_tensors="pt",
302
+ )[0]
303
+
304
+ # Update multi_modal_data to return
305
+ if pixel_values is not None:
306
+ multi_modal_data = {
307
+ "image": {
308
+ "pixel_values": pixel_values,
309
+ "image_token_id": image_token_id,
310
+ }
311
+ }
312
+ else:
313
+ multi_modal_data = {"image": {"image_embeds": image_data}}
314
+
315
+ return token_inputs(
316
+ prompt=prompt,
317
+ prompt_token_ids=new_prompt_token_ids,
318
+ multi_modal_data=multi_modal_data,
319
+ )
320
+
321
+ def input_mapper(
322
+ self,
323
+ ctx: InputContext,
324
+ data: object,
325
+ *,
326
+ max_dynamic_patch: Optional[int] = None,
327
+ ) -> MultiModalInputs:
328
+
329
+ # NOTE: Preprocessing for the image data is done in the
330
+ # 'input_processor' function during actual inference.
331
+ if isinstance(data, dict):
332
+ return MultiModalInputs(data)
333
+
334
+ # The section below is only used with dummy data during
335
+ # memory profiling.
336
+ hf_config = ctx.get_hf_config()
337
+
338
+ image_pixel_values_mapper = image_to_pixel_values_wrapper(
339
+ hf_config, max_dynamic_patch)
340
+
341
+ if isinstance(data, Image.Image):
342
+ pixel_values = image_pixel_values_mapper(data)
343
+ pixel_values = pixel_values.unsqueeze(0)
344
+
345
+ elif is_list_of(data, Image.Image):
346
+ hf_config.use_msac = False
347
+ pixel_values = [image_pixel_values_mapper(img) for img in data]
348
+
349
+ else:
350
+ return MultiModalInputs({"image_embeds": data})
351
+ model_config = ctx.model_config
352
+ tokenizer = cached_get_tokenizer(
353
+ model_config.tokenizer,
354
+ trust_remote_code=model_config.trust_remote_code,
355
+ )
356
+ image_token_id = tokenizer.encode(
357
+ self.img_context_token,
358
+ add_special_tokens=False,
359
+ return_tensors="pt",
360
+ )[0]
361
+
362
+ return MultiModalInputs({
363
+ "pixel_values": pixel_values,
364
+ "image_token_id": image_token_id
365
+ })
366
+
367
+
368
+input_pipeline = H2OVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
369
+
370
+
371
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
372
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
373
+@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
374
+@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
375
+class H2OVLChatModel(InternVLChatModel):
376
+
377
+ def _init_vision_model(
378
+ self,
379
+ config: PretrainedConfig,
380
+ quant_config: Optional[QuantizationConfig],
381
+ *,
382
+ is_mono: bool,
383
+ prefix: str,
384
+ ):
385
+ if not is_mono:
386
+ vision_feature_layer = config.select_layer
387
+ if vision_feature_layer < 0:
388
+ num_hidden_layers = (config.vision_config.num_hidden_layers +
389
+ vision_feature_layer + 1)
390
+ else:
391
+ num_hidden_layers = vision_feature_layer + 1
392
+
393
+ return InternVisionModel(
394
+ config.vision_config,
395
+ quant_config=quant_config,
396
+ num_hidden_layers_override=num_hidden_layers,
397
+ prefix=prefix,
398
+ )
399
+ else:
400
+ msg = "Monolith mode is not applicable to H2OVL"
401
+ raise NotImplementedError(msg)
vllm/model_executor/models/registry.py
+2
-1
@@ -128,6 +128,7 @@ _MULTIMODAL_MODELS = {
128
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
129
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
130
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
131
+ "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
132
"InternVLChatModel": ("internvl", "InternVLChatModel"),
133
"LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
134
"LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
@@ -482,4 +483,4 @@ def _run() -> None:
483
484
485
if __name__ == "__main__":
485
- _run()
486
+ _run()
\ No newline at end of file
vllm/transformers_utils/config.py
+2
@@ -19,6 +19,7 @@ from vllm.logger import init_logger
19
# yapf: disable
20
from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
21
EAGLEConfig, ExaoneConfig,
22
+ H2OVLChatConfig,
23
InternVLChatConfig, JAISConfig,
24
MedusaConfig, MllamaConfig,
25
MLPSpeculatorConfig, MPTConfig,
@@ -52,6 +53,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
53
"medusa": MedusaConfig,
54
"eagle": EAGLEConfig,
55
"exaone": ExaoneConfig,
56
+ "h2ovl_chat": H2OVLChatConfig,
57
"internvl_chat": InternVLChatConfig,
58
"nemotron": NemotronConfig,
59
"NVLM_D": NVLM_D_Config,
vllm/transformers_utils/configs/__init__.py
+3
-1
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
6
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7
# `FalconConfig` class from the official HuggingFace transformers library.
8
from vllm.transformers_utils.configs.falcon import RWConfig
9
+from vllm.transformers_utils.configs.h2ovl import H2OVLChatConfig
10
from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11
from vllm.transformers_utils.configs.jais import JAISConfig
12
from vllm.transformers_utils.configs.medusa import MedusaConfig
@@ -22,6 +23,7 @@ __all__ = [
23
"DbrxConfig",
24
"MPTConfig",
25
"RWConfig",
26
+ "H2OVLChatConfig",
27
"InternVLChatConfig",
28
"JAISConfig",
29
"MedusaConfig",
@@ -33,4 +35,4 @@ __all__ = [
35
"NVLM_D_Config",
36
"SolarConfig",
37
"UltravoxConfig",
36
-]
38
+]
\ No newline at end of file
vllm/transformers_utils/configs/h2ovl.py
new
+13
@@ -0,0 +1,13 @@
1
+# Adapted from
2
+# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/configuration_h2ovl_chat.py
3
+# --------------------------------------------------------
4
+# H2OVL-Mississippi
5
+# Copyright (c) 2024 H2O.AI
6
+# Licensed under Apache 2.0 License [see LICENSE for details]
7
+# --------------------------------------------------------
8
+
9
+from .internvl import InternVLChatConfig
10
+
11
+
12
+class H2OVLChatConfig(InternVLChatConfig):
13
+ model_type = "h2ovl_chat"