@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54597724f4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model] Add support for H2OVL-Mississippi models (#9747)

Signed-off-by: Shanshan Wang <shanshan.wang@h2o.ai> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>

shanshan wang committed Nov 3, 2024 at 18:15 UTC 54597724f4c6b52d50152f3cc46e86c101d9c820
12 files changed +698 -4
docs/source/models/supported_models.rst
+6
@@ -440,6 +440,12 @@ Text Generation
440 - :code:`THUDM/glm-4v-9b` etc.
441 -
442 - ✅︎
443 + * - :code:`H2OVLChatModel`
444 + - H2OVL
445 + - T + I\ :sup:`E+`
446 + - :code:`h2oai/h2ovl-mississippi-800m`, :code:`h2oai/h2ovl-mississippi-2b`, etc.
447 + -
448 + - ✅︎
449 * - :code:`InternVLChatModel`
450 - InternVL2
451 - T + I\ :sup:`E+`
examples/offline_inference_vision_language.py
+27 -1
@@ -176,6 +176,31 @@ def run_minicpmv(question: str, modality: str):
176 return llm, prompt, stop_token_ids
177
178
179 +# H2OVL-Mississippi
180 +def run_h2ovl(question: str, modality: str):
181 + assert modality == "image"
182 +
183 + model_name = "h2oai/h2ovl-mississippi-2b"
184 +
185 + llm = LLM(
186 + model=model_name,
187 + trust_remote_code=True,
188 + max_model_len=8192,
189 + )
190 +
191 + tokenizer = AutoTokenizer.from_pretrained(model_name,
192 + trust_remote_code=True)
193 + messages = [{'role': 'user', 'content': f"<image>\n{question}"}]
194 + prompt = tokenizer.apply_chat_template(messages,
195 + tokenize=False,
196 + add_generation_prompt=True)
197 +
198 + # Stop tokens for H2OVL-Mississippi
199 + # https://huggingface.co/h2oai/h2ovl-mississippi-2b
200 + stop_token_ids = [tokenizer.eos_token_id]
201 + return llm, prompt, stop_token_ids
202 +
203 +
204 # InternVL
205 def run_internvl(question: str, modality: str):
206 assert modality == "image"
@@ -363,6 +388,7 @@ model_example_map = {
388 "chameleon": run_chameleon,
389 "minicpmv": run_minicpmv,
390 "blip-2": run_blip2,
391 + "h2ovl_chat": run_h2ovl,
392 "internvl_chat": run_internvl,
393 "NVLM_D": run_nvlm_d,
394 "qwen_vl": run_qwen_vl,
@@ -475,4 +501,4 @@ if __name__ == "__main__":
501 default=16,
502 help='Number of frames to extract from the video.')
503 args = parser.parse_args()
478 - main(args)
504 + main(args)
\ No newline at end of file
examples/offline_inference_vision_language_multi_image.py
+35
@@ -107,6 +107,40 @@ def load_phi3v(question: str, image_urls: List[str]) -> ModelRequestData:
107 )
108
109
110 +def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
111 + model_name = "h2oai/h2ovl-mississippi-2b"
112 +
113 + llm = LLM(
114 + model=model_name,
115 + trust_remote_code=True,
116 + max_model_len=8192,
117 + limit_mm_per_prompt={"image": len(image_urls)},
118 + mm_processor_kwargs={"max_dynamic_patch": 4},
119 + )
120 +
121 + placeholders = "\n".join(f"Image-{i}: <image>\n"
122 + for i, _ in enumerate(image_urls, start=1))
123 + messages = [{'role': 'user', 'content': f"{placeholders}\n{question}"}]
124 +
125 + tokenizer = AutoTokenizer.from_pretrained(model_name,
126 + trust_remote_code=True)
127 + prompt = tokenizer.apply_chat_template(messages,
128 + tokenize=False,
129 + add_generation_prompt=True)
130 +
131 + # Stop tokens for H2OVL-Mississippi
132 + # https://huggingface.co/h2oai/h2ovl-mississippi-2b
133 + stop_token_ids = [tokenizer.eos_token_id]
134 +
135 + return ModelRequestData(
136 + llm=llm,
137 + prompt=prompt,
138 + stop_token_ids=stop_token_ids,
139 + image_data=[fetch_image(url) for url in image_urls],
140 + chat_template=None,
141 + )
142 +
143 +
144 def load_internvl(question: str, image_urls: List[str]) -> ModelRequestData:
145 model_name = "OpenGVLab/InternVL2-2B"
146
@@ -258,6 +292,7 @@ def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
292
293 model_example_map = {
294 "phi3_v": load_phi3v,
295 + "h2ovl_chat": load_h2onvl,
296 "internvl_chat": load_internvl,
297 "NVLM_D": load_nvlm_d,
298 "qwen2_vl": load_qwen2_vl,
tests/models/decoder_only/vision_language/test_h2ovl.py new
+130
@@ -0,0 +1,130 @@
1 +from typing import Optional, Tuple
2 +
3 +import pytest
4 +import torch
5 +from PIL.Image import Image
6 +from transformers import AutoConfig
7 +
8 +# Import the functions to test
9 +from vllm.model_executor.models.h2ovl import (calculate_num_blocks,
10 + image_to_pixel_values_wrapper)
11 +from vllm.multimodal.utils import rescale_image_size
12 +
13 +models = [
14 + "h2oai/h2ovl-mississippi-800m", # Replace with your actual model names
15 + "h2oai/h2ovl-mississippi-2b",
16 +]
17 +target_dtype = "bfloat16"
18 +
19 +
20 +def run_preprocessing_test(
21 + image: Image,
22 + config,
23 + max_dynamic_patch: Optional[int] = None,
24 +) -> Tuple[torch.Tensor, int]:
25 + """Test the image preprocessing and calculate expected blocks."""
26 +
27 + if max_dynamic_patch is None:
28 + max_dynamic_patch = config.max_dynamic_patch
29 +
30 + width, height = image.size
31 + use_MSAC = config.use_msac
32 +
33 + # Create the mapper function with the provided configuration
34 + mapper = image_to_pixel_values_wrapper(config, max_dynamic_patch, use_MSAC)
35 + pixel_values = mapper(image)
36 +
37 + # Calculate the expected number of blocks
38 + if use_MSAC:
39 + # First pass
40 + blocks1, _, _, aspect_ratio = calculate_num_blocks(
41 + width,
42 + height,
43 + config.min_dynamic_patch,
44 + max_dynamic_patch,
45 + config.vision_config.image_size,
46 + use_thumbnail=False, # Thumbnail is handled separately
47 + prior_aspect_ratio=None,
48 + )
49 +
50 + # Second pass
51 + blocks2, _, _, _ = calculate_num_blocks(
52 + width,
53 + height,
54 + config.min_dynamic_patch,
55 + max_dynamic_patch,
56 + config.vision_config.image_size,
57 + use_thumbnail=False,
58 + prior_aspect_ratio=aspect_ratio,
59 + )
60 +
61 + # Add thumbnail if use_thumbnail is True and total_blocks > 1
62 + if config.use_thumbnail:
63 + blocks1 += 1 if blocks1 > 1 else 0
64 + blocks2 += 1 if blocks2 > 1 else 0
65 +
66 + # Total blocks is the sum of blocks from both passes minus overlapping
67 + total_blocks = blocks1 + blocks2 - 1
68 +
69 + expected_blocks = total_blocks
70 +
71 + else:
72 + blocks, _, _, _ = calculate_num_blocks(
73 + width,
74 + height,
75 + config.min_dynamic_patch,
76 + max_dynamic_patch,
77 + config.vision_config.image_size,
78 + use_thumbnail=False,
79 + prior_aspect_ratio=None,
80 + )
81 + expected_blocks = blocks
82 +
83 + if config.use_thumbnail and expected_blocks > 1:
84 + expected_blocks += 1
85 +
86 + return pixel_values, expected_blocks
87 +
88 +
89 +@pytest.mark.parametrize("model_name", models)
90 +@pytest.mark.parametrize(
91 + "size_factors",
92 + [
93 + # Single-scale
94 + [1.0],
95 + # Single-scale, batched
96 + [1.0, 1.0, 1.0],
97 + # Multi-scale
98 + [0.25, 0.5, 1.0],
99 + ],
100 +)
101 +@pytest.mark.parametrize("max_dynamic_patch", [None, 2, 4, 8])
102 +def test_image_preprocessing(image_assets, model_name, size_factors,
103 + max_dynamic_patch):
104 + """Test image preprocessing pipeline with different configurations."""
105 + # Load the configuration from the model
106 + config = AutoConfig.from_pretrained(model_name, trust_remote_code=True)
107 +
108 + for asset in image_assets:
109 + image = asset.pil_image
110 + for factor in size_factors:
111 + scaled_image = rescale_image_size(image, factor)
112 +
113 + # Test preprocessing and get expected number of blocks
114 + pixel_values, expected_blocks = run_preprocessing_test(
115 + scaled_image, config, max_dynamic_patch)
116 +
117 + # Verify output shapes and properties
118 + actual_blocks = pixel_values.shape[0]
119 + assert actual_blocks == expected_blocks, (
120 + f"Expected {expected_blocks} blocks, got {actual_blocks}")
121 +
122 + # Check image dimensions
123 + expected_size = (
124 + 3, # Number of channels (C, H, W)
125 + config.vision_config.image_size,
126 + config.vision_config.image_size,
127 + )
128 + for img in pixel_values:
129 + assert img.shape == expected_size, (
130 + f"Expected image size {expected_size}, got {img.shape}")
tests/models/decoder_only/vision_language/test_models.py
+17
@@ -187,6 +187,23 @@ VLM_TEST_SETTINGS = {
187 marks=[large_gpu_mark(min_gb=48)],
188 patch_hf_runner=model_utils.glm_patch_hf_runner,
189 ),
190 + "h2ovl": VLMTestInfo(
191 + models = [
192 + "h2oai/h2ovl-mississippi-800m",
193 + "h2oai/h2ovl-mississippi-2b",
194 + ],
195 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
196 + prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>", # noqa: E501
197 + single_image_prompts=IMAGE_ASSETS.prompts({
198 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
199 + "cherry_blossom": "<image>\nWhat is the season?",
200 + }),
201 + multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
202 + max_model_len=8192,
203 + dtype="bfloat16",
204 + use_tokenizer_eos=True,
205 + patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
206 + ),
207 "intern_vl": VLMTestInfo(
208 models=[
209 "OpenGVLab/InternVL2-1B",
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+60
@@ -259,6 +259,66 @@ def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
259 return hf_model
260
261
262 +def h2ovl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
263 + """Patches and returns an instance of the HfRunner to use for H2OVL."""
264 +
265 + class H2OVLProcessor:
266 + """A simple processor for H2OVL models."""
267 +
268 + def __init__(self, hf_runner: HfRunner):
269 + self.num_image_token = hf_runner.model.num_image_token
270 + self.tokenizer = hf_runner.tokenizer
271 + self.dtype = hf_runner.model.dtype
272 +
273 + self.config = AutoConfig.from_pretrained(hf_runner.model_name,
274 + trust_remote_code=True)
275 + self.vision_config = self.config.vision_config
276 + self.use_thumbnail = self.config.use_thumbnail
277 + self.min_num = self.config.min_dynamic_patch
278 + self.max_num = self.config.max_dynamic_patch
279 + self.image_size = self.vision_config.image_size
280 +
281 + def __call__(self, text: str, images: Union[Image, List[Image]],
282 + **kwargs):
283 + # yapf: disable
284 + from vllm.model_executor.models.h2ovl import (
285 + IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
286 +
287 + # yapf: enable
288 + images = [images] if isinstance(images, Image) else images
289 + pixel_values = [
290 + image_to_pixel_values(image,
291 + self.image_size,
292 + self.min_num,
293 + self.max_num,
294 + self.use_thumbnail,
295 + use_MSAC=self.config.use_msac).to(
296 + self.dtype) for image in images
297 + ]
298 + num_patches_list = [
299 + pixel_value.shape[0] for pixel_value in pixel_values
300 + ]
301 + pixel_values = torch.cat(pixel_values, dim=0)
302 + for num_patches in num_patches_list:
303 + context_tokens = IMG_CONTEXT * self.num_image_token \
304 + * num_patches
305 + image_tokens = IMG_START + context_tokens + IMG_END
306 + text = text.replace('<image>', image_tokens, 1)
307 + prompt = self.tokenizer(text, return_tensors="pt")
308 + prompt.update({"pixel_values": pixel_values})
309 + return prompt
310 +
311 + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
312 + "<IMG_CONTEXT>")
313 + hf_model.model.img_context_token_id = img_context_token_id
314 + hf_model.processor = H2OVLProcessor(hf_model)
315 + hf_model.model.get_output_embeddings = lambda: \
316 + hf_model.model.language_model.get_output_embeddings()
317 + hf_model.model.generate = types.MethodType(_internvl_generate,
318 + hf_model.model)
319 + return hf_model
320 +
321 +
322 def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
323 """Patches and returns an instance of the HfRunner to use for InternVL."""
324
vllm/entrypoints/chat_utils.py
+2 -1
@@ -187,7 +187,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
187 if model_type.startswith("llava"):
188 return self._cached_token_str(self._tokenizer,
189 hf_config.image_token_index)
190 - if model_type in ("chameleon", "internvl_chat", "NVLM_D"):
190 + if model_type in ("chameleon", "internvl_chat", "NVLM_D",
191 + "h2ovl_chat"):
192 return "<image>"
193 if model_type == "mllama":
194 return "<|image|>"
vllm/model_executor/models/h2ovl.py new
+401
@@ -0,0 +1,401 @@
1 +# adapted from https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/modeling_h2ovl_chat.py
2 +# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/image_process.py
3 +# --------------------------------------------------------
4 +# H2OVL-Mississippi
5 +# Copyright (c) 2024 H2O.AI
6 +# Licensed under Apache 2.0 License [see LICENSE for details]
7 +# --------------------------------------------------------
8 +from functools import partial
9 +from typing import List, Optional, Tuple
10 +
11 +import torch
12 +from PIL import Image
13 +from transformers import PretrainedConfig
14 +
15 +from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, InputContext,
16 + token_inputs)
17 +from vllm.model_executor.layers.quantization import QuantizationConfig
18 +from vllm.multimodal import MULTIMODAL_REGISTRY
19 +from vllm.multimodal.base import MultiModalInputs
20 +from vllm.multimodal.utils import cached_get_tokenizer
21 +from vllm.utils import is_list_of
22 +
23 +from .intern_vit import InternVisionModel
24 +from .internvl import (IMG_CONTEXT, IMG_END, IMG_START, InternVLChatModel,
25 + InternVLInputPipeline, build_transform,
26 + find_closest_aspect_ratio, get_internvl_num_patches)
27 +
28 +
29 +# modified to include blocks generated in second pass
30 +def calculate_num_blocks(
31 + orig_width: int,
32 + orig_height: int,
33 + min_num: int,
34 + max_num: int,
35 + image_size: int,
36 + use_thumbnail: bool,
37 + prior_aspect_ratio=None,
38 +) -> Tuple[int, int, int, Tuple[int, int]]:
39 + aspect_ratio = orig_width / orig_height
40 +
41 + # calculate the existing image aspect ratio
42 + target_ratios = set((i, j) for n in range(min_num, max_num + 1)
43 + for i in range(1, n + 1) for j in range(1, n + 1)
44 + if i * j <= max_num and i * j >= min_num)
45 + target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
46 +
47 + # if prior_aspect_ratio is provided, filter the target ratios
48 + if prior_aspect_ratio is not None:
49 + target_ratios = [
50 + ratio for ratio in target_ratios if prior_aspect_ratio[0] %
51 + ratio[0] != 0 and prior_aspect_ratio[1] % ratio[1] != 0
52 + ]
53 +
54 + # find the closest aspect ratio to the target
55 + target_aspect_ratio = find_closest_aspect_ratio(aspect_ratio,
56 + target_ratios, orig_width,
57 + orig_height, image_size)
58 +
59 + # calculate the target width and height
60 + target_width = image_size * target_aspect_ratio[0]
61 + target_height = image_size * target_aspect_ratio[1]
62 + blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
63 + # add thumbnail image if num_blocks > 1
64 + if use_thumbnail and blocks > 1:
65 + blocks += 1
66 + return blocks, target_width, target_height, target_aspect_ratio
67 +
68 +
69 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
70 +# refactored to handle prior_aspect_ratio as optional
71 +def dynamic_preprocess(
72 + image: Image.Image,
73 + min_num: int,
74 + max_num: int,
75 + image_size: int,
76 + use_thumbnail: bool,
77 + prior_aspect_ratio: Optional[Tuple[int, int]] = None,
78 +) -> Tuple[List[Image.Image], Tuple[int, int]]:
79 + orig_width, orig_height = image.size
80 +
81 + # calculate the number of blocks based on prior aspect ratio if available
82 + blocks, target_width, target_height, target_aspect_ratio = (
83 + calculate_num_blocks(
84 + orig_width,
85 + orig_height,
86 + min_num,
87 + max_num,
88 + image_size,
89 + use_thumbnail=False,
90 + prior_aspect_ratio=prior_aspect_ratio,
91 + ))
92 + # resize the image
93 + resized_img = image.resize((target_width, target_height))
94 + processed_images = []
95 + for i in range(blocks):
96 + box = (
97 + (i % (target_width // image_size)) * image_size,
98 + (i // (target_width // image_size)) * image_size,
99 + ((i % (target_width // image_size)) + 1) * image_size,
100 + ((i // (target_width // image_size)) + 1) * image_size,
101 + )
102 + # split the image
103 + split_img = resized_img.crop(box)
104 + processed_images.append(split_img)
105 + assert len(processed_images) == blocks
106 + if use_thumbnail and len(processed_images) != 1:
107 + thumbnail_img = image.resize((image_size, image_size))
108 + processed_images.append(thumbnail_img)
109 + return processed_images, target_aspect_ratio
110 +
111 +
112 +def load_image(
113 + image: Image.Image,
114 + input_size=448,
115 + min_num=1,
116 + max_num=6,
117 + use_thumbnail=True,
118 + prior_aspect_ratio: Optional[Tuple[int, int]] = None,
119 +) -> Tuple[torch.Tensor, Tuple[int, int]]:
120 + transform = build_transform(input_size=input_size)
121 + images, target_aspect_ratio = dynamic_preprocess(
122 + image,
123 + image_size=input_size,
124 + use_thumbnail=use_thumbnail,
125 + min_num=min_num,
126 + max_num=max_num,
127 + prior_aspect_ratio=prior_aspect_ratio,
128 + )
129 + pixel_values = [transform(image) for image in images]
130 + pixel_values = torch.stack(pixel_values)
131 + return pixel_values, target_aspect_ratio
132 +
133 +
134 +# refactored to use the combined load_image function
135 +def image_to_pixel_values(
136 + image: Image.Image,
137 + input_size: int,
138 + min_num: int,
139 + max_num: int,
140 + use_thumbnail: bool,
141 + use_MSAC: bool,
142 +) -> torch.Tensor:
143 + # when MSAC is turned on, we need to process the image twice
144 + if use_MSAC:
145 + # first pass
146 + pixel_values, target_aspect_ratio = load_image(
147 + image,
148 + input_size=input_size,
149 + min_num=min_num,
150 + max_num=max_num,
151 + use_thumbnail=True,
152 + )
153 + # second pass
154 + pixel_values2, _ = load_image(
155 + image,
156 + input_size=input_size,
157 + min_num=min_num,
158 + max_num=max_num,
159 + prior_aspect_ratio=target_aspect_ratio,
160 + )
161 + # combine pixel values
162 + pixel_values = torch.cat(
163 + [pixel_values2[:-1], pixel_values[:-1], pixel_values2[-1:]], 0)
164 +
165 + else:
166 + pixel_values, _ = load_image(
167 + image,
168 + input_size=input_size,
169 + min_num=min_num,
170 + max_num=max_num,
171 + use_thumbnail=use_thumbnail,
172 + )
173 +
174 + return pixel_values
175 +
176 +
177 +def image_to_pixel_values_wrapper(hf_config: PretrainedConfig,
178 + max_dynamic_patch: Optional[int] = None,
179 + use_MSAC: Optional[bool] = None):
180 + image_size = hf_config.vision_config.image_size
181 + min_num = hf_config.min_dynamic_patch
182 + if max_dynamic_patch is None:
183 + max_dynamic_patch = hf_config.max_dynamic_patch
184 + if use_MSAC is None:
185 + use_MSAC = hf_config.use_msac
186 + use_thumbnail = hf_config.use_thumbnail
187 + return partial(
188 + image_to_pixel_values,
189 + input_size=image_size,
190 + min_num=min_num,
191 + max_num=max_dynamic_patch,
192 + use_thumbnail=use_thumbnail,
193 + use_MSAC=use_MSAC,
194 + )
195 +
196 +
197 +def get_max_internvl_image_tokens(ctx: InputContext,
198 + *,
199 + max_dynamic_patch: Optional[int] = None):
200 + """
201 + Calculate the maximum number of tokens with/without MSAC and thumbnail
202 + """
203 + hf_config = ctx.get_hf_config()
204 + use_thumbnail = hf_config.use_thumbnail
205 + use_MSAC = hf_config.use_msac
206 +
207 + if max_dynamic_patch is None:
208 + max_dynamic_patch = hf_config.max_dynamic_patch
209 +
210 + num_patches = get_internvl_num_patches(hf_config)
211 +
212 + coefficient = 2 if use_MSAC else 1
213 + num_blocks = coefficient * max_dynamic_patch + (1 if use_thumbnail else 0)
214 +
215 + return num_blocks * num_patches
216 +
217 +
218 +class H2OVLInputPipeline(InternVLInputPipeline):
219 + """
220 + Input pipeline for processing image and text data for the H2OVL model.
221 + """
222 +
223 + def input_processor(
224 + self,
225 + ctx: InputContext,
226 + inputs: DecoderOnlyInputs,
227 + *,
228 + max_dynamic_patch: Optional[int] = None,
229 + ) -> DecoderOnlyInputs:
230 + # get multi_modal_data
231 + multi_modal_data = inputs.get("multi_modal_data")
232 + if multi_modal_data is None or "image" not in multi_modal_data:
233 + return inputs
234 +
235 + model_config = ctx.model_config
236 + hf_config = ctx.get_hf_config()
237 + use_MSAC = hf_config.use_msac
238 +
239 + image_data = multi_modal_data["image"]
240 + num_patches = get_internvl_num_patches(hf_config)
241 +
242 + image_pixel_values_mapper = image_to_pixel_values_wrapper(
243 + hf_config, max_dynamic_patch=max_dynamic_patch)
244 +
245 + # single image
246 + if isinstance(image_data, Image.Image):
247 + pixel_values = image_pixel_values_mapper(image_data,
248 + use_MSAC=use_MSAC)
249 + num_blocks = pixel_values.shape[0]
250 + image_feature_sizes = [num_blocks * num_patches]
251 + pixel_values = pixel_values.unsqueeze(0)
252 +
253 + # multi images
254 + elif is_list_of(image_data, Image.Image):
255 + # Do not use MSAC for multi images
256 + image_feature_sizes = []
257 + pixel_values = [
258 + image_pixel_values_mapper(image, use_MSAC=False)
259 + for image in image_data
260 + ]
261 + for pixel_value in pixel_values:
262 + num_blocks = pixel_value.shape[0]
263 + image_feature_sizes.append(num_blocks * num_patches)
264 +
265 + # image embeddings as input
266 + elif isinstance(image_data, torch.Tensor):
267 + _, image_feature_size, _ = image_data.shape
268 + image_feature_sizes = [image_feature_size]
269 + pixel_values = None
270 +
271 + # multi-image image embeddings
272 + elif is_list_of(image_data, torch.Tensor):
273 +
274 + image_feature_sizes = []
275 + for image_embed in image_data:
276 + _, image_feature_size, _ = image_embed.shape
277 + image_feature_sizes.append(image_feature_size)
278 + pixel_values = None
279 +
280 + else:
281 + raise TypeError(f"Invalid image type: {type(image_data)}")
282 +
283 + tokenizer = cached_get_tokenizer(
284 + model_config.tokenizer,
285 + trust_remote_code=model_config.trust_remote_code,
286 + )
287 +
288 + prompt = inputs.get("prompt")
289 + prompt_token_ids = inputs["prompt_token_ids"]
290 + if prompt is None:
291 + prompt = tokenizer.decode(prompt_token_ids)
292 +
293 + new_prompt = self._expand_image_prompt(prompt, image_feature_sizes,
294 + num_patches)
295 + new_prompt_token_ids = tokenizer.encode(new_prompt)
296 +
297 + # Wrap image processing in input_processor to avoid duplication
298 + image_token_id = tokenizer.encode(
299 + self.img_context_token,
300 + add_special_tokens=False,
301 + return_tensors="pt",
302 + )[0]
303 +
304 + # Update multi_modal_data to return
305 + if pixel_values is not None:
306 + multi_modal_data = {
307 + "image": {
308 + "pixel_values": pixel_values,
309 + "image_token_id": image_token_id,
310 + }
311 + }
312 + else:
313 + multi_modal_data = {"image": {"image_embeds": image_data}}
314 +
315 + return token_inputs(
316 + prompt=prompt,
317 + prompt_token_ids=new_prompt_token_ids,
318 + multi_modal_data=multi_modal_data,
319 + )
320 +
321 + def input_mapper(
322 + self,
323 + ctx: InputContext,
324 + data: object,
325 + *,
326 + max_dynamic_patch: Optional[int] = None,
327 + ) -> MultiModalInputs:
328 +
329 + # NOTE: Preprocessing for the image data is done in the
330 + # 'input_processor' function during actual inference.
331 + if isinstance(data, dict):
332 + return MultiModalInputs(data)
333 +
334 + # The section below is only used with dummy data during
335 + # memory profiling.
336 + hf_config = ctx.get_hf_config()
337 +
338 + image_pixel_values_mapper = image_to_pixel_values_wrapper(
339 + hf_config, max_dynamic_patch)
340 +
341 + if isinstance(data, Image.Image):
342 + pixel_values = image_pixel_values_mapper(data)
343 + pixel_values = pixel_values.unsqueeze(0)
344 +
345 + elif is_list_of(data, Image.Image):
346 + hf_config.use_msac = False
347 + pixel_values = [image_pixel_values_mapper(img) for img in data]
348 +
349 + else:
350 + return MultiModalInputs({"image_embeds": data})
351 + model_config = ctx.model_config
352 + tokenizer = cached_get_tokenizer(
353 + model_config.tokenizer,
354 + trust_remote_code=model_config.trust_remote_code,
355 + )
356 + image_token_id = tokenizer.encode(
357 + self.img_context_token,
358 + add_special_tokens=False,
359 + return_tensors="pt",
360 + )[0]
361 +
362 + return MultiModalInputs({
363 + "pixel_values": pixel_values,
364 + "image_token_id": image_token_id
365 + })
366 +
367 +
368 +input_pipeline = H2OVLInputPipeline(IMG_START, IMG_END, IMG_CONTEXT)
369 +
370 +
371 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_pipeline.input_mapper)
372 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_internvl_image_tokens)
373 +@INPUT_REGISTRY.register_dummy_data(input_pipeline.dummy_data)
374 +@INPUT_REGISTRY.register_input_processor(input_pipeline.input_processor)
375 +class H2OVLChatModel(InternVLChatModel):
376 +
377 + def _init_vision_model(
378 + self,
379 + config: PretrainedConfig,
380 + quant_config: Optional[QuantizationConfig],
381 + *,
382 + is_mono: bool,
383 + prefix: str,
384 + ):
385 + if not is_mono:
386 + vision_feature_layer = config.select_layer
387 + if vision_feature_layer < 0:
388 + num_hidden_layers = (config.vision_config.num_hidden_layers +
389 + vision_feature_layer + 1)
390 + else:
391 + num_hidden_layers = vision_feature_layer + 1
392 +
393 + return InternVisionModel(
394 + config.vision_config,
395 + quant_config=quant_config,
396 + num_hidden_layers_override=num_hidden_layers,
397 + prefix=prefix,
398 + )
399 + else:
400 + msg = "Monolith mode is not applicable to H2OVL"
401 + raise NotImplementedError(msg)
vllm/model_executor/models/registry.py
+2 -1
@@ -128,6 +128,7 @@ _MULTIMODAL_MODELS = {
128 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
129 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
130 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
131 + "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
132 "InternVLChatModel": ("internvl", "InternVLChatModel"),
133 "LlavaForConditionalGeneration": ("llava", "LlavaForConditionalGeneration"),
134 "LlavaNextForConditionalGeneration": ("llava_next", "LlavaNextForConditionalGeneration"), # noqa: E501
@@ -482,4 +483,4 @@ def _run() -> None:
483
484
485 if __name__ == "__main__":
485 - _run()
486 + _run()
\ No newline at end of file
vllm/transformers_utils/config.py
+2
@@ -19,6 +19,7 @@ from vllm.logger import init_logger
19 # yapf: disable
20 from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
21 EAGLEConfig, ExaoneConfig,
22 + H2OVLChatConfig,
23 InternVLChatConfig, JAISConfig,
24 MedusaConfig, MllamaConfig,
25 MLPSpeculatorConfig, MPTConfig,
@@ -52,6 +53,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
53 "medusa": MedusaConfig,
54 "eagle": EAGLEConfig,
55 "exaone": ExaoneConfig,
56 + "h2ovl_chat": H2OVLChatConfig,
57 "internvl_chat": InternVLChatConfig,
58 "nemotron": NemotronConfig,
59 "NVLM_D": NVLM_D_Config,
vllm/transformers_utils/configs/__init__.py
+3 -1
@@ -6,6 +6,7 @@ from vllm.transformers_utils.configs.exaone import ExaoneConfig
6 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
7 # `FalconConfig` class from the official HuggingFace transformers library.
8 from vllm.transformers_utils.configs.falcon import RWConfig
9 +from vllm.transformers_utils.configs.h2ovl import H2OVLChatConfig
10 from vllm.transformers_utils.configs.internvl import InternVLChatConfig
11 from vllm.transformers_utils.configs.jais import JAISConfig
12 from vllm.transformers_utils.configs.medusa import MedusaConfig
@@ -22,6 +23,7 @@ __all__ = [
23 "DbrxConfig",
24 "MPTConfig",
25 "RWConfig",
26 + "H2OVLChatConfig",
27 "InternVLChatConfig",
28 "JAISConfig",
29 "MedusaConfig",
@@ -33,4 +35,4 @@ __all__ = [
35 "NVLM_D_Config",
36 "SolarConfig",
37 "UltravoxConfig",
36 -]
38 +]
\ No newline at end of file
vllm/transformers_utils/configs/h2ovl.py new
+13
@@ -0,0 +1,13 @@
1 +# Adapted from
2 +# https://huggingface.co/h2oai/h2ovl-mississippi-2b/blob/main/configuration_h2ovl_chat.py
3 +# --------------------------------------------------------
4 +# H2OVL-Mississippi
5 +# Copyright (c) 2024 H2O.AI
6 +# Licensed under Apache 2.0 License [see LICENSE for details]
7 +# --------------------------------------------------------
8 +
9 +from .internvl import InternVLChatConfig
10 +
11 +
12 +class H2OVLChatConfig(InternVLChatConfig):
13 + model_type = "h2ovl_chat"