@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7b86e7c9cd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Model] Add multi-image support for minicpmv (#7122)

Co-authored-by: hezhihui <hzh7269@modelbest.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Alphi committed Aug 5, 2024 at 09:23 UTC 7b86e7c9cd6541abdf5d083b0a8a98ee667a91d1
4 files changed +172 -37
tests/conftest.py
+3 -2
@@ -3,7 +3,7 @@ import gc
3 import os
4 import sys
5 from collections import UserList
6 -from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar
6 +from typing import Any, Dict, List, Optional, Tuple, TypedDict, TypeVar, Union
7
8 import pytest
9 import torch
@@ -508,7 +508,8 @@ class VllmRunner:
508 prompts: List[str],
509 max_tokens: int,
510 num_logprobs: int,
511 - images: Optional[List[Image.Image]] = None,
511 + images: Optional[Union[List[Image.Image],
512 + List[List[Image.Image]]]] = None,
513 stop_token_ids: Optional[List[int]] = None,
514 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
515 greedy_logprobs_params = SamplingParams(temperature=0.0,
tests/models/test_minicpmv.py
+133 -13
@@ -14,6 +14,18 @@ from .utils import check_logprobs_close
14
15 pytestmark = pytest.mark.vlm
16
17 +
18 +class NestedInputs(UserDict):
19 +
20 + def __init__(self, model_inputs: BatchFeature):
21 + super().__init__({"model_inputs": model_inputs})
22 +
23 + self.model_inputs = model_inputs
24 +
25 + def to(self, device: torch.types.Device):
26 + return NestedInputs(self.model_inputs.to(device))
27 +
28 +
29 # The image token is placed before "user" on purpose so that the test can pass
30 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
31 "stop_sign":
@@ -23,7 +35,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
35 "cherry_blossom":
36 "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
37 "(<image>./</image>)\nWhat is the season?<|eot_id|>" \
26 - "<|start_header_id|>assistant<|end_header_id|>\n\n"
38 + "<|start_header_id|>assistant<|end_header_id|>\n\n",
39 })
40
41 models = ["openbmb/MiniCPM-Llama3-V-2_5"]
@@ -94,22 +106,10 @@ def run_test(
106 ]
107
108 with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
97 -
98 - class NestedInputs(UserDict):
99 -
100 - def __init__(self, model_inputs: BatchFeature):
101 - super().__init__({"model_inputs": model_inputs})
102 -
103 - self.model_inputs = model_inputs
104 -
105 - def to(self, device: torch.types.Device):
106 - return NestedInputs(self.model_inputs.to(device))
107 -
109 hf_processor = hf_model.processor
110 hf_model.processor = lambda **kw: NestedInputs(
111 hf_processor(**kw) # type: ignore
112 )
112 -
113 hf_outputs_per_image = [
114 hf_model.generate_greedy_logprobs_limit(prompts,
115 max_tokens,
@@ -161,3 +161,123 @@ def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
161 num_logprobs=num_logprobs,
162 tensor_parallel_size=1,
163 )
164 +
165 +
166 +HF_MULTIIMAGE_IMAGE_PROMPT = \
167 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
168 + "(<image>./</image>)\n(<image>./</image>)\n" \
169 + "Describe these images.<|eot_id|>" \
170 + "<|start_header_id|>assistant<|end_header_id|>\n\n"
171 +
172 +
173 +def run_multi_image_test(
174 + hf_runner: Type[HfRunner],
175 + vllm_runner: Type[VllmRunner],
176 + image_assets: _ImageAssets,
177 + model: str,
178 + *,
179 + size_factors: List[float],
180 + dtype: str,
181 + max_tokens: int,
182 + num_logprobs: int,
183 + tensor_parallel_size: int,
184 + distributed_executor_backend: Optional[str] = None,
185 +):
186 + """Inference result should be the same between hf and vllm.
187 +
188 + All the image fixtures for the test is under tests/images.
189 + For huggingface runner, we provide the PIL images as input.
190 + For vllm runner, we provide MultiModalDataDict objects
191 + and corresponding vision language config as input.
192 + Note, the text input is also adjusted to abide by vllm contract.
193 + The text output is sanitized to be able to compare with hf.
194 + """
195 + images = [asset.pil_image for asset in image_assets]
196 +
197 + inputs_per_case = [
198 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
199 + [[rescale_image_size(image, factor) for image in images]
200 + for factor in size_factors])
201 + ]
202 +
203 + # NOTE: take care of the order. run vLLM first, and then run HF.
204 + # vLLM needs a fresh new process without cuda initialization.
205 + # if we run HF first, the cuda initialization will be done and it
206 + # will hurt multiprocessing backend with fork method (the default method).
207 +
208 + # max_model_len should be greater than image_feature_size
209 + with vllm_runner(model,
210 + max_model_len=4096,
211 + max_num_seqs=1,
212 + dtype=dtype,
213 + tensor_parallel_size=tensor_parallel_size,
214 + distributed_executor_backend=distributed_executor_backend,
215 + enforce_eager=True) as vllm_model:
216 + tokenizer = vllm_model.model.get_tokenizer()
217 + stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
218 + vllm_outputs_per_case = [
219 + vllm_model.generate_greedy_logprobs(prompts,
220 + max_tokens,
221 + num_logprobs=num_logprobs,
222 + images=images,
223 + stop_token_ids=stop_token_ids)
224 + for prompts, images in inputs_per_case
225 + ]
226 +
227 + with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
228 + hf_processor = hf_model.processor
229 + hf_model.processor = lambda **kw: NestedInputs(
230 + hf_processor(**kw) # type: ignore
231 + )
232 + hf_outputs_per_case = [
233 + hf_model.generate_greedy_logprobs_limit(prompts,
234 + max_tokens,
235 + num_logprobs=num_logprobs,
236 + images=images,
237 + tokenizer=tokenizer)
238 + for prompts, images in inputs_per_case
239 + ]
240 +
241 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
242 + vllm_outputs_per_case):
243 + check_logprobs_close(
244 + outputs_0_lst=[
245 + trunc_hf_output(hf_output) for hf_output in hf_outputs
246 + ],
247 + outputs_1_lst=vllm_outputs,
248 + name_0="hf",
249 + name_1="vllm",
250 + )
251 +
252 +
253 +@pytest.mark.parametrize("model", models)
254 +@pytest.mark.parametrize(
255 + "size_factors",
256 + [
257 + # No image
258 + [],
259 + # Single-scale
260 + [1.0],
261 + # Single-scale, batched
262 + [1.0, 1.0, 1.0],
263 + # Multi-scale
264 + [0.25, 0.5, 1.0],
265 + ],
266 +)
267 +@pytest.mark.parametrize("dtype", [target_dtype])
268 +@pytest.mark.parametrize("max_tokens", [128])
269 +@pytest.mark.parametrize("num_logprobs", [5])
270 +def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
271 + size_factors, dtype: str, max_tokens: int,
272 + num_logprobs: int) -> None:
273 + run_multi_image_test(
274 + hf_runner,
275 + vllm_runner,
276 + image_assets,
277 + model,
278 + size_factors=size_factors,
279 + dtype=dtype,
280 + max_tokens=max_tokens,
281 + num_logprobs=num_logprobs,
282 + tensor_parallel_size=1,
283 + )
vllm/model_executor/models/minicpmv.py
+35 -21
@@ -392,6 +392,20 @@ class Resampler2_5(BaseResampler):
392 return x
393
394
395 +def get_version_by_config(config: PretrainedConfig) -> Tuple[int, ...]:
396 + version_float = getattr(config, "version", None)
397 +
398 + # The old configs do not include version number
399 + # TODO: Remove this after the HF repos are updated
400 + if version_float is None:
401 + if config.hidden_size == 2304 and config.query_num == 64:
402 + return (2, 0)
403 + return (2, 5)
404 +
405 + version_str = str(version_float)
406 + return tuple(int(x) for x in version_str.split("."))
407 +
408 +
409 def get_max_minicpmv_image_tokens(ctx: InputContext):
410 hf_config = ctx.get_hf_config(PretrainedConfig)
411 return getattr(hf_config, "query_num", 64)
@@ -421,36 +435,43 @@ def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
435 multi_modal_data = llm_inputs.get("multi_modal_data")
436 if multi_modal_data is None or "image" not in multi_modal_data:
437 return llm_inputs
424 -
438 model_config = ctx.model_config
426 -
439 + version = get_version_by_config(model_config.hf_config)
440 tokenizer = cached_get_tokenizer(model_config.tokenizer,
441 trust_remote_code=True)
442 + image_processor = cached_get_image_processor(model_config.tokenizer)
443 +
444 + def get_placeholder(image_size: Tuple[int, int], num_image: int):
445 + if version == (2, 0) or version == (2, 5):
446 + return image_processor. \
447 + get_slice_image_placeholder(image_size)
448 + return image_processor. \
449 + get_slice_image_placeholder(image_size, num_image)
450
451 prompt = llm_inputs.get("prompt")
452 if prompt is None:
453 token_ids = llm_inputs.get("prompt_token_ids")
454 prompt = tokenizer.decode(token_ids)
434 - image_processor = cached_get_image_processor(model_config.tokenizer)
455
456 pattern = "(<image>./</image>)"
437 - image = multi_modal_data["image"]
457 + images = multi_modal_data["image"]
458 + if isinstance(images, Image.Image):
459 + images = [images]
460 image_tags = re.findall(pattern, prompt)
461
462 if len(image_tags) == 0:
463 new_token_ids = token_ids
464 new_prompt = prompt
465 else:
444 - if len(image_tags) > 1:
445 - logger.warning("Multiple image input is not supported yet, "
446 - "so any extra image tokens will be treated "
447 - "as plain text.")
448 -
466 text_chunks = prompt.split(pattern)
450 - new_prompt = (text_chunks[0] +
451 - image_processor.get_slice_image_placeholder(image.size) +
452 - "".join(text_chunks[1:]))
453 -
467 + new_prompt_chunks: List[str] = []
468 + for i in range(len(images)):
469 + new_prompt_chunks += [
470 + text_chunks[i],
471 + get_placeholder(images[i].size, i)
472 + ]
473 + new_prompt_chunks.append(text_chunks[-1])
474 + new_prompt = "".join(new_prompt_chunks)
475 new_token_ids = tokenizer.encode(new_prompt)
476
477 llm_inputs = LLMInputs(
@@ -478,14 +499,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsVision):
499 self.config = config
500 self.multimodal_config = multimodal_config
501
481 - if not hasattr(self.config, "version"):
482 - if self.config.hidden_size == 2304 and self.config.query_num == 64:
483 - self.version = (2, 0)
484 - else:
485 - self.version = (2, 5)
486 - else:
487 - self.version = str(self.config.version).split(".")
488 - self.version = tuple([int(x) for x in self.version])
502 + self.version = get_version_by_config(self.config)
503 self.llm = self.init_llm(config, cache_config, quant_config)
504 self.vpm = self.init_vision_module()
505 param_dtype = torch.get_default_dtype()
vllm/multimodal/image.py
+1 -1
@@ -113,7 +113,7 @@ class ImagePlugin(MultiModalPlugin):
113 def _default_input_mapper(self, ctx: InputContext,
114 data: object) -> MultiModalInputs:
115 model_config = ctx.model_config
116 - if isinstance(data, Image.Image):
116 + if isinstance(data, (Image.Image, list)):
117 image_processor = self._get_hf_image_processor(model_config)
118 if image_processor is None:
119 raise RuntimeError("No HuggingFace processor is available "