@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8aaf3d5347
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Model][VLM] Support multi-images inputs for Phi-3-vision models (#7783)

Isotr0py committed Aug 25, 2024 at 19:51 UTC 8aaf3d5347ad536de25869caa67b90e43f1ccd5b
2 files changed +168 -29
tests/models/test_phi3v.py
+111
@@ -21,6 +21,7 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
21 "cherry_blossom":
22 "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
23 })
24 +HF_MULTIIMAGE_IMAGE_PROMPT = "<|user|>\n<|image_1|>\n<|image_2|>\nDescribe these images.<|end|>\n<|assistant|>\n" # noqa: E501
25
26 models = ["microsoft/Phi-3.5-vision-instruct"]
27
@@ -184,3 +185,113 @@ def test_regression_7840(hf_runner, vllm_runner, image_assets, model,
185 num_logprobs=10,
186 tensor_parallel_size=1,
187 )
188 +
189 +
190 +def run_multi_image_test(
191 + hf_runner: Type[HfRunner],
192 + vllm_runner: Type[VllmRunner],
193 + images: List[Image.Image],
194 + model: str,
195 + *,
196 + size_factors: List[float],
197 + dtype: str,
198 + max_tokens: int,
199 + num_logprobs: int,
200 + tensor_parallel_size: int,
201 + distributed_executor_backend: Optional[str] = None,
202 +):
203 + """Inference result should be the same between hf and vllm.
204 +
205 + All the image fixtures for the test is under tests/images.
206 + For huggingface runner, we provide the PIL images as input.
207 + For vllm runner, we provide MultiModalDataDict objects
208 + and corresponding MultiModalConfig as input.
209 + Note, the text input is also adjusted to abide by vllm contract.
210 + The text output is sanitized to be able to compare with hf.
211 + """
212 +
213 + inputs_per_case = [
214 + ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
215 + [[rescale_image_size(image, factor) for image in images]
216 + for factor in size_factors])
217 + ]
218 +
219 + # NOTE: take care of the order. run vLLM first, and then run HF.
220 + # vLLM needs a fresh new process without cuda initialization.
221 + # if we run HF first, the cuda initialization will be done and it
222 + # will hurt multiprocessing backend with fork method (the default method).
223 +
224 + # max_model_len should be greater than image_feature_size
225 + with vllm_runner(model,
226 + max_model_len=4096,
227 + max_num_seqs=1,
228 + limit_mm_per_prompt={"image": len(images)},
229 + dtype=dtype,
230 + tensor_parallel_size=tensor_parallel_size,
231 + distributed_executor_backend=distributed_executor_backend,
232 + enforce_eager=True) as vllm_model:
233 + vllm_outputs_per_case = [
234 + vllm_model.generate_greedy_logprobs(prompts,
235 + max_tokens,
236 + num_logprobs=num_logprobs,
237 + images=images)
238 + for prompts, images in inputs_per_case
239 + ]
240 +
241 + hf_model_kwargs = {"_attn_implementation": "eager"}
242 + with hf_runner(model, dtype=dtype,
243 + model_kwargs=hf_model_kwargs) as hf_model:
244 + eos_token_id = hf_model.processor.tokenizer.eos_token_id
245 + hf_outputs_per_case = [
246 + hf_model.generate_greedy_logprobs_limit(prompts,
247 + max_tokens,
248 + num_logprobs=num_logprobs,
249 + images=images,
250 + eos_token_id=eos_token_id)
251 + for prompts, images in inputs_per_case
252 + ]
253 +
254 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_case,
255 + vllm_outputs_per_case):
256 + check_logprobs_close(
257 + outputs_0_lst=hf_outputs,
258 + outputs_1_lst=[
259 + vllm_to_hf_output(vllm_output, model)
260 + for vllm_output in vllm_outputs
261 + ],
262 + name_0="hf",
263 + name_1="vllm",
264 + )
265 +
266 +
267 +@pytest.mark.parametrize("model", models)
268 +@pytest.mark.parametrize(
269 + "size_factors",
270 + [
271 + # No image
272 + [],
273 + # Single-scale
274 + [1.0],
275 + # Single-scale, batched
276 + [1.0, 1.0, 1.0],
277 + # Multi-scale
278 + [0.25, 0.5, 1.0],
279 + ],
280 +)
281 +@pytest.mark.parametrize("dtype", [target_dtype])
282 +@pytest.mark.parametrize("max_tokens", [128])
283 +@pytest.mark.parametrize("num_logprobs", [5])
284 +def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
285 + size_factors, dtype: str, max_tokens: int,
286 + num_logprobs: int) -> None:
287 + run_multi_image_test(
288 + hf_runner,
289 + vllm_runner,
290 + [asset.pil_image for asset in image_assets],
291 + model,
292 + size_factors=size_factors,
293 + dtype=dtype,
294 + max_tokens=max_tokens,
295 + num_logprobs=num_logprobs,
296 + tensor_parallel_size=1,
297 + )
vllm/model_executor/models/phi3v.py
+57 -29
@@ -13,6 +13,7 @@
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 +import itertools
17 import re
18 from functools import lru_cache
19 from typing import (Any, Dict, Iterable, List, Literal, Mapping, Optional,
@@ -37,11 +38,11 @@ from vllm.model_executor.models.clip import CLIPVisionModel
38 from vllm.model_executor.models.llama import LlamaModel
39 from vllm.model_executor.sampling_metadata import SamplingMetadata
40 from vllm.multimodal import MULTIMODAL_REGISTRY
40 -from vllm.multimodal.utils import cached_get_tokenizer
41 +from vllm.multimodal.utils import cached_get_tokenizer, repeat_and_pad_token
42 from vllm.sequence import IntermediateTensors, SamplerOutput
43 +from vllm.utils import is_list_of
44
43 -from .clip import (dummy_image_for_clip, dummy_seq_data_for_clip,
44 - input_processor_for_clip)
45 +from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
46 from .interfaces import SupportsMultiModal
47 from .utils import merge_multimodal_embeddings
48
@@ -400,9 +401,20 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
401 image_data = multi_modal_data["image"]
402 if isinstance(image_data, Image.Image):
403 w, h = image_data.size
403 - image_feature_size = get_phi3v_image_feature_size(hf_config,
404 - input_width=w,
405 - input_height=h)
404 + image_feature_size = [
405 + get_phi3v_image_feature_size(hf_config,
406 + input_width=w,
407 + input_height=h)
408 + ]
409 + image_data = [image_data]
410 + elif is_list_of(image_data, Image.Image):
411 + image_feature_size = []
412 + for image in image_data:
413 + w, h = image.size
414 + image_feature_size.append(
415 + get_phi3v_image_feature_size(hf_config,
416 + input_width=w,
417 + input_height=h))
418 elif isinstance(image_data, torch.Tensor):
419 image_feature_size = image_data.shape[0]
420 else:
@@ -410,45 +422,61 @@ def input_processor_for_phi3v(ctx: InputContext, llm_inputs: LLMInputs):
422
423 prompt = llm_inputs.get("prompt")
424 if prompt is None:
425 + image_idx = []
426 new_prompt = None
427 else:
428 + image_idx = sorted(map(int, re.findall(r"<\|image_(\d+)\|>+", prompt)))
429 if prompt.count("<|image|>") > 0:
430 logger.warning("Please follow the prompt format that is "
431 "documented on HuggingFace which does not involve "
432 "repeating <|image|> tokens.")
419 - elif len(re.findall(r"(<\|image_\d+\|>)+", prompt)) > 1:
420 - logger.warning("Multiple image input is not supported yet, "
421 - "so any extra image tokens will be treated "
422 - "as plain text.")
423 -
433 + elif (num_image_tags := len(image_idx)) > 1:
434 + assert num_image_tags == len(
435 + image_data), "The count of image_placeholder not match image's"
436 new_prompt = prompt
437
426 - prompt_token_ids = llm_inputs["prompt_token_ids"]
427 - image_1_token_ids = _get_image_placeholder_token_ids(model_config, idx=1)
438 + prompt_token_ids = llm_inputs["prompt_token_ids"].copy()
439 +
440 + # masked place_holder with image token id
441 + for idx in image_idx:
442 + image_token_ids = _get_image_placeholder_token_ids(model_config,
443 + idx=idx)
444 + for i in range(len(prompt_token_ids) - len(image_token_ids) + 1):
445 + if prompt_token_ids[i:i + len(image_token_ids)] == image_token_ids:
446 + prompt_token_ids[i:i + len(image_token_ids)] = [
447 + _IMAGE_TOKEN_ID
448 + ] * len(image_token_ids)
449 + break
450 +
451 + # merge consecutive tag ids
452 + merged_token_ids: List[int] = []
453 + for is_placeholder, token_ids in itertools.groupby(
454 + prompt_token_ids, lambda x: x == _IMAGE_TOKEN_ID):
455 + if is_placeholder:
456 + merged_token_ids.append(_IMAGE_TOKEN_ID)
457 + else:
458 + merged_token_ids.extend(list(token_ids))
459
460 + # TODO: Move this to utils or integrate with clip.
461 new_token_ids: List[int] = []
430 - for i in range(len(prompt_token_ids) - len(image_1_token_ids) + 1):
431 - if prompt_token_ids[i:i + len(image_1_token_ids)] == image_1_token_ids:
432 - new_token_ids.append(_IMAGE_TOKEN_ID)
433 -
434 - # No need to further scan the list since we only replace once
435 - new_token_ids.extend(prompt_token_ids[i + len(image_1_token_ids):])
436 - break
462 + placeholder_idx = 0
463 + while merged_token_ids:
464 + token_id = merged_token_ids.pop(0)
465 + if token_id == _IMAGE_TOKEN_ID:
466 + new_token_ids.extend(
467 + repeat_and_pad_token(
468 + _IMAGE_TOKEN_ID,
469 + repeat_count=image_feature_size[placeholder_idx],
470 + ))
471 + placeholder_idx += 1
472 else:
438 - new_token_ids.append(prompt_token_ids[i])
473 + new_token_ids.append(token_id)
474
475 # NOTE: Create a defensive copy of the original inputs
476 llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
477 prompt=new_prompt,
478 multi_modal_data=multi_modal_data)
444 -
445 - return input_processor_for_clip(
446 - model_config,
447 - CLIP_VIT_LARGE_PATCH14_336_CONFIG,
448 - llm_inputs,
449 - image_token_id=_IMAGE_TOKEN_ID,
450 - image_feature_size_override=image_feature_size,
451 - )
479 + return llm_inputs
480
481
482 @MULTIMODAL_REGISTRY.register_image_input_mapper()