@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 31a08f5bd2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Add min_pixels / max_pixels to Qwen2VL as mm_processor_kwargs (#9612)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Oct 23, 2024 at 08:05 UTC 31a08f5bd231c2ac547e9bb6b6490282d2e76f83
3 files changed +236 -18
examples/offline_inference_vision_language.py
+5
@@ -267,6 +267,11 @@ def run_qwen2_vl(question: str, modality: str):
267 model=model_name,
268 max_model_len=8192,
269 max_num_seqs=5,
270 + # Note - mm_processor_kwargs can also be passed to generate/chat calls
271 + mm_processor_kwargs={
272 + "min_pixels": 28 * 28,
273 + "max_pixels": 1280 * 28 * 28,
274 + },
275 )
276
277 prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
tests/models/decoder_only/vision_language/test_qwen2_vl.py new
+160
@@ -0,0 +1,160 @@
1 +from typing import Any, Dict, Tuple
2 +
3 +import pytest
4 +import torch
5 +from PIL.Image import Image
6 +from transformers import AutoTokenizer
7 +
8 +from vllm.inputs import InputContext, token_inputs
9 +from vllm.multimodal import MultiModalRegistry
10 +
11 +from ....conftest import _ImageAssets
12 +from ...utils import build_model_context
13 +
14 +MODEL = "Qwen/Qwen2-VL-2B-Instruct"
15 +MIN_PIXELS = "min_pixels"
16 +MAX_PIXELS = "max_pixels"
17 +
18 +
19 +# Fixtures lazy import to avoid initializing CUDA during test collection
20 +# NOTE: Qwen2vl supports multiple input modalities, so it registers multiple
21 +# input mappers.
22 +@pytest.fixture()
23 +def image_input_mapper_for_qwen2_vl():
24 + from vllm.model_executor.models.qwen2_vl import (
25 + image_input_mapper_for_qwen2_vl)
26 + return image_input_mapper_for_qwen2_vl
27 +
28 +
29 +@pytest.fixture()
30 +def input_processor_for_qwen2_vl():
31 + from vllm.model_executor.models.qwen2_vl import (
32 + input_processor_for_qwen2_vl)
33 + return input_processor_for_qwen2_vl
34 +
35 +
36 +@pytest.fixture()
37 +def qwen2_vl_context() -> InputContext:
38 + return build_model_context(model_name=MODEL)
39 +
40 +
41 +@pytest.fixture()
42 +def get_max_qwen2_vl_image_tokens():
43 + from vllm.model_executor.models.qwen2_vl import (
44 + get_max_qwen2_vl_image_tokens)
45 + return get_max_qwen2_vl_image_tokens
46 +
47 +
48 +@pytest.fixture()
49 +def dummy_data_for_qwen2_vl():
50 + from vllm.model_executor.models.qwen2_vl import dummy_data_for_qwen2_vl
51 + return dummy_data_for_qwen2_vl
52 +
53 +
54 +@pytest.mark.parametrize("mm_processor_kwargs,expected_max_tokens", [
55 + ({}, 1225),
56 + ({
57 + MIN_PIXELS: 64**2,
58 + MAX_PIXELS: 512**2
59 + }, 324),
60 +])
61 +def test_qwen2_vl_max_image_tokens(get_max_qwen2_vl_image_tokens,
62 + qwen2_vl_context: InputContext,
63 + mm_processor_kwargs: Dict[str, Any],
64 + expected_max_tokens: int):
65 + """Ensure that the max token calc handles min/max pixels properly."""
66 + actual_max_tokens = get_max_qwen2_vl_image_tokens(qwen2_vl_context,
67 + **mm_processor_kwargs)
68 + assert actual_max_tokens == expected_max_tokens
69 +
70 +
71 +@pytest.mark.parametrize("mm_processor_kwargs,token_count,img_size", [
72 + [{}, 1225, (980, 980)],
73 + [{
74 + MIN_PIXELS: 64**2,
75 + MAX_PIXELS: 512**2
76 + }, 324, (504, 504)],
77 +])
78 +def test_qwen2_vl_dummy_data(dummy_data_for_qwen2_vl,
79 + qwen2_vl_context: InputContext,
80 + mm_processor_kwargs: Dict[str, Any],
81 + token_count: int, img_size: Tuple[int, int]):
82 + """Ensure that the dummy data handles min/max pixels properly."""
83 + seq_len = 3000
84 + hf_config = qwen2_vl_context.get_hf_config()
85 + image_token_id = hf_config.image_token_id
86 +
87 + # NOTE: video value is required, but isn't actually used
88 + # when making the dummy data except for error handling currently
89 + seq_data, mm_data = dummy_data_for_qwen2_vl(qwen2_vl_context, seq_len, {
90 + "image": 1,
91 + "video": 0
92 + }, **mm_processor_kwargs)
93 +
94 + # Ensure we have the right number of placeholders for min/max pixel values
95 + assert seq_data.get_token_ids().count(image_token_id) == token_count
96 +
97 + # Ensure the images were resized correctly
98 + image = mm_data["image"]
99 + assert isinstance(image, Image)
100 + assert image.size == img_size
101 +
102 +
103 +@pytest.mark.parametrize("mm_processor_kwargs,num_placeholders", [
104 + ({}, 1426),
105 + ({
106 + MIN_PIXELS: 64**2,
107 + MAX_PIXELS: 512**2
108 + }, 330),
109 +])
110 +def test_input_processor(input_processor_for_qwen2_vl,
111 + qwen2_vl_context: InputContext,
112 + image_assets: _ImageAssets, num_placeholders: int,
113 + mm_processor_kwargs: Dict[str, Any]):
114 + """Ensure that the image processor handles min/max pixels properly."""
115 + tokenizer = AutoTokenizer.from_pretrained(MODEL)
116 + prompt = "<|vision_start|><|image_pad|><|vision_end|>"
117 +
118 + image = image_assets[0].pil_image
119 + hf_config = qwen2_vl_context.get_hf_config()
120 + image_token_id = hf_config.image_token_id
121 +
122 + inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
123 + prompt=prompt,
124 + multi_modal_data={"image": [image]})
125 +
126 + processed_inputs = input_processor_for_qwen2_vl(qwen2_vl_context, inputs,
127 + **mm_processor_kwargs)
128 + assert processed_inputs["prompt_token_ids"].count(
129 + image_token_id) == num_placeholders
130 + assert len(processed_inputs["multi_modal_data"]["image"]) == 1
131 +
132 +
133 +@pytest.mark.parametrize("mm_processor_kwargs,pixels_shape", [
134 + ({}, [5704, 1176]),
135 + ({
136 + MIN_PIXELS: 64**2,
137 + MAX_PIXELS: 512**2
138 + }, [1320, 1176]),
139 +])
140 +def test_image_mapper_override(qwen2_vl_context: InputContext,
141 + image_assets: _ImageAssets,
142 + mm_processor_kwargs: Dict[str, Any],
143 + pixels_shape: Tuple[int, int]):
144 + """Ensure that the image mapper handles min/max pixels properly."""
145 + mm_registry = MultiModalRegistry()
146 + mm_registry.init_mm_limits_per_prompt(qwen2_vl_context.model_config)
147 +
148 + image = image_assets[0].pil_image
149 +
150 + mapped_output = mm_registry.map_input(
151 + qwen2_vl_context.model_config,
152 + {"image": image},
153 + mm_processor_kwargs=mm_processor_kwargs,
154 + )
155 +
156 + # Dimension 0 of pixel values should match the product of image_grid_thw
157 + actual_pixels_shape = mapped_output["pixel_values"].shape
158 + assert list(actual_pixels_shape) == pixels_shape
159 + assert actual_pixels_shape[0] == torch.prod(
160 + mapped_output["image_grid_thw"])
vllm/model_executor/models/qwen2_vl.py
+71 -18
@@ -549,6 +549,9 @@ def mm_input_mapper_for_qwen2_vl(
549 ctx: InputContext,
550 data: MultiModalData[object],
551 data_type_key: str,
552 + *,
553 + min_pixels: Optional[int] = None,
554 + max_pixels: Optional[int] = None,
555 ) -> MultiModalInputs:
556 """Input mapper for Qwen2-VL."""
557 if data_type_key == "image" and isinstance(data, dict):
@@ -557,8 +560,19 @@ def mm_input_mapper_for_qwen2_vl(
560 "image_grid_thw": data.get("image_grid_thw"),
561 })
562 model_config = ctx.model_config
563 + # Handle mm processor kwargs; we pass these at creation time
564 + # because preprocess() in transformers doesn't expose them
565 + mm_processor_kwargs = {}
566 + if min_pixels:
567 + mm_processor_kwargs["min_pixels"] = min_pixels
568 + if max_pixels:
569 + mm_processor_kwargs["max_pixels"] = max_pixels
570 +
571 image_processor = cached_get_image_processor(
561 - model_config.model, trust_remote_code=model_config.trust_remote_code)
572 + model_config.model,
573 + trust_remote_code=model_config.trust_remote_code,
574 + **mm_processor_kwargs,
575 + )
576 if image_processor is None:
577 raise RuntimeError("No HuggingFace processor is available "
578 "to process the image object")
@@ -631,25 +645,36 @@ def _get_max_image_info(
645 image_processor,
646 data_type_key: str = "image",
647 mm_count: int = 1,
648 + min_pixels: Optional[int] = None,
649 + max_pixels: Optional[int] = None,
650 ):
651 + # Limit min / max pixels unless they're explicitly provided
652 + if min_pixels is None:
653 + min_pixels = max(image_processor.min_pixels, 28 * 28)
654 + if max_pixels is None:
655 + max_pixels = min(image_processor.max_pixels, 1280 * 28 * 28)
656 +
657 return _get_vision_info(
658 image_processor,
659 height=9999999,
660 width=9999999,
639 -
640 - # Limit min / max pixels.
641 - min_pixels=max(image_processor.min_pixels, 28 * 28),
642 - max_pixels=min(image_processor.max_pixels, 1280 * 28 * 28),
661 + min_pixels=min_pixels,
662 + max_pixels=max_pixels,
663 data_type_key=data_type_key,
664 mm_count=mm_count,
665 )
666
667
648 -def get_max_qwen2_vl_mm_tokens(ctx: InputContext, data_type_key: str) -> int:
668 +def get_max_qwen2_vl_mm_tokens(ctx: InputContext,
669 + data_type_key: str,
670 + *,
671 + min_pixels=None,
672 + max_pixels=None) -> int:
673 image_processor = cached_get_image_processor(ctx.model_config.model)
674 max_resized_height, max_resized_width, max_llm_image_tokens = \
675 _get_max_image_info(image_processor, data_type_key=data_type_key,
652 - mm_count=1)
676 + mm_count=1, min_pixels=min_pixels,
677 + max_pixels=max_pixels)
678 return max_llm_image_tokens
679
680
@@ -660,14 +685,20 @@ get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
685
686
687 def dummy_data_for_qwen2_vl(
663 - ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
688 + ctx: InputContext,
689 + seq_len: int,
690 + mm_counts: Mapping[str, int],
691 + *,
692 + min_pixels: Optional[int] = None,
693 + max_pixels: Optional[int] = None
694 ) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
695 image_processor = cached_get_image_processor(ctx.model_config.model)
696
697 num_images = mm_counts["image"]
698 max_resized_height, max_resized_width, max_llm_image_tokens = \
699 _get_max_image_info(image_processor, data_type_key="image",
670 - mm_count=num_images)
700 + mm_count=num_images, min_pixels=min_pixels,
701 + max_pixels=max_pixels)
702 if seq_len - max_llm_image_tokens - 2 < 0:
703 raise RuntimeError(
704 f"Qwen2-VL cannot process {num_images} images in a prompt, "
@@ -678,10 +709,11 @@ def dummy_data_for_qwen2_vl(
709 num_videos = mm_counts["video"]
710 max_resized_height, max_resized_width, max_llm_video_tokens = \
711 _get_max_image_info(image_processor, data_type_key="video",
681 - mm_count=num_videos)
712 + mm_count=num_videos, min_pixels=min_pixels,
713 + max_pixels=max_pixels)
714 if seq_len - max_llm_video_tokens - 2 < 0:
715 raise RuntimeError(
684 - f"Qwen2-VL cannot process {num_images} videos in a prompt, "
716 + f"Qwen2-VL cannot process {num_videos} videos in a prompt, "
717 "please increase max_model_len or reduce video limit by "
718 "--limit-mm-per-prompt.")
719
@@ -706,6 +738,8 @@ def _get_llm_num_vision_tokens(
738 mm_inputs: list,
739 data_type_key: str,
740 image_processor,
741 + min_pixels: int,
742 + max_pixels: int,
743 ):
744 """Get number of vision tokens of multimodal inputs.
745
@@ -715,12 +749,13 @@ def _get_llm_num_vision_tokens(
749 image = to_numpy_array(mm_inputs[0])
750 input_data_format = infer_channel_dimension_format(image)
751 height, width = get_image_size(image, channel_dim=input_data_format)
752 +
753 _, _, llm_num_vision_tokens = _get_vision_info(
754 image_processor,
755 height=height,
756 width=width,
722 - min_pixels=image_processor.min_pixels,
723 - max_pixels=image_processor.max_pixels,
757 + min_pixels=min_pixels,
758 + max_pixels=max_pixels,
759 do_resize=image_processor.do_resize,
760 data_type_key=data_type_key,
761 mm_count=len(mm_inputs),
@@ -730,7 +765,8 @@ def _get_llm_num_vision_tokens(
765
766 def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
767 data_type_key: str, image_processor: Any,
733 - prompt_token_ids: List[int]) -> List[int]:
768 + prompt_token_ids: List[int], min_pixels: Optional[int],
769 + max_pixels: Optional[int]) -> List[int]:
770 """
771 Expand pad tokens for multi-modal inputs (e.g., images or videos).
772
@@ -741,6 +777,8 @@ def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
777 data_type_key (str): The type of the multi-modal input.
778 image_processor (Any): The image processor used to process the inputs.
779 prompt_token_ids (List[int]): The list of token IDs in the prompt.
780 + min_pixels (int): min pixels to used for img processing
781 + max_pixels (int): max pixels to be used for img processing
782
783 Returns:
784 List[int]: The list of token IDs for the multi-modal inputs.
@@ -757,6 +795,8 @@ def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
795 [data] if data_type_key == "image" else data,
796 data_type_key=data_type_key,
797 image_processor=image_processor,
798 + min_pixels=min_pixels,
799 + max_pixels=max_pixels,
800 )
801 if cnt == 0:
802 end_idx = indices[cnt]
@@ -773,6 +813,9 @@ def _expand_pad_tokens(inputs: list, token_id: int, make_batched_fn: Callable,
813 def input_processor_for_qwen2_vl(
814 ctx: InputContext,
815 inputs: DecoderOnlyInputs,
816 + *,
817 + min_pixels: Optional[int] = None,
818 + max_pixels: Optional[int] = None,
819 ) -> DecoderOnlyInputs:
820 multi_modal_data = inputs.get("multi_modal_data", None)
821 if multi_modal_data is None:
@@ -783,6 +826,10 @@ def input_processor_for_qwen2_vl(
826
827 processor = cached_get_processor(ctx.model_config.model)
828 image_processor = processor.image_processor
829 + # Apply processor kwarg overrides for image processor options
830 + min_pixels = min_pixels if min_pixels else image_processor.min_pixels
831 + max_pixels = max_pixels if max_pixels else image_processor.max_pixels
832 +
833 hf_config = ctx.get_hf_config(Qwen2VLConfig)
834
835 # To avoid redundant processing of vision objects (resize, rescale, etc.),
@@ -830,16 +877,22 @@ def input_processor_for_qwen2_vl(
877 else:
878 prompt_token_ids = _expand_pad_tokens(image_inputs,
879 hf_config.image_token_id,
833 - make_batched_images, "image",
880 + make_batched_images,
881 + "image",
882 image_processor,
835 - prompt_token_ids)
883 + prompt_token_ids,
884 + min_pixels=min_pixels,
885 + max_pixels=max_pixels)
886
887 if video_inputs is not None:
888 prompt_token_ids = _expand_pad_tokens(video_inputs,
889 hf_config.video_token_id,
840 - make_batched_videos, "video",
890 + make_batched_videos,
891 + "video",
892 image_processor,
842 - prompt_token_ids)
893 + prompt_token_ids,
894 + min_pixels=min_pixels,
895 + max_pixels=max_pixels)
896
897 return token_inputs(
898 prompt_token_ids=prompt_token_ids,