@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cc98f1e079
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[CI/Build] VLM Test Consolidation (#9372)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com>

Alex Brooks committed Oct 30, 2024 at 10:32 UTC cc98f1e0798cf2b5ea5bc5d0c565af2f884bf6e8
38 files changed +2381 -3096
.buildkite/test-pipeline.yaml
+5 -2
@@ -338,7 +338,10 @@ steps:
338 - tests/models/decoder_only/vision_language
339 commands:
340 - pytest -v -s models/decoder_only/audio_language
341 - - pytest -v -s models/decoder_only/vision_language
341 + # HACK - run phi3v tests separately to sidestep this transformers bug
342 + # https://github.com/huggingface/transformers/issues/34307
343 + - pytest -v -s models/decoder_only/vision_language/test_phi3v.py
344 + - pytest -v -s --ignore models/decoder_only/vision_language/test_phi3v.py models/decoder_only/vision_language
345
346 - label: Other Models Test # 6min
347 #mirror_hardwares: [amd]
@@ -413,7 +416,7 @@ steps:
416 # Avoid importing model tests that cause CUDA reinitialization error
417 - pytest models/encoder_decoder/language/test_bart.py -v -s -m distributed_2_gpus
418 - pytest models/encoder_decoder/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
416 - - pytest models/decoder_only/vision_language/test_broadcast.py -v -s -m distributed_2_gpus
419 + - pytest models/decoder_only/vision_language/test_models.py -v -s -m distributed_2_gpus
420 - pytest -v -s spec_decode/e2e/test_integration_dist_tp2.py
421 - pip install -e ./plugins/vllm_add_dummy_model
422 - pytest -v -s distributed/test_distributed_oot.py
tests/conftest.py
+3 -3
@@ -259,8 +259,7 @@ class HfRunner:
259 is_sentence_transformer: bool = False,
260 skip_tokenizer_init: bool = False,
261 auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM,
262 - postprocess_inputs: Callable[[BatchEncoding],
263 - BatchEncoding] = identity,
262 + postprocess_inputs: Callable[..., BatchEncoding] = identity,
263 ) -> None:
264 torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
265
@@ -303,6 +302,7 @@ class HfRunner:
302 if skip_tokenizer_init:
303 self.tokenizer = self.processor.tokenizer
304
305 + self.dtype = dtype
306 self.postprocess_inputs = postprocess_inputs
307
308 def get_inputs(
@@ -337,7 +337,7 @@ class HfRunner:
337 processor_kwargs["sampling_rate"] = sr
338
339 inputs = self.processor(**processor_kwargs)
340 - inputs = self.postprocess_inputs(inputs)
340 + inputs = self.postprocess_inputs(inputs, dtype=self.dtype)
341
342 all_inputs.append(inputs)
343
tests/engine/test_short_mm_context.py new
+29
@@ -0,0 +1,29 @@
1 +import pytest
2 +
3 +from ..conftest import IMAGE_ASSETS
4 +
5 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
6 + "stop_sign":
7 + "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
8 + "cherry_blossom":
9 + "USER: <image>\nWhat is the season?\nASSISTANT:",
10 +})
11 +
12 +models = ["llava-hf/llava-1.5-7b-hf"]
13 +
14 +
15 +@pytest.mark.parametrize("model", models)
16 +def test_context_length_too_short(vllm_runner, image_assets, model):
17 + images = [asset.pil_image for asset in image_assets]
18 +
19 + with pytest.raises(ValueError, match="too long to fit into the model"):
20 + vllm_model = vllm_runner(
21 + model,
22 + max_model_len=128, # LLaVA has a feature size of 576
23 + enforce_eager=True,
24 + )
25 +
26 + with vllm_model:
27 + vllm_model.generate_greedy([HF_IMAGE_PROMPTS[0]],
28 + max_tokens=1,
29 + images=[images[0]])
tests/models/decoder_only/audio_language/test_ultravox.py
+1 -1
@@ -92,7 +92,7 @@ def run_test(
92 for vllm_prompt, _, audio in prompts_and_audios
93 ]
94
95 - def process(hf_inputs: BatchEncoding):
95 + def process(hf_inputs: BatchEncoding, **kwargs):
96 hf_inputs["audio_values"] = hf_inputs["audio_values"] \
97 .to(torch_dtype) # type: ignore
98 return hf_inputs
tests/models/decoder_only/language/test_qwen.py new
+34
@@ -0,0 +1,34 @@
1 +"""Ensure that a text-only Qwen model can be run without throwing an error.
2 +We explicitly test this because Qwen is implemented as a multimodal and
3 +supports a visual encoder for models like Qwen-VL.
4 +"""
5 +from typing import List, Type
6 +
7 +import pytest
8 +
9 +from ....conftest import VllmRunner
10 +
11 +models = [
12 + "Qwen/Qwen-7B-Chat" # Has no visual encoder
13 +]
14 +
15 +
16 +@pytest.mark.parametrize("model", models)
17 +@pytest.mark.parametrize("dtype", ["bfloat16"])
18 +@pytest.mark.parametrize("max_tokens", [32])
19 +@pytest.mark.parametrize("num_logprobs", [5])
20 +def test_text_only_qwen_model_can_be_loaded_and_run(
21 + vllm_runner: Type[VllmRunner],
22 + example_prompts: List[str],
23 + model: str,
24 + *,
25 + dtype: str,
26 + max_tokens: int,
27 + num_logprobs: int,
28 +):
29 + with vllm_runner(model, dtype=dtype) as vllm_model:
30 + vllm_model.generate_greedy_logprobs(
31 + example_prompts,
32 + max_tokens,
33 + num_logprobs=num_logprobs,
34 + )
tests/models/decoder_only/vision_language/mm_processor_kwargs/__init__.py
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_llava_next.py new
+68
@@ -0,0 +1,68 @@
1 +import pytest
2 +
3 +from vllm.inputs import InputContext
4 +
5 +from ....utils import build_model_context
6 +
7 +
8 +@pytest.fixture()
9 +def get_max_llava_next_image_tokens():
10 + from vllm.model_executor.models.llava_next import (
11 + get_max_llava_next_image_tokens)
12 + return get_max_llava_next_image_tokens
13 +
14 +
15 +@pytest.fixture()
16 +def dummy_data_for_llava_next():
17 + from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
18 + return dummy_data_for_llava_next
19 +
20 +
21 +@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
22 + ([[336, 336]], 1176),
23 + ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
24 +])
25 +def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
26 + get_max_llava_next_image_tokens):
27 + ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
28 +
29 + # Update the config image_grid_pinpoints
30 + # and calculate the resulting max tokens
31 + ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
32 +
33 + actual_max_tokens = get_max_llava_next_image_tokens(
34 + InputContext(ctx.model_config))
35 +
36 + assert expected_max_tokens == actual_max_tokens
37 +
38 +
39 +@pytest.mark.parametrize(
40 + "gridpoints,expected_size",
41 + [
42 + # One point; it has to be the largest
43 + ([[336, 336]], (336, 336)),
44 + # Default for most llava next models; the 2x2 tile is the largest
45 + ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
46 + (672, 672)),
47 + # If two rectangular gridpoints are the same, the more vertical
48 + # one has the higher feature count due to newline features
49 + ([[336, 672], [672, 336]], (672, 336))
50 + ])
51 +def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
52 + gridpoints, expected_size):
53 + ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
54 +
55 + # Update the config image_grid_pinpoints
56 + ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
57 + seq_len = 5000 # bigger than the max feature size for any image
58 +
59 + seq_data, mm_data = dummy_data_for_llava_next(
60 + ctx,
61 + seq_len=seq_len,
62 + mm_counts={"image": 1},
63 + )
64 +
65 + # The dummy data dims should match the gridpoint with the biggest feat size
66 + assert mm_data["image"].height == expected_size[0]
67 + assert mm_data["image"].width == expected_size[1]
68 + assert len(seq_data.get_token_ids()) >= seq_len
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_phi3v.py new
+181
@@ -0,0 +1,181 @@
1 +"""Tests for phi3v's multimodal preprocessing kwargs."""
2 +from typing import Optional
3 +
4 +import pytest
5 +import torch
6 +from transformers import AutoImageProcessor, AutoTokenizer
7 +
8 +from vllm.inputs import InputContext, token_inputs
9 +from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
10 +from vllm.multimodal import MultiModalRegistry
11 +
12 +from .....conftest import _ImageAssets
13 +from ....utils import build_model_context
14 +
15 +models = ["microsoft/Phi-3.5-vision-instruct"]
16 +
17 +
18 +# Wrap lazy imports to avoid initializing CUDA during test collection
19 +@pytest.fixture()
20 +def input_processor_for_phi3v():
21 + from vllm.model_executor.models.phi3v import input_processor_for_phi3v
22 + return input_processor_for_phi3v
23 +
24 +
25 +@pytest.fixture()
26 +def dummy_data_for_phi3v():
27 + from vllm.model_executor.models.phi3v import dummy_data_for_phi3v
28 + return dummy_data_for_phi3v
29 +
30 +
31 +@pytest.fixture()
32 +def get_max_phi3v_image_tokens():
33 + from vllm.model_executor.models.phi3v import get_max_phi3v_image_tokens
34 + return get_max_phi3v_image_tokens
35 +
36 +
37 +@pytest.mark.parametrize("model", models)
38 +@pytest.mark.parametrize("num_crops", [4, 16, None])
39 +def test_input_mapper_override(model: str, image_assets: _ImageAssets,
40 + num_crops: Optional[int]):
41 + """Ensure that the [default] input mapper handles num_crops properly."""
42 + # We pass the processor kwargs here since for this model, we fall back to
43 + # the default mapper; this will fall back to the HF mapper and forward
44 + # mm_processor_kwargs to it.
45 + mm_processor_kwargs = {
46 + "num_crops": num_crops
47 + } if num_crops is not None else {}
48 + ctx = build_model_context(
49 + model_name=model,
50 + tokenizer_name=model,
51 + trust_remote_code=True,
52 + mm_processor_kwargs=mm_processor_kwargs,
53 + )
54 +
55 + hf_processor = AutoImageProcessor.from_pretrained(model,
56 + trust_remote_code=True,
57 + **mm_processor_kwargs)
58 +
59 + mm_registry = MultiModalRegistry()
60 + mm_registry.init_mm_limits_per_prompt(ctx.model_config)
61 +
62 + image = image_assets[0].pil_image
63 + hf_result = hf_processor.preprocess(
64 + image,
65 + return_tensors="pt",
66 + )
67 +
68 + vllm_result = mm_registry.map_input(
69 + ctx.model_config,
70 + {"image": image},
71 + )
72 +
73 + assert torch.all(hf_result["image_sizes"] == vllm_result["image_sizes"])
74 + assert torch.all(
75 + hf_result["num_img_tokens"] == vllm_result["num_img_tokens"])
76 +
77 + # For pixel values, the second axis should be the num_crops + 1
78 + # for the rescaled original image. The default value in VLLM falls
79 + # back to the HF config, which is why we compare to the processor num_crops
80 + assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
81 + assert vllm_result["pixel_values"].shape[1] == hf_processor.num_crops + 1
82 +
83 +
84 +@pytest.mark.parametrize("model", models)
85 +@pytest.mark.parametrize("num_crops,expected_max_tokens", [
86 + (4, 781),
87 + (16, 2653),
88 +])
89 +def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
90 + num_crops: int, expected_max_tokens: int):
91 + """Ensure get_max_phi3v_image_tokens handles num_crops properly."""
92 + # NOTE: mm_processor_kwargs on the context in this test is unused, since
93 + # this is testing the mapper directly. In practice, the processor kwargs
94 + # are wrapped in a closure when calling the max tokens func. We explicitly
95 + # do NOT use the mm_processor_kwargs in the model context here to ensure
96 + # that the max image tokens implementation is referencing a mix of the
97 + # kwargs to the function and the original mm_processor_kwargs in case
98 + # values are somehow updated and end up in a bad state.
99 + ctx = build_model_context(
100 + model_name=model,
101 + tokenizer_name=model,
102 + trust_remote_code=True,
103 + mm_processor_kwargs=None,
104 + )
105 +
106 + actual_max_tokens = get_max_phi3v_image_tokens(
107 + InputContext(ctx.model_config),
108 + num_crops=num_crops,
109 + )
110 +
111 + assert expected_max_tokens == actual_max_tokens
112 +
113 +
114 +@pytest.mark.parametrize("model", models)
115 +@pytest.mark.parametrize("num_crops,toks_per_img,num_imgs", [
116 + (4, 781, 1),
117 + (4, 781, 2),
118 + (16, 2653, 1),
119 + (16, 2653, 2),
120 +])
121 +def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
122 + toks_per_img: int, num_imgs: int):
123 + """Ensure dummy_data_for_phi3v handles num_crops properly."""
124 + # Same as the previous test - don't initialize mm_processor_kwargs
125 + # in this test and assume that the kwargs will be correctly expanded by
126 + # the partial when calling the dummy data func.
127 + ctx = build_model_context(
128 + model_name=model,
129 + tokenizer_name=model,
130 + trust_remote_code=True,
131 + mm_processor_kwargs=None,
132 + )
133 +
134 + sequence_data, _, = dummy_data_for_phi3v(
135 + ctx=ctx,
136 + seq_len=8192, # Should be bigger than num_imgs * toks_per_img
137 + mm_counts={"image": num_imgs},
138 + num_crops=num_crops,
139 + )
140 + # Ensure we have the right number of placeholders per num_crops size
141 + img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
142 + assert img_tok_count == toks_per_img * num_imgs
143 +
144 +
145 +@pytest.mark.parametrize("model", models)
146 +@pytest.mark.parametrize("num_crops,expected_toks_per_img,num_imgs", [
147 + (4, 757, 1),
148 + (4, 757, 2),
149 + (16, 1921, 1),
150 + (16, 1921, 2),
151 +])
152 +def test_input_processor_override(input_processor_for_phi3v,
153 + image_assets: _ImageAssets, model: str,
154 + num_crops: int, expected_toks_per_img: int,
155 + num_imgs: int):
156 + """Ensure input_processor_for_phi3v handles num_crops properly."""
157 + # Same as the previous test - don't initialize mm_processor_kwargs
158 + # in this test and assume that the kwargs will be correctly expanded by
159 + # the partial when calling the custom input processor.
160 + ctx = build_model_context(
161 + model_name=model,
162 + tokenizer_name=model,
163 + trust_remote_code=True,
164 + )
165 + tokenizer = AutoTokenizer.from_pretrained(model)
166 + # Build the image str / prompt based on the number of images we pass
167 + img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
168 + prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
169 + images = [image_assets[0].pil_image] * num_imgs
170 +
171 + inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
172 + prompt=prompt,
173 + multi_modal_data={"image": images})
174 +
175 + processed_inputs = input_processor_for_phi3v(ctx,
176 + inputs,
177 + num_crops=num_crops)
178 +
179 + # Ensure we have the right number of placeholders per num_crops size
180 + img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
181 + assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen.py new
+144
@@ -0,0 +1,144 @@
1 +"""Tests for Qwen's multimodal preprocessing kwargs."""
2 +from typing import Dict, List, Union
3 +
4 +import pytest
5 +import torch
6 +from PIL.Image import Image
7 +
8 +from vllm.inputs import InputContext, token_inputs
9 +from vllm.multimodal.base import MultiModalInputs
10 +from vllm.multimodal.utils import cached_get_tokenizer
11 +
12 +from .....conftest import IMAGE_ASSETS
13 +from ....utils import build_model_context
14 +
15 +### Multimodal preprocessing tests
16 +SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
17 +# These values are specific to Qwen-VL/Chat; we can get these from the model
18 +# config also, but they are hardcoded here to keep the parameterize/fixtures
19 +# easy to read.
20 +IMG_START_ID = 151857
21 +IMG_END_ID = 151858
22 +IMG_PAD_ID = 151859
23 +TOKS_PER_IMG = 256
24 +VIS_ENC_DIM = 4096
25 +IMG_SIZE = 448
26 +
27 +
28 +@pytest.fixture()
29 +def input_mapper_for_qwen():
30 + # Lazy import to avoid initializing CUDA during test collection
31 + from vllm.model_executor.models.qwen import input_mapper_for_qwen
32 + return input_mapper_for_qwen
33 +
34 +
35 +@pytest.fixture()
36 +def input_processor_for_qwen():
37 + # Lazy import to avoid initializing CUDA during test collection
38 + from vllm.model_executor.models.qwen import input_processor_for_qwen
39 + return input_processor_for_qwen
40 +
41 +
42 +@pytest.fixture()
43 +def qwen_vl_context() -> InputContext:
44 + """Get an InputContext for Qwen-VL."""
45 + return build_model_context(model_name="Qwen/Qwen-VL",
46 + trust_remote_code=True)
47 +
48 +
49 +# Happy path tests for single/multi-image scenarios for the multimodal
50 +# input processor and mapper, respectively
51 +@pytest.mark.parametrize("num_images", [1, 2])
52 +def test_input_processor_valid_mm_data(input_processor_for_qwen,
53 + qwen_vl_context: InputContext,
54 + num_images: int):
55 + """Happy cases for image inputs to Qwen's multimodal input processor."""
56 + prompt = "".join(
57 + [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
58 + inputs = token_inputs(
59 + prompt=prompt,
60 + # When processing multimodal data for a multimodal model, the qwen
61 + # input processor will overwrite the provided prompt_token_ids with
62 + # the image prompts
63 + prompt_token_ids=[],
64 + multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
65 + )
66 + proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
67 + assert isinstance(proc_inputs, dict)
68 +
69 + # Each image should have one start / stop and a fixed context of 256
70 + proc_tokens = proc_inputs["prompt_token_ids"]
71 + assert proc_tokens.count(IMG_START_ID) == num_images
72 + assert proc_tokens.count(IMG_END_ID) == num_images
73 + assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
74 +
75 +
76 +@pytest.mark.parametrize(
77 + "img_data,expected_shape",
78 + [
79 + # single / multi-image
80 + (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
81 + (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
82 + # single / multi-image embeddings
83 + (torch.rand(
84 + (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
85 + (torch.rand(
86 + (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
87 + (torch.rand(
88 + (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
89 + ])
90 +def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
91 + qwen_vl_context: InputContext,
92 + img_data: Union[torch.Tensor, List[Image],
93 + Image],
94 + expected_shape: List[int]):
95 + """Happy cases for image inputs to Qwen's multimodal input mapper."""
96 + mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
97 + # Ensure that we get the appropriately shaped pixel_values
98 + # for images and image embeddings, respectively.
99 + assert isinstance(mapped_img_data, MultiModalInputs)
100 + assert "pixel_values" in mapped_img_data
101 + assert mapped_img_data["pixel_values"].shape == expected_shape
102 +
103 +
104 +# Sad path tests for the multimodal input processor and mapper, respectively
105 +@pytest.mark.parametrize("mm_data", [
106 + {
107 + "image": torch.rand((5))
108 + },
109 + {
110 + "image": torch.rand((5, 5, 5, 5, 5))
111 + },
112 +])
113 +def test_input_processor_invalid_mm_data(input_processor_for_qwen,
114 + qwen_vl_context: InputContext,
115 + mm_data: Dict[str, torch.Tensor]):
116 + """Test sad cases validated in Qwen's multimodal input processor."""
117 + tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
118 + trust_remote_code=True)
119 + prompt = "Picture 1: <img></img>\n"
120 + prompt_token_ids = tokenizer.encode(prompt)
121 + inputs = token_inputs(prompt=prompt,
122 + prompt_token_ids=prompt_token_ids,
123 + multi_modal_data=mm_data)
124 + # Should fail since we have too many or too few dimensions for embeddings
125 + with pytest.raises(ValueError):
126 + input_processor_for_qwen(qwen_vl_context, inputs)
127 +
128 +
129 +@pytest.mark.parametrize(
130 + "img_data",
131 + [
132 + # Wrong context length
133 + torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
134 + # Wrong visual encoder output size
135 + torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
136 + ])
137 +def test_input_mapper_invalid_mm_data(
138 + input_mapper_for_qwen,
139 + qwen_vl_context: InputContext,
140 + img_data: Union[torch.Tensor, List[Image], Image],
141 +):
142 + """Sad cases validated in Qwen VL's multimodal input mapper."""
143 + with pytest.raises(ValueError):
144 + input_mapper_for_qwen(qwen_vl_context, img_data)
tests/models/decoder_only/vision_language/mm_processor_kwargs/test_qwen2_vl.py renamed
+2 -2
@@ -8,8 +8,8 @@ from transformers import AutoTokenizer
8 from vllm.inputs import InputContext, token_inputs
9 from vllm.multimodal import MultiModalRegistry
10
11 -from ....conftest import _ImageAssets
12 -from ...utils import build_model_context
11 +from .....conftest import _ImageAssets
12 +from ....utils import build_model_context
13
14 MODEL = "Qwen/Qwen2-VL-2B-Instruct"
15 MIN_PIXELS = "min_pixels"
tests/models/decoder_only/vision_language/test_blip2.py deleted
-101
@@ -1,101 +0,0 @@
1 -from typing import List, Optional, Tuple
2 -
3 -import pytest
4 -from transformers import AutoModelForVision2Seq, AutoTokenizer
5 -
6 -from vllm.multimodal.utils import rescale_image_size
7 -from vllm.sequence import SampleLogprobs
8 -
9 -from ....conftest import IMAGE_ASSETS
10 -from ...utils import check_logprobs_close
11 -
12 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13 - "stop_sign":
14 - "Question: What's the content of the image? Answer:",
15 - "cherry_blossom":
16 - "Question: What is the season? Answer:",
17 -})
18 -
19 -
20 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
21 - Optional[SampleLogprobs]],
22 - model: str):
23 - """Sanitize vllm output to be comparable with hf output."""
24 - _, output_str, out_logprobs = vllm_output
25 -
26 - hf_output_str = output_str + "\n"
27 -
28 - tokenizer = AutoTokenizer.from_pretrained(model)
29 - hf_output_ids = tokenizer.encode(hf_output_str)
30 - assert hf_output_ids[0] == tokenizer.bos_token_id
31 - hf_output_ids = hf_output_ids[1:]
32 -
33 - return hf_output_ids, hf_output_str, out_logprobs
34 -
35 -
36 -@pytest.mark.parametrize("model", ["Salesforce/blip2-opt-2.7b"])
37 -@pytest.mark.parametrize(
38 - "size_factors",
39 - [
40 - # No image
41 - [],
42 - # Single-scale
43 - [1.0],
44 - # Single-scale, batched
45 - [1.0, 1.0, 1.0],
46 - # Multi-scale
47 - [0.25, 0.5, 1.0],
48 - ],
49 -)
50 -@pytest.mark.parametrize("dtype", ["half"])
51 -@pytest.mark.parametrize("max_tokens", [128])
52 -@pytest.mark.parametrize("num_logprobs", [5])
53 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
54 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
55 - """Inference result should be the same between hf and vllm.
56 -
57 - All the image fixtures for the test are from IMAGE_ASSETS.
58 - For huggingface runner, we provide the PIL images as input.
59 - For vllm runner, we provide MultiModalData objects and corresponding
60 - MultiModalConfig as input.
61 - Note, the text input is also adjusted to abide by vllm contract.
62 - The text output is sanitized to be able to compare with hf.
63 - """
64 - images = [asset.pil_image for asset in image_assets]
65 -
66 - inputs_per_image = [(
67 - [prompt for _ in size_factors],
68 - [rescale_image_size(image, factor) for factor in size_factors],
69 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
70 -
71 - # max_model_len should be greater than image_feature_size
72 - with vllm_runner(model, dtype=dtype, enforce_eager=True) as vllm_model:
73 - vllm_outputs_per_image = [
74 - vllm_model.generate_greedy_logprobs(prompts,
75 - max_tokens,
76 - num_logprobs=num_logprobs,
77 - images=images)
78 - for prompts, images in inputs_per_image
79 - ]
80 -
81 - with hf_runner(model, dtype=dtype,
82 - auto_cls=AutoModelForVision2Seq) as hf_model:
83 - hf_outputs_per_image = [
84 - hf_model.generate_greedy_logprobs_limit(prompts,
85 - max_tokens,
86 - num_logprobs=num_logprobs,
87 - images=images)
88 - for prompts, images in inputs_per_image
89 - ]
90 -
91 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
92 - vllm_outputs_per_image):
93 - check_logprobs_close(
94 - outputs_0_lst=hf_outputs,
95 - outputs_1_lst=[
96 - vllm_to_hf_output(vllm_output, model)
97 - for vllm_output in vllm_outputs
98 - ],
99 - name_0="hf",
100 - name_1="vllm",
101 - )
tests/models/decoder_only/vision_language/test_broadcast.py deleted
-46
@@ -1,46 +0,0 @@
1 -import pytest
2 -import transformers
3 -
4 -from ....utils import multi_gpu_test
5 -
6 -
7 -@multi_gpu_test(num_gpus=2)
8 -@pytest.mark.parametrize("distributed_executor_backend", ["ray", "mp"])
9 -@pytest.mark.parametrize("model", [
10 - "llava-hf/llava-1.5-7b-hf",
11 - "llava-hf/llava-v1.6-mistral-7b-hf",
12 - "facebook/chameleon-7b",
13 -])
14 -def test_models(hf_runner, vllm_runner, image_assets,
15 - distributed_executor_backend, model) -> None:
16 -
17 - dtype = "half"
18 - max_tokens = 5
19 - num_logprobs = 5
20 - tensor_parallel_size = 2
21 -
22 - if model.startswith("llava-hf/llava-1.5"):
23 - from .test_llava import models, run_test
24 - elif model.startswith("llava-hf/llava-v1.6"):
25 - from .test_llava_next import models, run_test # type: ignore[no-redef]
26 - elif model.startswith("facebook/chameleon"):
27 - if transformers.__version__.startswith("4.46"):
28 - pytest.skip("Model broken in HF, "
29 - "see huggingface/transformers#34379")
30 - from .test_chameleon import models, run_test # type: ignore[no-redef]
31 - else:
32 - raise NotImplementedError(f"Unsupported model: {model}")
33 -
34 - run_test(
35 - hf_runner,
36 - vllm_runner,
37 - image_assets,
38 - model=models[0],
39 - # So that LLaVA-NeXT processor may return nested list
40 - size_factors=[0.25, 0.5, 1.0],
41 - dtype=dtype,
42 - max_tokens=max_tokens,
43 - num_logprobs=num_logprobs,
44 - tensor_parallel_size=tensor_parallel_size,
45 - distributed_executor_backend=distributed_executor_backend,
46 - )
tests/models/decoder_only/vision_language/test_chameleon.py deleted
-130
@@ -1,130 +0,0 @@
1 -from typing import List, Optional, Type
2 -
3 -import pytest
4 -import transformers
5 -from transformers import AutoModelForVision2Seq, BatchEncoding
6 -
7 -from vllm.multimodal.utils import rescale_image_size
8 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
9 -
10 -from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
11 -from ...utils import check_outputs_equal
12 -
13 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14 - "stop_sign":
15 - "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
16 - "cherry_blossom":
17 - "USER: <image>\nWhat is the season?\nASSISTANT:",
18 -})
19 -
20 -models = ["facebook/chameleon-7b"]
21 -
22 -
23 -def run_test(
24 - hf_runner: Type[HfRunner],
25 - vllm_runner: Type[VllmRunner],
26 - image_assets: _ImageAssets,
27 - model: str,
28 - *,
29 - size_factors: List[float],
30 - dtype: str,
31 - max_tokens: int,
32 - num_logprobs: int,
33 - tensor_parallel_size: int,
34 - distributed_executor_backend: Optional[str] = None,
35 -):
36 - """Inference result should be the same between hf and vllm.
37 -
38 - All the image fixtures for the test are from IMAGE_ASSETS.
39 - For huggingface runner, we provide the PIL images as input.
40 - For vllm runner, we provide MultiModalDataDict objects
41 - and corresponding vision language config as input.
42 - Note, the text input is also adjusted to abide by vllm contract.
43 - The text output is sanitized to be able to compare with hf.
44 - """
45 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
46 - images = [asset.pil_image for asset in image_assets]
47 -
48 - inputs_per_image = [(
49 - [prompt for _ in size_factors],
50 - [rescale_image_size(image, factor) for factor in size_factors],
51 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
52 -
53 - with vllm_runner(model,
54 - max_model_len=4096,
55 - dtype=dtype,
56 - tensor_parallel_size=tensor_parallel_size,
57 - distributed_executor_backend=distributed_executor_backend,
58 - enforce_eager=True) as vllm_model:
59 -
60 - vllm_outputs_per_image = [
61 - vllm_model.generate_greedy_logprobs(prompts,
62 - max_tokens,
63 - num_logprobs=num_logprobs,
64 - images=images)
65 - for prompts, images in inputs_per_image
66 - ]
67 -
68 - def process(hf_inputs: BatchEncoding):
69 - hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
70 - .to(torch_dtype) # type: ignore
71 - return hf_inputs
72 -
73 - with hf_runner(model,
74 - dtype=dtype,
75 - postprocess_inputs=process,
76 - auto_cls=AutoModelForVision2Seq) as hf_model:
77 - hf_outputs_per_image = [
78 - hf_model.generate_greedy_logprobs_limit(prompts,
79 - max_tokens,
80 - num_logprobs=num_logprobs,
81 - images=images)
82 - for prompts, images in inputs_per_image
83 - ]
84 -
85 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
86 - vllm_outputs_per_image):
87 - # HF Logprobs include image tokens, unlike vLLM, so we don't directly
88 - # compare them
89 - check_outputs_equal(
90 - outputs_0_lst=[outputs[:2] for outputs in hf_outputs],
91 - outputs_1_lst=[outputs[:2] for outputs in vllm_outputs],
92 - name_0="hf",
93 - name_1="vllm",
94 - )
95 -
96 -
97 -@pytest.mark.skipif(
98 - transformers.__version__.startswith("4.46.0"),
99 - reason="Model broken in HF, see huggingface/transformers#34379",
100 -)
101 -@pytest.mark.parametrize("model", models)
102 -@pytest.mark.parametrize(
103 - "size_factors",
104 - [
105 - # No image
106 - [],
107 - # Single-scale
108 - [1.0],
109 - # Single-scale, batched
110 - [1.0, 1.0, 1.0],
111 - # Multi-scale
112 - [0.25, 0.5, 1.0],
113 - ],
114 -)
115 -@pytest.mark.parametrize("dtype", ["bfloat16"])
116 -@pytest.mark.parametrize("max_tokens", [8])
117 -@pytest.mark.parametrize("num_logprobs", [5])
118 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
119 - dtype, max_tokens, num_logprobs) -> None:
120 - run_test(
121 - hf_runner,
122 - vllm_runner,
123 - image_assets,
124 - model,
125 - size_factors=size_factors,
126 - dtype=dtype,
127 - max_tokens=max_tokens,
128 - num_logprobs=num_logprobs,
129 - tensor_parallel_size=1,
130 - )
tests/models/decoder_only/vision_language/test_fuyu.py deleted
-139
@@ -1,139 +0,0 @@
1 -from typing import List, Optional, Tuple, Type
2 -
3 -import pytest
4 -
5 -from vllm.multimodal.utils import rescale_image_size
6 -from vllm.platforms import current_platform
7 -from vllm.sequence import SampleLogprobs
8 -
9 -from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
10 -from ...utils import check_logprobs_close
11 -
12 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13 - "stop_sign":
14 - "What's the content of the image?\n",
15 - "cherry_blossom":
16 - "What is the season?\n",
17 -})
18 -
19 -models = ["adept/fuyu-8b"]
20 -
21 -
22 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
23 - Optional[SampleLogprobs]]):
24 - """Sanitize vllm output to be comparable with hf output."""
25 - output_ids, output_str, out_logprobs = vllm_output
26 -
27 - hf_output_str = output_str.lstrip() + "|ENDOFTEXT|"
28 -
29 - return output_ids, hf_output_str, out_logprobs
30 -
31 -
32 -def run_test(
33 - hf_runner: Type[HfRunner],
34 - vllm_runner: Type[VllmRunner],
35 - image_assets: _ImageAssets,
36 - model: str,
37 - *,
38 - size_factors: List[float],
39 - dtype: str,
40 - max_tokens: int,
41 - num_logprobs: int,
42 - tensor_parallel_size: int,
43 - distributed_executor_backend: Optional[str] = None,
44 -):
45 - """Inference result should be the same between hf and vllm.
46 -
47 - All the image fixtures for the test are from IMAGE_ASSETS.
48 - For huggingface runner, we provide the PIL images as input.
49 - For vllm runner, we provide MultiModalDataDict objects
50 - and corresponding MultiModalConfig as input.
51 - Note, the text input is also adjusted to abide by vllm contract.
52 - The text output is sanitized to be able to compare with hf.
53 - """
54 - images = [asset.pil_image for asset in image_assets]
55 -
56 - inputs_per_image = [(
57 - [prompt for _ in size_factors],
58 - [rescale_image_size(image, factor) for factor in size_factors],
59 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
60 -
61 - # NOTE: take care of the order. run vLLM first, and then run HF.
62 - # vLLM needs a fresh new process without cuda initialization.
63 - # if we run HF first, the cuda initialization will be done and it
64 - # will hurt multiprocessing backend with fork method (the default method).
65 -
66 - # max_model_len should be greater than image_feature_size
67 - with vllm_runner(model,
68 - max_model_len=2048,
69 - max_num_seqs=2,
70 - dtype=dtype,
71 - tensor_parallel_size=tensor_parallel_size,
72 - distributed_executor_backend=distributed_executor_backend,
73 - enforce_eager=True) as vllm_model:
74 - vllm_outputs_per_image = [
75 - vllm_model.generate_greedy_logprobs(prompts,
76 - max_tokens,
77 - num_logprobs=num_logprobs,
78 - images=images)
79 - for prompts, images in inputs_per_image
80 - ]
81 -
82 - with hf_runner(model, dtype=dtype) as hf_model:
83 - eos_token_id = hf_model.processor.tokenizer.eos_token_id
84 - hf_outputs_per_image = [
85 - hf_model.generate_greedy_logprobs_limit(prompts,
86 - max_tokens,
87 - num_logprobs=num_logprobs,
88 - images=images,
89 - eos_token_id=eos_token_id)
90 - for prompts, images in inputs_per_image
91 - ]
92 -
93 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
94 - vllm_outputs_per_image):
95 - check_logprobs_close(
96 - outputs_0_lst=hf_outputs,
97 - outputs_1_lst=[
98 - vllm_to_hf_output(vllm_output) for vllm_output in vllm_outputs
99 - ],
100 - name_0="hf",
101 - name_1="vllm",
102 - )
103 -
104 -
105 -target_dtype = "half"
106 -if current_platform.is_cpu():
107 - target_dtype = "bfloat16"
108 -
109 -
110 -@pytest.mark.parametrize("model", models)
111 -@pytest.mark.parametrize(
112 - "size_factors",
113 - [
114 - # No image
115 - [],
116 - # Single-scale
117 - [0.25],
118 - # Single-scale, batched
119 - [0.25, 0.25, 0.25],
120 - # Multi-scale
121 - [0.25, 0.2, 0.15],
122 - ],
123 -)
124 -@pytest.mark.parametrize("dtype", [target_dtype])
125 -@pytest.mark.parametrize("max_tokens", [128])
126 -@pytest.mark.parametrize("num_logprobs", [10])
127 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
128 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
129 - run_test(
130 - hf_runner,
131 - vllm_runner,
132 - image_assets,
133 - model,
134 - size_factors=size_factors,
135 - dtype=dtype,
136 - max_tokens=max_tokens,
137 - num_logprobs=num_logprobs,
138 - tensor_parallel_size=1,
139 - )
tests/models/decoder_only/vision_language/test_glm4.py deleted
-133
@@ -1,133 +0,0 @@
1 -from typing import List, Optional, Tuple, Type
2 -
3 -import pytest
4 -
5 -from vllm.multimodal.utils import rescale_image_size
6 -from vllm.transformers_utils.tokenizer import patch_padding_side
7 -
8 -from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
9 -from ....utils import large_gpu_test
10 -from ...utils import check_logprobs_close
11 -
12 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
13 - "stop_sign":
14 - "What's the content of the image?",
15 - "cherry_blossom":
16 - "What is the season?",
17 -})
18 -
19 -models = ["THUDM/glm-4v-9b"]
20 -target_dtype = "bfloat16"
21 -
22 -
23 -def run_test(
24 - hf_runner: Type[HfRunner],
25 - vllm_runner: Type[VllmRunner],
26 - inputs: List[Tuple[List[str], PromptImageInput]],
27 - model: str,
28 - *,
29 - dtype: str,
30 - max_tokens: int,
31 - num_logprobs: int,
32 - mm_limit: int,
33 - tensor_parallel_size: int,
34 - distributed_executor_backend: Optional[str] = None,
35 -):
36 - # max_model_len should be greater than image_feature_size
37 - with vllm_runner(model,
38 - max_model_len=2048,
39 - max_num_seqs=2,
40 - dtype=dtype,
41 - limit_mm_per_prompt={"image": mm_limit},
42 - tensor_parallel_size=tensor_parallel_size,
43 - distributed_executor_backend=distributed_executor_backend,
44 - enforce_eager=True) as vllm_model:
45 - stop_token_ids = [151329, 151336, 151338]
46 - vllm_outputs_per_image = [
47 - vllm_model.generate_greedy_logprobs(prompts,
48 - max_tokens,
49 - num_logprobs=num_logprobs,
50 - images=images,
51 - stop_token_ids=stop_token_ids)
52 - for prompts, images in inputs
53 - ]
54 -
55 - with hf_runner(model, dtype=dtype) as hf_model:
56 - hf_processor = hf_model.processor
57 - patch_padding_side(hf_processor)
58 -
59 - def processor(*args, text="", images=None, **kwargs):
60 - if images is None:
61 - return hf_processor(*args, **kwargs)
62 -
63 - return hf_processor.apply_chat_template(
64 - [{
65 - "role": "user",
66 - "image": images,
67 - "content": text
68 - }],
69 - add_generation_prompt=True,
70 - tokenize=True,
71 - return_dict=True,
72 - **kwargs,
73 - )
74 -
75 - hf_model.processor = processor
76 - hf_model.model.get_output_embeddings = lambda: \
77 - hf_model.model.transformer.output_layer
78 - hf_outputs_per_image = [
79 - hf_model.generate_greedy_logprobs_limit(
80 - prompts,
81 - max_tokens,
82 - num_logprobs=num_logprobs,
83 - images=images,
84 - ) for prompts, images in inputs
85 - ]
86 -
87 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
88 - vllm_outputs_per_image):
89 - check_logprobs_close(
90 - outputs_0_lst=hf_outputs,
91 - outputs_1_lst=vllm_outputs,
92 - name_0="hf",
93 - name_1="vllm",
94 - )
95 -
96 -
97 -@large_gpu_test(min_gb=48)
98 -@pytest.mark.parametrize("model", models)
99 -@pytest.mark.parametrize(
100 - "size_factors",
101 - [
102 - # No image
103 - [],
104 - # Single-scale
105 - [1.0],
106 - # Single-scale, batched
107 - [1.0, 1.0, 1.0],
108 - # Multi-scale
109 - [0.25, 0.5, 1.0],
110 - ],
111 -)
112 -@pytest.mark.parametrize("dtype", [target_dtype])
113 -@pytest.mark.parametrize("max_tokens", [128])
114 -@pytest.mark.parametrize("num_logprobs", [5])
115 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
116 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
117 - images = [asset.pil_image for asset in image_assets]
118 -
119 - inputs_per_image = [(
120 - [prompt for _ in size_factors],
121 - [rescale_image_size(image, factor) for factor in size_factors],
122 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
123 - run_test(
124 - hf_runner,
125 - vllm_runner,
126 - inputs_per_image,
127 - model,
128 - dtype=dtype,
129 - max_tokens=max_tokens,
130 - num_logprobs=num_logprobs,
131 - mm_limit=1,
132 - tensor_parallel_size=1,
133 - )
tests/models/decoder_only/vision_language/test_internvl.py
+2 -288
@@ -1,15 +1,11 @@
1 -import types
2 -from typing import List, Optional, Tuple, Type, Union
1 +from typing import List, Optional, Tuple, Type
2
3 import pytest
4 import torch
6 -from PIL.Image import Image
7 -from transformers import AutoConfig
5
6 from vllm.multimodal.utils import rescale_image_size
7
11 -from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12 - _ImageAssets)
8 +from ....conftest import IMAGE_ASSETS, VllmRunner, _ImageAssets
9 from ...utils import check_logprobs_close
10
11 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
@@ -18,171 +14,6 @@ HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14 "cherry_blossom":
15 "<|im_start|>User\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
16 })
21 -HF_MULTIIMAGE_IMAGE_PROMPT = "<|im_start|>User\nImage-1: <image>\nImage-2: <image>\nDescribe the two images in short.<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
22 -
23 -models = [
24 - "OpenGVLab/InternVL2-1B",
25 - "OpenGVLab/InternVL2-2B",
26 - # NOTE: Mono-InternVL-2B doesn't work with fp16,
27 - # it will result NaN during inference.
28 - # See: https://huggingface.co/OpenGVLab/Mono-InternVL-2B/discussions/9
29 - "OpenGVLab/Mono-InternVL-2B",
30 - # Broken due to outdated implementation of Phi-3
31 - # See: https://huggingface.co/OpenGVLab/InternVL2-4B/discussions/3
32 - # "OpenGVLab/InternVL2-4B",
33 -]
34 -target_dtype = "bfloat16"
35 -
36 -
37 -# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B/blob/main/modeling_internvl_chat.py
38 -def generate(
39 - self,
40 - pixel_values: torch.FloatTensor,
41 - input_ids: torch.FloatTensor,
42 - attention_mask: Optional[torch.LongTensor] = None,
43 - **generate_kwargs,
44 -) -> torch.LongTensor:
45 - """Generate method for InternVL2 model without fixed use_cache."""
46 - assert self.img_context_token_id is not None
47 - vit_embeds = self.extract_feature(pixel_values)
48 - input_embeds = self.language_model.get_input_embeddings()(input_ids)
49 - B, N, C = input_embeds.shape
50 - input_embeds = input_embeds.reshape(B * N, C)
51 -
52 - input_ids = input_ids.reshape(B * N)
53 - selected = (input_ids == self.img_context_token_id)
54 - assert selected.sum() != 0
55 - input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
56 -
57 - input_embeds = input_embeds.reshape(B, N, C)
58 -
59 - forward_kwargs = dict(
60 - inputs_embeds=input_embeds,
61 - attention_mask=attention_mask,
62 - )
63 - if getattr(self, "use_visual_token_mask", False):
64 - visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype)
65 - forward_kwargs["visual_token_mask"] = visual_token_mask
66 - outputs = self.language_model.generate(
67 - **forward_kwargs,
68 - **generate_kwargs,
69 - )
70 -
71 - return outputs
72 -
73 -
74 -def run_test(
75 - hf_runner: Type[HfRunner],
76 - vllm_runner: Type[VllmRunner],
77 - inputs: List[Tuple[List[str], PromptImageInput]],
78 - model: str,
79 - *,
80 - dtype: str,
81 - max_tokens: int,
82 - num_logprobs: int,
83 - mm_limit: int,
84 - tensor_parallel_size: int,
85 - distributed_executor_backend: Optional[str] = None,
86 -):
87 - """Inference result should be the same between hf and vllm.
88 -
89 - All the image fixtures for the test are from IMAGE_ASSETS.
90 - For huggingface runner, we provide the PIL images as input.
91 - For vllm runner, we provide MultiModalDataDict objects
92 - and corresponding MultiModalConfig as input.
93 - Note, the text input is also adjusted to abide by vllm contract.
94 - The text output is sanitized to be able to compare with hf.
95 - """
96 -
97 - # NOTE: take care of the order. run vLLM first, and then run HF.
98 - # vLLM needs a fresh new process without cuda initialization.
99 - # if we run HF first, the cuda initialization will be done and it
100 - # will hurt multiprocessing backend with fork method (the default method).
101 -
102 - class InternVLProcessor:
103 - """A simple processor for InternVL2 which misses a processor."""
104 -
105 - def __init__(self, hf_runner: HfRunner):
106 - self.num_image_token = hf_runner.model.num_image_token
107 - self.tokenizer = hf_runner.tokenizer
108 - self.dtype = hf_runner.model.dtype
109 -
110 - self.config = AutoConfig.from_pretrained(hf_runner.model_name,
111 - trust_remote_code=True)
112 - self.vision_config = self.config.vision_config
113 - self.use_thumbnail = self.config.use_thumbnail
114 - self.min_num = self.config.min_dynamic_patch
115 - self.max_num = self.config.max_dynamic_patch
116 - self.image_size = self.vision_config.image_size
117 -
118 - def __call__(self, text: str, images: Union[Image, List[Image]],
119 - **kwargs):
120 - from vllm.model_executor.models.internvl import (
121 - IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
122 - images = [images] if isinstance(images, Image) else images
123 - pixel_values = [
124 - image_to_pixel_values(image, self.image_size, self.min_num,
125 - self.max_num,
126 - self.use_thumbnail).to(self.dtype)
127 - for image in images
128 - ]
129 - num_patches_list = [
130 - pixel_value.shape[0] for pixel_value in pixel_values
131 - ]
132 - pixel_values = torch.cat(pixel_values, dim=0)
133 - for num_patches in num_patches_list:
134 - context_tokens = IMG_CONTEXT * self.num_image_token \
135 - * num_patches
136 - image_tokens = IMG_START + context_tokens + IMG_END
137 - text = text.replace('<image>', image_tokens, 1)
138 - prompt = self.tokenizer(text, return_tensors="pt")
139 - prompt.update({"pixel_values": pixel_values})
140 - return prompt
141 -
142 - # max_model_len should be greater than image_feature_size
143 - with vllm_runner(model,
144 - max_model_len=4096,
145 - dtype=dtype,
146 - limit_mm_per_prompt={"image": mm_limit},
147 - tensor_parallel_size=tensor_parallel_size,
148 - distributed_executor_backend=distributed_executor_backend,
149 - enforce_eager=True) as vllm_model:
150 - vllm_outputs_per_image = [
151 - vllm_model.generate_greedy_logprobs(prompts,
152 - max_tokens,
153 - num_logprobs=num_logprobs,
154 - images=images)
155 - for prompts, images in inputs
156 - ]
157 -
158 - with hf_runner(model, dtype=dtype) as hf_model:
159 - img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
160 - "<IMG_CONTEXT>")
161 - hf_model.model.img_context_token_id = img_context_token_id
162 - hf_model.processor = InternVLProcessor(hf_model)
163 - hf_model.model.get_output_embeddings = lambda: \
164 - hf_model.model.language_model.get_output_embeddings()
165 - hf_model.model.generate = types.MethodType(generate, hf_model.model)
166 - eos_token_id = hf_model.tokenizer.eos_token_id
167 - hf_outputs_per_image = [
168 - hf_model.generate_greedy_logprobs_limit(prompts,
169 - max_tokens,
170 - num_logprobs=num_logprobs,
171 - images=hf_images,
172 - eos_token_id=eos_token_id)
173 - for prompts, hf_images in inputs
174 - ]
175 -
176 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
177 - vllm_outputs_per_image):
178 - # TODO: Check whether using original CLIPVisionModel can improve
179 - # consistency against HF
180 - check_logprobs_close(
181 - outputs_0_lst=hf_outputs,
182 - outputs_1_lst=vllm_outputs,
183 - name_0="hf",
184 - name_1="vllm",
185 - )
17
18
19 def run_awq_test(
@@ -253,123 +84,6 @@ def run_awq_test(
84 )
85
86
256 -@pytest.mark.parametrize("model", models)
257 -@pytest.mark.parametrize(
258 - "size_factors",
259 - [
260 - # No image
261 - [],
262 - # Single-scale
263 - [1.0],
264 - # Single-scale, batched
265 - [1.0, 1.0, 1.0],
266 - # Multi-scale
267 - [0.25, 0.5, 1.0],
268 - ],
269 -)
270 -@pytest.mark.parametrize("dtype", [target_dtype])
271 -@pytest.mark.parametrize("max_tokens", [128])
272 -@pytest.mark.parametrize("num_logprobs", [5])
273 -@torch.inference_mode()
274 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
275 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
276 - images = [asset.pil_image for asset in image_assets]
277 -
278 - inputs_per_image = [(
279 - [prompt for _ in size_factors],
280 - [rescale_image_size(image, factor) for factor in size_factors],
281 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
282 -
283 - run_test(
284 - hf_runner,
285 - vllm_runner,
286 - inputs_per_image,
287 - model,
288 - dtype=dtype,
289 - max_tokens=max_tokens,
290 - num_logprobs=num_logprobs,
291 - mm_limit=1,
292 - tensor_parallel_size=1,
293 - )
294 -
295 -
296 -@pytest.mark.parametrize("model", models)
297 -@pytest.mark.parametrize(
298 - "size_factors",
299 - [
300 - # No image
301 - [],
302 - # Single-scale
303 - [1.0],
304 - # Single-scale, batched
305 - [1.0, 1.0, 1.0],
306 - # Multi-scale
307 - [0.5, 0.75, 1.0],
308 - ],
309 -)
310 -@pytest.mark.parametrize("dtype", [target_dtype])
311 -@pytest.mark.parametrize("max_tokens", [128])
312 -@pytest.mark.parametrize("num_logprobs", [5])
313 -@torch.inference_mode()
314 -def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
315 - size_factors, dtype: str, max_tokens: int,
316 - num_logprobs: int) -> None:
317 - images = [asset.pil_image for asset in image_assets]
318 -
319 - inputs_per_case = [
320 - ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
321 - [[rescale_image_size(image, factor) for image in images]
322 - for factor in size_factors])
323 - ]
324 -
325 - run_test(
326 - hf_runner,
327 - vllm_runner,
328 - inputs_per_case,
329 - model,
330 - dtype=dtype,
331 - max_tokens=max_tokens,
332 - num_logprobs=num_logprobs,
333 - mm_limit=2,
334 - tensor_parallel_size=1,
335 - )
336 -
337 -
338 -@pytest.mark.parametrize("model", ["OpenGVLab/InternVL2-2B"])
339 -@pytest.mark.parametrize("size_factors", [[0.5, 1.0]])
340 -@pytest.mark.parametrize("dtype", [target_dtype])
341 -@pytest.mark.parametrize("max_tokens", [128])
342 -@pytest.mark.parametrize("num_logprobs", [5])
343 -@torch.inference_mode()
344 -def test_different_num_patches(hf_runner, vllm_runner, image_assets, model,
345 - size_factors, dtype: str, max_tokens: int,
346 - num_logprobs: int) -> None:
347 - images = [asset.pil_image.resize((896, 896)) for asset in image_assets]
348 -
349 - inputs_batching = [(
350 - [prompt for _ in size_factors],
351 - [rescale_image_size(image, factor) for factor in size_factors],
352 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
353 -
354 - inputs_multi_images = [
355 - ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
356 - [[rescale_image_size(image, factor) for image in images]
357 - for factor in size_factors])
358 - ]
359 - for inputs in [inputs_batching, inputs_multi_images]:
360 - run_test(
361 - hf_runner,
362 - vllm_runner,
363 - inputs,
364 - model,
365 - dtype=dtype,
366 - max_tokens=max_tokens,
367 - num_logprobs=num_logprobs,
368 - mm_limit=2,
369 - tensor_parallel_size=1,
370 - )
371 -
372 -
87 @pytest.mark.parametrize(
88 "models", [("OpenGVLab/InternVL2-2B", "OpenGVLab/InternVL2-2B-AWQ")])
89 @pytest.mark.parametrize(
tests/models/decoder_only/vision_language/test_llava.py deleted
-313
@@ -1,313 +0,0 @@
1 -from typing import List, Optional, Tuple, Type, overload
2 -
3 -import pytest
4 -from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 - BatchEncoding)
6 -
7 -from vllm.multimodal.utils import rescale_image_size
8 -from vllm.sequence import SampleLogprobs
9 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
10 -
11 -from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12 - _ImageAssets)
13 -from ...utils import check_logprobs_close
14 -
15 -_LIMIT_IMAGE_PER_PROMPT = 4
16 -
17 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 - "stop_sign":
19 - "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
20 - "cherry_blossom":
21 - "USER: <image>\nWhat is the season?\nASSISTANT:",
22 -})
23 -
24 -models = [
25 - "llava-hf/llava-1.5-7b-hf",
26 - # TODO: Get this model to produce meaningful output in vLLM
27 - # "TIGER-Lab/Mantis-8B-siglip-llama3",
28 -]
29 -
30 -
31 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
32 - Optional[SampleLogprobs]],
33 - model: str):
34 - """Sanitize vllm output to be comparable with hf output."""
35 - output_ids, output_str, out_logprobs = vllm_output
36 -
37 - config = AutoConfig.from_pretrained(model)
38 - image_token_id = config.image_token_index
39 -
40 - tokenizer = AutoTokenizer.from_pretrained(model)
41 - eos_token_id = tokenizer.eos_token_id
42 -
43 - hf_output_ids = [
44 - token_id for idx, token_id in enumerate(output_ids)
45 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
46 - ]
47 -
48 - assert output_str[0] == " "
49 - hf_output_str = output_str[1:]
50 - if hf_output_ids[-1] == eos_token_id:
51 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
52 -
53 - return hf_output_ids, hf_output_str, out_logprobs
54 -
55 -
56 -@overload
57 -def run_test(
58 - hf_runner: Type[HfRunner],
59 - vllm_runner: Type[VllmRunner],
60 - image_assets: _ImageAssets,
61 - model: str,
62 - *,
63 - size_factors: List[float],
64 - dtype: str,
65 - max_tokens: int,
66 - num_logprobs: int,
67 - tensor_parallel_size: int,
68 - distributed_executor_backend: Optional[str] = None,
69 -):
70 - ...
71 -
72 -
73 -@overload
74 -def run_test(
75 - hf_runner: Type[HfRunner],
76 - vllm_runner: Type[VllmRunner],
77 - image_assets: _ImageAssets,
78 - model: str,
79 - *,
80 - sizes: List[Tuple[int, int]],
81 - dtype: str,
82 - max_tokens: int,
83 - num_logprobs: int,
84 - tensor_parallel_size: int,
85 - distributed_executor_backend: Optional[str] = None,
86 -):
87 - ...
88 -
89 -
90 -def run_test(
91 - hf_runner: Type[HfRunner],
92 - vllm_runner: Type[VllmRunner],
93 - image_assets: _ImageAssets,
94 - model: str,
95 - *,
96 - size_factors: Optional[List[float]] = None,
97 - sizes: Optional[List[Tuple[int, int]]] = None,
98 - dtype: str,
99 - max_tokens: int,
100 - num_logprobs: int,
101 - tensor_parallel_size: int,
102 - distributed_executor_backend: Optional[str] = None,
103 -):
104 - images = [asset.pil_image for asset in image_assets]
105 -
106 - if size_factors is not None:
107 - inputs_per_image = [(
108 - [prompt for _ in size_factors],
109 - [rescale_image_size(image, factor) for factor in size_factors],
110 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
111 - elif sizes is not None:
112 - inputs_per_image = [(
113 - [prompt for _ in sizes],
114 - [image.resize(size) for size in sizes],
115 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
116 - else:
117 - raise ValueError("You must provide either `size_factors` or `sizes`")
118 -
119 - _run_test(hf_runner,
120 - vllm_runner,
121 - inputs_per_image,
122 - model,
123 - dtype=dtype,
124 - max_tokens=max_tokens,
125 - num_logprobs=num_logprobs,
126 - tensor_parallel_size=tensor_parallel_size,
127 - distributed_executor_backend=distributed_executor_backend)
128 -
129 -
130 -def _run_test(
131 - hf_runner: Type[HfRunner],
132 - vllm_runner: Type[VllmRunner],
133 - inputs: List[Tuple[List[str], PromptImageInput]],
134 - model: str,
135 - *,
136 - dtype: str,
137 - max_tokens: int,
138 - num_logprobs: int,
139 - tensor_parallel_size: int,
140 - distributed_executor_backend: Optional[str] = None,
141 -):
142 - """Inference result should be the same between hf and vllm.
143 -
144 - All the image fixtures for the test are from IMAGE_ASSETS.
145 - For huggingface runner, we provide the PIL images as input.
146 - For vllm runner, we provide MultiModalDataDict objects
147 - and corresponding MultiModalConfig as input.
148 - Note, the text input is also adjusted to abide by vllm contract.
149 - The text output is sanitized to be able to compare with hf.
150 - """
151 - # NOTE: For local use; this isn't tested in CI yet (see TODO above)
152 - if model.startswith("TIGER-Lab/Mantis"):
153 - from mantis.models.mllava import MLlavaProcessor
154 -
155 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
156 - mantis_processor = MLlavaProcessor.from_pretrained(
157 - model, torch_dtype=torch_dtype)
158 - assert isinstance(mantis_processor, MLlavaProcessor)
159 - else:
160 - mantis_processor = None
161 -
162 - # NOTE: take care of the order. run vLLM first, and then run HF.
163 - # vLLM needs a fresh new process without cuda initialization.
164 - # if we run HF first, the cuda initialization will be done and it
165 - # will hurt multiprocessing backend with fork method (the default method).
166 -
167 - # max_model_len should be greater than image_feature_size
168 - with vllm_runner(model,
169 - dtype=dtype,
170 - max_model_len=4096,
171 - tensor_parallel_size=tensor_parallel_size,
172 - distributed_executor_backend=distributed_executor_backend,
173 - enforce_eager=True,
174 - limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
175 - }) as vllm_model:
176 - vllm_outputs_per_image = [
177 - vllm_model.generate_greedy_logprobs(prompts,
178 - max_tokens,
179 - num_logprobs=num_logprobs,
180 - images=images)
181 - for prompts, images in inputs
182 - ]
183 -
184 - if mantis_processor is not None:
185 -
186 - def process(hf_inputs: BatchEncoding):
187 - hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
188 - .to(torch_dtype) # type: ignore
189 - return hf_inputs
190 - else:
191 -
192 - def process(hf_inputs: BatchEncoding):
193 - return hf_inputs
194 -
195 - with hf_runner(model,
196 - dtype=dtype,
197 - postprocess_inputs=process,
198 - auto_cls=AutoModelForVision2Seq) as hf_model:
199 - hf_outputs_per_image = [
200 - hf_model.generate_greedy_logprobs_limit(prompts,
201 - max_tokens,
202 - num_logprobs=num_logprobs,
203 - images=images)
204 - for prompts, images in inputs
205 - ]
206 -
207 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
208 - vllm_outputs_per_image):
209 - # TODO: Check whether using original CLIPVisionModel can improve
210 - # consistency against HF
211 - check_logprobs_close(
212 - outputs_0_lst=hf_outputs,
213 - outputs_1_lst=[
214 - vllm_to_hf_output(vllm_output, model)
215 - for vllm_output in vllm_outputs
216 - ],
217 - name_0="hf",
218 - name_1="vllm",
219 - )
220 -
221 -
222 -@pytest.mark.parametrize("model", models)
223 -@pytest.mark.parametrize(
224 - "size_factors",
225 - [
226 - # No image
227 - [],
228 - # Single-scale
229 - [1.0],
230 - # Single-scale, batched
231 - [1.0, 1.0, 1.0],
232 - # Multi-scale
233 - [0.25, 0.5, 1.0],
234 - ],
235 -)
236 -@pytest.mark.parametrize("dtype", ["half"])
237 -@pytest.mark.parametrize("max_tokens", [128])
238 -@pytest.mark.parametrize("num_logprobs", [5])
239 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
240 - dtype, max_tokens, num_logprobs) -> None:
241 - run_test(
242 - hf_runner,
243 - vllm_runner,
244 - image_assets,
245 - model,
246 - size_factors=size_factors,
247 - dtype=dtype,
248 - max_tokens=max_tokens,
249 - num_logprobs=num_logprobs,
250 - tensor_parallel_size=1,
251 - )
252 -
253 -
254 -@pytest.mark.parametrize("model", models)
255 -@pytest.mark.parametrize("dtype", ["half"])
256 -@pytest.mark.parametrize("max_tokens", [128])
257 -@pytest.mark.parametrize("num_logprobs", [5])
258 -def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
259 - model, dtype, max_tokens,
260 - num_logprobs) -> None:
261 - stop_sign = image_assets[0].pil_image
262 - cherry_blossom = image_assets[1].pil_image
263 -
264 - inputs = [(
265 - [
266 - "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
267 - "USER: <image><image>\nDescribe 2 images.\nASSISTANT:",
268 - "USER: <image><image><image><image>\nDescribe 4 images.\nASSISTANT:", # noqa: E501
269 - "USER: <image>\nWhat is the season?\nASSISTANT:",
270 - ],
271 - [
272 - [stop_sign, cherry_blossom],
273 - # Images with different sizes and aspect-ratios
274 - [
275 - rescale_image_size(stop_sign, 0.1),
276 - stop_sign,
277 - ],
278 - [
279 - stop_sign,
280 - rescale_image_size(stop_sign, 0.25),
281 - cherry_blossom.resize((183, 488)),
282 - cherry_blossom.resize((488, 183))
283 - ],
284 - cherry_blossom,
285 - ])]
286 -
287 - _run_test(
288 - hf_runner,
289 - vllm_runner,
290 - inputs,
291 - model,
292 - dtype=dtype,
293 - max_tokens=max_tokens,
294 - num_logprobs=num_logprobs,
295 - tensor_parallel_size=1,
296 - )
297 -
298 -
299 -@pytest.mark.parametrize("model", models)
300 -def test_context_length_too_short(vllm_runner, image_assets, model):
301 - images = [asset.pil_image for asset in image_assets]
302 -
303 - with pytest.raises(ValueError, match="too long to fit into the model"):
304 - vllm_model = vllm_runner(
305 - model,
306 - max_model_len=128, # LLaVA has a feature size of 576
307 - enforce_eager=True,
308 - )
309 -
310 - with vllm_model:
311 - vllm_model.generate_greedy([HF_IMAGE_PROMPTS[0]],
312 - max_tokens=1,
313 - images=[images[0]])
tests/models/decoder_only/vision_language/test_llava_image_embeds.py deleted
-158
@@ -1,158 +0,0 @@
1 -from typing import List, Optional, Tuple, Type
2 -
3 -import pytest
4 -from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5 -
6 -from vllm.sequence import SampleLogprobs
7 -
8 -from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9 -from ...utils import check_logprobs_close
10 -
11 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
12 - "stop_sign":
13 - "USER: <image>\nWhat's the content of the image?\nASSISTANT:",
14 - "cherry_blossom":
15 - "USER: <image>\nWhat is the season?\nASSISTANT:",
16 -})
17 -
18 -models = [
19 - "llava-hf/llava-1.5-7b-hf",
20 -]
21 -
22 -
23 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
24 - Optional[SampleLogprobs]],
25 - model: str):
26 - """Sanitize vllm output to be comparable with hf output."""
27 - output_ids, output_str, out_logprobs = vllm_output
28 -
29 - config = AutoConfig.from_pretrained(model)
30 - image_token_id = config.image_token_index
31 -
32 - tokenizer = AutoTokenizer.from_pretrained(model)
33 - eos_token_id = tokenizer.eos_token_id
34 -
35 - hf_output_ids = [
36 - token_id for idx, token_id in enumerate(output_ids)
37 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
38 - ]
39 -
40 - assert output_str[0] == " "
41 - hf_output_str = output_str[1:]
42 - if hf_output_ids[-1] == eos_token_id:
43 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
44 -
45 - return hf_output_ids, hf_output_str, out_logprobs
46 -
47 -
48 -def run_test(
49 - hf_runner: Type[HfRunner],
50 - vllm_runner: Type[VllmRunner],
51 - image_assets: _ImageAssets,
52 - model: str,
53 - *,
54 - size_factors: List[float],
55 - dtype: str,
56 - max_tokens: int,
57 - num_logprobs: int,
58 - tensor_parallel_size: int,
59 - distributed_executor_backend: Optional[str] = None,
60 -):
61 - """Inference result should be the same between hf and vllm.
62 -
63 - All the image fixtures for the test are from IMAGE_ASSETS.
64 - For huggingface runner, we provide the PIL images as input.
65 - For vllm runner, we provide MultiModalDataDict objects
66 - and corresponding vision language config as input.
67 - Note, the text input is also adjusted to abide by vllm contract.
68 - The text output is sanitized to be able to compare with hf.
69 - """
70 -
71 - # vLLM to load from image embeddings
72 - vllm_images = [asset.image_embeds for asset in image_assets]
73 -
74 - # transformers to load from PIL images
75 - hf_images = [asset.pil_image for asset in image_assets]
76 -
77 - vllm_inputs_per_image = [(
78 - [prompt for _ in size_factors],
79 - [image for _ in size_factors],
80 - ) for image, prompt in zip(vllm_images, HF_IMAGE_PROMPTS)]
81 -
82 - hf_inputs_per_image = [(
83 - [prompt for _ in size_factors],
84 - [image for _ in size_factors],
85 - ) for image, prompt in zip(hf_images, HF_IMAGE_PROMPTS)]
86 -
87 - # NOTE: take care of the order. run vLLM first, and then run HF.
88 - # vLLM needs a fresh new process without cuda initialization.
89 - # if we run HF first, the cuda initialization will be done and it
90 - # will hurt multiprocessing backend with fork method (the default method).
91 -
92 - # max_model_len should be greater than image_feature_size
93 - with vllm_runner(model,
94 - dtype=dtype,
95 - tensor_parallel_size=tensor_parallel_size,
96 - distributed_executor_backend=distributed_executor_backend,
97 - enforce_eager=True) as vllm_model:
98 - vllm_outputs_per_image = [
99 - vllm_model.generate_greedy_logprobs(prompts,
100 - max_tokens,
101 - num_logprobs=num_logprobs,
102 - images=images)
103 - for prompts, images in vllm_inputs_per_image
104 - ]
105 -
106 - with hf_runner(model, dtype=dtype,
107 - auto_cls=AutoModelForVision2Seq) as hf_model:
108 - hf_outputs_per_image = [
109 - hf_model.generate_greedy_logprobs_limit(prompts,
110 - max_tokens,
111 - num_logprobs=num_logprobs,
112 - images=images)
113 - for prompts, images in hf_inputs_per_image
114 - ]
115 -
116 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
117 - vllm_outputs_per_image):
118 - # TODO: Check whether using original CLIPVisionModel can improve
119 - # consistency against HF
120 - check_logprobs_close(
121 - outputs_0_lst=hf_outputs,
122 - outputs_1_lst=[
123 - vllm_to_hf_output(vllm_output, model)
124 - for vllm_output in vllm_outputs
125 - ],
126 - name_0="hf",
127 - name_1="vllm",
128 - )
129 -
130 -
131 -@pytest.mark.parametrize("model", models)
132 -@pytest.mark.parametrize(
133 - "size_factors",
134 - [
135 - # No image
136 - [],
137 - # Single-scale
138 - [1.0],
139 - # Single-scale, batched
140 - [1.0, 1.0, 1.0],
141 - ],
142 -)
143 -@pytest.mark.parametrize("dtype", ["half"])
144 -@pytest.mark.parametrize("max_tokens", [128])
145 -@pytest.mark.parametrize("num_logprobs", [5])
146 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
147 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
148 - run_test(
149 - hf_runner,
150 - vllm_runner,
151 - image_assets,
152 - model,
153 - size_factors=size_factors,
154 - dtype=dtype,
155 - max_tokens=max_tokens,
156 - num_logprobs=num_logprobs,
157 - tensor_parallel_size=1,
158 - )
tests/models/decoder_only/vision_language/test_llava_next.py deleted
-347
@@ -1,347 +0,0 @@
1 -from typing import List, Optional, Tuple, Type, overload
2 -
3 -import pytest
4 -from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5 -
6 -from vllm.inputs import InputContext
7 -from vllm.multimodal.utils import rescale_image_size
8 -from vllm.sequence import SampleLogprobs
9 -
10 -from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
11 - _ImageAssets)
12 -from ...utils import build_model_context, check_logprobs_close
13 -
14 -_LIMIT_IMAGE_PER_PROMPT = 4
15 -
16 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
17 - "stop_sign":
18 - "[INST] <image>\nWhat's the content of the image? [/INST]",
19 - "cherry_blossom":
20 - "[INST] <image>\nWhat is the season? [/INST]",
21 -})
22 -
23 -models = ["llava-hf/llava-v1.6-mistral-7b-hf"]
24 -
25 -
26 -@pytest.fixture()
27 -def get_max_llava_next_image_tokens():
28 - from vllm.model_executor.models.llava_next import (
29 - get_max_llava_next_image_tokens)
30 - return get_max_llava_next_image_tokens
31 -
32 -
33 -@pytest.fixture()
34 -def dummy_data_for_llava_next():
35 - from vllm.model_executor.models.llava_next import dummy_data_for_llava_next
36 - return dummy_data_for_llava_next
37 -
38 -
39 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
40 - Optional[SampleLogprobs]],
41 - model: str):
42 - """Sanitize vllm output to be comparable with hf output."""
43 - output_ids, output_str, out_logprobs = vllm_output
44 -
45 - config = AutoConfig.from_pretrained(model)
46 - image_token_id = config.image_token_index
47 -
48 - tokenizer = AutoTokenizer.from_pretrained(model)
49 - eos_token_id = tokenizer.eos_token_id
50 -
51 - hf_output_ids = [
52 - token_id for idx, token_id in enumerate(output_ids)
53 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
54 - ]
55 -
56 - assert output_str[0] == " "
57 - hf_output_str = output_str[1:]
58 - if hf_output_ids[-1] == eos_token_id:
59 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
60 -
61 - return hf_output_ids, hf_output_str, out_logprobs
62 -
63 -
64 -@overload
65 -def run_test(
66 - hf_runner: Type[HfRunner],
67 - vllm_runner: Type[VllmRunner],
68 - image_assets: _ImageAssets,
69 - model: str,
70 - *,
71 - size_factors: List[float],
72 - dtype: str,
73 - max_tokens: int,
74 - num_logprobs: int,
75 - tensor_parallel_size: int,
76 - distributed_executor_backend: Optional[str] = None,
77 -):
78 - ...
79 -
80 -
81 -@overload
82 -def run_test(
83 - hf_runner: Type[HfRunner],
84 - vllm_runner: Type[VllmRunner],
85 - image_assets: _ImageAssets,
86 - model: str,
87 - *,
88 - sizes: List[Tuple[int, int]],
89 - dtype: str,
90 - max_tokens: int,
91 - num_logprobs: int,
92 - tensor_parallel_size: int,
93 - distributed_executor_backend: Optional[str] = None,
94 -):
95 - ...
96 -
97 -
98 -def run_test(
99 - hf_runner: Type[HfRunner],
100 - vllm_runner: Type[VllmRunner],
101 - image_assets: _ImageAssets,
102 - model: str,
103 - *,
104 - size_factors: Optional[List[float]] = None,
105 - sizes: Optional[List[Tuple[int, int]]] = None,
106 - dtype: str,
107 - max_tokens: int,
108 - num_logprobs: int,
109 - tensor_parallel_size: int,
110 - distributed_executor_backend: Optional[str] = None,
111 -):
112 - images = [asset.pil_image for asset in image_assets]
113 -
114 - if size_factors is not None:
115 - inputs_per_image = [(
116 - [prompt for _ in size_factors],
117 - [rescale_image_size(image, factor) for factor in size_factors],
118 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
119 - elif sizes is not None:
120 - inputs_per_image = [(
121 - [prompt for _ in sizes],
122 - [image.resize(size) for size in sizes],
123 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
124 - else:
125 - raise ValueError("You must provide either `size_factors` or `sizes`")
126 -
127 - _run_test(hf_runner,
128 - vllm_runner,
129 - inputs_per_image,
130 - model,
131 - dtype=dtype,
132 - max_tokens=max_tokens,
133 - num_logprobs=num_logprobs,
134 - tensor_parallel_size=tensor_parallel_size,
135 - distributed_executor_backend=distributed_executor_backend)
136 -
137 -
138 -def _run_test(
139 - hf_runner: Type[HfRunner],
140 - vllm_runner: Type[VllmRunner],
141 - inputs: List[Tuple[List[str], PromptImageInput]],
142 - model: str,
143 - dtype: str,
144 - max_tokens: int,
145 - num_logprobs: int,
146 - tensor_parallel_size: int,
147 - distributed_executor_backend: Optional[str] = None,
148 -):
149 - # max_model_len should be greater than image_feature_size
150 - with vllm_runner(model,
151 - dtype=dtype,
152 - max_model_len=10240,
153 - tensor_parallel_size=tensor_parallel_size,
154 - distributed_executor_backend=distributed_executor_backend,
155 - enforce_eager=True,
156 - limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
157 - }) as vllm_model:
158 - vllm_outputs_per_image = [
159 - vllm_model.generate_greedy_logprobs(prompts,
160 - max_tokens,
161 - num_logprobs=num_logprobs,
162 - images=images)
163 - for prompts, images in inputs
164 - ]
165 -
166 - with hf_runner(model, dtype=dtype,
167 - auto_cls=AutoModelForVision2Seq) as hf_model:
168 - hf_outputs_per_image = [
169 - hf_model.generate_greedy_logprobs_limit(prompts,
170 - max_tokens,
171 - num_logprobs=num_logprobs,
172 - images=images)
173 - for prompts, images in inputs
174 - ]
175 -
176 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
177 - vllm_outputs_per_image):
178 - # TODO: Check whether using original CLIPVisionModel can improve
179 - # consistency against HF
180 - check_logprobs_close(
181 - outputs_0_lst=hf_outputs,
182 - outputs_1_lst=[
183 - vllm_to_hf_output(vllm_output, model)
184 - for vllm_output in vllm_outputs
185 - ],
186 - name_0="hf",
187 - name_1="vllm",
188 - )
189 -
190 -
191 -@pytest.mark.parametrize("model", models)
192 -@pytest.mark.parametrize(
193 - "size_factors",
194 - [
195 - # No image
196 - [],
197 - # Single-scale
198 - [1.0],
199 - # Single-scale, batched
200 - [1.0, 1.0, 1.0],
201 - # Multi-scale
202 - [0.25, 0.5, 1.0],
203 - ],
204 -)
205 -@pytest.mark.parametrize("dtype", ["half"])
206 -@pytest.mark.parametrize("max_tokens", [128])
207 -@pytest.mark.parametrize("num_logprobs", [5])
208 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
209 - dtype, max_tokens, num_logprobs) -> None:
210 - """Inference result should be the same between hf and vllm.
211 -
212 - All the image fixtures for the test are from IMAGE_ASSETS.
213 - For huggingface runner, we provide the PIL images as input.
214 - For vllm runner, we provide MultiModalDataDict objects
215 - and corresponding MultiModalConfig as input.
216 - Note, the text input is also adjusted to abide by vllm contract.
217 - The text output is sanitized to be able to compare with hf.
218 - """
219 - run_test(
220 - hf_runner,
221 - vllm_runner,
222 - image_assets,
223 - model,
224 - size_factors=size_factors,
225 - dtype=dtype,
226 - max_tokens=max_tokens,
227 - num_logprobs=num_logprobs,
228 - tensor_parallel_size=1,
229 - )
230 -
231 -
232 -@pytest.mark.parametrize("model", models)
233 -@pytest.mark.parametrize(
234 - "sizes",
235 - [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
236 -)
237 -@pytest.mark.parametrize("dtype", ["half"])
238 -@pytest.mark.parametrize("max_tokens", [128])
239 -@pytest.mark.parametrize("num_logprobs", [5])
240 -def test_models_fixed_sizes(hf_runner, vllm_runner, image_assets, model, sizes,
241 - dtype, max_tokens, num_logprobs) -> None:
242 - run_test(
243 - hf_runner,
244 - vllm_runner,
245 - image_assets,
246 - model,
247 - sizes=sizes,
248 - dtype=dtype,
249 - max_tokens=max_tokens,
250 - num_logprobs=num_logprobs,
251 - tensor_parallel_size=1,
252 - )
253 -
254 -
255 -@pytest.mark.parametrize("model", models)
256 -@pytest.mark.parametrize("dtype", ["half"])
257 -@pytest.mark.parametrize("max_tokens", [128])
258 -@pytest.mark.parametrize("num_logprobs", [5])
259 -def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
260 - model, dtype, max_tokens,
261 - num_logprobs) -> None:
262 - stop_sign = image_assets[0].pil_image
263 - cherry_blossom = image_assets[1].pil_image
264 -
265 - inputs = [(
266 - [
267 - "[INST] <image><image>\nDescribe 2 images. [/INST]",
268 - "[INST] <image><image>\nDescribe 2 images. [/INST]",
269 - "[INST] <image><image><image><image>\nDescribe 4 images. [/INST]",
270 - "[INST] <image>\nWhat is the season? [/INST]"
271 - ],
272 - [
273 - [stop_sign, cherry_blossom],
274 - # Images with different sizes and aspect-ratios
275 - [
276 - rescale_image_size(stop_sign, 0.1),
277 - stop_sign,
278 - ],
279 - [
280 - stop_sign,
281 - rescale_image_size(stop_sign, 0.25),
282 - cherry_blossom.resize((183, 488)),
283 - cherry_blossom.resize((488, 183))
284 - ],
285 - cherry_blossom,
286 - ])]
287 -
288 - _run_test(
289 - hf_runner,
290 - vllm_runner,
291 - inputs,
292 - model,
293 - dtype=dtype,
294 - max_tokens=max_tokens,
295 - num_logprobs=num_logprobs,
296 - tensor_parallel_size=1,
297 - )
298 -
299 -
300 -@pytest.mark.parametrize("gridpoints,expected_max_tokens", [
301 - ([[336, 336]], 1176),
302 - ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]], 2928),
303 -])
304 -def test_get_max_llava_next_image_tokens(gridpoints, expected_max_tokens,
305 - get_max_llava_next_image_tokens):
306 - ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
307 -
308 - # Update the config image_grid_pinpoints
309 - # and calculate the resulting max tokens
310 - ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
311 -
312 - actual_max_tokens = get_max_llava_next_image_tokens(
313 - InputContext(ctx.model_config))
314 -
315 - assert expected_max_tokens == actual_max_tokens
316 -
317 -
318 -@pytest.mark.parametrize(
319 - "gridpoints,expected_size",
320 - [
321 - # One point; it has to be the largest
322 - ([[336, 336]], (336, 336)),
323 - # Default for most llava next models; the 2x2 tile is the largest
324 - ([[336, 672], [672, 336], [672, 672], [1008, 336], [336, 1008]],
325 - (672, 672)),
326 - # If two rectangular gridpoints are the same, the more vertical
327 - # one has the higher feature count due to newline features
328 - ([[336, 672], [672, 336]], (672, 336))
329 - ])
330 -def test_dummy_data_for_llava_next_feature_size(dummy_data_for_llava_next,
331 - gridpoints, expected_size):
332 - ctx = build_model_context(model_name="llava-hf/llava-v1.6-mistral-7b-hf")
333 -
334 - # Update the config image_grid_pinpoints
335 - ctx.model_config.hf_config.image_grid_pinpoints = gridpoints
336 - seq_len = 5000 # bigger than the max feature size for any image
337 -
338 - seq_data, mm_data = dummy_data_for_llava_next(
339 - ctx,
340 - seq_len=seq_len,
341 - mm_counts={"image": 1},
342 - )
343 -
344 - # The dummy data dims should match the gridpoint with the biggest feat size
345 - assert mm_data["image"].height == expected_size[0]
346 - assert mm_data["image"].width == expected_size[1]
347 - assert len(seq_data.get_token_ids()) >= seq_len
tests/models/decoder_only/vision_language/test_llava_next_video.py deleted
-226
@@ -1,226 +0,0 @@
1 -from typing import List, Optional, Tuple, Type, overload
2 -
3 -import pytest
4 -from transformers import AutoConfig, AutoModelForVision2Seq, AutoTokenizer
5 -
6 -from vllm.multimodal.utils import (rescale_video_size, resize_video,
7 - sample_frames_from_video)
8 -from vllm.sequence import SampleLogprobs
9 -
10 -from ....conftest import VIDEO_ASSETS, HfRunner, VllmRunner, _VideoAssets
11 -from ...utils import check_logprobs_close
12 -
13 -_PREFACE = (
14 - "A chat between a curious human and an artificial intelligence assistant. "
15 - "The assistant gives helpful, detailed, and polite answers to the human's "
16 - "questions.")
17 -
18 -HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
19 - "sample_demo_1":
20 - f"{_PREFACE}USER: <video>\nWhy is this video funny? ASSISTANT:"
21 -})
22 -
23 -models = ["llava-hf/LLaVA-NeXT-Video-7B-hf"]
24 -
25 -
26 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
27 - Optional[SampleLogprobs]],
28 - model: str):
29 - """Sanitize vllm output to be comparable with hf output."""
30 - output_ids, output_str, out_logprobs = vllm_output
31 -
32 - config = AutoConfig.from_pretrained(model)
33 - video_token_id = config.video_token_index
34 -
35 - tokenizer = AutoTokenizer.from_pretrained(model)
36 - eos_token_id = tokenizer.eos_token_id
37 -
38 - hf_output_ids = [
39 - token_id for idx, token_id in enumerate(output_ids)
40 - if token_id != video_token_id or output_ids[idx - 1] != video_token_id
41 - ]
42 -
43 - assert output_str[0] == " "
44 - hf_output_str = output_str[1:]
45 - if hf_output_ids[-1] == eos_token_id:
46 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
47 -
48 - return hf_output_ids, hf_output_str, out_logprobs
49 -
50 -
51 -@overload
52 -def run_test(
53 - hf_runner: Type[HfRunner],
54 - vllm_runner: Type[VllmRunner],
55 - video_assets: _VideoAssets,
56 - model: str,
57 - *,
58 - size_factors: List[float],
59 - dtype: str,
60 - max_tokens: int,
61 - num_logprobs: int,
62 - num_frames: int,
63 - tensor_parallel_size: int,
64 - distributed_executor_backend: Optional[str] = None,
65 -):
66 - ...
67 -
68 -
69 -@overload
70 -def run_test(
71 - hf_runner: Type[HfRunner],
72 - vllm_runner: Type[VllmRunner],
73 - video_assets: _VideoAssets,
74 - model: str,
75 - *,
76 - sizes: List[Tuple[int, int]],
77 - dtype: str,
78 - max_tokens: int,
79 - num_logprobs: int,
80 - num_frames: int,
81 - tensor_parallel_size: int,
82 - distributed_executor_backend: Optional[str] = None,
83 -):
84 - ...
85 -
86 -
87 -def run_test(
88 - hf_runner: Type[HfRunner],
89 - vllm_runner: Type[VllmRunner],
90 - video_assets: _VideoAssets,
91 - model: str,
92 - *,
93 - size_factors: Optional[List[float]] = None,
94 - sizes: Optional[List[Tuple[int, int]]] = None,
95 - dtype: str,
96 - max_tokens: int,
97 - num_logprobs: int,
98 - num_frames: int,
99 - tensor_parallel_size: int,
100 - distributed_executor_backend: Optional[str] = None,
101 -):
102 - videos = [
103 - sample_frames_from_video(asset.np_ndarrays, num_frames)
104 - for asset in video_assets
105 - ]
106 -
107 - if size_factors is not None:
108 - inputs_per_video = [(
109 - [prompt for _ in size_factors],
110 - [rescale_video_size(video, factor) for factor in size_factors],
111 - ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
112 - elif sizes is not None:
113 - inputs_per_video = [(
114 - [prompt for _ in sizes],
115 - [resize_video(video, size) for size in sizes],
116 - ) for video, prompt in zip(videos, HF_VIDEO_PROMPTS)]
117 - else:
118 - raise ValueError("You must provide either `size_factors` or `sizes`")
119 -
120 - # max_model_len should be greater than image_feature_size
121 - with vllm_runner(model,
122 - dtype=dtype,
123 - max_model_len=4096,
124 - tensor_parallel_size=tensor_parallel_size,
125 - distributed_executor_backend=distributed_executor_backend,
126 - enforce_eager=True) as vllm_model:
127 - vllm_outputs_per_video = [
128 - vllm_model.generate_greedy_logprobs(prompts,
129 - max_tokens,
130 - num_logprobs=num_logprobs,
131 - videos=videos)
132 - for prompts, videos in inputs_per_video
133 - ]
134 -
135 - with hf_runner(model, dtype=dtype,
136 - auto_cls=AutoModelForVision2Seq) as hf_model:
137 - hf_outputs_per_video = [
138 - hf_model.generate_greedy_logprobs_limit(prompts,
139 - max_tokens,
140 - num_logprobs=num_logprobs,
141 - videos=videos)
142 - for prompts, videos in inputs_per_video
143 - ]
144 -
145 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_video,
146 - vllm_outputs_per_video):
147 - # TODO: Check whether using original CLIPVisionModel can improve
148 - # consistency against HF
149 - check_logprobs_close(
150 - outputs_0_lst=hf_outputs,
151 - outputs_1_lst=[
152 - vllm_to_hf_output(vllm_output, model)
153 - for vllm_output in vllm_outputs
154 - ],
155 - name_0="hf",
156 - name_1="vllm",
157 - )
158 -
159 -
160 -@pytest.mark.parametrize("model", models)
161 -@pytest.mark.parametrize(
162 - "size_factors",
163 - [
164 - # No video
165 - [],
166 - # Single-scale
167 - [1.0],
168 - # Single-scale, batched
169 - [1.0, 1.0, 1.0],
170 - # Multi-scale
171 - [0.25, 0.5, 1.0],
172 - ],
173 -)
174 -@pytest.mark.parametrize("dtype", ["half"])
175 -@pytest.mark.parametrize("max_tokens", [128])
176 -@pytest.mark.parametrize("num_logprobs", [5])
177 -@pytest.mark.parametrize("num_frames", [16])
178 -def test_models(hf_runner, vllm_runner, video_assets, model, size_factors,
179 - dtype, max_tokens, num_logprobs, num_frames) -> None:
180 - """Inference result should be the same between hf and vllm.
181 -
182 - All the image fixtures for the test is under tests/videos.
183 - For huggingface runner, we provide the np.ndarray as input.
184 - For vllm runner, we provide MultiModalDataDict objects
185 - and corresponding MultiModalConfig as input.
186 - Note, the text input is also adjusted to abide by vllm contract.
187 - The text output is sanitized to be able to compare with hf.
188 - """
189 - run_test(
190 - hf_runner,
191 - vllm_runner,
192 - video_assets,
193 - model,
194 - size_factors=size_factors,
195 - dtype=dtype,
196 - max_tokens=max_tokens,
197 - num_logprobs=num_logprobs,
198 - num_frames=num_frames,
199 - tensor_parallel_size=1,
200 - )
201 -
202 -
203 -@pytest.mark.parametrize("model", models)
204 -@pytest.mark.parametrize(
205 - "sizes",
206 - [[(1669, 2560), (2560, 1669), (183, 488), (488, 183)]],
207 -)
208 -@pytest.mark.parametrize("dtype", ["half"])
209 -@pytest.mark.parametrize("max_tokens", [128])
210 -@pytest.mark.parametrize("num_logprobs", [5])
211 -@pytest.mark.parametrize("num_frames", [16])
212 -def test_models_fixed_sizes(hf_runner, vllm_runner, video_assets, model, sizes,
213 - dtype, max_tokens, num_logprobs,
214 - num_frames) -> None:
215 - run_test(
216 - hf_runner,
217 - vllm_runner,
218 - video_assets,
219 - model,
220 - sizes=sizes,
221 - dtype=dtype,
222 - max_tokens=max_tokens,
223 - num_logprobs=num_logprobs,
224 - num_frames=num_frames,
225 - tensor_parallel_size=1,
226 - )
tests/models/decoder_only/vision_language/test_llava_onevision.py deleted
-272
@@ -1,272 +0,0 @@
1 -from typing import List, Optional, Tuple, Type
2 -
3 -import pytest
4 -from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
5 - BatchEncoding)
6 -
7 -from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
8 - resize_video, sample_frames_from_video)
9 -from vllm.sequence import SampleLogprobs
10 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
11 -
12 -from ....conftest import (VIDEO_ASSETS, HfRunner, PromptImageInput,
13 - PromptVideoInput, VllmRunner)
14 -from ...utils import check_logprobs_close
15 -
16 -# Video test
17 -HF_VIDEO_PROMPTS = VIDEO_ASSETS.prompts({
18 - "sample_demo_1":
19 - "<|im_start|>user\n<video>\nwhy is this video funny?<|im_end|>\n<|im_start|>assistant\n" # noqa: E501
20 -})
21 -
22 -models = ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"]
23 -
24 -
25 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
26 - Optional[SampleLogprobs]],
27 - model: str):
28 - """Sanitize vllm output to be comparable with hf output."""
29 - output_ids, output_str, out_logprobs = vllm_output
30 -
31 - config = AutoConfig.from_pretrained(model)
32 - video_token_id = config.video_token_index
33 -
34 - tokenizer = AutoTokenizer.from_pretrained(model)
35 - eos_token_id = tokenizer.eos_token_id
36 -
37 - hf_output_ids = [
38 - token_id for idx, token_id in enumerate(output_ids)
39 - if token_id != video_token_id or output_ids[idx - 1] != video_token_id
40 - ]
41 -
42 - hf_output_str = output_str
43 - if hf_output_ids[-1] == eos_token_id:
44 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
45 -
46 - return hf_output_ids, hf_output_str, out_logprobs
47 -
48 -
49 -# Video test
50 -_LIMIT_VIDEO_PER_PROMPT = 4
51 -
52 -
53 -def run_video_test(
54 - hf_runner: Type[HfRunner],
55 - vllm_runner: Type[VllmRunner],
56 - inputs: List[Tuple[List[str], PromptVideoInput]],
57 - model: str,
58 - *,
59 - dtype: str,
60 - max_tokens: int,
61 - num_logprobs: int,
62 - num_frames: int,
63 - tensor_parallel_size: int,
64 - distributed_executor_backend: Optional[str] = None,
65 -):
66 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
67 - with vllm_runner(model,
68 - dtype=dtype,
69 - max_model_len=16384,
70 - tensor_parallel_size=tensor_parallel_size,
71 - distributed_executor_backend=distributed_executor_backend,
72 - enforce_eager=True,
73 - limit_mm_per_prompt={"video": _LIMIT_VIDEO_PER_PROMPT
74 - }) as vllm_model:
75 - vllm_outputs_per_input = [
76 - vllm_model.generate_greedy_logprobs(prompts,
77 - max_tokens,
78 - num_logprobs=num_logprobs,
79 - videos=videos)
80 - for prompts, videos in inputs
81 - ]
82 -
83 - def process(hf_inputs: BatchEncoding):
84 - hf_inputs["pixel_values_videos"] = hf_inputs["pixel_values_videos"] \
85 - .to(torch_dtype) # type: ignore
86 - return hf_inputs
87 -
88 - with hf_runner(model,
89 - dtype=dtype,
90 - postprocess_inputs=process,
91 - auto_cls=AutoModelForVision2Seq) as hf_model:
92 - hf_outputs_per_input = [
93 - hf_model.generate_greedy_logprobs_limit(prompts,
94 - max_tokens,
95 - num_logprobs=num_logprobs,
96 - videos=videos)
97 - for prompts, videos in inputs
98 - ]
99 -
100 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_input,
101 - vllm_outputs_per_input):
102 - # TODO: Check whether using original CLIPVisionModel can improve
103 - # consistency against HF
104 - check_logprobs_close(
105 - outputs_0_lst=hf_outputs,
106 - outputs_1_lst=[
107 - vllm_to_hf_output(vllm_output, model)
108 - for vllm_output in vllm_outputs
109 - ],
110 - name_0="hf",
111 - name_1="vllm",
112 - )
113 -
114 -
115 -@pytest.mark.parametrize("model", models)
116 -@pytest.mark.parametrize("dtype", ["half"])
117 -@pytest.mark.parametrize("max_tokens", [128])
118 -@pytest.mark.parametrize("num_logprobs", [5])
119 -@pytest.mark.parametrize("num_frames", [16])
120 -def test_models_multiple_video_inputs(hf_runner, vllm_runner, video_assets,
121 - model, dtype, max_tokens, num_logprobs,
122 - num_frames) -> None:
123 - video = sample_frames_from_video(video_assets[0].np_ndarrays, num_frames)
124 - inputs = [(
125 - [
126 - "<|im_start|>user <video><video>\nDescribe 2 videos. \
127 - <|im_end|><|im_start|>assistant\n",
128 - "<|im_start|>user <video><video>\nDescribe 2 videos. \
129 - <|im_end|><|im_start|>assistant\n",
130 - "<|im_start|>user <video><video><video><video>\nDescribe 4 videos. \
131 - <|im_end|><|im_start|>assistant\n",
132 - "<|im_start|>user <video>\nwhy is this video funny? \
133 - <|im_end|><|im_start|>assistant\n",
134 - ],
135 - [
136 - [video, video],
137 - # Images with different sizes and aspect-ratios
138 - [
139 - rescale_video_size(video, 0.1),
140 - video,
141 - ],
142 - [
143 - video,
144 - rescale_video_size(video, 0.25),
145 - resize_video(video, (183, 488)),
146 - resize_video(video, (488, 183))
147 - ],
148 - video,
149 - ])]
150 - run_video_test(
151 - hf_runner,
152 - vllm_runner,
153 - inputs,
154 - model,
155 - dtype=dtype,
156 - max_tokens=max_tokens,
157 - num_logprobs=num_logprobs,
158 - tensor_parallel_size=1,
159 - num_frames=num_frames,
160 - )
161 -
162 -
163 -# Image test
164 -_LIMIT_IMAGE_PER_PROMPT = 4
165 -
166 -
167 -def run_image_test(
168 - hf_runner: Type[HfRunner],
169 - vllm_runner: Type[VllmRunner],
170 - inputs: List[Tuple[List[str], PromptImageInput]],
171 - model: str,
172 - dtype: str,
173 - max_tokens: int,
174 - num_logprobs: int,
175 - tensor_parallel_size: int,
176 - distributed_executor_backend: Optional[str] = None,
177 -):
178 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
179 -
180 - # max_model_len should be greater than image_feature_size
181 - with vllm_runner(model,
182 - dtype=dtype,
183 - max_model_len=16384,
184 - max_num_seqs=2,
185 - tensor_parallel_size=tensor_parallel_size,
186 - distributed_executor_backend=distributed_executor_backend,
187 - enforce_eager=True,
188 - limit_mm_per_prompt={"image": _LIMIT_IMAGE_PER_PROMPT
189 - }) as vllm_model:
190 - vllm_outputs_per_image = [
191 - vllm_model.generate_greedy_logprobs(prompts,
192 - max_tokens,
193 - num_logprobs=num_logprobs,
194 - images=images)
195 - for prompts, images in inputs
196 - ]
197 -
198 - def process(hf_inputs: BatchEncoding):
199 - hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
200 - .to(torch_dtype) # type: ignore
201 - return hf_inputs
202 -
203 - with hf_runner(model,
204 - dtype=dtype,
205 - postprocess_inputs=process,
206 - auto_cls=AutoModelForVision2Seq) as hf_model:
207 - hf_outputs_per_image = [
208 - hf_model.generate_greedy_logprobs_limit(prompts,
209 - max_tokens,
210 - num_logprobs=num_logprobs,
211 - images=images)
212 - for prompts, images in inputs
213 - ]
214 -
215 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
216 - vllm_outputs_per_image):
217 - # TODO: Check whether using original CLIPVisionModel can improve
218 - # consistency against HF
219 - check_logprobs_close(
220 - outputs_0_lst=hf_outputs,
221 - outputs_1_lst=[
222 - vllm_to_hf_output(vllm_output, model)
223 - for vllm_output in vllm_outputs
224 - ],
225 - name_0="hf",
226 - name_1="vllm",
227 - )
228 -
229 -
230 -@pytest.mark.parametrize("model", models)
231 -@pytest.mark.parametrize("dtype", ["half"])
232 -@pytest.mark.parametrize("max_tokens", [128])
233 -@pytest.mark.parametrize("num_logprobs", [5])
234 -def test_models_multiple_image_inputs(hf_runner, vllm_runner, image_assets,
235 - model, dtype, max_tokens,
236 - num_logprobs) -> None:
237 - stop_sign = image_assets[0].pil_image
238 - cherry_blossom = image_assets[1].pil_image
239 -
240 - inputs = [(
241 - [
242 - "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
243 - "<|im_start|>user\n<image><image>\nDescribe 2 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
244 - "<|im_start|>user\n<image><image><image><image>\nDescribe 4 images.<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
245 - "<|im_start|>user\n<image>\nWhat is the season?<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
246 - ],
247 - [
248 - [stop_sign, cherry_blossom],
249 - # Images with different sizes and aspect-ratios
250 - [
251 - rescale_image_size(stop_sign, 0.1),
252 - stop_sign,
253 - ],
254 - [
255 - stop_sign,
256 - rescale_image_size(stop_sign, 0.25),
257 - cherry_blossom.resize((183, 488)),
258 - cherry_blossom.resize((488, 183))
259 - ],
260 - cherry_blossom,
261 - ])]
262 -
263 - run_image_test(
264 - hf_runner,
265 - vllm_runner,
266 - inputs,
267 - model,
268 - dtype=dtype,
269 - max_tokens=max_tokens,
270 - num_logprobs=num_logprobs,
271 - tensor_parallel_size=1,
272 - )
tests/models/decoder_only/vision_language/test_minicpmv.py deleted
-199
@@ -1,199 +0,0 @@
1 -from typing import List, Optional, Tuple, Type, Union
2 -
3 -import pytest
4 -import torch
5 -import torch.types
6 -from PIL import Image
7 -from transformers import BatchEncoding
8 -
9 -from vllm.multimodal.utils import rescale_image_size
10 -from vllm.sequence import SampleLogprobs
11 -
12 -from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner
13 -from ...utils import check_logprobs_close
14 -
15 -# The image token is placed before "user" on purpose so that the test can pass
16 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
17 - "stop_sign":
18 - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
19 - "(<image>./</image>)\nWhat's the content of the image?<|eot_id|>" \
20 - "<|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
21 - "cherry_blossom":
22 - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
23 - "(<image>./</image>)\nWhat is the season?<|eot_id|>" \
24 - "<|start_header_id|>assistant<|end_header_id|>\n\n",
25 -})
26 -HF_MULTIIMAGE_IMAGE_PROMPT = \
27 - "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
28 - "(<image>./</image>)\n(<image>./</image>)\n" \
29 - "Describe these images.<|eot_id|>" \
30 - "<|start_header_id|>assistant<|end_header_id|>\n\n"
31 -
32 -models = ["openbmb/MiniCPM-Llama3-V-2_5"]
33 -
34 -
35 -def _wrap_inputs(hf_inputs: BatchEncoding):
36 - return {"model_inputs": hf_inputs}
37 -
38 -
39 -def trunc_hf_output(hf_output: Tuple[List[int], str,
40 - Optional[SampleLogprobs]]):
41 - output_ids, output_str, out_logprobs = hf_output
42 - if output_str.endswith("<|eot_id|>"):
43 - output_str = output_str.split("<|eot_id|>")[0]
44 - return output_ids, output_str, out_logprobs
45 -
46 -
47 -target_dtype = "half"
48 -
49 -
50 -def run_test(
51 - hf_runner: Type[HfRunner],
52 - vllm_runner: Type[VllmRunner],
53 - inputs: List[Tuple[List[str], Union[List[Image.Image],
54 - List[List[Image.Image]]]]],
55 - model: str,
56 - *,
57 - dtype: str,
58 - max_tokens: int,
59 - num_logprobs: int,
60 - mm_limit: int,
61 - tensor_parallel_size: int,
62 - distributed_executor_backend: Optional[str] = None,
63 -):
64 - """Inference result should be the same between hf and vllm.
65 -
66 - All the image fixtures for the test are from IMAGE_ASSETS.
67 - For huggingface runner, we provide the PIL images as input.
68 - For vllm runner, we provide MultiModalDataDict objects
69 - and corresponding MultiModalConfig as input.
70 - Note, the text input is also adjusted to abide by vllm contract.
71 - The text output is sanitized to be able to compare with hf.
72 - """
73 -
74 - # NOTE: take care of the order. run vLLM first, and then run HF.
75 - # vLLM needs a fresh new process without cuda initialization.
76 - # if we run HF first, the cuda initialization will be done and it
77 - # will hurt multiprocessing backend with fork method (the default method).
78 -
79 - # max_model_len should be greater than image_feature_size
80 - with vllm_runner(model,
81 - max_model_len=4096,
82 - max_num_seqs=2,
83 - dtype=dtype,
84 - limit_mm_per_prompt={"image": mm_limit},
85 - tensor_parallel_size=tensor_parallel_size,
86 - distributed_executor_backend=distributed_executor_backend,
87 - enforce_eager=True) as vllm_model:
88 - tokenizer = vllm_model.model.get_tokenizer()
89 - stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
90 - vllm_outputs_per_image = [
91 - vllm_model.generate_greedy_logprobs(prompts,
92 - max_tokens,
93 - num_logprobs=num_logprobs,
94 - images=images,
95 - stop_token_ids=stop_token_ids)
96 - for prompts, images in inputs
97 - ]
98 -
99 - hf_model = hf_runner(model, dtype=dtype, postprocess_inputs=_wrap_inputs)
100 - with hf_model, torch.no_grad():
101 - hf_outputs_per_image = [
102 - hf_model.generate_greedy_logprobs_limit(prompts,
103 - max_tokens,
104 - num_logprobs=num_logprobs,
105 - images=images,
106 - tokenizer=tokenizer)
107 - for prompts, images in inputs
108 - ]
109 -
110 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
111 - vllm_outputs_per_image):
112 - check_logprobs_close(
113 - outputs_0_lst=[
114 - trunc_hf_output(hf_output) for hf_output in hf_outputs
115 - ],
116 - outputs_1_lst=vllm_outputs,
117 - name_0="hf",
118 - name_1="vllm",
119 - )
120 -
121 -
122 -@pytest.mark.parametrize("model", models)
123 -@pytest.mark.parametrize(
124 - "size_factors",
125 - [
126 - # No image
127 - [],
128 - # Single-scale
129 - [1.0],
130 - # Single-scale, batched
131 - [1.0, 1.0, 1.0],
132 - # Multi-scale
133 - [0.25, 0.5, 1.0],
134 - ],
135 -)
136 -@pytest.mark.parametrize("dtype", [target_dtype])
137 -@pytest.mark.parametrize("max_tokens", [128])
138 -@pytest.mark.parametrize("num_logprobs", [5])
139 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
140 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
141 - images = [asset.pil_image for asset in image_assets]
142 -
143 - inputs_per_image = [(
144 - [prompt for _ in size_factors],
145 - [rescale_image_size(image, factor) for factor in size_factors],
146 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
147 -
148 - run_test(
149 - hf_runner,
150 - vllm_runner,
151 - inputs_per_image,
152 - model,
153 - dtype=dtype,
154 - max_tokens=max_tokens,
155 - num_logprobs=num_logprobs,
156 - mm_limit=1,
157 - tensor_parallel_size=1,
158 - )
159 -
160 -
161 -@pytest.mark.parametrize("model", models)
162 -@pytest.mark.parametrize(
163 - "size_factors",
164 - [
165 - # No image
166 - [],
167 - # Single-scale
168 - [1.0],
169 - # Single-scale, batched
170 - [1.0, 1.0, 1.0],
171 - # Multi-scale
172 - [0.25, 0.5, 1.0],
173 - ],
174 -)
175 -@pytest.mark.parametrize("dtype", [target_dtype])
176 -@pytest.mark.parametrize("max_tokens", [128])
177 -@pytest.mark.parametrize("num_logprobs", [5])
178 -def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
179 - size_factors, dtype: str, max_tokens: int,
180 - num_logprobs: int) -> None:
181 - images = [asset.pil_image for asset in image_assets]
182 -
183 - inputs_per_case = [
184 - ([HF_MULTIIMAGE_IMAGE_PROMPT for _ in size_factors],
185 - [[rescale_image_size(image, factor) for image in images]
186 - for factor in size_factors])
187 - ]
188 -
189 - run_test(
190 - hf_runner,
191 - vllm_runner,
192 - inputs_per_case,
193 - model,
194 - dtype=dtype,
195 - max_tokens=max_tokens,
196 - num_logprobs=num_logprobs,
197 - mm_limit=2,
198 - tensor_parallel_size=1,
199 - )
tests/models/decoder_only/vision_language/test_models.py new
+594
@@ -0,0 +1,594 @@
1 +"""Common tests for testing .generate() functionality for single / multiple
2 +image, embedding, and video support for different VLMs in vLLM.
3 +"""
4 +import os
5 +from pathlib import PosixPath
6 +from typing import Type
7 +
8 +import pytest
9 +import transformers
10 +from transformers import AutoModelForVision2Seq
11 +
12 +from vllm.platforms import current_platform
13 +from vllm.utils import cuda_device_count_stateless, identity
14 +
15 +from ....conftest import (IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets,
16 + _VideoAssets)
17 +from ....utils import fork_new_process_for_each_test, large_gpu_mark
18 +from ...utils import check_outputs_equal
19 +from .vlm_utils import custom_inputs, model_utils, runners
20 +from .vlm_utils.case_filtering import get_parametrized_options
21 +from .vlm_utils.types import (CustomTestOptions, ExpandableVLMTestArgs,
22 + VLMTestInfo, VLMTestType)
23 +
24 +# This hack is needed for phi3v & paligemma models
25 +# ROCm Triton FA can run into shared memory issues with these models,
26 +# use other backends in the meantime
27 +# FIXME (mattwong, gshtrasb, hongxiayan)
28 +if current_platform.is_rocm():
29 + os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
30 +
31 +# yapf: disable
32 +COMMON_BROADCAST_SETTINGS = {
33 + "test_type": VLMTestType.IMAGE,
34 + "dtype": "half",
35 + "max_tokens": 5,
36 + "tensor_parallel_size": 2,
37 + "image_size_factors": [(.25, 0.5, 1.0)],
38 + "distributed_executor_backend": (
39 + "ray",
40 + "mp",
41 + )
42 +}
43 +
44 +### Test configuration for specific models
45 +# NOTE: The convention of the test settings below is to lead each test key
46 +# with the name of the model arch used in the test, using underscores in place
47 +# of hyphens; this makes it more convenient to filter tests for a specific kind
48 +# of model. For example....
49 +#
50 +# To run all test types for a specific key:
51 +# use the k flag to substring match with a leading square bracket; if the
52 +# model arch happens to be a substring of another one, you can add a
53 +# trailing hyphen. E.g.,
54 +# - pytest $TEST_FILE -k "[llava-"
55 +# prevents matching on "[llava_next-" & will match just the enabled cases
56 +# for llava, i.e., single image, image embedding, and custom input tests.
57 +#
58 +# To run a test for a Test Info for just one of multiple models:
59 +# use the k flag to substring match the model name, e.g.,
60 +# - pytest $TEST_FILE -k OpenGVLab/InternVL2-1B
61 +# prevents matching on nGVLab/InternVL2-2B.
62 +#
63 +# You can also combine substrings to match more granularly.
64 +# ex 1:
65 +# pytest $TEST_FILE -k "test_single_image and OpenGVLab/InternVL2-1B"
66 +# will run only test_single_image* for OpenGVLab/InternVL2-1B; this would
67 +# match both wrappers for single image tests, since it also matches
68 +# test_single_image_heavy (which forks if we have a distributed backend)
69 +# ex 2:
70 +# pytest $TEST_FILE -k "[llava- or [intern_vl-"
71 +# will run all of the tests for only llava & internvl.
72 +#
73 +# NOTE you can add --collect-only to any of the above commands to see
74 +# which cases would be selected and deselected by pytest. In general,
75 +# this is a good idea for checking your command first, since tests are slow.
76 +
77 +VLM_TEST_SETTINGS = {
78 + "blip2": VLMTestInfo(
79 + models=["Salesforce/blip2-opt-2.7b"],
80 + test_type=VLMTestType.IMAGE,
81 + prompt_formatter=lambda img_prompt: f"Question: {img_prompt} Answer:",
82 + img_idx_to_prompt=lambda idx: "",
83 + auto_cls=AutoModelForVision2Seq,
84 + vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
85 + ),
86 + "chameleon": VLMTestInfo(
87 + models=["facebook/chameleon-7b"],
88 + test_type=VLMTestType.IMAGE,
89 + prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
90 + max_model_len=4096,
91 + auto_cls=AutoModelForVision2Seq,
92 + postprocess_inputs=model_utils.get_key_type_post_processor(
93 + "pixel_values"
94 + ),
95 + # For chameleon, we only compare the sequences
96 + vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
97 + hf_output_post_proc = lambda hf_output, model: hf_output[:2],
98 + comparator=check_outputs_equal,
99 + max_tokens=8,
100 + dtype="bfloat16",
101 + marks=[
102 + pytest.mark.skipif(
103 + transformers.__version__.startswith("4.46"),
104 + reason="Model broken in HF, see huggingface/transformers#34379"
105 + )
106 + ]
107 + ),
108 + "fuyu": VLMTestInfo(
109 + models=["adept/fuyu-8b"],
110 + test_type=VLMTestType.IMAGE,
111 + prompt_formatter=lambda img_prompt: f"{img_prompt}\n",
112 + img_idx_to_prompt=lambda idx: "",
113 + max_model_len=2048,
114 + max_num_seqs=2,
115 + use_tokenizer_eos=True,
116 + vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
117 + num_logprobs=10,
118 + dtype="bfloat16" if current_platform.is_cpu() else "half",
119 + image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
120 + ),
121 + "glm4": VLMTestInfo(
122 + models=["THUDM/glm-4v-9b"],
123 + test_type=VLMTestType.IMAGE,
124 + prompt_formatter=identity,
125 + img_idx_to_prompt=lambda idx: "",
126 + max_model_len=2048,
127 + max_num_seqs=2,
128 + dtype="bfloat16",
129 + get_stop_token_ids=lambda tok: [151329, 151336, 151338],
130 + marks=[large_gpu_mark(min_gb=48)],
131 + patch_hf_runner=model_utils.glm_patch_hf_runner,
132 + ),
133 + "intern_vl": VLMTestInfo(
134 + models=[
135 + "OpenGVLab/InternVL2-1B",
136 + "OpenGVLab/InternVL2-2B",
137 + "OpenGVLab/Mono-InternVL-2B",
138 + ],
139 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
140 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
141 + single_image_prompts=IMAGE_ASSETS.prompts({
142 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
143 + "cherry_blossom": "<image>\nWhat is the season?",
144 + }),
145 + multi_image_prompt="Image-1: <image>\nImage-2: <image>\nDescribe the two images in short.", # noqa: E501
146 + max_model_len=4096,
147 + # NOTE: Mono-InternVL-2B doesn't work with fp16,
148 + # it will result NaN during inference.
149 + # See: https://huggingface.co/OpenGVLab/Mono-InternVL-2B/discussions/9
150 + dtype="bfloat16",
151 + use_tokenizer_eos=True,
152 + patch_hf_runner=model_utils.internvl_patch_hf_runner,
153 + ),
154 + "llava": VLMTestInfo(
155 + models=["llava-hf/llava-1.5-7b-hf"],
156 + test_type=(
157 + VLMTestType.EMBEDDING,
158 + VLMTestType.IMAGE,
159 + VLMTestType.CUSTOM_INPUTS
160 + ),
161 + prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
162 + convert_assets_to_embeddings=model_utils.get_llava_embeddings,
163 + max_model_len=4096,
164 + auto_cls=AutoModelForVision2Seq,
165 + vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
166 + custom_test_opts=[CustomTestOptions(
167 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
168 + formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
169 + ),
170 + limit_mm_per_prompt={"image": 4},
171 + )],
172 + ),
173 + "llava_next": VLMTestInfo(
174 + models=["llava-hf/llava-v1.6-mistral-7b-hf"],
175 + test_type=(VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
176 + prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
177 + max_model_len=10240,
178 + auto_cls=AutoModelForVision2Seq,
179 + vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
180 + custom_test_opts=[CustomTestOptions(
181 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
182 + formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]"
183 + ),
184 + limit_mm_per_prompt={"image": 4},
185 + )],
186 + # Llava-next tests fixed sizes & the default size factors
187 + image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
188 + ),
189 + "llava_one_vision": VLMTestInfo(
190 + models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
191 + test_type=VLMTestType.CUSTOM_INPUTS,
192 + prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
193 + dtype="half",
194 + num_video_frames=16,
195 + max_model_len=16384,
196 + postprocess_inputs=model_utils.get_key_type_post_processor(
197 + "pixel_values_videos"
198 + ),
199 + auto_cls=AutoModelForVision2Seq,
200 + vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
201 + # Llava-one-vision tests fixed sizes & the default size factors
202 + image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
203 + runner_mm_key="videos",
204 + custom_test_opts=[CustomTestOptions(
205 + inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
206 + formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
207 + ),
208 + limit_mm_per_prompt={"video": 4},
209 + )],
210 + ),
211 + # FIXME
212 + "llava_next_video": VLMTestInfo(
213 + models=["llava-hf/LLaVA-NeXT-Video-7B-hf"],
214 + test_type=VLMTestType.VIDEO,
215 + prompt_formatter=lambda vid_prompt: f"USER: {vid_prompt} ASSISTANT:",
216 + num_video_frames=16,
217 + max_model_len=4096,
218 + auto_cls=AutoModelForVision2Seq,
219 + vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
220 + image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
221 + runner_mm_key="videos",
222 + marks=[
223 + pytest.mark.skip(reason="LLava next video tests currently fail.")
224 + ],
225 + ),
226 + "minicpmv": VLMTestInfo(
227 + models=["openbmb/MiniCPM-Llama3-V-2_5"],
228 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
229 + prompt_formatter=lambda img_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{img_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
230 + img_idx_to_prompt=lambda idx: "(<image>./</image>)\n",
231 + max_model_len=4096,
232 + max_num_seqs=2,
233 + get_stop_token_ids=lambda tok: [tok.eos_id, tok.eot_id],
234 + postprocess_inputs=model_utils.wrap_inputs_post_processor,
235 + hf_output_post_proc=model_utils.minicmpv_trunc_hf_output,
236 + ),
237 + "paligemma": VLMTestInfo(
238 + models=["google/paligemma-3b-mix-224"],
239 + test_type=VLMTestType.IMAGE,
240 + prompt_formatter=identity,
241 + img_idx_to_prompt = lambda idx: "",
242 + # Paligemma uses its own sample prompts because the default one fails
243 + single_image_prompts=IMAGE_ASSETS.prompts({
244 + "stop_sign": "caption es",
245 + "cherry_blossom": "What is in the picture?",
246 + }),
247 + auto_cls=AutoModelForVision2Seq,
248 + postprocess_inputs=model_utils.get_key_type_post_processor(
249 + "pixel_values"
250 + ),
251 + vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
252 + dtype="half" if current_platform.is_rocm() else ("half", "float"),
253 + ),
254 + # Tests for phi3v currently live in another file because of a bug in
255 + # transformers. Once this issue is fixed, we can enable them here instead.
256 + # https://github.com/huggingface/transformers/issues/34307
257 + # "phi3v": VLMTestInfo(
258 + # models=["microsoft/Phi-3.5-vision-instruct"],
259 + # test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
260 + # prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|end|>\n<|assistant|>\n", # noqa: E501
261 + # img_idx_to_prompt=lambda idx: f"<|image_{idx}|>\n",
262 + # max_model_len=4096,
263 + # max_num_seqs=2,
264 + # task="generate",
265 + # # use eager mode for hf runner since phi3v didn't work with flash_attn
266 + # model_kwargs={"_attn_implementation": "eager"},
267 + # use_tokenizer_eos=True,
268 + # vllm_output_post_proc=model_utils.phi3v_vllm_to_hf_output,
269 + # num_logprobs=10,
270 + # ),
271 + "qwen": VLMTestInfo(
272 + models=["Qwen/Qwen-VL"],
273 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
274 + prompt_formatter=identity,
275 + img_idx_to_prompt=lambda idx: f"Picture {idx}: <img></img>\n",
276 + max_model_len=1024,
277 + max_num_seqs=2,
278 + vllm_output_post_proc=model_utils.qwen_vllm_to_hf_output,
279 + prompt_path_encoder=model_utils.qwen_prompt_path_encoder,
280 + ),
281 + ### Tensor parallel / multi-gpu broadcast tests
282 + "broadcast-chameleon": VLMTestInfo(
283 + models=["facebook/chameleon-7b"],
284 + prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
285 + max_model_len=4096,
286 + auto_cls=AutoModelForVision2Seq,
287 + postprocess_inputs=model_utils.get_key_type_post_processor(
288 + "pixel_values"
289 + ),
290 + vllm_output_post_proc = lambda vllm_output, model: vllm_output[:2],
291 + hf_output_post_proc = lambda hf_output, model: hf_output[:2],
292 + comparator=check_outputs_equal,
293 + marks=[
294 + pytest.mark.distributed_2_gpus,
295 + pytest.mark.skipif(
296 + cuda_device_count_stateless() < 2,
297 + reason="Need at least 2 GPUs to run the test.",
298 + ),
299 + pytest.mark.skipif(
300 + transformers.__version__.startswith("4.46"),
301 + reason="Model broken in HF, see huggingface/transformers#34379"
302 + )
303 + ],
304 + **COMMON_BROADCAST_SETTINGS # type: ignore
305 + ),
306 + "broadcast-llava": VLMTestInfo(
307 + models=["llava-hf/llava-1.5-7b-hf"],
308 + prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
309 + max_model_len=4096,
310 + auto_cls=AutoModelForVision2Seq,
311 + vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
312 + marks=[
313 + pytest.mark.distributed_2_gpus,
314 + pytest.mark.skipif(
315 + cuda_device_count_stateless() < 2,
316 + reason="Need at least 2 GPUs to run the test.",
317 + )
318 + ],
319 + **COMMON_BROADCAST_SETTINGS # type: ignore
320 + ),
321 + "broadcast-llava_next": VLMTestInfo(
322 + models=["llava-hf/llava-v1.6-mistral-7b-hf"],
323 + prompt_formatter=lambda img_prompt: f"[INST] {img_prompt} [/INST]",
324 + max_model_len=10240,
325 + auto_cls=AutoModelForVision2Seq,
326 + vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
327 + marks=[
328 + pytest.mark.distributed_2_gpus,
329 + pytest.mark.skipif(
330 + cuda_device_count_stateless() < 2,
331 + reason="Need at least 2 GPUs to run the test.",
332 + )
333 + ],
334 + **COMMON_BROADCAST_SETTINGS # type: ignore
335 + ),
336 + ### Custom input edge-cases for specific models
337 + "intern_vl-diff-patches": VLMTestInfo(
338 + models=["OpenGVLab/InternVL2-2B"],
339 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
340 + test_type=VLMTestType.CUSTOM_INPUTS,
341 + max_model_len=4096,
342 + dtype="bfloat16" if current_platform.is_cpu() else "half",
343 + use_tokenizer_eos=True,
344 + patch_hf_runner=model_utils.internvl_patch_hf_runner,
345 + custom_test_opts=[
346 + CustomTestOptions(
347 + inputs=inp,
348 + limit_mm_per_prompt={"image": 2},
349 + ) for inp in custom_inputs.different_patch_input_cases_internvl()
350 + ],
351 + ),
352 + "llava_one_vision-multiple-images": VLMTestInfo(
353 + models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
354 + test_type=VLMTestType.CUSTOM_INPUTS,
355 + max_model_len=16384,
356 + max_num_seqs=2,
357 + dtype="half",
358 + postprocess_inputs=model_utils.get_key_type_post_processor(
359 + "pixel_values"
360 + ),
361 + auto_cls=AutoModelForVision2Seq,
362 + vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
363 + custom_test_opts=[CustomTestOptions(
364 + inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
365 + formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
366 + ),
367 + limit_mm_per_prompt={"image": 4},
368 + )],
369 + ),
370 +}
371 +# yapf: enable
372 +
373 +
374 +### Test wrappers
375 +# Wrappers around the core test running func for:
376 +# - single image
377 +# - multi-image
378 +# - image embeddings
379 +# - video
380 +# - custom inputs
381 +@pytest.mark.parametrize("model_type,test_case",
382 + get_parametrized_options(
383 + VLM_TEST_SETTINGS,
384 + test_type=VLMTestType.IMAGE,
385 + fork_new_process_for_each_test=False,
386 + ))
387 +def test_single_image_models(tmp_path: PosixPath, model_type: str,
388 + test_case: ExpandableVLMTestArgs,
389 + hf_runner: Type[HfRunner],
390 + vllm_runner: Type[VllmRunner],
391 + image_assets: _ImageAssets):
392 + model_test_info = VLM_TEST_SETTINGS[model_type]
393 + runners.run_single_image_test(
394 + tmp_path=tmp_path,
395 + model_test_info=model_test_info,
396 + test_case=test_case,
397 + hf_runner=hf_runner,
398 + vllm_runner=vllm_runner,
399 + image_assets=image_assets,
400 + )
401 +
402 +
403 +@pytest.mark.parametrize("model_type,test_case",
404 + get_parametrized_options(
405 + VLM_TEST_SETTINGS,
406 + test_type=VLMTestType.MULTI_IMAGE,
407 + fork_new_process_for_each_test=False,
408 + ))
409 +def test_multi_image_models(tmp_path: PosixPath, model_type: str,
410 + test_case: ExpandableVLMTestArgs,
411 + hf_runner: Type[HfRunner],
412 + vllm_runner: Type[VllmRunner],
413 + image_assets: _ImageAssets):
414 + model_test_info = VLM_TEST_SETTINGS[model_type]
415 + runners.run_multi_image_test(
416 + tmp_path=tmp_path,
417 + model_test_info=model_test_info,
418 + test_case=test_case,
419 + hf_runner=hf_runner,
420 + vllm_runner=vllm_runner,
421 + image_assets=image_assets,
422 + )
423 +
424 +
425 +@pytest.mark.parametrize("model_type,test_case",
426 + get_parametrized_options(
427 + VLM_TEST_SETTINGS,
428 + test_type=VLMTestType.EMBEDDING,
429 + fork_new_process_for_each_test=False,
430 + ))
431 +def test_image_embedding_models(model_type: str,
432 + test_case: ExpandableVLMTestArgs,
433 + hf_runner: Type[HfRunner],
434 + vllm_runner: Type[VllmRunner],
435 + image_assets: _ImageAssets):
436 + model_test_info = VLM_TEST_SETTINGS[model_type]
437 + runners.run_embedding_test(
438 + model_test_info=model_test_info,
439 + test_case=test_case,
440 + hf_runner=hf_runner,
441 + vllm_runner=vllm_runner,
442 + image_assets=image_assets,
443 + )
444 +
445 +
446 +@pytest.mark.parametrize("model_type,test_case",
447 + get_parametrized_options(
448 + VLM_TEST_SETTINGS,
449 + test_type=VLMTestType.VIDEO,
450 + fork_new_process_for_each_test=False,
451 + ))
452 +def test_video_models(model_type: str, test_case: ExpandableVLMTestArgs,
453 + hf_runner: Type[HfRunner], vllm_runner: Type[VllmRunner],
454 + video_assets: _VideoAssets):
455 + model_test_info = VLM_TEST_SETTINGS[model_type]
456 + runners.run_video_test(
457 + model_test_info=model_test_info,
458 + test_case=test_case,
459 + hf_runner=hf_runner,
460 + vllm_runner=vllm_runner,
461 + video_assets=video_assets,
462 + )
463 +
464 +
465 +@pytest.mark.parametrize("model_type,test_case",
466 + get_parametrized_options(
467 + VLM_TEST_SETTINGS,
468 + test_type=VLMTestType.CUSTOM_INPUTS,
469 + fork_new_process_for_each_test=False,
470 + ))
471 +def test_custom_inputs_models(
472 + model_type: str,
473 + test_case: ExpandableVLMTestArgs,
474 + hf_runner: Type[HfRunner],
475 + vllm_runner: Type[VllmRunner],
476 +):
477 + model_test_info = VLM_TEST_SETTINGS[model_type]
478 + runners.run_custom_inputs_test(
479 + model_test_info=model_test_info,
480 + test_case=test_case,
481 + hf_runner=hf_runner,
482 + vllm_runner=vllm_runner,
483 + )
484 +
485 +
486 +#### Tests filtering for things running each test as a new process
487 +@pytest.mark.parametrize("model_type,test_case",
488 + get_parametrized_options(
489 + VLM_TEST_SETTINGS,
490 + test_type=VLMTestType.IMAGE,
491 + fork_new_process_for_each_test=True,
492 + ))
493 +@fork_new_process_for_each_test
494 +def test_single_image_models_heavy(tmp_path: PosixPath, model_type: str,
495 + test_case: ExpandableVLMTestArgs,
496 + hf_runner: Type[HfRunner],
497 + vllm_runner: Type[VllmRunner],
498 + image_assets: _ImageAssets):
499 + model_test_info = VLM_TEST_SETTINGS[model_type]
500 + runners.run_single_image_test(
501 + tmp_path=tmp_path,
502 + model_test_info=model_test_info,
503 + test_case=test_case,
504 + hf_runner=hf_runner,
505 + vllm_runner=vllm_runner,
506 + image_assets=image_assets,
507 + )
508 +
509 +
510 +@pytest.mark.parametrize("model_type,test_case",
511 + get_parametrized_options(
512 + VLM_TEST_SETTINGS,
513 + test_type=VLMTestType.MULTI_IMAGE,
514 + fork_new_process_for_each_test=True,
515 + ))
516 +@fork_new_process_for_each_test
517 +def test_multi_image_models_heavy(tmp_path: PosixPath, model_type: str,
518 + test_case: ExpandableVLMTestArgs,
519 + hf_runner: Type[HfRunner],
520 + vllm_runner: Type[VllmRunner],
521 + image_assets: _ImageAssets):
522 + model_test_info = VLM_TEST_SETTINGS[model_type]
523 + runners.run_multi_image_test(
524 + tmp_path=tmp_path,
525 + model_test_info=model_test_info,
526 + test_case=test_case,
527 + hf_runner=hf_runner,
528 + vllm_runner=vllm_runner,
529 + image_assets=image_assets,
530 + )
531 +
532 +
533 +@pytest.mark.parametrize("model_type,test_case",
534 + get_parametrized_options(
535 + VLM_TEST_SETTINGS,
536 + test_type=VLMTestType.EMBEDDING,
537 + fork_new_process_for_each_test=True,
538 + ))
539 +@fork_new_process_for_each_test
540 +def test_image_embedding_models_heavy(model_type: str,
541 + test_case: ExpandableVLMTestArgs,
542 + hf_runner: Type[HfRunner],
543 + vllm_runner: Type[VllmRunner],
544 + image_assets: _ImageAssets):
545 + model_test_info = VLM_TEST_SETTINGS[model_type]
546 + runners.run_embedding_test(
547 + model_test_info=model_test_info,
548 + test_case=test_case,
549 + hf_runner=hf_runner,
550 + vllm_runner=vllm_runner,
551 + image_assets=image_assets,
552 + )
553 +
554 +
555 +@pytest.mark.parametrize("model_type,test_case",
556 + get_parametrized_options(
557 + VLM_TEST_SETTINGS,
558 + test_type=VLMTestType.VIDEO,
559 + fork_new_process_for_each_test=True,
560 + ))
561 +def test_video_models_heavy(model_type: str, test_case: ExpandableVLMTestArgs,
562 + hf_runner: Type[HfRunner],
563 + vllm_runner: Type[VllmRunner],
564 + video_assets: _VideoAssets):
565 + model_test_info = VLM_TEST_SETTINGS[model_type]
566 + runners.run_video_test(
567 + model_test_info=model_test_info,
568 + test_case=test_case,
569 + hf_runner=hf_runner,
570 + vllm_runner=vllm_runner,
571 + video_assets=video_assets,
572 + )
573 +
574 +
575 +@pytest.mark.parametrize("model_type,test_case",
576 + get_parametrized_options(
577 + VLM_TEST_SETTINGS,
578 + test_type=VLMTestType.CUSTOM_INPUTS,
579 + fork_new_process_for_each_test=True,
580 + ))
581 +@fork_new_process_for_each_test
582 +def test_custom_inputs_models_heavy(
583 + model_type: str,
584 + test_case: ExpandableVLMTestArgs,
585 + hf_runner: Type[HfRunner],
586 + vllm_runner: Type[VllmRunner],
587 +):
588 + model_test_info = VLM_TEST_SETTINGS[model_type]
589 + runners.run_custom_inputs_test(
590 + model_test_info=model_test_info,
591 + test_case=test_case,
592 + hf_runner=hf_runner,
593 + vllm_runner=vllm_runner,
594 + )
tests/models/decoder_only/vision_language/test_paligemma.py deleted
-174
@@ -1,174 +0,0 @@
1 -import os
2 -from typing import List, Optional, Tuple, Type
3 -
4 -import pytest
5 -from transformers import (AutoConfig, AutoModelForVision2Seq, AutoTokenizer,
6 - BatchEncoding)
7 -
8 -from vllm.multimodal.utils import rescale_image_size
9 -from vllm.platforms import current_platform
10 -from vllm.sequence import SampleLogprobs
11 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
12 -
13 -from ....conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
14 -from ...utils import check_logprobs_close
15 -
16 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
17 - "stop_sign":
18 - "caption es",
19 - "cherry_blossom":
20 - "What is in the picture?",
21 -})
22 -
23 -models = ["google/paligemma-3b-mix-224"]
24 -
25 -# ROCm Triton FA can run into compilation issues with these models due to,
26 -# excessive use of shared memory. Use other backends in the meantime.
27 -# FIXME (mattwong, gshtrasb, hongxiayan)
28 -if current_platform.is_rocm():
29 - os.environ["VLLM_USE_TRITON_FLASH_ATTN"] = "0"
30 -
31 -
32 -def vllm_to_hf_output(vllm_output: Tuple[List[int], str,
33 - Optional[SampleLogprobs]],
34 - model: str):
35 - """Sanitize vllm output to be comparable with hf output."""
36 - output_ids, output_str, out_logprobs = vllm_output
37 -
38 - config = AutoConfig.from_pretrained(model)
39 - image_token_id = config.image_token_index
40 -
41 - tokenizer = AutoTokenizer.from_pretrained(model)
42 - eos_token_id = tokenizer.eos_token_id
43 -
44 - hf_output_ids = [
45 - token_id for idx, token_id in enumerate(output_ids)
46 - if token_id != image_token_id or output_ids[idx - 1] != image_token_id
47 - ]
48 -
49 - hf_output_str = output_str
50 -
51 - if hf_output_ids[-1] == eos_token_id:
52 - hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
53 -
54 - return hf_output_ids, hf_output_str, out_logprobs
55 -
56 -
57 -def run_test(
58 - hf_runner: Type[HfRunner],
59 - vllm_runner: Type[VllmRunner],
60 - image_assets: _ImageAssets,
61 - model: str,
62 - *,
63 - size_factors: List[float],
64 - dtype: str,
65 - max_tokens: int,
66 - num_logprobs: int,
67 - tensor_parallel_size: int,
68 - distributed_executor_backend: Optional[str] = None,
69 -):
70 - """Inference result should be the same between hf and vllm.
71 -
72 - All the image fixtures for the test are from IMAGE_ASSETS.
73 - For huggingface runner, we provide the PIL images as input.
74 - For vllm runner, we provide MultiModalDataDict objects
75 - and corresponding MultiModalConfig as input.
76 - Note, the text input is also adjusted to abide by vllm contract.
77 - The text output is sanitized to be able to compare with hf.
78 - """
79 - torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
80 - images = [asset.pil_image for asset in image_assets]
81 -
82 - inputs_per_image = [(
83 - [prompt for _ in size_factors],
84 - [rescale_image_size(image, factor) for factor in size_factors],
85 - ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
86 -
87 - # NOTE: take care of the order. run vLLM first, and then run HF.
88 - # vLLM needs a fresh new process without cuda initialization.
89 - # if we run HF first, the cuda initialization will be done and it
90 - # will hurt multiprocessing backend with fork method (the default method).
91 -
92 - # max_model_len should be greater than image_feature_size
93 - with vllm_runner(model,
94 - dtype=dtype,
95 - tensor_parallel_size=tensor_parallel_size,
96 - distributed_executor_backend=distributed_executor_backend,
97 - enforce_eager=True) as vllm_model:
98 - vllm_outputs_per_image = [
99 - vllm_model.generate_greedy_logprobs(prompts,
100 - max_tokens,
101 - num_logprobs=num_logprobs,
102 - images=images)
103 - for prompts, images in inputs_per_image
104 - ]
105 -
106 - def process(hf_inputs: BatchEncoding):
107 - hf_inputs["pixel_values"] = hf_inputs["pixel_values"] \
108 - .to(torch_dtype) # type: ignore
109 - return hf_inputs
110 -
111 - with hf_runner(model,
112 - dtype=dtype,
113 - postprocess_inputs=process,
114 - auto_cls=AutoModelForVision2Seq) as hf_model:
115 - hf_outputs_per_image = [
116 - hf_model.generate_greedy_logprobs_limit(prompts,
117 - max_tokens,
118 - num_logprobs=num_logprobs,
119 - images=images)
120 - for prompts, images in inputs_per_image
121 - ]
122 -
123 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
124 - vllm_outputs_per_image):
125 -
126 - check_logprobs_close(
127 - outputs_0_lst=hf_outputs,
128 - outputs_1_lst=[
129 - vllm_to_hf_output(vllm_output, model)
130 - for vllm_output in vllm_outputs
131 - ],
132 - name_0="hf",
133 - name_1="vllm",
134 - )
135 -
136 -
137 -@pytest.mark.parametrize("model", models)
138 -@pytest.mark.parametrize(
139 - "size_factors",
140 - [
141 - # No image
142 - [],
143 - # Single-scale
144 - [1.0],
145 - # Single-scale, batched
146 - [1.0, 1.0, 1.0],
147 - # Multi-scale
148 - [0.25, 0.5, 1.0],
149 - ],
150 -)
151 -@pytest.mark.parametrize("dtype", [
152 - pytest.param(
153 - "float",
154 - marks=pytest.mark.skipif(
155 - current_platform.is_rocm(),
156 - reason=
157 - "ROCm FA does not yet fully support 32-bit precision on PaliGemma")
158 - ), "half"
159 -])
160 -@pytest.mark.parametrize("max_tokens", [128])
161 -@pytest.mark.parametrize("num_logprobs", [5])
162 -def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
163 - dtype: str, max_tokens: int, num_logprobs: int) -> None:
164 - run_test(
165 - hf_runner,
166 - vllm_runner,
167 - image_assets,
168 - model,
169 - size_factors=size_factors,
170 - dtype=dtype,
171 - max_tokens=max_tokens,
172 - num_logprobs=num_logprobs,
173 - tensor_parallel_size=1,
174 - )
tests/models/decoder_only/vision_language/test_phi3v.py
+7 -178
@@ -3,19 +3,14 @@ import re
3 from typing import List, Optional, Tuple, Type
4
5 import pytest
6 -import torch
7 -from transformers import AutoImageProcessor, AutoTokenizer
6 +from transformers import AutoTokenizer
7
9 -from vllm.inputs import InputContext, token_inputs
10 -from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
11 -from vllm.multimodal import MultiModalRegistry
8 from vllm.multimodal.utils import rescale_image_size
9 from vllm.platforms import current_platform
10 from vllm.sequence import SampleLogprobs
11
16 -from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
17 - _ImageAssets)
18 -from ...utils import build_model_context, check_logprobs_close
12 +from ....conftest import IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner
13 +from ...utils import check_logprobs_close
14
15 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
16 "stop_sign":
@@ -81,12 +76,15 @@ def run_test(
76 Note, the text input is also adjusted to abide by vllm contract.
77 The text output is sanitized to be able to compare with hf.
78 """
79 + # HACK - this is an attempted workaround for the following bug
80 + # https://github.com/huggingface/transformers/issues/34307
81 + from transformers import AutoImageProcessor # noqa: F401
82 + from transformers import AutoProcessor # noqa: F401
83
84 # NOTE: take care of the order. run vLLM first, and then run HF.
85 # vLLM needs a fresh new process without cuda initialization.
86 # if we run HF first, the cuda initialization will be done and it
87 # will hurt multiprocessing backend with fork method (the default method).
89 -
88 # max_model_len should be greater than image_feature_size
89 with vllm_runner(model,
90 task="generate",
@@ -236,172 +234,3 @@ def test_multi_images_models(hf_runner, vllm_runner, image_assets, model,
234 mm_limit=2,
235 tensor_parallel_size=1,
236 )
239 -
240 -
241 -### Fast tests for correctness in processor_kwarg override handling
242 -
243 -
244 -# Wrap lazy imports to avoid initializing CUDA during test collection
245 -@pytest.fixture()
246 -def input_processor_for_phi3v():
247 - from vllm.model_executor.models.phi3v import input_processor_for_phi3v
248 - return input_processor_for_phi3v
249 -
250 -
251 -@pytest.fixture()
252 -def dummy_data_for_phi3v():
253 - from vllm.model_executor.models.phi3v import dummy_data_for_phi3v
254 - return dummy_data_for_phi3v
255 -
256 -
257 -@pytest.fixture()
258 -def get_max_phi3v_image_tokens():
259 - from vllm.model_executor.models.phi3v import get_max_phi3v_image_tokens
260 - return get_max_phi3v_image_tokens
261 -
262 -
263 -@pytest.mark.parametrize("model", models)
264 -@pytest.mark.parametrize("num_crops", [4, 16, None])
265 -def test_input_mapper_override(model: str, image_assets: _ImageAssets,
266 - num_crops: Optional[int]):
267 - """Ensure that the [default] input mapper handles num_crops properly."""
268 - # We pass the processor kwargs here since for this model, we fall back to
269 - # the default mapper; this will fall back to the HF mapper and forward
270 - # mm_processor_kwargs to it.
271 - mm_processor_kwargs = {
272 - "num_crops": num_crops
273 - } if num_crops is not None else {}
274 - ctx = build_model_context(
275 - model_name=model,
276 - tokenizer_name=model,
277 - trust_remote_code=True,
278 - mm_processor_kwargs=mm_processor_kwargs,
279 - )
280 -
281 - hf_processor = AutoImageProcessor.from_pretrained(model,
282 - trust_remote_code=True,
283 - **mm_processor_kwargs)
284 -
285 - mm_registry = MultiModalRegistry()
286 - mm_registry.init_mm_limits_per_prompt(ctx.model_config)
287 -
288 - image = image_assets[0].pil_image
289 - hf_result = hf_processor.preprocess(
290 - image,
291 - return_tensors="pt",
292 - )
293 -
294 - vllm_result = mm_registry.map_input(
295 - ctx.model_config,
296 - {"image": image},
297 - )
298 -
299 - assert torch.all(hf_result["image_sizes"] == vllm_result["image_sizes"])
300 - assert torch.all(
301 - hf_result["num_img_tokens"] == vllm_result["num_img_tokens"])
302 -
303 - # For pixel values, the second axis should be the num_crops + 1
304 - # for the rescaled original image. The default value in VLLM falls
305 - # back to the HF config, which is why we compare to the processor num_crops
306 - assert torch.all(hf_result["pixel_values"] == vllm_result["pixel_values"])
307 - assert vllm_result["pixel_values"].shape[1] == hf_processor.num_crops + 1
308 -
309 -
310 -@pytest.mark.parametrize("model", models)
311 -@pytest.mark.parametrize("num_crops,expected_max_tokens", [
312 - (4, 781),
313 - (16, 2653),
314 -])
315 -def test_max_tokens_override(get_max_phi3v_image_tokens, model: str,
316 - num_crops: int, expected_max_tokens: int):
317 - """Ensure get_max_phi3v_image_tokens handles num_crops properly."""
318 - # NOTE: mm_processor_kwargs on the context in this test is unused, since
319 - # this is testing the mapper directly. In practice, the processor kwargs
320 - # are wrapped in a closure when calling the max tokens func. We explicitly
321 - # do NOT use the mm_processor_kwargs in the model context here to ensure
322 - # that the max image tokens implementation is referencing a mix of the
323 - # kwargs to the function and the original mm_processor_kwargs in case
324 - # values are somehow updated and end up in a bad state.
325 - ctx = build_model_context(
326 - model_name=model,
327 - tokenizer_name=model,
328 - trust_remote_code=True,
329 - mm_processor_kwargs=None,
330 - )
331 -
332 - actual_max_tokens = get_max_phi3v_image_tokens(
333 - InputContext(ctx.model_config),
334 - num_crops=num_crops,
335 - )
336 -
337 - assert expected_max_tokens == actual_max_tokens
338 -
339 -
340 -@pytest.mark.parametrize("model", models)
341 -@pytest.mark.parametrize("num_crops,toks_per_img,num_imgs", [
342 - (4, 781, 1),
343 - (4, 781, 2),
344 - (16, 2653, 1),
345 - (16, 2653, 2),
346 -])
347 -def test_dummy_data_override(dummy_data_for_phi3v, model: str, num_crops: int,
348 - toks_per_img: int, num_imgs: int):
349 - """Ensure dummy_data_for_phi3v handles num_crops properly."""
350 - # Same as the previous test - don't initialize mm_processor_kwargs
351 - # in this test and assume that the kwargs will be correctly expanded by
352 - # the partial when calling the dummy data func.
353 - ctx = build_model_context(
354 - model_name=model,
355 - tokenizer_name=model,
356 - trust_remote_code=True,
357 - mm_processor_kwargs=None,
358 - )
359 -
360 - sequence_data, _, = dummy_data_for_phi3v(
361 - ctx=ctx,
362 - seq_len=8192, # Should be bigger than num_imgs * toks_per_img
363 - mm_counts={"image": num_imgs},
364 - num_crops=num_crops,
365 - )
366 - # Ensure we have the right number of placeholders per num_crops size
367 - img_tok_count = sequence_data.get_token_ids().count(_IMAGE_TOKEN_ID)
368 - assert img_tok_count == toks_per_img * num_imgs
369 -
370 -
371 -@pytest.mark.parametrize("model", models)
372 -@pytest.mark.parametrize("num_crops,expected_toks_per_img,num_imgs", [
373 - (4, 757, 1),
374 - (4, 757, 2),
375 - (16, 1921, 1),
376 - (16, 1921, 2),
377 -])
378 -def test_input_processor_override(input_processor_for_phi3v,
379 - image_assets: _ImageAssets, model: str,
380 - num_crops: int, expected_toks_per_img: int,
381 - num_imgs: int):
382 - """Ensure input_processor_for_phi3v handles num_crops properly."""
383 - # Same as the previous test - don't initialize mm_processor_kwargs
384 - # in this test and assume that the kwargs will be correctly expanded by
385 - # the partial when calling the custom input processor.
386 - ctx = build_model_context(
387 - model_name=model,
388 - tokenizer_name=model,
389 - trust_remote_code=True,
390 - )
391 - tokenizer = AutoTokenizer.from_pretrained(model)
392 - # Build the image str / prompt based on the number of images we pass
393 - img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
394 - prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
395 - images = [image_assets[0].pil_image] * num_imgs
396 -
397 - inputs = token_inputs(prompt_token_ids=tokenizer.encode(prompt),
398 - prompt=prompt,
399 - multi_modal_data={"image": images})
400 -
401 - processed_inputs = input_processor_for_phi3v(ctx,
402 - inputs,
403 - num_crops=num_crops)
404 -
405 - # Ensure we have the right number of placeholders per num_crops size
406 - img_tok_count = processed_inputs["prompt_token_ids"].count(_IMAGE_TOKEN_ID)
407 - assert img_tok_count == expected_toks_per_img * num_imgs
tests/models/decoder_only/vision_language/test_qwen.py deleted
-374
@@ -1,374 +0,0 @@
1 -import pathlib
2 -from typing import Dict, List, Optional, Tuple, Type, Union
3 -
4 -import pytest
5 -import torch
6 -from PIL.Image import Image
7 -
8 -from vllm.inputs import InputContext, token_inputs
9 -from vllm.multimodal.base import MultiModalInputs
10 -from vllm.multimodal.utils import cached_get_tokenizer, rescale_image_size
11 -
12 -from ....conftest import (IMAGE_ASSETS, HfRunner, ImageAsset, PromptImageInput,
13 - VllmRunner, _ImageAssets)
14 -from ...utils import build_model_context, check_logprobs_close
15 -
16 -text_only_models = [
17 - "Qwen/Qwen-7B-Chat" # Has no visual component
18 -]
19 -
20 -multimodal_models = ["Qwen/Qwen-VL"]
21 -
22 -HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
23 - "stop_sign":
24 - "Picture 1: <img></img>\nWhat's the content of the image?: ",
25 - "cherry_blossom":
26 - "Picture 1: <img></img>\nWhat is the season?: ",
27 -})
28 -
29 -HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nCan you compare these images?\n" # noqa: E501
30 -HF_MULTIIMAGE_IMAGE_PROMPT = "Picture 1: <img></img>\nPicture 2: <img></img>\nDescribe the two images in detail.\n" # noqa: E501
31 -### Multimodal preprocessing tests
32 -SAMPLE_IMAGE = IMAGE_ASSETS[0].pil_image
33 -# These values are specific to Qwen-VL/Chat; we can get these from the model
34 -# config also, but they are hardcoded here to keep the parameterize/fixtures
35 -# easy to read.
36 -IMG_START_ID = 151857
37 -IMG_END_ID = 151858
38 -IMG_PAD_ID = 151859
39 -TOKS_PER_IMG = 256
40 -VIS_ENC_DIM = 4096
41 -IMG_SIZE = 448
42 -
43 -
44 -@pytest.fixture()
45 -def input_mapper_for_qwen():
46 - # Lazy import to avoid initializing CUDA during test collection
47 - from vllm.model_executor.models.qwen import input_mapper_for_qwen
48 - return input_mapper_for_qwen
49 -
50 -
51 -@pytest.fixture()
52 -def input_processor_for_qwen():
53 - # Lazy import to avoid initializing CUDA during test collection
54 - from vllm.model_executor.models.qwen import input_processor_for_qwen
55 - return input_processor_for_qwen
56 -
57 -
58 -@pytest.fixture()
59 -def qwen_vl_context() -> InputContext:
60 - """Get an InputContext for Qwen-VL."""
61 - return build_model_context(model_name="Qwen/Qwen-VL",
62 - trust_remote_code=True)
63 -
64 -
65 -# Happy path tests for single/multi-image scenarios for the multimodal
66 -# input processor and mapper, respectively
67 -@pytest.mark.parametrize("num_images", [1, 2])
68 -def test_input_processor_valid_mm_data(input_processor_for_qwen,
69 - qwen_vl_context: InputContext,
70 - num_images: int):
71 - """Happy cases for image inputs to Qwen's multimodal input processor."""
72 - prompt = "".join(
73 - [f"Picture {num}: <img></img>\n" for num in range(1, num_images + 1)])
74 - inputs = token_inputs(
75 - prompt=prompt,
76 - # When processing multimodal data for a multimodal model, the qwen
77 - # input processor will overwrite the provided prompt_token_ids with
78 - # the image prompts
79 - prompt_token_ids=[],
80 - multi_modal_data={"image": torch.rand(num_images, TOKS_PER_IMG, 4096)},
81 - )
82 - proc_inputs = input_processor_for_qwen(qwen_vl_context, inputs)
83 - assert isinstance(proc_inputs, dict)
84 -
85 - # Each image should have one start / stop and a fixed context of 256
86 - proc_tokens = proc_inputs["prompt_token_ids"]
87 - assert proc_tokens.count(IMG_START_ID) == num_images
88 - assert proc_tokens.count(IMG_END_ID) == num_images
89 - assert proc_tokens.count(IMG_PAD_ID) == num_images * TOKS_PER_IMG
90 -
91 -
92 -@pytest.mark.parametrize(
93 - "img_data,expected_shape",
94 - [
95 - # single / multi-image
96 - (SAMPLE_IMAGE, (1, 3, IMG_SIZE, IMG_SIZE)),
97 - (2 * [SAMPLE_IMAGE], (2, 3, IMG_SIZE, IMG_SIZE)),
98 - # single / multi-image embeddings
99 - (torch.rand(
100 - (TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
101 - (torch.rand(
102 - (1, TOKS_PER_IMG, VIS_ENC_DIM)), (1, TOKS_PER_IMG, VIS_ENC_DIM)),
103 - (torch.rand(
104 - (2, TOKS_PER_IMG, VIS_ENC_DIM)), (2, TOKS_PER_IMG, VIS_ENC_DIM)),
105 - ])
106 -def test_input_mapper_valid_mm_data(input_mapper_for_qwen,
107 - qwen_vl_context: InputContext,
108 - img_data: Union[torch.Tensor, List[Image],
109 - Image],
110 - expected_shape: List[int]):
111 - """Happy cases for image inputs to Qwen's multimodal input mapper."""
112 - mapped_img_data = input_mapper_for_qwen(qwen_vl_context, img_data)
113 - # Ensure that we get the appropriately shaped pixel_values
114 - # for images and image embeddings, respectively.
115 - assert isinstance(mapped_img_data, MultiModalInputs)
116 - assert "pixel_values" in mapped_img_data
117 - assert mapped_img_data["pixel_values"].shape == expected_shape
118 -
119 -
120 -# Sad path tests for the multimodal input processor and mapper, respectively
121 -@pytest.mark.parametrize("mm_data", [
122 - {
123 - "image": torch.rand((5))
124 - },
125 - {
126 - "image": torch.rand((5, 5, 5, 5, 5))
127 - },
128 -])
129 -def test_input_processor_invalid_mm_data(input_processor_for_qwen,
130 - qwen_vl_context: InputContext,
131 - mm_data: Dict[str, torch.Tensor]):
132 - """Test sad cases validated in Qwen's multimodal input processor."""
133 - tokenizer = cached_get_tokenizer(qwen_vl_context.model_config.tokenizer,
134 - trust_remote_code=True)
135 - prompt = "Picture 1: <img></img>\n"
136 - prompt_token_ids = tokenizer.encode(prompt)
137 - inputs = token_inputs(prompt=prompt,
138 - prompt_token_ids=prompt_token_ids,
139 - multi_modal_data=mm_data)
140 - # Should fail since we have too many or too few dimensions for embeddings
141 - with pytest.raises(ValueError):
142 - input_processor_for_qwen(qwen_vl_context, inputs)
143 -
144 -
145 -@pytest.mark.parametrize(
146 - "img_data",
147 - [
148 - # Wrong context length
149 - torch.rand((1, TOKS_PER_IMG + 10, VIS_ENC_DIM)),
150 - # Wrong visual encoder output size
151 - torch.rand((1, TOKS_PER_IMG, VIS_ENC_DIM + 10)),
152 - ])
153 -def test_input_mapper_invalid_mm_data(
154 - input_mapper_for_qwen,
155 - qwen_vl_context: InputContext,
156 - img_data: Union[torch.Tensor, List[Image], Image],
157 -):
158 - """Sad cases validated in Qwen VL's multimodal input mapper."""
159 - with pytest.raises(ValueError):
160 - input_mapper_for_qwen(qwen_vl_context, img_data)
161 -
162 -
163 -### End-to-end generation tests
164 -def get_prompt_with_path(tmp_path: pathlib.PosixPath, prompt: str,
165 - assets: Union[_ImageAssets, List[ImageAsset]]) -> str:
166 - """Given a temporary dir path, export one or more image assets into the
167 - tempdir & replace its contents with the local path to the string so that
168 - the HF version of Qwen-VL can resolve the path and load the image ni its
169 - forward() call.
170 -
171 - Args:
172 - tmp_path: Tempdir for test under consideration.
173 - prompt: Prompt with image placeholders.
174 - assets: List of image assets whose len equals the num placeholders.
175 - """
176 - # Ensure that the number of placeholders matches the number of assets;
177 - # If this is not true, the test is probably written incorrectly.
178 - assert prompt.count("<img></img>") == len(assets)
179 -
180 - # Replace the placeholders with local paths to the exported assets
181 - for asset in assets:
182 - image_tmp_path = tmp_path / f"{asset.name}.jpg"
183 - asset.pil_image.save(image_tmp_path)
184 - prompt = prompt.replace(
185 - "<img></img>",
186 - f"<img>{image_tmp_path}</img>",
187 - 1,
188 - )
189 - return prompt
190 -
191 -
192 -def run_test(
193 - hf_runner: Type[HfRunner],
194 - vllm_runner: Type[VllmRunner],
195 - inputs: List[Tuple[List[str], PromptImageInput]],
196 - model: str,
197 - *,
198 - dtype: str,
199 - max_tokens: int,
200 - num_logprobs: int,
201 - mm_limit: int,
202 - tensor_parallel_size: int,
203 - distributed_executor_backend: Optional[str] = None,
204 -):
205 - """Inference result should be the same between hf and vllm.
206 -
207 - All the image fixtures for the test is under tests/images.
208 - For huggingface runner, we provide the PIL images as input.
209 - For vllm runner, we provide MultiModalDataDict objects
210 - and corresponding MultiModalConfig as input.
211 - Note, the text input is also adjusted to abide by vllm contract.
212 - The text output is sanitized to be able to compare with hf.
213 - """
214 -
215 - # NOTE: take care of the order. run vLLM first, and then run HF.
216 - # vLLM needs a fresh new process without cuda initialization.
217 - # if we run HF first, the cuda initialization will be done and it
218 - # will hurt multiprocessing backend with fork method (the default method).
219 -
220 - # max_model_len should be greater than image_feature_size
221 - # Qwen encodes each image into a fixed content size of 256
222 - with vllm_runner(model,
223 - max_model_len=1024,
224 - max_num_seqs=2,
225 - dtype=dtype,
226 - limit_mm_per_prompt={"image": mm_limit},
227 - tensor_parallel_size=tensor_parallel_size,
228 - distributed_executor_backend=distributed_executor_backend,
229 - enforce_eager=True) as vllm_model:
230 - vllm_outputs_per_image = [
231 - vllm_model.generate_greedy_logprobs(prompts,
232 - max_tokens,
233 - num_logprobs=num_logprobs,
234 - images=images)
235 - for prompts, images in inputs
236 - ]
237 -
238 - with hf_runner(model, dtype=dtype) as hf_model:
239 - hf_outputs_per_image = [
240 - hf_model.generate_greedy_logprobs_limit(prompts,
241 - max_tokens,
242 - num_logprobs=num_logprobs,
243 - images=images)
244 - for prompts, images in inputs
245 - ]
246 -
247 - for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
248 - vllm_outputs_per_image):
249 -
250 - check_logprobs_close(
251 - outputs_0_lst=hf_outputs,
252 - outputs_1_lst=vllm_outputs,
253 - name_0="hf",
254 - name_1="vllm",
255 - )
256 -
257 -
258 -@pytest.mark.parametrize("model", multimodal_models)
259 -@pytest.mark.parametrize(
260 - "size_factors",
261 - [
262 - # No image
263 - [],
264 - # Single-scale
265 - [1.0],
266 - # Single-scale, batched
267 - [1.0, 1.0, 1.0],
268 - # Multi-scale
269 - [0.25, 0.5, 1.0],
270 - ],
271 -)
272 -@pytest.mark.parametrize("dtype", ["bfloat16"])
273 -@pytest.mark.parametrize("max_tokens", [8])
274 -@pytest.mark.parametrize("num_logprobs", [5])
275 -def test_multimodal_models_single_image(tmp_path: pathlib.PosixPath,
276 - hf_runner: Type[HfRunner],
277 - vllm_runner: Type[VllmRunner],
278 - image_assets: _ImageAssets, model: str,
279 - size_factors: List[float], dtype: str,
280 - max_tokens: int,
281 - num_logprobs: int) -> None:
282 - """Tests multimodal models with single image prompts."""
283 - images = [asset.pil_image for asset in image_assets]
284 -
285 - prompts = [
286 - get_prompt_with_path(tmp_path, prompt, [asset])
287 - for prompt, asset in zip(HF_IMAGE_PROMPTS, image_assets)
288 - ]
289 -
290 - inputs = [(
291 - [prompt for _ in size_factors],
292 - [rescale_image_size(image, factor) for factor in size_factors],
293 - ) for image, prompt in zip(images, prompts)]
294 -
295 - run_test(
296 - hf_runner,
297 - vllm_runner,
298 - inputs,
299 - model,
300 - dtype=dtype,
301 - max_tokens=max_tokens,
302 - num_logprobs=num_logprobs,
303 - mm_limit=1,
304 - tensor_parallel_size=1,
305 - )
306 -
307 -
308 -@pytest.mark.parametrize("model", multimodal_models)
309 -@pytest.mark.parametrize(
310 - "size_factors",
311 - [
312 - # No image
313 - [],
314 - # Single-scale
315 - [1.0],
316 - # Single-scale, batched
317 - [1.0, 1.0, 1.0],
318 - # Multi-scale
319 - [0.25, 0.5, 1.0],
320 - ],
321 -)
322 -@pytest.mark.parametrize("dtype", ["bfloat16"])
323 -@pytest.mark.parametrize("max_tokens", [128])
324 -@pytest.mark.parametrize("num_logprobs", [5])
325 -def test_multimodal_models_multi_image(tmp_path: pathlib.PosixPath,
326 - hf_runner: Type[HfRunner],
327 - vllm_runner: Type[VllmRunner],
328 - image_assets: _ImageAssets, model: str,
329 - size_factors: List[float], dtype: str,
330 - max_tokens: int,
331 - num_logprobs: int) -> None:
332 - """Tests multimodal models with multi-image prompts."""
333 - images = [asset.pil_image for asset in image_assets]
334 - # Put all of the images into one prompt.
335 - prompt = get_prompt_with_path(tmp_path, HF_MULTIIMAGE_IMAGE_PROMPT,
336 - image_assets)
337 - inputs = [([prompt for _ in size_factors],
338 - [[rescale_image_size(image, factor) for image in images]
339 - for factor in size_factors])]
340 -
341 - run_test(
342 - hf_runner,
343 - vllm_runner,
344 - inputs,
345 - model,
346 - dtype=dtype,
347 - max_tokens=max_tokens,
348 - num_logprobs=num_logprobs,
349 - mm_limit=2,
350 - tensor_parallel_size=1,
351 - )
352 -
353 -
354 -# Ensure that a text-only Qwen model can still be loaded and
355 -# used for inference in VLLM without throwing.
356 -@pytest.mark.parametrize("model", text_only_models)
357 -@pytest.mark.parametrize("dtype", ["bfloat16"])
358 -@pytest.mark.parametrize("max_tokens", [32])
359 -@pytest.mark.parametrize("num_logprobs", [5])
360 -def test_text_only_qwen_model_can_be_loaded_and_run(
361 - vllm_runner: Type[VllmRunner],
362 - example_prompts: List[str],
363 - model: str,
364 - *,
365 - dtype: str,
366 - max_tokens: int,
367 - num_logprobs: int,
368 -):
369 - with vllm_runner(model, dtype=dtype) as vllm_model:
370 - vllm_model.generate_greedy_logprobs(
371 - example_prompts,
372 - max_tokens,
373 - num_logprobs=num_logprobs,
374 - )
tests/models/decoder_only/vision_language/vlm_utils/__init__.py
tests/models/decoder_only/vision_language/vlm_utils/builders.py new
+235
@@ -0,0 +1,235 @@
1 +"""Helpers for building inputs that can be leveraged for different test types.
2 +"""
3 +from pathlib import PosixPath
4 +from typing import Callable, Iterable, List, Optional, Tuple, Union
5 +
6 +import torch
7 +
8 +from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
9 + resize_video, sample_frames_from_video)
10 +
11 +from .....conftest import _ImageAssets, _VideoAssets
12 +from .types import (SINGLE_IMAGE_BASE_PROMPTS, TEST_IMG_PLACEHOLDER,
13 + TEST_VIDEO_PLACEHOLDER, VIDEO_BASE_PROMPT,
14 + ImageSizeWrapper, SizeType, VLMTestInfo)
15 +
16 +
17 +def replace_test_placeholder(prompt: str, img_idx_to_prompt: Callable[[int],
18 + str],
19 + test_placeholder: str) -> str:
20 + """Given a prompt, replaces each test placeholder with the
21 + model-specific tag.
22 + """
23 + prompt_segments = prompt.split(test_placeholder)
24 + img_prompt = prompt_segments[0]
25 + for placeholder_idx, next_seg in enumerate(prompt_segments[1:], start=1):
26 + img_prompt += img_idx_to_prompt(placeholder_idx)
27 + img_prompt += next_seg
28 + return img_prompt
29 +
30 +
31 +def get_model_prompts(base_prompts: Iterable[str],
32 + img_idx_to_prompt: Optional[Callable[[int], str]],
33 + video_idx_to_prompt: Optional[Callable[[int], str]],
34 + prompt_formatter: Callable[[str], str]) -> List[str]:
35 + """Given a model-agnostic base prompt and test configuration for a model(s)
36 + to be tested, update the media placeholders and apply the prompt formatting
37 + to get the test prompt string for this model.
38 +
39 + Example for phi3v, given the base_prompt: "<image>What is the season?"
40 + 1. Replace img placeholder(s)
41 + -> "<|image_1|>\nWhat is the season?"
42 + 2. Apply prompt formatter:
43 + -> <|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n
44 + """
45 + assert isinstance(base_prompts, (list, tuple))
46 + model_prompts = []
47 + for base_prompt in base_prompts:
48 + # Replace the multimodal placeholders in the base prompt with
49 + # the correct ones for the model that we are testing
50 + if img_idx_to_prompt:
51 + base_prompt = replace_test_placeholder(base_prompt,
52 + img_idx_to_prompt,
53 + TEST_IMG_PLACEHOLDER)
54 +
55 + if video_idx_to_prompt:
56 + base_prompt = replace_test_placeholder(base_prompt,
57 + video_idx_to_prompt,
58 + TEST_VIDEO_PLACEHOLDER)
59 +
60 + # Apply the prompt formatter to wrap the base prompt with
61 + # the correct media placeholders to get the model test prompt
62 + model_prompt = prompt_formatter(base_prompt)
63 + model_prompts.append(model_prompt)
64 + return model_prompts
65 +
66 +
67 +def build_single_image_inputs_from_test_info(
68 + test_info: VLMTestInfo,
69 + image_assets: _ImageAssets,
70 + size_wrapper: ImageSizeWrapper,
71 + tmp_path: Optional[PosixPath] = None):
72 + if test_info.prompt_formatter is None:
73 + raise ValueError(
74 + "Prompt formatter must be set to build single image inputs")
75 +
76 + model_prompts = get_model_prompts(test_info.single_image_prompts,
77 + test_info.img_idx_to_prompt,
78 + test_info.video_idx_to_prompt,
79 + test_info.prompt_formatter)
80 +
81 + # For models that require a local path / URL encoded in the image; export
82 + # assets and encode into tmp_path for this test. This should be avoided
83 + # where possible (currently needed for Qwen-VL).
84 + if test_info.prompt_path_encoder is not None:
85 + if tmp_path is None:
86 + raise ValueError("Prompt path encoder requires setting local path")
87 + model_prompts = [
88 + test_info.prompt_path_encoder(tmp_path, prompt, [asset])
89 + for prompt, asset in zip(model_prompts, image_assets)
90 + ]
91 +
92 + images = [asset.pil_image for asset in image_assets]
93 + assert len(images) == len(model_prompts)
94 + return build_single_image_inputs(images, model_prompts, size_wrapper)
95 +
96 +
97 +def build_single_image_inputs(images, model_prompts,
98 + size_wrapper: ImageSizeWrapper):
99 + # For every image / prompt pair, get a pair containing two lists of
100 + # length size_factors, where the first contains duplicates of the model
101 + # prompt [str], and the second contains copies of the image after being
102 + # scaled by one of the size factors.
103 + #
104 + # NOTE: rescaling preserves the image aspect ratio.
105 + return [(
106 + [prompt for _ in size_wrapper.data],
107 + [
108 + apply_image_size_scaling(image, size, size_wrapper.type)
109 + for size in size_wrapper.data
110 + ],
111 + ) for image, prompt in zip(images, model_prompts)]
112 +
113 +
114 +def build_multi_image_inputs_from_test_info(
115 + test_info: VLMTestInfo,
116 + image_assets: _ImageAssets,
117 + size_wrapper: ImageSizeWrapper,
118 + tmp_path: Optional[PosixPath] = None):
119 + if test_info.prompt_formatter is None:
120 + raise ValueError(
121 + "Prompt formatter must be set to build multi image inputs")
122 +
123 + model_prompts = get_model_prompts([test_info.multi_image_prompt],
124 + test_info.img_idx_to_prompt,
125 + test_info.video_idx_to_prompt,
126 + test_info.prompt_formatter)
127 +
128 + if test_info.prompt_path_encoder is not None:
129 + if tmp_path is None:
130 + raise ValueError("Prompt path encoder requires setting local path")
131 + model_prompts = [
132 + test_info.prompt_path_encoder(tmp_path, model_prompt, image_assets)
133 + for model_prompt in model_prompts
134 + ]
135 +
136 + images = [asset.pil_image for asset in image_assets]
137 +
138 + # Currently, we only have one multi-image list & one multi-image prompt
139 + return build_multi_image_inputs(
140 + image_lists=[images],
141 + model_prompts=model_prompts,
142 + size_wrapper=size_wrapper,
143 + )
144 +
145 +
146 +def build_multi_image_inputs(image_lists, model_prompts,
147 + size_wrapper: ImageSizeWrapper):
148 + return [(
149 + [prompt for _ in size_wrapper.data],
150 + [[
151 + apply_image_size_scaling(image, size, size_wrapper.type)
152 + for image in images
153 + ] for size in size_wrapper.data],
154 + ) for images, prompt in zip(image_lists, model_prompts)]
155 +
156 +
157 +def build_embedding_inputs_from_test_info(
158 + test_info: VLMTestInfo,
159 + image_assets: _ImageAssets,
160 + size_wrapper: ImageSizeWrapper,
161 +):
162 + # These conditions will always be true if invoked through filtering,
163 + # but we still check them in case this is ever called directly
164 + if test_info.prompt_formatter is None:
165 + raise ValueError(
166 + "Prompt formatter must be set to build image embedding inputs")
167 + if size_wrapper.type != SizeType.SIZE_FACTOR or not \
168 + all(factor == 1.0 for factor in size_wrapper.data):
169 + raise ValueError("Embedding tests require constant (1.0) size factors")
170 + if test_info.convert_assets_to_embeddings is None:
171 + raise ValueError("No conversion func for getting embeddings found")
172 +
173 + model_prompts = get_model_prompts(
174 + SINGLE_IMAGE_BASE_PROMPTS,
175 + test_info.img_idx_to_prompt,
176 + test_info.video_idx_to_prompt,
177 + test_info.prompt_formatter,
178 + )
179 +
180 + images = [asset.pil_image for asset in image_assets]
181 + embeds = test_info.convert_assets_to_embeddings(image_assets)
182 + assert len(images) == len(model_prompts)
183 +
184 + inputs = build_single_image_inputs(images, model_prompts, size_wrapper)
185 + vllm_embeddings = build_single_image_inputs(embeds, model_prompts,
186 + size_wrapper)
187 + return inputs, vllm_embeddings
188 +
189 +
190 +def build_video_inputs_from_test_info(
191 + test_info: VLMTestInfo,
192 + video_assets: _VideoAssets,
193 + size_wrapper: ImageSizeWrapper,
194 + num_frames: int,
195 +):
196 + if test_info.prompt_formatter is None:
197 + raise ValueError("Prompt formatter must be set to build video inputs")
198 + model_prompts = get_model_prompts(
199 + [VIDEO_BASE_PROMPT],
200 + test_info.img_idx_to_prompt,
201 + test_info.video_idx_to_prompt,
202 + test_info.prompt_formatter,
203 + )
204 +
205 + sampled_vids = [
206 + sample_frames_from_video(asset.np_ndarrays, num_frames)
207 + for asset in video_assets
208 + ]
209 +
210 + video_scaler = (resize_video if size_wrapper.type == SizeType.FIXED_SIZE
211 + else rescale_video_size)
212 +
213 + return [(
214 + [prompt for _ in size_wrapper.data],
215 + [video_scaler(video, size) for size in size_wrapper.data],
216 + ) for video, prompt in zip(sampled_vids, model_prompts)]
217 +
218 +
219 +def apply_image_size_scaling(image, size: Union[float, Tuple[int, int]],
220 + size_type: SizeType):
221 + """Applies a size scaler to one image; this can be a an image size factor,
222 + which scales the image while maintaining the aspect ratio"""
223 + # Special case for embeddings; if it's a tensor, it's only valid if we
224 + # are considering size factors at constant scale, i.e., we just clone
225 + # the tensor
226 + if isinstance(image, torch.Tensor):
227 + assert size_type == SizeType.SIZE_FACTOR and size == 1
228 + return image
229 + if size_type == SizeType.SIZE_FACTOR:
230 + # We have a list of image size factors
231 + return rescale_image_size(image, size)
232 + elif size_type == SizeType.FIXED_SIZE:
233 + # We have a list of fixed sizes
234 + return image.resize(size)
235 + raise ValueError("ImageSizeWrapper type must be FIXED_SIZE or SIZE_FACTOR")
tests/models/decoder_only/vision_language/vlm_utils/case_filtering.py new
+157
@@ -0,0 +1,157 @@
1 +"""Utils for determining which subset of model tests belong to a specific
2 +modality, getting all combinations (similar to pytest's parametrization),
3 +handling multimodal placeholder substitution, and so on.
4 +"""
5 +import itertools
6 +from collections import OrderedDict
7 +from typing import Dict, Iterable, Tuple
8 +
9 +import pytest
10 +
11 +from .types import (EMBEDDING_SIZE_FACTORS, ExpandableVLMTestArgs,
12 + ImageSizeWrapper, SizeType, VLMTestInfo, VLMTestType)
13 +
14 +
15 +def get_filtered_test_settings(test_settings: Dict[str, VLMTestInfo],
16 + test_type: VLMTestType,
17 + fork_per_test: bool) -> Dict[str, VLMTestInfo]:
18 + """Given the dict of potential test settings to run, return a subdict
19 + of tests who have the current test type enabled with the matching val for
20 + fork_per_test.
21 + """
22 +
23 + def matches_test_type(test_info: VLMTestInfo, test_type: VLMTestType):
24 + return test_info.test_type == test_type or (
25 + isinstance(test_info.test_type, Iterable)
26 + and test_type in test_info.test_type)
27 +
28 + matching_tests = {}
29 + for test_name, test_info in test_settings.items():
30 + # Otherwise check if the test has the right type & keep if it does
31 + if matches_test_type(test_info, test_type):
32 + # Embedding tests need to have a conversion func in their test info
33 + if matches_test_type(test_info, VLMTestType.EMBEDDING):
34 + assert test_info.convert_assets_to_embeddings is not None
35 + # Custom test inputs need to explicitly define the mm limit/inputs
36 + if matches_test_type(test_info, VLMTestType.CUSTOM_INPUTS):
37 + assert (test_info.custom_test_opts is not None
38 + and isinstance(test_info.custom_test_opts, Iterable))
39 + # For all types besides custom inputs, we need a prompt formatter
40 + else:
41 + assert test_info.prompt_formatter is not None
42 +
43 + # Everything looks okay; keep if this is has correct proc handling
44 + if (test_info.distributed_executor_backend
45 + is not None) == fork_per_test:
46 + matching_tests[test_name] = test_info
47 +
48 + return matching_tests
49 +
50 +
51 +def get_parametrized_options(test_settings: Dict[str, VLMTestInfo],
52 + test_type: VLMTestType,
53 + fork_new_process_for_each_test: bool):
54 + """Converts all of our VLMTestInfo into an expanded list of parameters.
55 + This is similar to nesting pytest parametrize calls, but done directly
56 + through an itertools product so that each test can set things like
57 + size factors etc, while still running in isolated test cases.
58 + """
59 + matching_tests = get_filtered_test_settings(
60 + test_settings, test_type, fork_new_process_for_each_test)
61 +
62 + # Ensure that something is wrapped as an iterable it's not already
63 + ensure_wrapped = lambda e: e if isinstance(e, (list, tuple)) else (e, )
64 +
65 + def get_model_type_cases(model_type: str, test_info: VLMTestInfo):
66 + # This is essentially the same as nesting a bunch of mark.parametrize
67 + # decorators, but we do it programmatically to allow overrides for on
68 + # a per-model basis, while still being able to execute each of these
69 + # as individual test cases in pytest.
70 + iter_kwargs = OrderedDict([
71 + ("model", ensure_wrapped(test_info.models)),
72 + ("max_tokens", ensure_wrapped(test_info.max_tokens)),
73 + ("num_logprobs", ensure_wrapped(test_info.num_logprobs)),
74 + ("dtype", ensure_wrapped(test_info.dtype)),
75 + ("distributed_executor_backend",
76 + ensure_wrapped(test_info.distributed_executor_backend)),
77 + ])
78 +
79 + # num_frames is video only
80 + if test_type == VLMTestType.VIDEO:
81 + iter_kwargs["num_video_frames"] = ensure_wrapped(
82 + test_info.num_video_frames)
83 +
84 + # No sizes passed for custom inputs, since inputs are directly provided
85 + if test_type != VLMTestType.CUSTOM_INPUTS:
86 + wrapped_sizes = get_wrapped_test_sizes(test_info, test_type)
87 + if wrapped_sizes is None:
88 + raise ValueError(
89 + f"Sizes must be set for test type {test_type}")
90 + iter_kwargs["size_wrapper"] = wrapped_sizes
91 +
92 + #Otherwise expand the custom test options instead
93 + else:
94 + if test_info.custom_test_opts is None:
95 + raise ValueError("Test has type CUSTOM_INPUTS, but none given")
96 + iter_kwargs["custom_test_opts"] = test_info.custom_test_opts
97 +
98 + # yapf: disable
99 + # Wrap all model cases in a pytest parameter & pass marks through
100 + return [
101 + pytest.param(
102 + model_type,
103 + ExpandableVLMTestArgs(
104 + **{k: v for k, v in zip(iter_kwargs.keys(), case)}
105 + ),
106 + marks=test_info.marks if test_info.marks is not None else []
107 + ) for case in list(itertools.product(*iter_kwargs.values()))
108 + ]
109 + # yapf: enable
110 +
111 + # Get a list per model type, where each entry contains a tuple of all of
112 + # that model type's cases, then flatten them into the top level so that
113 + # we can consume them in one mark.parametrize call.
114 + cases_by_model_type = [
115 + get_model_type_cases(model_type, test_info)
116 + for model_type, test_info in matching_tests.items()
117 + ]
118 + return list(itertools.chain(*cases_by_model_type))
119 +
120 +
121 +def get_wrapped_test_sizes(
122 + test_info: VLMTestInfo,
123 + test_type: VLMTestType) -> Tuple[ImageSizeWrapper, ...]:
124 + """Given a test info which may have size factors or fixed sizes, wrap them
125 + and combine them into an iterable, each of which will be used in parameter
126 + expansion.
127 +
128 + Args:
129 + test_info: Test configuration to be expanded.
130 + test_type: The type of test being filtered for.
131 + """
132 + # If it is an embedding test, we always use the EMBEDDING_SIZE_FACTORS
133 + if test_type == VLMTestType.EMBEDDING:
134 + return tuple([
135 + ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=factor)
136 + for factor in EMBEDDING_SIZE_FACTORS
137 + ])
138 + # Custom inputs have preprocessed inputs
139 + elif test_type == VLMTestType.CUSTOM_INPUTS:
140 + return tuple()
141 +
142 + size_factors = test_info.image_size_factors \
143 + if test_info.image_size_factors else []
144 + fixed_sizes = test_info.image_sizes \
145 + if test_info.image_sizes else []
146 +
147 + wrapped_factors = [
148 + ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=factor)
149 + for factor in size_factors
150 + ]
151 +
152 + wrapped_sizes = [
153 + ImageSizeWrapper(type=SizeType.FIXED_SIZE, data=size)
154 + for size in fixed_sizes
155 + ]
156 +
157 + return tuple(wrapped_factors + wrapped_sizes)
tests/models/decoder_only/vision_language/vlm_utils/core.py new
+141
@@ -0,0 +1,141 @@
1 +"""Core test implementation to be shared across modalities."""
2 +from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
3 +
4 +import torch
5 +from PIL.Image import Image
6 +from transformers import AutoTokenizer, BatchEncoding
7 +from transformers.models.auto.auto_factory import _BaseAutoModelClass
8 +
9 +from .....conftest import HfRunner, VllmRunner
10 +from .types import RunnerOutput
11 +
12 +
13 +def run_test(
14 + *,
15 + hf_runner: Type[HfRunner],
16 + vllm_runner: Type[VllmRunner],
17 + inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]],
18 + model: str,
19 + dtype: str,
20 + max_tokens: int,
21 + num_logprobs: int,
22 + enforce_eager: bool,
23 + max_model_len: int,
24 + max_num_seqs: int,
25 + hf_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]],
26 + vllm_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]],
27 + auto_cls: Type[_BaseAutoModelClass],
28 + use_tokenizer_eos: bool,
29 + postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
30 + comparator: Callable[..., None],
31 + get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]],
32 + limit_mm_per_prompt: Dict[str, int],
33 + model_kwargs: Optional[Dict[str, Any]],
34 + patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
35 + task: str = "auto",
36 + runner_mm_key: str = "images",
37 + distributed_executor_backend: Optional[str] = None,
38 + tensor_parallel_size: int = 1,
39 + vllm_embeddings: Optional[torch.Tensor] = None,
40 +):
41 + """Modality agnostic test test executor for comparing HF/vLLM outputs."""
42 + # In the case of embeddings, vLLM takes separate input tensors
43 + vllm_inputs = vllm_embeddings if vllm_embeddings is not None else inputs
44 + tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
45 +
46 + vllm_outputs_per_mm = []
47 + hf_outputs_per_mm = []
48 +
49 + # NOTE: take care of the order. run vLLM first, and then run HF.
50 + # vLLM needs a fresh new process without cuda initialization.
51 + # if we run HF first, the cuda initialization will be done and it
52 + # will hurt multiprocessing backend with fork method (the default method).
53 + vllm_kwargs = {}
54 + if get_stop_token_ids is not None:
55 + vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
56 +
57 + with vllm_runner(model,
58 + max_model_len=max_model_len,
59 + max_num_seqs=max_num_seqs,
60 + dtype=dtype,
61 + limit_mm_per_prompt=limit_mm_per_prompt,
62 + tensor_parallel_size=tensor_parallel_size,
63 + distributed_executor_backend=distributed_executor_backend,
64 + enforce_eager=enforce_eager,
65 + task=task) as vllm_model:
66 + for prompts, media in vllm_inputs:
67 + vllm_kwargs[runner_mm_key] = media
68 + vllm_output = vllm_model.generate_greedy_logprobs(
69 + prompts, max_tokens, num_logprobs=num_logprobs, **vllm_kwargs)
70 + vllm_outputs_per_mm.append(vllm_output)
71 +
72 + hf_model = hf_runner(model,
73 + dtype=dtype,
74 + auto_cls=auto_cls,
75 + postprocess_inputs=postprocess_inputs,
76 + model_kwargs=model_kwargs)
77 +
78 + # Some models need to patch things like the model processor, e.g., internvl
79 + if patch_hf_runner is not None:
80 + hf_model = patch_hf_runner(hf_model)
81 +
82 + # Some models need to explicitly pass the eos_token_id off the tokenizer or
83 + # processor for a good comparison; currently assume processor/tokenizer
84 + # agree on the EOS, and pull it off the tokenizer if requested.
85 + hf_kwargs = {}
86 + if use_tokenizer_eos:
87 + hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
88 +
89 + with hf_model, torch.no_grad():
90 + for prompts, media in inputs:
91 + hf_kwargs[runner_mm_key] = media
92 + hf_output = hf_model.generate_greedy_logprobs_limit(
93 + prompts,
94 + max_tokens,
95 + num_logprobs=num_logprobs,
96 + tokenizer=tokenizer,
97 + **hf_kwargs)
98 + hf_outputs_per_mm.append(hf_output)
99 +
100 + # Apply output processing / sanitation to the vLLM and HF runner results
101 + hf_outputs_per_mm, vllm_outputs_per_mm = process_runner_outputs(
102 + model,
103 + first_runner_outputs=hf_outputs_per_mm,
104 + second_runner_outputs=vllm_outputs_per_mm,
105 + first_runner_processor=hf_output_post_proc,
106 + second_runner_processor=vllm_output_post_proc,
107 + )
108 +
109 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_mm,
110 + vllm_outputs_per_mm):
111 + # This is usually check_logprobs_close, but it's passed through to
112 + # allow things like check_outputs_equal where needed
113 + comparator(
114 + outputs_0_lst=hf_outputs,
115 + outputs_1_lst=vllm_outputs,
116 + name_0="hf",
117 + name_1="vllm",
118 + )
119 +
120 +
121 +def process_runner_outputs(
122 + model,
123 + first_runner_outputs,
124 + second_runner_outputs,
125 + first_runner_processor=None,
126 + second_runner_processor=None,
127 +):
128 + """Applies the runner processor(s) to the runner outputs, if any."""
129 + if first_runner_processor is not None:
130 + first_runner_outputs = process_outputs(first_runner_processor, model,
131 + first_runner_outputs)
132 + if second_runner_processor is not None:
133 + second_runner_outputs = process_outputs(second_runner_processor, model,
134 + second_runner_outputs)
135 + return first_runner_outputs, second_runner_outputs
136 +
137 +
138 +def process_outputs(output_processor, model, outputs_per_image):
139 + """Applies a model specific post-processor function to a runner's output"""
140 + return [[output_processor(res, model) for res in outputs]
141 + for outputs in outputs_per_image]
tests/models/decoder_only/vision_language/vlm_utils/custom_inputs.py new
+102
@@ -0,0 +1,102 @@
1 +"""Custom input builders for edge-cases in different models."""
2 +from typing import Callable
3 +
4 +from vllm.multimodal.utils import (rescale_image_size, rescale_video_size,
5 + resize_video, sample_frames_from_video)
6 +
7 +from .....conftest import IMAGE_ASSETS, VIDEO_ASSETS
8 +from .builders import build_multi_image_inputs, build_single_image_inputs
9 +from .types import ImageSizeWrapper, SizeType
10 +
11 +
12 +def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
13 + """Builds inputs for multi-image (varied sizes/aspect ratio) testing.
14 +
15 + Args:
16 + formatter: model-specific prompt formatter.
17 + """
18 + stop_sign = IMAGE_ASSETS[0].pil_image
19 + cherry_blossom = IMAGE_ASSETS[1].pil_image
20 +
21 + # Apply the selected formatter to the base prompts
22 + img_prompts = [
23 + "<image><image>\nDescribe 2 images.",
24 + "<image><image>\nDescribe 2 images.",
25 + "<image><image><image><image>\nDescribe 4 images.",
26 + "<image>\nWhat is the season?",
27 + ]
28 + formatted_prompts = [formatter(prompt) for prompt in img_prompts]
29 +
30 + return [(
31 + formatted_prompts,
32 + [
33 + [stop_sign, cherry_blossom],
34 + # Images with different sizes and aspect-ratios
35 + [
36 + rescale_image_size(stop_sign, 0.1),
37 + stop_sign,
38 + ],
39 + [
40 + stop_sign,
41 + rescale_image_size(stop_sign, 0.25),
42 + cherry_blossom.resize((183, 488)),
43 + cherry_blossom.resize((488, 183))
44 + ],
45 + cherry_blossom,
46 + ])]
47 +
48 +
49 +def multi_video_multi_aspect_ratio_inputs(formatter: Callable[[str], str],
50 + num_frames: int = 16):
51 + """Builds inputs for multi-video (varied sizes/aspect ratio) testing.
52 +
53 + Args:
54 + formatter: model-specific prompt formatter.
55 + """
56 + video = sample_frames_from_video(VIDEO_ASSETS[0].np_ndarrays, num_frames)
57 + # Apply the selected formatter to the base prompts
58 + video_prompts = [
59 + "<video><video>\nDescribe 2 videos.",
60 + "<video><video>\nDescribe 2 videos.",
61 + "<video><video><video><video>\nDescribe 4 videos.",
62 + "<video>\nWhy is this video funny?",
63 + ]
64 + formatted_prompts = [formatter(prompt) for prompt in video_prompts]
65 +
66 + return [(
67 + formatted_prompts,
68 + [
69 + [video, video],
70 + # Videos with different sizes and aspect-ratios
71 + [
72 + rescale_video_size(video, 0.1),
73 + video,
74 + ],
75 + [
76 + video,
77 + rescale_video_size(video, 0.25),
78 + resize_video(video, (183, 488)),
79 + resize_video(video, (488, 183))
80 + ],
81 + video,
82 + ])]
83 +
84 +
85 +def different_patch_input_cases_internvl():
86 + images = [asset.pil_image.resize((896, 896)) for asset in IMAGE_ASSETS]
87 + formatter = lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
88 + single_img_prompts = [
89 + "<image>\nWhat's the content in the center of the image?",
90 + "<image>\nWhat is the season?",
91 + ]
92 + multi_img_prompts = [
93 + "Image-1: <image>\nImage-2: <image>\nDescribe the two images in detail.\n", # noqa: E501
94 + ]
95 + formatted_sprompts = [formatter(prompt) for prompt in single_img_prompts]
96 + formatted_mprompts = [formatter(prompt) for prompt in multi_img_prompts]
97 +
98 + wrapped_sf = ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=[0.5, 1.0])
99 + return [
100 + build_single_image_inputs(images, formatted_sprompts, wrapped_sf),
101 + build_multi_image_inputs([images], formatted_mprompts, wrapped_sf),
102 + ]
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py new
+338
@@ -0,0 +1,338 @@
1 +"""Common utility functions relating to different models that are useful
2 +for manipulating the input / output of HF & vLLM test runners, which are
3 +typically specific to a small subset of models.
4 +"""
5 +import re
6 +import types
7 +from pathlib import PosixPath
8 +from typing import Callable, List, Optional, Tuple, Union
9 +
10 +import torch
11 +from PIL.Image import Image
12 +from transformers import AutoConfig, AutoTokenizer, BatchEncoding
13 +
14 +from vllm.sequence import SampleLogprobs
15 +from vllm.transformers_utils.tokenizer import patch_padding_side
16 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
17 +
18 +from .....conftest import HfRunner, ImageAsset, _ImageAssets
19 +from .types import RunnerOutput
20 +
21 +
22 +####### vLLM output processors functions
23 +def blip2_vllm_to_hf_output(vllm_output: RunnerOutput,
24 + model: str) -> RunnerOutput:
25 + """Sanitize vllm output [blip2 models] to be comparable with hf output."""
26 + _, output_str, out_logprobs = vllm_output
27 +
28 + hf_output_str = output_str + "\n"
29 +
30 + tokenizer = AutoTokenizer.from_pretrained(model)
31 + hf_output_ids = tokenizer.encode(hf_output_str)
32 + assert hf_output_ids[0] == tokenizer.bos_token_id
33 + hf_output_ids = hf_output_ids[1:]
34 +
35 + return hf_output_ids, hf_output_str, out_logprobs
36 +
37 +
38 +def fuyu_vllm_to_hf_output(vllm_output: RunnerOutput,
39 + model: str) -> RunnerOutput:
40 + """Sanitize vllm output [fuyu models] to be comparable with hf output."""
41 + output_ids, output_str, out_logprobs = vllm_output
42 +
43 + hf_output_str = output_str.lstrip() + "|ENDOFTEXT|"
44 +
45 + return output_ids, hf_output_str, out_logprobs
46 +
47 +
48 +def qwen_vllm_to_hf_output(
49 + vllm_output: RunnerOutput,
50 + model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:
51 + """Sanitize vllm output [qwen models] to be comparable with hf output."""
52 + output_ids, output_str, out_logprobs = vllm_output
53 +
54 + hf_output_str = output_str + "<|endoftext|>"
55 +
56 + return output_ids, hf_output_str, out_logprobs
57 +
58 +
59 +def llava_image_vllm_to_hf_output(vllm_output: RunnerOutput,
60 + model: str) -> RunnerOutput:
61 + config = AutoConfig.from_pretrained(model)
62 + mm_token_id = config.image_token_index
63 + return _llava_vllm_to_hf_output(vllm_output, model, mm_token_id)
64 +
65 +
66 +def llava_video_vllm_to_hf_output(
67 + vllm_output: RunnerOutput,
68 + model: str) -> Tuple[List[int], str, Optional[SampleLogprobs]]:
69 + config = AutoConfig.from_pretrained(model)
70 + mm_token_id = config.video_token_index
71 + return _llava_vllm_to_hf_output(vllm_output, model, mm_token_id)
72 +
73 +
74 +def _llava_vllm_to_hf_output(vllm_output: RunnerOutput, model: str,
75 + mm_token_id: int) -> RunnerOutput:
76 + """Sanitize vllm output [Llava models] to be comparable with hf output."""
77 + output_ids, output_str, out_logprobs = vllm_output
78 +
79 + tokenizer = AutoTokenizer.from_pretrained(model)
80 + eos_token_id = tokenizer.eos_token_id
81 +
82 + hf_output_ids = [
83 + token_id for idx, token_id in enumerate(output_ids)
84 + if token_id != mm_token_id or output_ids[idx - 1] != mm_token_id
85 + ]
86 +
87 + assert output_str[0] == " "
88 + hf_output_str = output_str[1:]
89 + if hf_output_ids[-1] == eos_token_id:
90 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
91 +
92 + return hf_output_ids, hf_output_str, out_logprobs
93 +
94 +
95 +def llava_onevision_vllm_to_hf_output(vllm_output: RunnerOutput,
96 + model: str) -> RunnerOutput:
97 + """Sanitize vllm output [llava-onevision] to compare with hf output."""
98 + output_ids, output_str, out_logprobs = vllm_output
99 +
100 + config = AutoConfig.from_pretrained(model)
101 + video_token_id = config.video_token_index
102 +
103 + tokenizer = AutoTokenizer.from_pretrained(model)
104 + eos_token_id = tokenizer.eos_token_id
105 +
106 + hf_output_ids = [
107 + token_id for idx, token_id in enumerate(output_ids)
108 + if token_id != video_token_id or output_ids[idx - 1] != video_token_id
109 + ]
110 +
111 + hf_output_str = output_str
112 + if hf_output_ids[-1] == eos_token_id:
113 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
114 +
115 + return hf_output_ids, hf_output_str, out_logprobs
116 +
117 +
118 +def phi3v_vllm_to_hf_output(vllm_output: RunnerOutput,
119 + model: str) -> RunnerOutput:
120 + """Sanitize vllm output [phi3v] to be comparable with hf output."""
121 + _, output_str, out_logprobs = vllm_output
122 +
123 + output_str_without_image = re.sub(r"(<\|image_\d+\|>)+", "", output_str)
124 + assert output_str_without_image[0] == " "
125 + output_str_without_image = output_str_without_image[1:]
126 +
127 + hf_output_str = output_str_without_image + "<|end|><|endoftext|>"
128 +
129 + tokenizer = AutoTokenizer.from_pretrained(model)
130 + hf_output_ids = tokenizer.encode(output_str_without_image)
131 + assert hf_output_ids[0] == 1
132 + hf_output_ids = hf_output_ids[1:]
133 +
134 + return hf_output_ids, hf_output_str, out_logprobs
135 +
136 +
137 +def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput,
138 + model: str) -> RunnerOutput:
139 + """Sanitize vllm output to be comparable with hf output."""
140 + output_ids, output_str, out_logprobs = vllm_output
141 +
142 + config = AutoConfig.from_pretrained(model)
143 + image_token_id = config.image_token_index
144 +
145 + tokenizer = AutoTokenizer.from_pretrained(model)
146 + eos_token_id = tokenizer.eos_token_id
147 +
148 + hf_output_ids = [
149 + token_id for idx, token_id in enumerate(output_ids)
150 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
151 + ]
152 +
153 + hf_output_str = output_str
154 +
155 + if hf_output_ids[-1] == eos_token_id:
156 + hf_output_str = hf_output_str + tokenizer.decode(eos_token_id)
157 +
158 + return hf_output_ids, hf_output_str, out_logprobs
159 +
160 +
161 +####### Post-processors for HF outputs
162 +def minicmpv_trunc_hf_output(hf_output: RunnerOutput,
163 + model: str) -> RunnerOutput:
164 + output_ids, output_str, out_logprobs = hf_output
165 + if output_str.endswith("<|eot_id|>"):
166 + output_str = output_str.split("<|eot_id|>")[0]
167 + return output_ids, output_str, out_logprobs
168 +
169 +
170 +####### Functions for converting image assets to embeddings
171 +def get_llava_embeddings(image_assets: _ImageAssets):
172 + return [asset.image_embeds for asset in image_assets]
173 +
174 +
175 +####### postprocessors to run on HF BatchEncoding
176 +def get_key_type_post_processor(
177 + hf_inp_key: str) -> Callable[[BatchEncoding, str], BatchEncoding]:
178 + """Gets a handle to a post processor which converts a given key into a
179 + target data type."""
180 +
181 + def process(hf_inputs: BatchEncoding, dtype: str):
182 + torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[dtype]
183 + hf_inputs[hf_inp_key] = hf_inputs[hf_inp_key].to(torch_dtype)
184 + return hf_inputs
185 +
186 + return process
187 +
188 +
189 +def wrap_inputs_post_processor(hf_inputs: BatchEncoding, dtype: str):
190 + return {"model_inputs": hf_inputs}
191 +
192 +
193 +####### Prompt path encoders for models that need models on disk
194 +def qwen_prompt_path_encoder(
195 + tmp_path: PosixPath, prompt: str, assets: Union[List[ImageAsset],
196 + _ImageAssets]) -> str:
197 + """Given a temporary dir path, export one or more image assets into the
198 + tempdir & replace its contents with the local path to the string so that
199 + the HF version of Qwen-VL can resolve the path and load the image in its
200 + forward() call.
201 +
202 + Args:
203 + tmp_path: Tempdir for test under consideration.
204 + prompt: Prompt with image placeholders.
205 + assets: List of image assets whose len equals the num placeholders.
206 + """
207 + # Ensure that the number of placeholders matches the number of assets;
208 + # If this is not true, the test is probably written incorrectly.
209 + assert prompt.count("<img></img>") == len(assets)
210 +
211 + # Replace the placeholders with local paths to the exported assets
212 + for asset in assets:
213 + image_tmp_path = tmp_path / f"{asset.name}.jpg"
214 + asset.pil_image.save(image_tmp_path)
215 + prompt = prompt.replace(
216 + "<img></img>",
217 + f"<img>{image_tmp_path}</img>",
218 + 1,
219 + )
220 + return prompt
221 +
222 +
223 +####### Model-specific HuggingFace runner patchers
224 +def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
225 + """Patches and returns an instance of the HfRunner to use for GLM4."""
226 + hf_processor = hf_model.processor
227 + patch_padding_side(hf_processor)
228 +
229 + def processor(*args, text="", images=None, **kwargs):
230 + if images is None:
231 + return hf_processor(*args, **kwargs)
232 +
233 + return hf_processor.apply_chat_template(
234 + [{
235 + "role": "user",
236 + "image": images,
237 + "content": text
238 + }],
239 + add_generation_prompt=True,
240 + tokenize=True,
241 + return_dict=True,
242 + **kwargs,
243 + )
244 +
245 + hf_model.processor = processor
246 + hf_model.model.get_output_embeddings = lambda: \
247 + hf_model.model.transformer.output_layer
248 + return hf_model
249 +
250 +
251 +def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
252 + """Patches and returns an instance of the HfRunner to use for InternVL."""
253 +
254 + class InternVLProcessor:
255 + """A simple processor for InternVL2 which misses a processor."""
256 +
257 + def __init__(self, hf_runner: HfRunner):
258 + self.num_image_token = hf_runner.model.num_image_token
259 + self.tokenizer = hf_runner.tokenizer
260 + self.dtype = hf_runner.model.dtype
261 +
262 + self.config = AutoConfig.from_pretrained(hf_runner.model_name,
263 + trust_remote_code=True)
264 + self.vision_config = self.config.vision_config
265 + self.use_thumbnail = self.config.use_thumbnail
266 + self.min_num = self.config.min_dynamic_patch
267 + self.max_num = self.config.max_dynamic_patch
268 + self.image_size = self.vision_config.image_size
269 +
270 + def __call__(self, text: str, images: Union[Image, List[Image]],
271 + **kwargs):
272 + from vllm.model_executor.models.internvl import (
273 + IMG_CONTEXT, IMG_END, IMG_START, image_to_pixel_values)
274 + images = [images] if isinstance(images, Image) else images
275 + pixel_values = [
276 + image_to_pixel_values(image, self.image_size, self.min_num,
277 + self.max_num,
278 + self.use_thumbnail).to(self.dtype)
279 + for image in images
280 + ]
281 + num_patches_list = [
282 + pixel_value.shape[0] for pixel_value in pixel_values
283 + ]
284 + pixel_values = torch.cat(pixel_values, dim=0)
285 + for num_patches in num_patches_list:
286 + context_tokens = IMG_CONTEXT * self.num_image_token \
287 + * num_patches
288 + image_tokens = IMG_START + context_tokens + IMG_END
289 + text = text.replace('<image>', image_tokens, 1)
290 + prompt = self.tokenizer(text, return_tensors="pt")
291 + prompt.update({"pixel_values": pixel_values})
292 + return prompt
293 +
294 + img_context_token_id = hf_model.tokenizer.convert_tokens_to_ids(
295 + "<IMG_CONTEXT>")
296 + hf_model.model.img_context_token_id = img_context_token_id
297 + hf_model.processor = InternVLProcessor(hf_model)
298 + hf_model.model.get_output_embeddings = lambda: \
299 + hf_model.model.language_model.get_output_embeddings()
300 + hf_model.model.generate = types.MethodType(_internvl_generate,
301 + hf_model.model)
302 + return hf_model
303 +
304 +
305 +def _internvl_generate(
306 + self,
307 + pixel_values: torch.FloatTensor,
308 + input_ids: torch.FloatTensor,
309 + attention_mask: Optional[torch.LongTensor] = None,
310 + **generate_kwargs,
311 +) -> torch.LongTensor:
312 + """Generate method for InternVL2 model without fixed use_cache."""
313 + assert self.img_context_token_id is not None
314 + vit_embeds = self.extract_feature(pixel_values)
315 + input_embeds = self.language_model.get_input_embeddings()(input_ids)
316 + B, N, C = input_embeds.shape
317 + input_embeds = input_embeds.reshape(B * N, C)
318 +
319 + input_ids = input_ids.reshape(B * N)
320 + selected = (input_ids == self.img_context_token_id)
321 + assert selected.sum() != 0
322 + input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
323 +
324 + input_embeds = input_embeds.reshape(B, N, C)
325 +
326 + forward_kwargs = dict(
327 + inputs_embeds=input_embeds,
328 + attention_mask=attention_mask,
329 + )
330 + if getattr(self, "use_visual_token_mask", False):
331 + visual_token_mask = selected.reshape(B, N, 1).to(input_embeds.dtype)
332 + forward_kwargs["visual_token_mask"] = visual_token_mask
333 + outputs = self.language_model.generate(
334 + **forward_kwargs,
335 + **generate_kwargs,
336 + )
337 +
338 + return outputs
tests/models/decoder_only/vision_language/vlm_utils/runners.py new
+130
@@ -0,0 +1,130 @@
1 +"""Entrypoints for wrapping the core run_test implementation for specific test
2 +types / modalities.
3 +"""
4 +from pathlib import PosixPath
5 +from typing import Type
6 +
7 +from .....conftest import HfRunner, VllmRunner, _ImageAssets, _VideoAssets
8 +from . import builders, core
9 +from .types import ExpandableVLMTestArgs, VLMTestInfo
10 +
11 +
12 +####### Entrypoints for running different test types
13 +def run_single_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
14 + test_case: ExpandableVLMTestArgs,
15 + hf_runner: Type[HfRunner],
16 + vllm_runner: Type[VllmRunner],
17 + image_assets: _ImageAssets):
18 + assert test_case.size_wrapper is not None
19 + inputs = builders.build_single_image_inputs_from_test_info(
20 + model_test_info, image_assets, test_case.size_wrapper, tmp_path)
21 +
22 + core.run_test(
23 + hf_runner=hf_runner,
24 + vllm_runner=vllm_runner,
25 + inputs=inputs,
26 + model=test_case.model,
27 + dtype=test_case.dtype,
28 + max_tokens=test_case.max_tokens,
29 + num_logprobs=test_case.num_logprobs,
30 + limit_mm_per_prompt={"image": 1},
31 + distributed_executor_backend=test_case.distributed_executor_backend,
32 + **model_test_info.get_non_parametrized_runner_kwargs())
33 +
34 +
35 +def run_multi_image_test(*, tmp_path: PosixPath, model_test_info: VLMTestInfo,
36 + test_case: ExpandableVLMTestArgs,
37 + hf_runner: Type[HfRunner],
38 + vllm_runner: Type[VllmRunner],
39 + image_assets: _ImageAssets):
40 + assert test_case.size_wrapper is not None
41 + inputs = builders.build_multi_image_inputs_from_test_info(
42 + model_test_info, image_assets, test_case.size_wrapper, tmp_path)
43 +
44 + core.run_test(
45 + hf_runner=hf_runner,
46 + vllm_runner=vllm_runner,
47 + inputs=inputs,
48 + model=test_case.model,
49 + dtype=test_case.dtype,
50 + max_tokens=test_case.max_tokens,
51 + num_logprobs=test_case.num_logprobs,
52 + limit_mm_per_prompt={"image": len(image_assets)},
53 + distributed_executor_backend=test_case.distributed_executor_backend,
54 + **model_test_info.get_non_parametrized_runner_kwargs())
55 +
56 +
57 +def run_embedding_test(*, model_test_info: VLMTestInfo,
58 + test_case: ExpandableVLMTestArgs,
59 + hf_runner: Type[HfRunner],
60 + vllm_runner: Type[VllmRunner],
61 + image_assets: _ImageAssets):
62 + assert test_case.size_wrapper is not None
63 + inputs, vllm_embeddings = builders.build_embedding_inputs_from_test_info(
64 + model_test_info, image_assets, test_case.size_wrapper)
65 +
66 + core.run_test(
67 + hf_runner=hf_runner,
68 + vllm_runner=vllm_runner,
69 + inputs=inputs,
70 + model=test_case.model,
71 + dtype=test_case.dtype,
72 + max_tokens=test_case.max_tokens,
73 + num_logprobs=test_case.num_logprobs,
74 + limit_mm_per_prompt={"image": 1},
75 + vllm_embeddings=vllm_embeddings,
76 + distributed_executor_backend=test_case.distributed_executor_backend,
77 + **model_test_info.get_non_parametrized_runner_kwargs())
78 +
79 +
80 +def run_video_test(
81 + *,
82 + model_test_info: VLMTestInfo,
83 + test_case: ExpandableVLMTestArgs,
84 + hf_runner: Type[HfRunner],
85 + vllm_runner: Type[VllmRunner],
86 + video_assets: _VideoAssets,
87 +):
88 + assert test_case.size_wrapper is not None
89 + assert test_case.num_video_frames is not None
90 + inputs = builders.build_video_inputs_from_test_info(
91 + model_test_info, video_assets, test_case.size_wrapper,
92 + test_case.num_video_frames)
93 +
94 + core.run_test(
95 + hf_runner=hf_runner,
96 + vllm_runner=vllm_runner,
97 + inputs=inputs,
98 + model=test_case.model,
99 + dtype=test_case.dtype,
100 + max_tokens=test_case.max_tokens,
101 + num_logprobs=test_case.num_logprobs,
102 + limit_mm_per_prompt={"video": len(video_assets)},
103 + distributed_executor_backend=test_case.distributed_executor_backend,
104 + **model_test_info.get_non_parametrized_runner_kwargs())
105 +
106 +
107 +def run_custom_inputs_test(*, model_test_info: VLMTestInfo,
108 + test_case: ExpandableVLMTestArgs,
109 + hf_runner: Type[HfRunner],
110 + vllm_runner: Type[VllmRunner]):
111 + # Custom test cases can provide inputs directly, but they need to
112 + # explicitly provided a CustomTestConfig, which wraps the inputs and
113 + # the limit_mm_per_prompt
114 + assert test_case.custom_test_opts is not None
115 +
116 + inputs = test_case.custom_test_opts.inputs
117 + limit_mm_per_prompt = test_case.custom_test_opts.limit_mm_per_prompt
118 + assert inputs is not None and limit_mm_per_prompt is not None
119 +
120 + core.run_test(
121 + hf_runner=hf_runner,
122 + vllm_runner=vllm_runner,
123 + inputs=inputs,
124 + model=test_case.model,
125 + dtype=test_case.dtype,
126 + max_tokens=test_case.max_tokens,
127 + num_logprobs=test_case.num_logprobs,
128 + limit_mm_per_prompt=limit_mm_per_prompt,
129 + distributed_executor_backend=test_case.distributed_executor_backend,
130 + **model_test_info.get_non_parametrized_runner_kwargs())
tests/models/decoder_only/vision_language/vlm_utils/types.py new
+187
@@ -0,0 +1,187 @@
1 +"""Types for writing multimodal model tests."""
2 +from enum import Enum
3 +from pathlib import PosixPath
4 +from typing import (Any, Callable, Dict, Iterable, List, NamedTuple, Optional,
5 + Tuple, Type, Union)
6 +
7 +import torch
8 +from PIL.Image import Image
9 +from pytest import MarkDecorator
10 +from transformers import AutoModelForCausalLM, AutoTokenizer, BatchEncoding
11 +from transformers.models.auto.auto_factory import _BaseAutoModelClass
12 +
13 +from vllm.sequence import SampleLogprobs
14 +from vllm.utils import identity
15 +
16 +from .....conftest import IMAGE_ASSETS, HfRunner, ImageAsset, _ImageAssets
17 +from ....utils import check_logprobs_close
18 +
19 +# meta image tag; will be replaced by the appropriate tag for the model
20 +TEST_IMG_PLACEHOLDER = "<vlm_image>"
21 +TEST_VIDEO_PLACEHOLDER = "<vlm_video>"
22 +
23 +# yapf: disable
24 +SINGLE_IMAGE_BASE_PROMPTS = IMAGE_ASSETS.prompts({
25 + "stop_sign": f"{TEST_IMG_PLACEHOLDER}What's the content of the image?",
26 + "cherry_blossom": f"{TEST_IMG_PLACEHOLDER}What is the season?",
27 +})
28 +
29 +MULTI_IMAGE_BASE_PROMPT = f"Image-1: {TEST_IMG_PLACEHOLDER}Image-2: {TEST_IMG_PLACEHOLDER}Describe the two images in detail.\n" # noqa: E501
30 +VIDEO_BASE_PROMPT = f"{TEST_VIDEO_PLACEHOLDER}Why is this video funny?"
31 +
32 +
33 +IMAGE_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0), (0.25, 0.5, 1.0)]
34 +EMBEDDING_SIZE_FACTORS = [(), (1.0, ), (1.0, 1.0, 1.0)]
35 +RunnerOutput = Tuple[List[int], str, Optional[SampleLogprobs]]
36 +# yapf: enable
37 +
38 +
39 +class VLMTestType(Enum):
40 + IMAGE = 1
41 + MULTI_IMAGE = 2
42 + EMBEDDING = 3
43 + VIDEO = 4
44 + CUSTOM_INPUTS = 5
45 +
46 +
47 +class SizeType(Enum):
48 + SIZE_FACTOR = 1
49 + FIXED_SIZE = 2
50 +
51 +
52 +class CustomTestOptions(NamedTuple):
53 + inputs: List[Tuple[List[str], List[Union[List[Image], Image]]]]
54 + limit_mm_per_prompt: Dict[str, int]
55 +
56 +
57 +class ImageSizeWrapper(NamedTuple):
58 + type: SizeType
59 + # A size factor is a wrapper of 0+ floats,
60 + # while a fixed size contains an iterable of integer pairs
61 + data: Union[Iterable[float], Iterable[Tuple[int, int]]]
62 +
63 +
64 +class VLMTestInfo(NamedTuple):
65 + """Holds the configuration for 1+ tests for one model architecture."""
66 +
67 + models: Union[List[str]]
68 + test_type: Union[VLMTestType, Iterable[VLMTestType]]
69 +
70 + # Should be None only if this is a CUSTOM_INPUTS test
71 + prompt_formatter: Optional[Callable[[str], str]] = None
72 + img_idx_to_prompt: Callable[[int], str] = lambda idx: "<image>\n"
73 + video_idx_to_prompt: Callable[[int], str] = lambda idx: "<video>\n"
74 +
75 + # Most models work on the single / multi-image prompts above, but in some
76 + # cases the log prob check fails, e.g., for paligemma. We allow passing
77 + # an override for the single image prompts / multi-image prompt for this
78 + # reason.
79 + single_image_prompts: Iterable[str] = SINGLE_IMAGE_BASE_PROMPTS
80 + multi_image_prompt: str = MULTI_IMAGE_BASE_PROMPT
81 +
82 + # Function for converting ImageAssets to image embeddings;
83 + # We need to define this explicitly for embedding tests
84 + convert_assets_to_embeddings: Optional[Callable[[_ImageAssets],
85 + torch.Tensor]] = None
86 +
87 + # Exposed options for vLLM runner; we change these in a several tests,
88 + # but the defaults are derived from VllmRunner & the engine defaults
89 + # These settings are chosen to avoid OOMs when running in the CI
90 + enforce_eager: bool = True
91 + max_model_len: int = 1024
92 + max_num_seqs: int = 256
93 + task: str = "auto"
94 + tensor_parallel_size: int = 1
95 +
96 + # Optional callable which gets a list of token IDs from the model tokenizer
97 + get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]] = None
98 +
99 + # Exposed options for HF runner
100 + model_kwargs: Optional[Dict[str, Any]] = None
101 + # Indicates we should explicitly pass the EOS from the tokeniezr
102 + use_tokenizer_eos: bool = False
103 + auto_cls: Type[_BaseAutoModelClass] = AutoModelForCausalLM
104 + # Callable to pass to the HF runner to run on inputs; for now, we also pass
105 + # the data type to input post processing, because almost all of the uses of
106 + # postprocess_inputs are to fix the data types of BatchEncoding values.
107 + postprocess_inputs: Callable[[BatchEncoding, str],
108 + BatchEncoding] = identity
109 + patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]] = None
110 +
111 + # Post processors that if defined, will run oun the outputs of the
112 + # vLLM and HF runner, respectively (useful for sanitization, etc).
113 + vllm_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]] = None
114 + hf_output_post_proc: Optional[Callable[[RunnerOutput, str], Any]] = None
115 +
116 + # Consumes the output of the callables above and checks if they're equal
117 + comparator: Callable[..., None] = check_logprobs_close
118 +
119 + # Default expandable params per test; these defaults can be overridden in
120 + # instances of this object; the complete set of test cases for the model
121 + # is all combinations of .models + all fields below
122 + max_tokens: Union[int, Tuple[int]] = 128
123 + num_logprobs: Union[int, Tuple[int]] = 5
124 + dtype: Union[str, Iterable[str]] = "half"
125 + distributed_executor_backend: Optional[Union[str, Iterable[str]]] = None
126 + # Only expanded in video tests
127 + num_video_frames: Union[int, Tuple[int]] = 16
128 +
129 + # Fixed image sizes / image size factors; most tests use image_size_factors
130 + # The values provided for these two fields will be stacked and expanded
131 + # such that each model will consider each image size factor / image size
132 + # once per tests (much like concatenating and wrapping in one parametrize
133 + # call)
134 + image_size_factors: Iterable[Iterable[float]] = IMAGE_SIZE_FACTORS
135 + image_sizes: Optional[Iterable[Iterable[Tuple[int, int]]]] = None
136 +
137 + # Hack for updating a prompt to take into a local path; currently only used
138 + # for Qwen-VL, which requires encoding the image path / url into the prompt
139 + # for HF runner
140 + prompt_path_encoder: Optional[
141 + Callable[[PosixPath, str, Union[List[ImageAsset], _ImageAssets]],
142 + str]] = None # noqa: E501
143 +
144 + # kwarg to pass multimodal data in as to vllm/hf runner instances
145 + runner_mm_key: str = "images"
146 +
147 + # Allows configuring a test to run with custom inputs
148 + custom_test_opts: Optional[List[CustomTestOptions]] = None
149 +
150 + marks: Optional[List[MarkDecorator]] = None
151 +
152 + def get_non_parametrized_runner_kwargs(self):
153 + """Returns a dictionary of expandable kwargs for items that are used
154 + in all test types, which are NOT used when creating the parametrized
155 + test cases.
156 + """
157 + return {
158 + "enforce_eager": self.enforce_eager,
159 + "max_model_len": self.max_model_len,
160 + "max_num_seqs": self.max_num_seqs,
161 + "task": self.task,
162 + "hf_output_post_proc": self.hf_output_post_proc,
163 + "vllm_output_post_proc": self.vllm_output_post_proc,
164 + "auto_cls": self.auto_cls,
165 + "use_tokenizer_eos": self.use_tokenizer_eos,
166 + "postprocess_inputs": self.postprocess_inputs,
167 + "comparator": self.comparator,
168 + "get_stop_token_ids": self.get_stop_token_ids,
169 + "model_kwargs": self.model_kwargs,
170 + "patch_hf_runner": self.patch_hf_runner,
171 + "runner_mm_key": self.runner_mm_key,
172 + }
173 +
174 +
175 +class ExpandableVLMTestArgs(NamedTuple):
176 + """The expanded kwargs which correspond to a single test case."""
177 + model: str
178 + max_tokens: int
179 + num_logprobs: int
180 + dtype: str
181 + distributed_executor_backend: Optional[str]
182 + # Sizes are used for everything except for custom input tests
183 + size_wrapper: Optional[ImageSizeWrapper] = None
184 + # Video only
185 + num_video_frames: Optional[int] = None
186 + # Custom inputs only
187 + custom_test_opts: Optional[CustomTestOptions] = None
tests/models/embedding/vision_language/test_llava_next.py
+2
@@ -85,6 +85,8 @@ def _run_test(
85 )
86
87
88 +# FIXME
89 +@pytest.mark.skip(reason="LLava next embedding tests currently fail")
90 @pytest.mark.parametrize("model", MODELS)
91 @pytest.mark.parametrize("dtype", ["half"])
92 def test_models_text(
tests/models/encoder_decoder/vision_language/test_mllama.py
+1 -1
@@ -192,7 +192,7 @@ def _run_test(
192 for prompts, images in inputs
193 ]
194
195 - def process(hf_inputs: BatchEncoding):
195 + def process(hf_inputs: BatchEncoding, **kwargs):
196 return hf_inputs
197
198 with hf_runner(model,
tests/utils.py
+16 -8
@@ -561,12 +561,11 @@ def fork_new_process_for_each_test(
561 return wrapper
562
563
564 -def large_gpu_test(*, min_gb: int):
565 - """
566 - Decorate a test to be skipped if no GPU is available or it does not have
567 - sufficient memory.
568 -
569 - Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
564 +def large_gpu_mark(min_gb: int) -> pytest.MarkDecorator:
565 + """Gets a pytest skipif mark, which triggers ig the the device doesn't have
566 + meet a minimum memory requirement in gb; can be leveraged via
567 + @large_gpu_test to skip tests in environments without enough resources, or
568 + called when filtering tests to run directly.
569 """
570 try:
571 if current_platform.is_cpu():
@@ -578,14 +577,23 @@ def large_gpu_test(*, min_gb: int):
577 f"An error occurred when finding the available memory: {e}",
578 stacklevel=2,
579 )
581 -
580 memory_gb = 0
581
584 - test_skipif = pytest.mark.skipif(
582 + return pytest.mark.skipif(
583 memory_gb < min_gb,
584 reason=f"Need at least {memory_gb}GB GPU memory to run the test.",
585 )
586
587 +
588 +def large_gpu_test(*, min_gb: int):
589 + """
590 + Decorate a test to be skipped if no GPU is available or it does not have
591 + sufficient memory.
592 +
593 + Currently, the CI machine uses L4 GPU which has 24 GB VRAM.
594 + """
595 + test_skipif = large_gpu_mark(min_gb)
596 +
597 def wrapper(f: Callable[_P, None]) -> Callable[_P, None]:
598 return test_skipif(f)
599
vllm/utils.py
+2 -1
@@ -977,7 +977,8 @@ def enable_trace_function_call_for_thread() -> None:
977
978
979 # `functools` helpers
980 -def identity(value: T) -> T:
980 +def identity(value: T, **kwargs) -> T:
981 + """Returns the first provided value."""
982 return value
983
984