@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6984c02a27
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[CI/Build] Refactor image test assets (#5821)

Cyrus Leung committed Jun 26, 2024 at 16:02 UTC 6984c02a2735d4d08426d2c426c34b6d73bee89e
5 files changed +127 -92
tests/conftest.py
+70 -41
@@ -1,7 +1,12 @@
1 import contextlib
2 import gc
3 import os
4 -from typing import Any, Dict, List, Optional, Tuple, TypeVar
4 +from collections import UserList
5 +from dataclasses import dataclass
6 +from functools import cached_property
7 +from pathlib import Path
8 +from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
9 + TypeVar)
10
11 import pytest
12 import torch
@@ -28,21 +33,8 @@ _TEST_DIR = os.path.dirname(__file__)
33 _TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
34 _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
35
31 -# Multi modal related
32 -# You can use `.buildkite/download-images.sh` to download the assets
33 -PIXEL_VALUES_FILES = [
34 - os.path.join(_TEST_DIR, "images", filename) for filename in
35 - ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
36 -]
37 -IMAGE_FEATURES_FILES = [
38 - os.path.join(_TEST_DIR, "images", filename) for filename in
39 - ["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
40 -]
41 -IMAGE_FILES = [
42 - os.path.join(_TEST_DIR, "images", filename)
43 - for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
44 -]
45 -assert len(PIXEL_VALUES_FILES) == len(IMAGE_FEATURES_FILES) == len(IMAGE_FILES)
36 +_IMAGE_DIR = Path(_TEST_DIR) / "images"
37 +"""You can use `.buildkite/download-images.sh` to download the assets."""
38
39
40 def _read_prompts(filename: str) -> List[str]:
@@ -51,6 +43,63 @@ def _read_prompts(filename: str) -> List[str]:
43 return prompts
44
45
46 +@dataclass(frozen=True)
47 +class ImageAsset:
48 + name: Literal["stop_sign", "cherry_blossom"]
49 +
50 + @cached_property
51 + def pixel_values(self) -> torch.Tensor:
52 + return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
53 +
54 + @cached_property
55 + def image_features(self) -> torch.Tensor:
56 + return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
57 +
58 + @cached_property
59 + def pil_image(self) -> Image.Image:
60 + return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
61 +
62 + def for_hf(self) -> Image.Image:
63 + return self.pil_image
64 +
65 + def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
66 + image_input_type = vision_config.image_input_type
67 + ImageInputType = VisionLanguageConfig.ImageInputType
68 +
69 + if image_input_type == ImageInputType.IMAGE_FEATURES:
70 + return ImageFeatureData(self.image_features)
71 + if image_input_type == ImageInputType.PIXEL_VALUES:
72 + return ImagePixelData(self.pil_image)
73 +
74 + raise NotImplementedError
75 +
76 +
77 +class _ImageAssetPrompts(TypedDict):
78 + stop_sign: str
79 + cherry_blossom: str
80 +
81 +
82 +class _ImageAssets(UserList[ImageAsset]):
83 +
84 + def __init__(self) -> None:
85 + super().__init__(
86 + [ImageAsset("stop_sign"),
87 + ImageAsset("cherry_blossom")])
88 +
89 + def prompts(self, prompts: _ImageAssetPrompts) -> List[str]:
90 + """
91 + Convenience method to define the prompt for each test image.
92 +
93 + The order of the returned prompts matches the order of the
94 + assets when iterating through this object.
95 + """
96 + return [prompts["stop_sign"], prompts["cherry_blossom"]]
97 +
98 +
99 +IMAGE_ASSETS = _ImageAssets()
100 +"""Singleton instance of :class:`_ImageAssets`."""
101 +
102 +
103 def cleanup():
104 destroy_model_parallel()
105 destroy_distributed_environment()
@@ -81,31 +130,6 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
130 cleanup()
131
132
84 -@pytest.fixture(scope="session")
85 -def hf_images() -> List[Image.Image]:
86 - return [Image.open(filename) for filename in IMAGE_FILES]
87 -
88 -
89 -@pytest.fixture()
90 -def vllm_images(request) -> List[MultiModalData]:
91 - vision_language_config = request.getfixturevalue("model_and_config")[1]
92 - if vision_language_config.image_input_type == (
93 - VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
94 - return [
95 - ImageFeatureData(torch.load(filename))
96 - for filename in IMAGE_FEATURES_FILES
97 - ]
98 - else:
99 - return [
100 - ImagePixelData(Image.open(filename)) for filename in IMAGE_FILES
101 - ]
102 -
103 -
104 -@pytest.fixture()
105 -def vllm_image_tensors(request) -> List[torch.Tensor]:
106 - return [torch.load(filename) for filename in PIXEL_VALUES_FILES]
107 -
108 -
133 @pytest.fixture
134 def example_prompts() -> List[str]:
135 prompts = []
@@ -122,6 +146,11 @@ def example_long_prompts() -> List[str]:
146 return prompts
147
148
149 +@pytest.fixture(scope="session")
150 +def image_assets() -> _ImageAssets:
151 + return IMAGE_ASSETS
152 +
153 +
154 _STR_DTYPE_TO_TORCH_DTYPE = {
155 "half": torch.half,
156 "bfloat16": torch.bfloat16,
tests/models/test_llava.py
+14 -12
@@ -5,17 +5,17 @@ from transformers import AutoTokenizer
5
6 from vllm.config import VisionLanguageConfig
7
8 -from ..conftest import IMAGE_FILES
8 +from ..conftest import IMAGE_ASSETS
9
10 pytestmark = pytest.mark.vlm
11
12 # The image token is placed before "user" on purpose so that the test can pass
13 -HF_IMAGE_PROMPTS = [
13 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14 + "stop_sign":
15 "<image>\nUSER: What's the content of the image?\nASSISTANT:",
16 + "cherry_blossom":
17 "<image>\nUSER: What is the season?\nASSISTANT:",
16 -]
17 -
18 -assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
18 +})
19
20
21 def iter_llava_configs(model_name: str):
@@ -49,28 +49,28 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
49 x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
50 It also reduces `output_str` from "<image><image>bla" to "bla".
51 """
52 - input_ids, output_str = vllm_output
52 + output_ids, output_str = vllm_output
53 image_token_id = vlm_config.image_token_id
54
55 tokenizer = AutoTokenizer.from_pretrained(model_id)
56 image_token_str = tokenizer.decode(image_token_id)
57
58 - hf_input_ids = [
59 - input_id for idx, input_id in enumerate(input_ids)
60 - if input_id != image_token_id or input_ids[idx - 1] != image_token_id
58 + hf_output_ids = [
59 + token_id for idx, token_id in enumerate(output_ids)
60 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
61 ]
62 hf_output_str = output_str \
63 .replace(image_token_str * vlm_config.image_feature_size, "")
64
65 - return hf_input_ids, hf_output_str
65 + return hf_output_ids, hf_output_str
66
67
68 # TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
69 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
70 @pytest.mark.parametrize("dtype", ["half"])
71 @pytest.mark.parametrize("max_tokens", [128])
72 -def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
73 - model_and_config, dtype: str, max_tokens: int) -> None:
72 +def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
73 + dtype: str, max_tokens: int) -> None:
74 """Inference result should be the same between hf and vllm.
75
76 All the image fixtures for the test is under tests/images.
@@ -81,6 +81,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
81 The text output is sanitized to be able to compare with hf.
82 """
83 model_id, vlm_config = model_and_config
84 + hf_images = [asset.for_hf() for asset in image_assets]
85 + vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
86
87 with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
88 hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_llava_next.py
+16 -14
@@ -5,7 +5,7 @@ from transformers import AutoTokenizer
5
6 from vllm.config import VisionLanguageConfig
7
8 -from ..conftest import IMAGE_FILES
8 +from ..conftest import IMAGE_ASSETS
9
10 pytestmark = pytest.mark.vlm
11
@@ -15,12 +15,12 @@ _PREFACE = (
15 "questions.")
16
17 # The image token is placed before "user" on purpose so that the test can pass
18 -HF_IMAGE_PROMPTS = [
19 - f"{_PREFACE} <image>\nUSER: What's the content of the image? ASSISTANT:",
20 - f"{_PREFACE} <image>\nUSER: What is the season? ASSISTANT:",
21 -]
22 -
23 -assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
18 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19 + "stop_sign":
20 + f"{_PREFACE} <image>\nUSER: What's the content of the image?\nASSISTANT:",
21 + "cherry_blossom":
22 + f"{_PREFACE} <image>\nUSER: What is the season?\nASSISTANT:",
23 +})
24
25
26 def iter_llava_next_configs(model_name: str):
@@ -56,20 +56,20 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
56 x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
57 It also reduces `output_str` from "<image><image>bla" to "bla".
58 """
59 - input_ids, output_str = vllm_output
59 + output_ids, output_str = vllm_output
60 image_token_id = vlm_config.image_token_id
61
62 tokenizer = AutoTokenizer.from_pretrained(model_id)
63 image_token_str = tokenizer.decode(image_token_id)
64
65 - hf_input_ids = [
66 - input_id for idx, input_id in enumerate(input_ids)
67 - if input_id != image_token_id or input_ids[idx - 1] != image_token_id
65 + hf_output_ids = [
66 + token_id for idx, token_id in enumerate(output_ids)
67 + if token_id != image_token_id or output_ids[idx - 1] != image_token_id
68 ]
69 hf_output_str = output_str \
70 .replace(image_token_str * vlm_config.image_feature_size, " ")
71
72 - return hf_input_ids, hf_output_str
72 + return hf_output_ids, hf_output_str
73
74
75 @pytest.mark.xfail(
@@ -78,8 +78,8 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
78 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
79 @pytest.mark.parametrize("dtype", ["half"])
80 @pytest.mark.parametrize("max_tokens", [128])
81 -def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82 - model_and_config, dtype: str, max_tokens: int) -> None:
81 +def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
82 + dtype: str, max_tokens: int) -> None:
83 """Inference result should be the same between hf and vllm.
84
85 All the image fixtures for the test is under tests/images.
@@ -90,6 +90,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
90 The text output is sanitized to be able to compare with hf.
91 """
92 model_id, vlm_config = model_and_config
93 + hf_images = [asset.for_hf() for asset in image_assets]
94 + vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
95
96 with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
97 hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_phi3v.py
+15 -13
@@ -6,17 +6,17 @@ from transformers import AutoTokenizer
6 from vllm.config import VisionLanguageConfig
7 from vllm.utils import is_cpu
8
9 -from ..conftest import IMAGE_FILES
9 +from ..conftest import IMAGE_ASSETS
10
11 pytestmark = pytest.mark.vlm
12
13 # The image token is placed before "user" on purpose so that the test can pass
14 -HF_IMAGE_PROMPTS = [
14 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15 + "stop_sign":
16 "<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
16 - "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
17 -]
18 -
19 -assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
17 + "cherry_blossom":
18 + "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n", # noqa: E501
19 +})
20
21
22 def iter_phi3v_configs(model_name: str):
@@ -50,22 +50,22 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
50 x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
51 It also reduces `output_str` from "<image><image>bla" to "bla".
52 """
53 - input_ids, output_str = vllm_output
53 + output_ids, output_str = vllm_output
54 image_token_id = vlm_config.image_token_id
55
56 tokenizer = AutoTokenizer.from_pretrained(model_id)
57 image_token_str = tokenizer.decode(image_token_id)
58
59 - hf_input_ids = [
60 - input_id if input_id != image_token_id else 0
61 - for idx, input_id in enumerate(input_ids)
59 + hf_output_ids = [
60 + token_id if token_id != image_token_id else 0
61 + for idx, token_id in enumerate(output_ids)
62 ]
63 hf_output_str = output_str \
64 .replace(image_token_str * vlm_config.image_feature_size, "") \
65 .replace("<s>", " ").replace("<|user|>", "") \
66 .replace("<|end|>\n<|assistant|>", " ")
67
68 - return hf_input_ids, hf_output_str
68 + return hf_output_ids, hf_output_str
69
70
71 target_dtype = "half"
@@ -82,8 +82,8 @@ if is_cpu():
82 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
83 @pytest.mark.parametrize("dtype", [target_dtype])
84 @pytest.mark.parametrize("max_tokens", [128])
85 -def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
86 - model_and_config, dtype: str, max_tokens: int) -> None:
85 +def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
86 + dtype: str, max_tokens: int) -> None:
87 """Inference result should be the same between hf and vllm.
88
89 All the image fixtures for the test is under tests/images.
@@ -94,6 +94,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
94 The text output is sanitized to be able to compare with hf.
95 """
96 model_id, vlm_config = model_and_config
97 + hf_images = [asset.for_hf() for asset in image_assets]
98 + vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
99
100 # use eager mode for hf runner, since phi3_v didn't work with flash_attn
101 hf_model_kwargs = {"_attn_implementation": "eager"}
tests/multimodal/test_processor.py
+12 -12
@@ -10,7 +10,7 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
10
11
12 @pytest.mark.parametrize("dtype", ["half", "float"])
13 -def test_clip_image_processor(hf_images, dtype):
13 +def test_clip_image_processor(image_assets, dtype):
14 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
15 IMAGE_HEIGHT = IMAGE_WIDTH = 560
16
@@ -35,13 +35,13 @@ def test_clip_image_processor(hf_images, dtype):
35 image_processor_revision=None,
36 )
37
38 - for image in hf_images:
38 + for asset in image_assets:
39 hf_result = hf_processor.preprocess(
40 - image,
40 + asset.pil_image,
41 return_tensors="pt",
42 ).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
43 vllm_result = MULTIMODAL_REGISTRY.process_input(
44 - ImagePixelData(image),
44 + ImagePixelData(asset.pil_image),
45 model_config=model_config,
46 vlm_config=vlm_config,
47 )
@@ -59,7 +59,7 @@ def test_clip_image_processor(hf_images, dtype):
59 reason="Inconsistent image processor being used due to lack "
60 "of support for dynamic image token replacement")
61 @pytest.mark.parametrize("dtype", ["half", "float"])
62 -def test_llava_next_image_processor(hf_images, dtype):
62 +def test_llava_next_image_processor(image_assets, dtype):
63 MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
64 IMAGE_HEIGHT = IMAGE_WIDTH = 560
65
@@ -84,13 +84,13 @@ def test_llava_next_image_processor(hf_images, dtype):
84 image_processor_revision=None,
85 )
86
87 - for image in hf_images:
87 + for asset in image_assets:
88 hf_result = hf_processor.preprocess(
89 - image,
89 + asset.pil_image,
90 return_tensors="pt",
91 ).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
92 vllm_result = MULTIMODAL_REGISTRY.process_input(
93 - ImagePixelData(image),
93 + ImagePixelData(asset.pil_image),
94 model_config=model_config,
95 vlm_config=vlm_config,
96 )
@@ -107,7 +107,7 @@ def test_llava_next_image_processor(hf_images, dtype):
107 @pytest.mark.xfail(
108 reason="Example image pixels were not processed using HuggingFace")
109 @pytest.mark.parametrize("dtype", ["float"])
110 -def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
110 +def test_image_pixel_types(image_assets, dtype):
111 MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
112 IMAGE_HEIGHT = IMAGE_WIDTH = 560
113
@@ -129,14 +129,14 @@ def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
129 image_processor_revision=None,
130 )
131
132 - for image, tensor in zip(hf_images, vllm_image_tensors):
132 + for asset in image_assets:
133 image_result = MULTIMODAL_REGISTRY.process_input(
134 - ImagePixelData(image),
134 + ImagePixelData(asset.pil_image),
135 model_config=model_config,
136 vlm_config=vlm_config,
137 )
138 tensor_result = MULTIMODAL_REGISTRY.process_input(
139 - ImagePixelData(tensor),
139 + ImagePixelData(asset.pixel_values),
140 model_config=model_config,
141 vlm_config=vlm_config,
142 )