A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[CI/Build] Refactor image test assets (#5821)
Cyrus Leung committed
Jun 26, 2024 at 16:02 UTC
6984c02a2735d4d08426d2c426c34b6d73bee89e
5 files changed
+127
-92
tests/conftest.py
+70
-41
@@ -1,7 +1,12 @@
1
import contextlib
2
import gc
3
import os
4
-from typing import Any, Dict, List, Optional, Tuple, TypeVar
4
+from collections import UserList
5
+from dataclasses import dataclass
6
+from functools import cached_property
7
+from pathlib import Path
8
+from typing import (Any, Dict, List, Literal, Optional, Tuple, TypedDict,
9
+ TypeVar)
10
11
import pytest
12
import torch
@@ -28,21 +33,8 @@ _TEST_DIR = os.path.dirname(__file__)
33
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
34
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
35
31
-# Multi modal related
32
-# You can use `.buildkite/download-images.sh` to download the assets
33
-PIXEL_VALUES_FILES = [
34
- os.path.join(_TEST_DIR, "images", filename) for filename in
35
- ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
36
-]
37
-IMAGE_FEATURES_FILES = [
38
- os.path.join(_TEST_DIR, "images", filename) for filename in
39
- ["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
40
-]
41
-IMAGE_FILES = [
42
- os.path.join(_TEST_DIR, "images", filename)
43
- for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
44
-]
45
-assert len(PIXEL_VALUES_FILES) == len(IMAGE_FEATURES_FILES) == len(IMAGE_FILES)
36
+_IMAGE_DIR = Path(_TEST_DIR) / "images"
37
+"""You can use `.buildkite/download-images.sh` to download the assets."""
38
39
40
def _read_prompts(filename: str) -> List[str]:
@@ -51,6 +43,63 @@ def _read_prompts(filename: str) -> List[str]:
43
return prompts
44
45
46
+@dataclass(frozen=True)
47
+class ImageAsset:
48
+ name: Literal["stop_sign", "cherry_blossom"]
49
+
50
+ @cached_property
51
+ def pixel_values(self) -> torch.Tensor:
52
+ return torch.load(_IMAGE_DIR / f"{self.name}_pixel_values.pt")
53
+
54
+ @cached_property
55
+ def image_features(self) -> torch.Tensor:
56
+ return torch.load(_IMAGE_DIR / f"{self.name}_image_features.pt")
57
+
58
+ @cached_property
59
+ def pil_image(self) -> Image.Image:
60
+ return Image.open(_IMAGE_DIR / f"{self.name}.jpg")
61
+
62
+ def for_hf(self) -> Image.Image:
63
+ return self.pil_image
64
+
65
+ def for_vllm(self, vision_config: VisionLanguageConfig) -> MultiModalData:
66
+ image_input_type = vision_config.image_input_type
67
+ ImageInputType = VisionLanguageConfig.ImageInputType
68
+
69
+ if image_input_type == ImageInputType.IMAGE_FEATURES:
70
+ return ImageFeatureData(self.image_features)
71
+ if image_input_type == ImageInputType.PIXEL_VALUES:
72
+ return ImagePixelData(self.pil_image)
73
+
74
+ raise NotImplementedError
75
+
76
+
77
+class _ImageAssetPrompts(TypedDict):
78
+ stop_sign: str
79
+ cherry_blossom: str
80
+
81
+
82
+class _ImageAssets(UserList[ImageAsset]):
83
+
84
+ def __init__(self) -> None:
85
+ super().__init__(
86
+ [ImageAsset("stop_sign"),
87
+ ImageAsset("cherry_blossom")])
88
+
89
+ def prompts(self, prompts: _ImageAssetPrompts) -> List[str]:
90
+ """
91
+ Convenience method to define the prompt for each test image.
92
+
93
+ The order of the returned prompts matches the order of the
94
+ assets when iterating through this object.
95
+ """
96
+ return [prompts["stop_sign"], prompts["cherry_blossom"]]
97
+
98
+
99
+IMAGE_ASSETS = _ImageAssets()
100
+"""Singleton instance of :class:`_ImageAssets`."""
101
+
102
+
103
def cleanup():
104
destroy_model_parallel()
105
destroy_distributed_environment()
@@ -81,31 +130,6 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
130
cleanup()
131
132
84
-@pytest.fixture(scope="session")
85
-def hf_images() -> List[Image.Image]:
86
- return [Image.open(filename) for filename in IMAGE_FILES]
87
-
88
-
89
-@pytest.fixture()
90
-def vllm_images(request) -> List[MultiModalData]:
91
- vision_language_config = request.getfixturevalue("model_and_config")[1]
92
- if vision_language_config.image_input_type == (
93
- VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
94
- return [
95
- ImageFeatureData(torch.load(filename))
96
- for filename in IMAGE_FEATURES_FILES
97
- ]
98
- else:
99
- return [
100
- ImagePixelData(Image.open(filename)) for filename in IMAGE_FILES
101
- ]
102
-
103
-
104
-@pytest.fixture()
105
-def vllm_image_tensors(request) -> List[torch.Tensor]:
106
- return [torch.load(filename) for filename in PIXEL_VALUES_FILES]
107
-
108
-
133
@pytest.fixture
134
def example_prompts() -> List[str]:
135
prompts = []
@@ -122,6 +146,11 @@ def example_long_prompts() -> List[str]:
146
return prompts
147
148
149
+@pytest.fixture(scope="session")
150
+def image_assets() -> _ImageAssets:
151
+ return IMAGE_ASSETS
152
+
153
+
154
_STR_DTYPE_TO_TORCH_DTYPE = {
155
"half": torch.half,
156
"bfloat16": torch.bfloat16,
tests/models/test_llava.py
+14
-12
@@ -5,17 +5,17 @@ from transformers import AutoTokenizer
5
6
from vllm.config import VisionLanguageConfig
7
8
-from ..conftest import IMAGE_FILES
8
+from ..conftest import IMAGE_ASSETS
9
10
pytestmark = pytest.mark.vlm
11
12
# The image token is placed before "user" on purpose so that the test can pass
13
-HF_IMAGE_PROMPTS = [
13
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
14
+ "stop_sign":
15
"<image>\nUSER: What's the content of the image?\nASSISTANT:",
16
+ "cherry_blossom":
17
"<image>\nUSER: What is the season?\nASSISTANT:",
16
-]
17
-
18
-assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
18
+})
19
20
21
def iter_llava_configs(model_name: str):
@@ -49,28 +49,28 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
49
x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
50
It also reduces `output_str` from "<image><image>bla" to "bla".
51
"""
52
- input_ids, output_str = vllm_output
52
+ output_ids, output_str = vllm_output
53
image_token_id = vlm_config.image_token_id
54
55
tokenizer = AutoTokenizer.from_pretrained(model_id)
56
image_token_str = tokenizer.decode(image_token_id)
57
58
- hf_input_ids = [
59
- input_id for idx, input_id in enumerate(input_ids)
60
- if input_id != image_token_id or input_ids[idx - 1] != image_token_id
58
+ hf_output_ids = [
59
+ token_id for idx, token_id in enumerate(output_ids)
60
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
61
]
62
hf_output_str = output_str \
63
.replace(image_token_str * vlm_config.image_feature_size, "")
64
65
- return hf_input_ids, hf_output_str
65
+ return hf_output_ids, hf_output_str
66
67
68
# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
69
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
70
@pytest.mark.parametrize("dtype", ["half"])
71
@pytest.mark.parametrize("max_tokens", [128])
72
-def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
73
- model_and_config, dtype: str, max_tokens: int) -> None:
72
+def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
73
+ dtype: str, max_tokens: int) -> None:
74
"""Inference result should be the same between hf and vllm.
75
76
All the image fixtures for the test is under tests/images.
@@ -81,6 +81,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
81
The text output is sanitized to be able to compare with hf.
82
"""
83
model_id, vlm_config = model_and_config
84
+ hf_images = [asset.for_hf() for asset in image_assets]
85
+ vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
86
87
with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
88
hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_llava_next.py
+16
-14
@@ -5,7 +5,7 @@ from transformers import AutoTokenizer
5
6
from vllm.config import VisionLanguageConfig
7
8
-from ..conftest import IMAGE_FILES
8
+from ..conftest import IMAGE_ASSETS
9
10
pytestmark = pytest.mark.vlm
11
@@ -15,12 +15,12 @@ _PREFACE = (
15
"questions.")
16
17
# The image token is placed before "user" on purpose so that the test can pass
18
-HF_IMAGE_PROMPTS = [
19
- f"{_PREFACE} <image>\nUSER: What's the content of the image? ASSISTANT:",
20
- f"{_PREFACE} <image>\nUSER: What is the season? ASSISTANT:",
21
-]
22
-
23
-assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
18
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19
+ "stop_sign":
20
+ f"{_PREFACE} <image>\nUSER: What's the content of the image?\nASSISTANT:",
21
+ "cherry_blossom":
22
+ f"{_PREFACE} <image>\nUSER: What is the season?\nASSISTANT:",
23
+})
24
25
26
def iter_llava_next_configs(model_name: str):
@@ -56,20 +56,20 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
56
x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
57
It also reduces `output_str` from "<image><image>bla" to "bla".
58
"""
59
- input_ids, output_str = vllm_output
59
+ output_ids, output_str = vllm_output
60
image_token_id = vlm_config.image_token_id
61
62
tokenizer = AutoTokenizer.from_pretrained(model_id)
63
image_token_str = tokenizer.decode(image_token_id)
64
65
- hf_input_ids = [
66
- input_id for idx, input_id in enumerate(input_ids)
67
- if input_id != image_token_id or input_ids[idx - 1] != image_token_id
65
+ hf_output_ids = [
66
+ token_id for idx, token_id in enumerate(output_ids)
67
+ if token_id != image_token_id or output_ids[idx - 1] != image_token_id
68
]
69
hf_output_str = output_str \
70
.replace(image_token_str * vlm_config.image_feature_size, " ")
71
72
- return hf_input_ids, hf_output_str
72
+ return hf_output_ids, hf_output_str
73
74
75
@pytest.mark.xfail(
@@ -78,8 +78,8 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
78
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
79
@pytest.mark.parametrize("dtype", ["half"])
80
@pytest.mark.parametrize("max_tokens", [128])
81
-def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
82
- model_and_config, dtype: str, max_tokens: int) -> None:
81
+def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
82
+ dtype: str, max_tokens: int) -> None:
83
"""Inference result should be the same between hf and vllm.
84
85
All the image fixtures for the test is under tests/images.
@@ -90,6 +90,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
90
The text output is sanitized to be able to compare with hf.
91
"""
92
model_id, vlm_config = model_and_config
93
+ hf_images = [asset.for_hf() for asset in image_assets]
94
+ vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
95
96
with hf_runner(model_id, dtype=dtype, is_vision_model=True) as hf_model:
97
hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
tests/models/test_phi3v.py
+15
-13
@@ -6,17 +6,17 @@ from transformers import AutoTokenizer
6
from vllm.config import VisionLanguageConfig
7
from vllm.utils import is_cpu
8
9
-from ..conftest import IMAGE_FILES
9
+from ..conftest import IMAGE_ASSETS
10
11
pytestmark = pytest.mark.vlm
12
13
# The image token is placed before "user" on purpose so that the test can pass
14
-HF_IMAGE_PROMPTS = [
14
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
15
+ "stop_sign":
16
"<|user|>\n<|image_1|>\nWhat's the content of the image?<|end|>\n<|assistant|>\n", # noqa: E501
16
- "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n",
17
-]
18
-
19
-assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
17
+ "cherry_blossom":
18
+ "<|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n", # noqa: E501
19
+})
20
21
22
def iter_phi3v_configs(model_name: str):
@@ -50,22 +50,22 @@ def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
50
x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
51
It also reduces `output_str` from "<image><image>bla" to "bla".
52
"""
53
- input_ids, output_str = vllm_output
53
+ output_ids, output_str = vllm_output
54
image_token_id = vlm_config.image_token_id
55
56
tokenizer = AutoTokenizer.from_pretrained(model_id)
57
image_token_str = tokenizer.decode(image_token_id)
58
59
- hf_input_ids = [
60
- input_id if input_id != image_token_id else 0
61
- for idx, input_id in enumerate(input_ids)
59
+ hf_output_ids = [
60
+ token_id if token_id != image_token_id else 0
61
+ for idx, token_id in enumerate(output_ids)
62
]
63
hf_output_str = output_str \
64
.replace(image_token_str * vlm_config.image_feature_size, "") \
65
.replace("<s>", " ").replace("<|user|>", "") \
66
.replace("<|end|>\n<|assistant|>", " ")
67
68
- return hf_input_ids, hf_output_str
68
+ return hf_output_ids, hf_output_str
69
70
71
target_dtype = "half"
@@ -82,8 +82,8 @@ if is_cpu():
82
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
83
@pytest.mark.parametrize("dtype", [target_dtype])
84
@pytest.mark.parametrize("max_tokens", [128])
85
-def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
86
- model_and_config, dtype: str, max_tokens: int) -> None:
85
+def test_models(hf_runner, vllm_runner, image_assets, model_and_config,
86
+ dtype: str, max_tokens: int) -> None:
87
"""Inference result should be the same between hf and vllm.
88
89
All the image fixtures for the test is under tests/images.
@@ -94,6 +94,8 @@ def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
94
The text output is sanitized to be able to compare with hf.
95
"""
96
model_id, vlm_config = model_and_config
97
+ hf_images = [asset.for_hf() for asset in image_assets]
98
+ vllm_images = [asset.for_vllm(vlm_config) for asset in image_assets]
99
100
# use eager mode for hf runner, since phi3_v didn't work with flash_attn
101
hf_model_kwargs = {"_attn_implementation": "eager"}
tests/multimodal/test_processor.py
+12
-12
@@ -10,7 +10,7 @@ from ..conftest import _STR_DTYPE_TO_TORCH_DTYPE
10
11
12
@pytest.mark.parametrize("dtype", ["half", "float"])
13
-def test_clip_image_processor(hf_images, dtype):
13
+def test_clip_image_processor(image_assets, dtype):
14
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
15
IMAGE_HEIGHT = IMAGE_WIDTH = 560
16
@@ -35,13 +35,13 @@ def test_clip_image_processor(hf_images, dtype):
35
image_processor_revision=None,
36
)
37
38
- for image in hf_images:
38
+ for asset in image_assets:
39
hf_result = hf_processor.preprocess(
40
- image,
40
+ asset.pil_image,
41
return_tensors="pt",
42
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
43
vllm_result = MULTIMODAL_REGISTRY.process_input(
44
- ImagePixelData(image),
44
+ ImagePixelData(asset.pil_image),
45
model_config=model_config,
46
vlm_config=vlm_config,
47
)
@@ -59,7 +59,7 @@ def test_clip_image_processor(hf_images, dtype):
59
reason="Inconsistent image processor being used due to lack "
60
"of support for dynamic image token replacement")
61
@pytest.mark.parametrize("dtype", ["half", "float"])
62
-def test_llava_next_image_processor(hf_images, dtype):
62
+def test_llava_next_image_processor(image_assets, dtype):
63
MODEL_NAME = "llava-hf/llava-v1.6-34b-hf"
64
IMAGE_HEIGHT = IMAGE_WIDTH = 560
65
@@ -84,13 +84,13 @@ def test_llava_next_image_processor(hf_images, dtype):
84
image_processor_revision=None,
85
)
86
87
- for image in hf_images:
87
+ for asset in image_assets:
88
hf_result = hf_processor.preprocess(
89
- image,
89
+ asset.pil_image,
90
return_tensors="pt",
91
).to(dtype=_STR_DTYPE_TO_TORCH_DTYPE[dtype])
92
vllm_result = MULTIMODAL_REGISTRY.process_input(
93
- ImagePixelData(image),
93
+ ImagePixelData(asset.pil_image),
94
model_config=model_config,
95
vlm_config=vlm_config,
96
)
@@ -107,7 +107,7 @@ def test_llava_next_image_processor(hf_images, dtype):
107
@pytest.mark.xfail(
108
reason="Example image pixels were not processed using HuggingFace")
109
@pytest.mark.parametrize("dtype", ["float"])
110
-def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
110
+def test_image_pixel_types(image_assets, dtype):
111
MODEL_NAME = "llava-hf/llava-1.5-7b-hf"
112
IMAGE_HEIGHT = IMAGE_WIDTH = 560
113
@@ -129,14 +129,14 @@ def test_image_pixel_types(hf_images, vllm_image_tensors, dtype):
129
image_processor_revision=None,
130
)
131
132
- for image, tensor in zip(hf_images, vllm_image_tensors):
132
+ for asset in image_assets:
133
image_result = MULTIMODAL_REGISTRY.process_input(
134
- ImagePixelData(image),
134
+ ImagePixelData(asset.pil_image),
135
model_config=model_config,
136
vlm_config=vlm_config,
137
)
138
tensor_result = MULTIMODAL_REGISTRY.process_input(
139
- ImagePixelData(tensor),
139
+ ImagePixelData(asset.pixel_values),
140
model_config=model_config,
141
vlm_config=vlm_config,
142
)