A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[CI/Build] Update vision tests (#5307)
Cyrus Leung committed
Jun 6, 2024 at 18:17 UTC
89c920785f97821fd81547048a82c7c2a4f7a2c2
6 files changed
+90
-88
.buildkite/test-pipeline.yaml
+2
-3
@@ -93,14 +93,13 @@ steps:
93
- label: Models Test
94
#mirror_hardwares: [amd]
95
commands:
96
- - bash ../.buildkite/download-images.sh
97
- - pytest -v -s models --ignore=models/test_llava.py
96
+ - pytest -v -s models -m \"not llava\"
97
98
- label: Llava Test
99
mirror_hardwares: [amd]
100
commands:
101
- bash ../.buildkite/download-images.sh
103
- - pytest -v -s models/test_llava.py
102
+ - pytest -v -s models -m llava
103
104
- label: Prefix Caching Test
105
mirror_hardwares: [amd]
pyproject.toml
+1
@@ -71,4 +71,5 @@ markers = [
71
"skip_global_cleanup",
72
"llm: run tests for vLLM API only",
73
"openai: run tests for OpenAI API only",
74
+ "llava: run tests for LLaVA models only",
75
]
tests/conftest.py
+8
-27
@@ -29,24 +29,19 @@ _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
29
30
# Multi modal related
31
# You can use `.buildkite/download-images.sh` to download the assets
32
-_PIXEL_VALUES_FILES = [
32
+PIXEL_VALUES_FILES = [
33
os.path.join(_TEST_DIR, "images", filename) for filename in
34
["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
35
]
36
-_IMAGE_FEATURES_FILES = [
36
+IMAGE_FEATURES_FILES = [
37
os.path.join(_TEST_DIR, "images", filename) for filename in
38
["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
39
]
40
-_IMAGE_FILES = [
40
+IMAGE_FILES = [
41
os.path.join(_TEST_DIR, "images", filename)
42
for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
43
]
44
-_IMAGE_PROMPTS = [
45
- "<image>\nUSER: What's the content of the image?\nASSISTANT:",
46
- "<image>\nUSER: What is the season?\nASSISTANT:"
47
-]
48
-assert len(_PIXEL_VALUES_FILES) == len(_IMAGE_FEATURES_FILES) == len(
49
- _IMAGE_FILES) == len(_IMAGE_PROMPTS)
44
+assert len(PIXEL_VALUES_FILES) == len(IMAGE_FEATURES_FILES) == len(IMAGE_FILES)
45
46
47
def _read_prompts(filename: str) -> List[str]:
@@ -84,14 +79,9 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
79
cleanup()
80
81
87
-@pytest.fixture(scope="session")
88
-def hf_image_prompts() -> List[str]:
89
- return _IMAGE_PROMPTS
90
-
91
-
82
@pytest.fixture(scope="session")
83
def hf_images() -> List[Image.Image]:
94
- return [Image.open(filename) for filename in _IMAGE_FILES]
84
+ return [Image.open(filename) for filename in IMAGE_FILES]
85
86
87
@pytest.fixture()
@@ -101,26 +91,17 @@ def vllm_images(request) -> List[MultiModalData]:
91
VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
92
return [
93
ImageFeatureData(torch.load(filename))
104
- for filename in _IMAGE_FEATURES_FILES
94
+ for filename in IMAGE_FEATURES_FILES
95
]
96
else:
97
return [
108
- ImagePixelData(Image.open(filename)) for filename in _IMAGE_FILES
98
+ ImagePixelData(Image.open(filename)) for filename in IMAGE_FILES
99
]
100
101
102
@pytest.fixture()
103
def vllm_image_tensors(request) -> List[torch.Tensor]:
114
- return [torch.load(filename) for filename in _PIXEL_VALUES_FILES]
115
-
116
-
117
-@pytest.fixture()
118
-def vllm_image_prompts(request) -> List[str]:
119
- vision_language_config = request.getfixturevalue("model_and_config")[1]
120
- return [
121
- "<image>" * (vision_language_config.image_feature_size - 1) + p
122
- for p in _IMAGE_PROMPTS
123
- ]
104
+ return [torch.load(filename) for filename in PIXEL_VALUES_FILES]
105
106
107
@pytest.fixture
tests/models/test_llava.py
+42
-55
@@ -1,14 +1,22 @@
1
-import gc
2
-from dataclasses import fields
3
-from enum import Enum
4
-from typing import Any, Dict, List, Tuple
1
+from typing import List, Tuple
2
3
import pytest
7
-import torch
4
from transformers import AutoTokenizer
5
6
from vllm.config import VisionLanguageConfig
7
8
+from ..conftest import IMAGE_FILES
9
+
10
+pytestmark = pytest.mark.llava
11
+
12
+# The image token is placed before "user" on purpose so that the test can pass
13
+HF_IMAGE_PROMPTS = [
14
+ "<image>\nUSER: What's the content of the image?\nASSISTANT:",
15
+ "<image>\nUSER: What is the season?\nASSISTANT:",
16
+]
17
+
18
+assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
19
+
20
21
def iter_llava_configs(model_name: str):
22
image_hw_to_feature_size = {
@@ -36,53 +44,35 @@ model_and_vl_config = [
44
]
45
46
39
-def as_dict(vlm_config: VisionLanguageConfig) -> Dict[str, Any]:
40
- """Flatten vision language config to pure args.
41
-
42
- Compatible with what llm entrypoint expects.
43
- """
44
- result = {}
45
- for field in fields(vlm_config):
46
- value = getattr(vlm_config, field.name)
47
- if isinstance(value, Enum):
48
- result[field.name] = value.name.lower()
49
- elif isinstance(value, tuple):
50
- result[field.name] = ",".join([str(item) for item in value])
51
- else:
52
- result[field.name] = value
53
-
54
- result["disable_image_processor"] = vlm_config.image_processor is None
55
-
56
- return result
57
-
58
-
59
-def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
60
- vision_language_config: VisionLanguageConfig,
61
- model_id: str):
47
+def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
48
+ vlm_config: VisionLanguageConfig, model_id: str):
49
"""Sanitize vllm output to be comparable with hf output.
50
The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
51
x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
52
It also reduces `output_str` from "<image><image>bla" to "bla".
53
"""
67
- tokenizer = AutoTokenizer.from_pretrained(model_id)
68
- image_token_str = tokenizer.decode(vision_language_config.image_token_id)
69
- image_token_str_len = len(image_token_str)
54
input_ids, output_str = vllm_output
71
- sanitized_input_ids = input_ids[0:2] + input_ids[2 + vision_language_config
72
- .image_feature_size - 1:]
73
- sanitzied_output_str = output_str[vision_language_config.
74
- image_feature_size *
75
- image_token_str_len:]
76
- return sanitized_input_ids, sanitzied_output_str
55
+ image_token_id = vlm_config.image_token_id
56
+
57
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
58
+ image_token_str = tokenizer.decode(image_token_id)
59
60
+ hf_input_ids = [
61
+ input_id for idx, input_id in enumerate(input_ids)
62
+ if input_id != image_token_id or input_ids[idx - 1] != image_token_id
63
+ ]
64
+ hf_output_str = output_str \
65
+ .replace(image_token_str * vlm_config.image_feature_size, "")
66
79
-@pytest.mark.parametrize("worker_use_ray", [False])
67
+ return hf_input_ids, hf_output_str
68
+
69
+
70
+# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
71
@pytest.mark.parametrize("model_and_config", model_and_vl_config)
72
@pytest.mark.parametrize("dtype", ["half"])
73
@pytest.mark.parametrize("max_tokens", [128])
83
-def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
84
- vllm_image_prompts, vllm_images, model_and_config, dtype: str,
85
- max_tokens: int, worker_use_ray: bool) -> None:
74
+def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
75
+ model_and_config, dtype: str, max_tokens: int) -> None:
76
"""Inference result should be the same between hf and vllm.
77
78
All the image fixtures for the test is under tests/images.
@@ -92,36 +82,33 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
82
Note, the text input is also adjusted to abide by vllm contract.
83
The text output is sanitized to be able to compare with hf.
84
"""
95
- model_id, vision_language_config = model_and_config
85
+ model_id, vlm_config = model_and_config
86
87
hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
98
- hf_outputs = hf_model.generate_greedy(hf_image_prompts,
88
+ hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89
max_tokens,
90
images=hf_images)
91
del hf_model
92
93
+ vllm_image_prompts = [
94
+ p.replace("<image>", "<image>" * vlm_config.image_feature_size)
95
+ for p in HF_IMAGE_PROMPTS
96
+ ]
97
+
98
vllm_model = vllm_runner(model_id,
99
dtype=dtype,
105
- worker_use_ray=worker_use_ray,
100
enforce_eager=True,
107
- **as_dict(vision_language_config))
101
+ **vlm_config.as_cli_args_dict())
102
vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
103
max_tokens,
104
images=vllm_images)
105
del vllm_model
106
113
- gc.collect()
114
- torch.cuda.empty_cache()
115
-
116
- for i in range(len(hf_image_prompts)):
107
+ for i in range(len(HF_IMAGE_PROMPTS)):
108
hf_output_ids, hf_output_str = hf_outputs[i]
118
- vllm_output_ids, vllm_output_str = sanitize_vllm_output(
119
- vllm_outputs[i], vision_language_config, model_id)
109
+ vllm_output_ids, vllm_output_str = vllm_to_hf_output(
110
+ vllm_outputs[i], vlm_config, model_id)
111
assert hf_output_str == vllm_output_str, (
112
f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
113
assert hf_output_ids == vllm_output_ids, (
114
f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
124
-
125
-
126
-# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
127
-# (Requires multiple GPUs)
vllm/config.py
+21
-1
@@ -1,7 +1,8 @@
1
import enum
2
import json
3
from dataclasses import dataclass, field, fields
4
-from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
4
+from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
5
+ Union)
6
7
import torch
8
from transformers import PretrainedConfig
@@ -1114,6 +1115,25 @@ class VisionLanguageConfig:
1115
f"Expecting to choose from "
1116
f"{[x.name for x in cls.ImageInputType]}.") from e
1117
1118
+ def as_cli_args_dict(self) -> Dict[str, Any]:
1119
+ """Flatten vision language config to pure args.
1120
+
1121
+ Compatible with what llm entrypoint expects.
1122
+ """
1123
+ result: Dict[str, Any] = {}
1124
+ for f in fields(self):
1125
+ value = getattr(self, f.name)
1126
+ if isinstance(value, enum.Enum):
1127
+ result[f.name] = value.name.lower()
1128
+ elif isinstance(value, tuple):
1129
+ result[f.name] = ",".join([str(item) for item in value])
1130
+ else:
1131
+ result[f.name] = value
1132
+
1133
+ result["disable_image_processor"] = self.image_processor is None
1134
+
1135
+ return result
1136
+
1137
1138
_STR_DTYPE_TO_TORCH_DTYPE = {
1139
"half": torch.float16,
vllm/multimodal/image.py
+16
-2
@@ -75,6 +75,14 @@ class ImagePixelData(MultiModalData):
75
76
self.image = image
77
78
+ def __repr__(self) -> str:
79
+ image = self.image
80
+ if isinstance(image, Image.Image):
81
+ return f"{type(self).__name__}(image={image})"
82
+
83
+ return (f"{type(self).__name__}(image=torch.Tensor(shape="
84
+ f"{image.shape}, dtype={image.dtype}))")
85
+
86
87
class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
88
@@ -96,10 +104,10 @@ class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
104
self, data: ImagePixelData, model_config: ModelConfig,
105
vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
106
image = data.image
99
- image_processor = self._get_hf_image_processor(model_config,
100
- vlm_config)
107
108
if isinstance(image, Image.Image):
109
+ image_processor = self._get_hf_image_processor(
110
+ model_config, vlm_config)
111
if image_processor is None:
112
raise RuntimeError("No HuggingFace processor is available"
113
"to process the image object")
@@ -127,6 +135,12 @@ class ImageFeatureData(MultiModalData):
135
def __init__(self, image_features: torch.Tensor) -> None:
136
self.image_features = image_features
137
138
+ def __repr__(self) -> str:
139
+ image_features = self.image_features
140
+
141
+ return (f"{type(self).__name__}(image_features=torch.Tensor(shape="
142
+ f"{image_features.shape}, dtype={image_features.dtype}))")
143
+
144
145
class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
146