@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89c920785f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[CI/Build] Update vision tests (#5307)

Cyrus Leung committed Jun 6, 2024 at 18:17 UTC 89c920785f97821fd81547048a82c7c2a4f7a2c2
6 files changed +90 -88
.buildkite/test-pipeline.yaml
+2 -3
@@ -93,14 +93,13 @@ steps:
93 - label: Models Test
94 #mirror_hardwares: [amd]
95 commands:
96 - - bash ../.buildkite/download-images.sh
97 - - pytest -v -s models --ignore=models/test_llava.py
96 + - pytest -v -s models -m \"not llava\"
97
98 - label: Llava Test
99 mirror_hardwares: [amd]
100 commands:
101 - bash ../.buildkite/download-images.sh
103 - - pytest -v -s models/test_llava.py
102 + - pytest -v -s models -m llava
103
104 - label: Prefix Caching Test
105 mirror_hardwares: [amd]
pyproject.toml
+1
@@ -71,4 +71,5 @@ markers = [
71 "skip_global_cleanup",
72 "llm: run tests for vLLM API only",
73 "openai: run tests for OpenAI API only",
74 + "llava: run tests for LLaVA models only",
75 ]
tests/conftest.py
+8 -27
@@ -29,24 +29,19 @@ _LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
29
30 # Multi modal related
31 # You can use `.buildkite/download-images.sh` to download the assets
32 -_PIXEL_VALUES_FILES = [
32 +PIXEL_VALUES_FILES = [
33 os.path.join(_TEST_DIR, "images", filename) for filename in
34 ["stop_sign_pixel_values.pt", "cherry_blossom_pixel_values.pt"]
35 ]
36 -_IMAGE_FEATURES_FILES = [
36 +IMAGE_FEATURES_FILES = [
37 os.path.join(_TEST_DIR, "images", filename) for filename in
38 ["stop_sign_image_features.pt", "cherry_blossom_image_features.pt"]
39 ]
40 -_IMAGE_FILES = [
40 +IMAGE_FILES = [
41 os.path.join(_TEST_DIR, "images", filename)
42 for filename in ["stop_sign.jpg", "cherry_blossom.jpg"]
43 ]
44 -_IMAGE_PROMPTS = [
45 - "<image>\nUSER: What's the content of the image?\nASSISTANT:",
46 - "<image>\nUSER: What is the season?\nASSISTANT:"
47 -]
48 -assert len(_PIXEL_VALUES_FILES) == len(_IMAGE_FEATURES_FILES) == len(
49 - _IMAGE_FILES) == len(_IMAGE_PROMPTS)
44 +assert len(PIXEL_VALUES_FILES) == len(IMAGE_FEATURES_FILES) == len(IMAGE_FILES)
45
46
47 def _read_prompts(filename: str) -> List[str]:
@@ -84,14 +79,9 @@ def cleanup_fixture(should_do_global_cleanup_after_test: bool):
79 cleanup()
80
81
87 -@pytest.fixture(scope="session")
88 -def hf_image_prompts() -> List[str]:
89 - return _IMAGE_PROMPTS
90 -
91 -
82 @pytest.fixture(scope="session")
83 def hf_images() -> List[Image.Image]:
94 - return [Image.open(filename) for filename in _IMAGE_FILES]
84 + return [Image.open(filename) for filename in IMAGE_FILES]
85
86
87 @pytest.fixture()
@@ -101,26 +91,17 @@ def vllm_images(request) -> List[MultiModalData]:
91 VisionLanguageConfig.ImageInputType.IMAGE_FEATURES):
92 return [
93 ImageFeatureData(torch.load(filename))
104 - for filename in _IMAGE_FEATURES_FILES
94 + for filename in IMAGE_FEATURES_FILES
95 ]
96 else:
97 return [
108 - ImagePixelData(Image.open(filename)) for filename in _IMAGE_FILES
98 + ImagePixelData(Image.open(filename)) for filename in IMAGE_FILES
99 ]
100
101
102 @pytest.fixture()
103 def vllm_image_tensors(request) -> List[torch.Tensor]:
114 - return [torch.load(filename) for filename in _PIXEL_VALUES_FILES]
115 -
116 -
117 -@pytest.fixture()
118 -def vllm_image_prompts(request) -> List[str]:
119 - vision_language_config = request.getfixturevalue("model_and_config")[1]
120 - return [
121 - "<image>" * (vision_language_config.image_feature_size - 1) + p
122 - for p in _IMAGE_PROMPTS
123 - ]
104 + return [torch.load(filename) for filename in PIXEL_VALUES_FILES]
105
106
107 @pytest.fixture
tests/models/test_llava.py
+42 -55
@@ -1,14 +1,22 @@
1 -import gc
2 -from dataclasses import fields
3 -from enum import Enum
4 -from typing import Any, Dict, List, Tuple
1 +from typing import List, Tuple
2
3 import pytest
7 -import torch
4 from transformers import AutoTokenizer
5
6 from vllm.config import VisionLanguageConfig
7
8 +from ..conftest import IMAGE_FILES
9 +
10 +pytestmark = pytest.mark.llava
11 +
12 +# The image token is placed before "user" on purpose so that the test can pass
13 +HF_IMAGE_PROMPTS = [
14 + "<image>\nUSER: What's the content of the image?\nASSISTANT:",
15 + "<image>\nUSER: What is the season?\nASSISTANT:",
16 +]
17 +
18 +assert len(HF_IMAGE_PROMPTS) == len(IMAGE_FILES)
19 +
20
21 def iter_llava_configs(model_name: str):
22 image_hw_to_feature_size = {
@@ -36,53 +44,35 @@ model_and_vl_config = [
44 ]
45
46
39 -def as_dict(vlm_config: VisionLanguageConfig) -> Dict[str, Any]:
40 - """Flatten vision language config to pure args.
41 -
42 - Compatible with what llm entrypoint expects.
43 - """
44 - result = {}
45 - for field in fields(vlm_config):
46 - value = getattr(vlm_config, field.name)
47 - if isinstance(value, Enum):
48 - result[field.name] = value.name.lower()
49 - elif isinstance(value, tuple):
50 - result[field.name] = ",".join([str(item) for item in value])
51 - else:
52 - result[field.name] = value
53 -
54 - result["disable_image_processor"] = vlm_config.image_processor is None
55 -
56 - return result
57 -
58 -
59 -def sanitize_vllm_output(vllm_output: Tuple[List[int], str],
60 - vision_language_config: VisionLanguageConfig,
61 - model_id: str):
47 +def vllm_to_hf_output(vllm_output: Tuple[List[int], str],
48 + vlm_config: VisionLanguageConfig, model_id: str):
49 """Sanitize vllm output to be comparable with hf output.
50 The function reduces `input_ids` from 1, 32000, 32000, ..., 32000,
51 x1, x2, x3 ... to 1, 32000, x1, x2, x3 ...
52 It also reduces `output_str` from "<image><image>bla" to "bla".
53 """
67 - tokenizer = AutoTokenizer.from_pretrained(model_id)
68 - image_token_str = tokenizer.decode(vision_language_config.image_token_id)
69 - image_token_str_len = len(image_token_str)
54 input_ids, output_str = vllm_output
71 - sanitized_input_ids = input_ids[0:2] + input_ids[2 + vision_language_config
72 - .image_feature_size - 1:]
73 - sanitzied_output_str = output_str[vision_language_config.
74 - image_feature_size *
75 - image_token_str_len:]
76 - return sanitized_input_ids, sanitzied_output_str
55 + image_token_id = vlm_config.image_token_id
56 +
57 + tokenizer = AutoTokenizer.from_pretrained(model_id)
58 + image_token_str = tokenizer.decode(image_token_id)
59
60 + hf_input_ids = [
61 + input_id for idx, input_id in enumerate(input_ids)
62 + if input_id != image_token_id or input_ids[idx - 1] != image_token_id
63 + ]
64 + hf_output_str = output_str \
65 + .replace(image_token_str * vlm_config.image_feature_size, "")
66
79 -@pytest.mark.parametrize("worker_use_ray", [False])
67 + return hf_input_ids, hf_output_str
68 +
69 +
70 +# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
71 @pytest.mark.parametrize("model_and_config", model_and_vl_config)
72 @pytest.mark.parametrize("dtype", ["half"])
73 @pytest.mark.parametrize("max_tokens", [128])
83 -def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
84 - vllm_image_prompts, vllm_images, model_and_config, dtype: str,
85 - max_tokens: int, worker_use_ray: bool) -> None:
74 +def test_models(hf_runner, vllm_runner, hf_images, vllm_images,
75 + model_and_config, dtype: str, max_tokens: int) -> None:
76 """Inference result should be the same between hf and vllm.
77
78 All the image fixtures for the test is under tests/images.
@@ -92,36 +82,33 @@ def test_models(hf_runner, vllm_runner, hf_image_prompts, hf_images,
82 Note, the text input is also adjusted to abide by vllm contract.
83 The text output is sanitized to be able to compare with hf.
84 """
95 - model_id, vision_language_config = model_and_config
85 + model_id, vlm_config = model_and_config
86
87 hf_model = hf_runner(model_id, dtype=dtype, is_vision_model=True)
98 - hf_outputs = hf_model.generate_greedy(hf_image_prompts,
88 + hf_outputs = hf_model.generate_greedy(HF_IMAGE_PROMPTS,
89 max_tokens,
90 images=hf_images)
91 del hf_model
92
93 + vllm_image_prompts = [
94 + p.replace("<image>", "<image>" * vlm_config.image_feature_size)
95 + for p in HF_IMAGE_PROMPTS
96 + ]
97 +
98 vllm_model = vllm_runner(model_id,
99 dtype=dtype,
105 - worker_use_ray=worker_use_ray,
100 enforce_eager=True,
107 - **as_dict(vision_language_config))
101 + **vlm_config.as_cli_args_dict())
102 vllm_outputs = vllm_model.generate_greedy(vllm_image_prompts,
103 max_tokens,
104 images=vllm_images)
105 del vllm_model
106
113 - gc.collect()
114 - torch.cuda.empty_cache()
115 -
116 - for i in range(len(hf_image_prompts)):
107 + for i in range(len(HF_IMAGE_PROMPTS)):
108 hf_output_ids, hf_output_str = hf_outputs[i]
118 - vllm_output_ids, vllm_output_str = sanitize_vllm_output(
119 - vllm_outputs[i], vision_language_config, model_id)
109 + vllm_output_ids, vllm_output_str = vllm_to_hf_output(
110 + vllm_outputs[i], vlm_config, model_id)
111 assert hf_output_str == vllm_output_str, (
112 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
113 assert hf_output_ids == vllm_output_ids, (
114 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
124 -
125 -
126 -# TODO: Add test for `tensor_parallel_size` [ref: PR #3883]
127 -# (Requires multiple GPUs)
vllm/config.py
+21 -1
@@ -1,7 +1,8 @@
1 import enum
2 import json
3 from dataclasses import dataclass, field, fields
4 -from typing import TYPE_CHECKING, ClassVar, List, Optional, Tuple, Union
4 +from typing import (TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Tuple,
5 + Union)
6
7 import torch
8 from transformers import PretrainedConfig
@@ -1114,6 +1115,25 @@ class VisionLanguageConfig:
1115 f"Expecting to choose from "
1116 f"{[x.name for x in cls.ImageInputType]}.") from e
1117
1118 + def as_cli_args_dict(self) -> Dict[str, Any]:
1119 + """Flatten vision language config to pure args.
1120 +
1121 + Compatible with what llm entrypoint expects.
1122 + """
1123 + result: Dict[str, Any] = {}
1124 + for f in fields(self):
1125 + value = getattr(self, f.name)
1126 + if isinstance(value, enum.Enum):
1127 + result[f.name] = value.name.lower()
1128 + elif isinstance(value, tuple):
1129 + result[f.name] = ",".join([str(item) for item in value])
1130 + else:
1131 + result[f.name] = value
1132 +
1133 + result["disable_image_processor"] = self.image_processor is None
1134 +
1135 + return result
1136 +
1137
1138 _STR_DTYPE_TO_TORCH_DTYPE = {
1139 "half": torch.float16,
vllm/multimodal/image.py
+16 -2
@@ -75,6 +75,14 @@ class ImagePixelData(MultiModalData):
75
76 self.image = image
77
78 + def __repr__(self) -> str:
79 + image = self.image
80 + if isinstance(image, Image.Image):
81 + return f"{type(self).__name__}(image={image})"
82 +
83 + return (f"{type(self).__name__}(image=torch.Tensor(shape="
84 + f"{image.shape}, dtype={image.dtype}))")
85 +
86
87 class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
88
@@ -96,10 +104,10 @@ class ImagePixelPlugin(MultiModalPlugin[ImagePixelData]):
104 self, data: ImagePixelData, model_config: ModelConfig,
105 vlm_config: VisionLanguageConfig) -> Dict[str, torch.Tensor]:
106 image = data.image
99 - image_processor = self._get_hf_image_processor(model_config,
100 - vlm_config)
107
108 if isinstance(image, Image.Image):
109 + image_processor = self._get_hf_image_processor(
110 + model_config, vlm_config)
111 if image_processor is None:
112 raise RuntimeError("No HuggingFace processor is available"
113 "to process the image object")
@@ -127,6 +135,12 @@ class ImageFeatureData(MultiModalData):
135 def __init__(self, image_features: torch.Tensor) -> None:
136 self.image_features = image_features
137
138 + def __repr__(self) -> str:
139 + image_features = self.image_features
140 +
141 + return (f"{type(self).__name__}(image_features=torch.Tensor(shape="
142 + f"{image_features.shape}, dtype={image_features.dtype}))")
143 +
144
145 class ImageFeaturePlugin(MultiModalPlugin[ImageFeatureData]):
146