A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model]: add some tests for aria model (#10770)
Signed-off-by: xffxff <1247714429@qq.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
zhou fan committed
Dec 2, 2024 at 13:36 UTC
ef31eabc68099ff2f64bbe5f42dc06101451a18d
4 files changed
+51
-3
tests/conftest.py
+5
-1
@@ -656,6 +656,7 @@ class VllmRunner:
656
model_name: str,
657
task: TaskOption = "auto",
658
tokenizer_name: Optional[str] = None,
659
+ tokenizer_mode: str = "auto",
660
# Use smaller max model length, otherwise bigger model cannot run due
661
# to kv cache size limit.
662
max_model_len: int = 1024,
@@ -672,6 +673,7 @@ class VllmRunner:
673
model=model_name,
674
task=task,
675
tokenizer=tokenizer_name,
676
+ tokenizer_mode=tokenizer_mode,
677
trust_remote_code=True,
678
dtype=dtype,
679
swap_space=swap_space,
@@ -842,6 +844,7 @@ class VllmRunner:
844
audios: Optional[PromptAudioInput] = None,
845
videos: Optional[PromptVideoInput] = None,
846
stop_token_ids: Optional[List[int]] = None,
847
+ stop: Optional[List[str]] = None,
848
) -> Union[List[TokensTextLogprobs],
849
List[TokensTextLogprobsPromptLogprobs]]:
850
greedy_logprobs_params = SamplingParams(
@@ -849,7 +852,8 @@ class VllmRunner:
852
max_tokens=max_tokens,
853
logprobs=num_logprobs,
854
prompt_logprobs=num_prompt_logprobs,
852
- stop_token_ids=stop_token_ids)
855
+ stop_token_ids=stop_token_ids,
856
+ stop=stop)
857
858
return self.generate_w_logprobs(prompts,
859
greedy_logprobs_params,
tests/models/decoder_only/vision_language/test_models.py
+30
@@ -8,6 +8,7 @@ from typing import Type
8
import pytest
9
import transformers
10
from transformers import AutoModelForVision2Seq
11
+from transformers.utils import is_flash_attn_2_available
12
13
from vllm.platforms import current_platform
14
from vllm.utils import cuda_device_count_stateless, identity
@@ -134,6 +135,35 @@ VLM_TEST_SETTINGS = {
135
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
136
),
137
#### Extended model tests
138
+ "aria": VLMTestInfo(
139
+ models=["rhymes-ai/Aria"],
140
+ tokenizer_mode="slow",
141
+ test_type=(
142
+ VLMTestType.IMAGE,
143
+ VLMTestType.MULTI_IMAGE,
144
+ ),
145
+ dtype="bfloat16",
146
+ prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
147
+ img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
148
+ max_model_len=4096,
149
+ max_num_seqs=2,
150
+ single_image_prompts=IMAGE_ASSETS.prompts({
151
+ "stop_sign": "<vlm_image>Please describe the image shortly.",
152
+ "cherry_blossom": "<vlm_image>Please infer the season with reason.",
153
+ }),
154
+ multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
155
+ postprocess_inputs=model_utils.get_key_type_post_processor("pixel_values"),
156
+ stop_str=["<|im_end|>"],
157
+ image_size_factors=[(0.10, 0.15)],
158
+ max_tokens=64,
159
+ marks=[
160
+ pytest.mark.skipif(
161
+ not is_flash_attn_2_available(),
162
+ reason="Model needs flash-attn for numeric convergence.",
163
+ ),
164
+ large_gpu_mark(min_gb=64),
165
+ ],
166
+ ),
167
"blip2": VLMTestInfo(
168
models=["Salesforce/blip2-opt-2.7b"],
169
test_type=VLMTestType.IMAGE,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+9
-2
@@ -29,6 +29,8 @@ def run_test(
29
postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
30
comparator: Callable[..., None],
31
get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]],
32
+ stop_str: Optional[List[str]],
33
+ tokenizer_mode: str,
34
limit_mm_per_prompt: Dict[str, int],
35
model_kwargs: Optional[Dict[str, Any]],
36
patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
@@ -50,11 +52,14 @@ def run_test(
52
# vLLM needs a fresh new process without cuda initialization.
53
# if we run HF first, the cuda initialization will be done and it
54
# will hurt multiprocessing backend with fork method (the default method).
53
- vllm_kwargs = {}
55
+ vllm_kwargs: Dict[str, Any] = {}
56
if get_stop_token_ids is not None:
57
vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
58
+ if stop_str:
59
+ vllm_kwargs["stop"] = stop_str
60
61
with vllm_runner(model,
62
+ tokenizer_mode=tokenizer_mode,
63
max_model_len=max_model_len,
64
max_num_seqs=max_num_seqs,
65
dtype=dtype,
@@ -85,6 +90,8 @@ def run_test(
90
hf_kwargs = {}
91
if use_tokenizer_eos:
92
hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
93
+ if stop_str:
94
+ hf_kwargs["stop_strings"] = stop_str
95
96
with hf_model, torch.no_grad():
97
for prompts, media in inputs:
@@ -138,4 +145,4 @@ def process_runner_outputs(
145
def process_outputs(output_processor, model, outputs_per_image):
146
"""Applies a model specific post-processor function to a runner's output"""
147
return [[output_processor(res, model) for res in outputs]
141
- for outputs in outputs_per_image]
148
+ for outputs in outputs_per_image]
\ No newline at end of file
tests/models/decoder_only/vision_language/vlm_utils/types.py
+7
@@ -97,6 +97,9 @@ class VLMTestInfo(NamedTuple):
97
98
# Optional callable which gets a list of token IDs from the model tokenizer
99
get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]] = None
100
+ # Optional list of strings to stop generation, useful when stop tokens are
101
+ # not special tokens in the tokenizer
102
+ stop_str: Optional[List[str]] = None
103
104
# Exposed options for HF runner
105
model_kwargs: Optional[Dict[str, Any]] = None
@@ -148,6 +151,8 @@ class VLMTestInfo(NamedTuple):
151
152
marks: Optional[List[MarkDecorator]] = None
153
154
+ tokenizer_mode: str = "auto"
155
+
156
def get_non_parametrized_runner_kwargs(self):
157
"""Returns a dictionary of expandable kwargs for items that are used
158
in all test types, which are NOT used when creating the parametrized
@@ -166,8 +171,10 @@ class VLMTestInfo(NamedTuple):
171
"postprocess_inputs": self.postprocess_inputs,
172
"comparator": self.comparator,
173
"get_stop_token_ids": self.get_stop_token_ids,
174
+ "stop_str": self.stop_str,
175
"model_kwargs": self.model_kwargs,
176
"patch_hf_runner": self.patch_hf_runner,
177
+ "tokenizer_mode": self.tokenizer_mode
178
}
179
180