@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ef31eabc68
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model]: add some tests for aria model (#10770)

Signed-off-by: xffxff <1247714429@qq.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

zhou fan committed Dec 2, 2024 at 13:36 UTC ef31eabc68099ff2f64bbe5f42dc06101451a18d
4 files changed +51 -3
tests/conftest.py
+5 -1
@@ -656,6 +656,7 @@ class VllmRunner:
656 model_name: str,
657 task: TaskOption = "auto",
658 tokenizer_name: Optional[str] = None,
659 + tokenizer_mode: str = "auto",
660 # Use smaller max model length, otherwise bigger model cannot run due
661 # to kv cache size limit.
662 max_model_len: int = 1024,
@@ -672,6 +673,7 @@ class VllmRunner:
673 model=model_name,
674 task=task,
675 tokenizer=tokenizer_name,
676 + tokenizer_mode=tokenizer_mode,
677 trust_remote_code=True,
678 dtype=dtype,
679 swap_space=swap_space,
@@ -842,6 +844,7 @@ class VllmRunner:
844 audios: Optional[PromptAudioInput] = None,
845 videos: Optional[PromptVideoInput] = None,
846 stop_token_ids: Optional[List[int]] = None,
847 + stop: Optional[List[str]] = None,
848 ) -> Union[List[TokensTextLogprobs],
849 List[TokensTextLogprobsPromptLogprobs]]:
850 greedy_logprobs_params = SamplingParams(
@@ -849,7 +852,8 @@ class VllmRunner:
852 max_tokens=max_tokens,
853 logprobs=num_logprobs,
854 prompt_logprobs=num_prompt_logprobs,
852 - stop_token_ids=stop_token_ids)
855 + stop_token_ids=stop_token_ids,
856 + stop=stop)
857
858 return self.generate_w_logprobs(prompts,
859 greedy_logprobs_params,
tests/models/decoder_only/vision_language/test_models.py
+30
@@ -8,6 +8,7 @@ from typing import Type
8 import pytest
9 import transformers
10 from transformers import AutoModelForVision2Seq
11 +from transformers.utils import is_flash_attn_2_available
12
13 from vllm.platforms import current_platform
14 from vllm.utils import cuda_device_count_stateless, identity
@@ -134,6 +135,35 @@ VLM_TEST_SETTINGS = {
135 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
136 ),
137 #### Extended model tests
138 + "aria": VLMTestInfo(
139 + models=["rhymes-ai/Aria"],
140 + tokenizer_mode="slow",
141 + test_type=(
142 + VLMTestType.IMAGE,
143 + VLMTestType.MULTI_IMAGE,
144 + ),
145 + dtype="bfloat16",
146 + prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
147 + img_idx_to_prompt=lambda idx: "<fim_prefix><|img|><fim_suffix>\n",
148 + max_model_len=4096,
149 + max_num_seqs=2,
150 + single_image_prompts=IMAGE_ASSETS.prompts({
151 + "stop_sign": "<vlm_image>Please describe the image shortly.",
152 + "cherry_blossom": "<vlm_image>Please infer the season with reason.",
153 + }),
154 + multi_image_prompt="<vlm_image><vlm_image>Describe the two images shortly.", # noqa: E501
155 + postprocess_inputs=model_utils.get_key_type_post_processor("pixel_values"),
156 + stop_str=["<|im_end|>"],
157 + image_size_factors=[(0.10, 0.15)],
158 + max_tokens=64,
159 + marks=[
160 + pytest.mark.skipif(
161 + not is_flash_attn_2_available(),
162 + reason="Model needs flash-attn for numeric convergence.",
163 + ),
164 + large_gpu_mark(min_gb=64),
165 + ],
166 + ),
167 "blip2": VLMTestInfo(
168 models=["Salesforce/blip2-opt-2.7b"],
169 test_type=VLMTestType.IMAGE,
tests/models/decoder_only/vision_language/vlm_utils/core.py
+9 -2
@@ -29,6 +29,8 @@ def run_test(
29 postprocess_inputs: Callable[[BatchEncoding], BatchEncoding],
30 comparator: Callable[..., None],
31 get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]],
32 + stop_str: Optional[List[str]],
33 + tokenizer_mode: str,
34 limit_mm_per_prompt: Dict[str, int],
35 model_kwargs: Optional[Dict[str, Any]],
36 patch_hf_runner: Optional[Callable[[HfRunner], HfRunner]],
@@ -50,11 +52,14 @@ def run_test(
52 # vLLM needs a fresh new process without cuda initialization.
53 # if we run HF first, the cuda initialization will be done and it
54 # will hurt multiprocessing backend with fork method (the default method).
53 - vllm_kwargs = {}
55 + vllm_kwargs: Dict[str, Any] = {}
56 if get_stop_token_ids is not None:
57 vllm_kwargs["stop_token_ids"] = get_stop_token_ids(tokenizer)
58 + if stop_str:
59 + vllm_kwargs["stop"] = stop_str
60
61 with vllm_runner(model,
62 + tokenizer_mode=tokenizer_mode,
63 max_model_len=max_model_len,
64 max_num_seqs=max_num_seqs,
65 dtype=dtype,
@@ -85,6 +90,8 @@ def run_test(
90 hf_kwargs = {}
91 if use_tokenizer_eos:
92 hf_kwargs["eos_token_id"] = tokenizer.eos_token_id
93 + if stop_str:
94 + hf_kwargs["stop_strings"] = stop_str
95
96 with hf_model, torch.no_grad():
97 for prompts, media in inputs:
@@ -138,4 +145,4 @@ def process_runner_outputs(
145 def process_outputs(output_processor, model, outputs_per_image):
146 """Applies a model specific post-processor function to a runner's output"""
147 return [[output_processor(res, model) for res in outputs]
141 - for outputs in outputs_per_image]
148 + for outputs in outputs_per_image]
\ No newline at end of file
tests/models/decoder_only/vision_language/vlm_utils/types.py
+7
@@ -97,6 +97,9 @@ class VLMTestInfo(NamedTuple):
97
98 # Optional callable which gets a list of token IDs from the model tokenizer
99 get_stop_token_ids: Optional[Callable[[AutoTokenizer], List[int]]] = None
100 + # Optional list of strings to stop generation, useful when stop tokens are
101 + # not special tokens in the tokenizer
102 + stop_str: Optional[List[str]] = None
103
104 # Exposed options for HF runner
105 model_kwargs: Optional[Dict[str, Any]] = None
@@ -148,6 +151,8 @@ class VLMTestInfo(NamedTuple):
151
152 marks: Optional[List[MarkDecorator]] = None
153
154 + tokenizer_mode: str = "auto"
155 +
156 def get_non_parametrized_runner_kwargs(self):
157 """Returns a dictionary of expandable kwargs for items that are used
158 in all test types, which are NOT used when creating the parametrized
@@ -166,8 +171,10 @@ class VLMTestInfo(NamedTuple):
171 "postprocess_inputs": self.postprocess_inputs,
172 "comparator": self.comparator,
173 "get_stop_token_ids": self.get_stop_token_ids,
174 + "stop_str": self.stop_str,
175 "model_kwargs": self.model_kwargs,
176 "patch_hf_runner": self.patch_hf_runner,
177 + "tokenizer_mode": self.tokenizer_mode
178 }
179
180