@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 261 lines 8.09 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4 import pytest
5
6 from vllm.assets.image import ImageAsset
7 from vllm.platforms import current_platform
8
9 from ....conftest import VIDEO_ASSETS
10 from .vlm_utils.builders import sample_frames_with_video_metadata
11
12 models = ["Qwen/Qwen2.5-VL-3B-Instruct"]
13 target_dtype = "bfloat16"
14
15 VIDEO_PLACEHOLDER = "<|vision_start|><|video_pad|><|vision_end|>"
16 IMAGE_PLACEHOLDER = "<|vision_start|><|image_pad|><|vision_end|>"
17
18
19 def qwen2_5_vl_chat_template(*query):
20 return f"<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\n{''.join(query)}<|im_end|><|im_start|>assistant\n" # noqa: E501
21
22
23 VIDEO_PROMPTS = VIDEO_ASSETS.prompts(
24 {
25 "baby_reading": qwen2_5_vl_chat_template(
26 VIDEO_PLACEHOLDER,
27 "Describe this video with a short sentence ",
28 "(no more than 20 words)",
29 ),
30 }
31 )
32
33
34 WINDOW_ATTN_IMAGE_PROMPT = qwen2_5_vl_chat_template(
35 IMAGE_PLACEHOLDER,
36 "Describe the image.",
37 )
38 IMAGE_ONLY_LIMIT_MM_PER_PROMPT = {"image": 1, "video": 0}
39
40
41 def _window_attention_regression_image():
42 # image from regression issue: https://github.com/vllm-project/vllm/issues/15122
43 image = ImageAsset("hato").pil_image
44 return image.resize((image.width // 2, image.height // 2))
45
46
47 def _encoder_cudagraph_config(*, max_vision_items: int) -> dict:
48 return {
49 "cudagraph_mm_encoder": True,
50 "encoder_cudagraph_max_vision_items_per_batch": max_vision_items,
51 }
52
53
54 @pytest.mark.core_model
55 @pytest.mark.parametrize("model", models)
56 @pytest.mark.parametrize(
57 "video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
58 )
59 @pytest.mark.parametrize("num_frames", [16])
60 @pytest.mark.parametrize("dtype", [target_dtype])
61 @pytest.mark.parametrize("max_tokens", [128])
62 @pytest.mark.parametrize(
63 "use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
64 )
65 def test_qwen2_5_vl_evs_functionality(
66 vllm_runner,
67 video_assets,
68 model,
69 video_pruning_rate: float,
70 num_frames: int,
71 dtype: str,
72 max_tokens: int,
73 use_bytecode_hook: bool,
74 monkeypatch,
75 ) -> None:
76 """Test EVS (Efficient Video Sampling) functionality with different
77 pruning rates.
78 """
79 # Set the environment variable for this test
80 monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
81
82 # Sample frames from video assets
83 sampled_vids = [
84 sample_frames_with_video_metadata(
85 (asset.np_ndarrays, asset.metadata), num_frames
86 )
87 for asset in video_assets
88 ]
89
90 prompts = [VIDEO_PROMPTS[0]]
91 videos = [sampled_vids[0]]
92
93 # Initialize model with EVS configuration
94 with vllm_runner(
95 model,
96 runner="generate",
97 max_model_len=4000,
98 dtype=dtype,
99 limit_mm_per_prompt={"video": 1},
100 video_pruning_rate=video_pruning_rate,
101 ) as vllm_model:
102 # Generate output - this should not crash
103 outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
104
105 # Basic validation that we got a response
106 assert len(outputs) == 1
107 output_ids, output_text = outputs[0]
108
109 # Ensure we got some output
110 assert len(output_ids) > 0
111 assert len(output_text) > 0
112
113 # Ensure the output is a string
114 assert isinstance(output_text, str)
115
116
117 @pytest.mark.core_model
118 @pytest.mark.parametrize("model", models)
119 @pytest.mark.parametrize(
120 "video_pruning_rate", [0.0] if current_platform.is_cpu() else [0.0, 0.75]
121 )
122 @pytest.mark.parametrize("num_frames", [16])
123 @pytest.mark.parametrize("dtype", [target_dtype])
124 @pytest.mark.parametrize("max_tokens", [128])
125 @pytest.mark.parametrize(
126 "use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
127 )
128 def test_qwen2_5_vl_evs_batched_videos(
129 vllm_runner,
130 video_assets,
131 model,
132 video_pruning_rate: float,
133 num_frames: int,
134 dtype: str,
135 max_tokens: int,
136 use_bytecode_hook: bool,
137 monkeypatch,
138 ) -> None:
139 """Test EVS functionality with batched videos.
140
141 This test validates that:
142 1. The model handles batched video inputs correctly with EVS
143 2. Both pruning configurations work with multiple videos
144 3. The model doesn't crash when processing multiple videos simultaneously
145 """
146 # Set the environment variable for this test
147 monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
148 # Sample frames from video assets
149 sampled_vids = [
150 sample_frames_with_video_metadata(
151 (asset.np_ndarrays, asset.metadata), num_frames
152 )
153 for asset in video_assets
154 ]
155
156 # Test batched videos
157 prompts = [VIDEO_PROMPTS[0], VIDEO_PROMPTS[0]]
158 videos = [sampled_vids[0], sampled_vids[0]] # Use same video twice for testing
159
160 # Initialize model with EVS configuration
161 with vllm_runner(
162 model,
163 runner="generate",
164 max_model_len=4000,
165 max_num_seqs=2,
166 dtype=dtype,
167 limit_mm_per_prompt={"video": 2},
168 tensor_parallel_size=1,
169 video_pruning_rate=video_pruning_rate,
170 ) as vllm_model:
171 # Generate output - this should not crash
172 outputs = vllm_model.generate_greedy(prompts, max_tokens, videos=videos)
173
174 # Basic validation that we got responses for both videos
175 assert len(outputs) == 2
176
177 for output_ids, output_text in outputs:
178 # Ensure we got some output for each video
179 assert len(output_ids) > 0
180 assert len(output_text) > 0
181
182 # Ensure the output is a string
183 assert isinstance(output_text, str)
184
185
186 @pytest.mark.core_model
187 @pytest.mark.parametrize("model", models)
188 @pytest.mark.parametrize("dtype", [target_dtype])
189 @pytest.mark.parametrize("max_tokens", [128])
190 @pytest.mark.parametrize(
191 "use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
192 )
193 def test_qwen2_5_vl_window_attention_image(
194 vllm_runner,
195 model,
196 dtype: str,
197 max_tokens: int,
198 use_bytecode_hook: bool,
199 monkeypatch,
200 ) -> None:
201 """Regression test for Qwen2.5 window-attention image path."""
202 monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
203
204 prompt = [WINDOW_ATTN_IMAGE_PROMPT]
205 images = [[_window_attention_regression_image()]]
206
207 with vllm_runner(
208 model,
209 runner="generate",
210 max_model_len=4096,
211 dtype=dtype,
212 limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
213 compilation_config=_encoder_cudagraph_config(max_vision_items=1),
214 ) as vllm_model:
215 outputs = vllm_model.generate_greedy(prompt, max_tokens, images=images)
216
217 assert len(outputs) == 1
218 output_ids, output_text = outputs[0]
219 assert len(output_ids) > 0
220 assert len(output_text) > 0
221 assert isinstance(output_text, str)
222
223
224 @pytest.mark.core_model
225 @pytest.mark.parametrize("model", models)
226 @pytest.mark.parametrize("dtype", [target_dtype])
227 @pytest.mark.parametrize("max_tokens", [128])
228 @pytest.mark.parametrize(
229 "use_bytecode_hook", [True] if current_platform.is_cpu() else [True, False]
230 )
231 def test_qwen2_5_vl_window_attention_image_batch(
232 vllm_runner,
233 model,
234 dtype: str,
235 max_tokens: int,
236 use_bytecode_hook: bool,
237 monkeypatch,
238 ) -> None:
239 """Regression test window-attention with a small image batch."""
240 monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1" if use_bytecode_hook else "0")
241
242 image = _window_attention_regression_image()
243 prompts = [WINDOW_ATTN_IMAGE_PROMPT, WINDOW_ATTN_IMAGE_PROMPT]
244 images = [[image], [image]]
245
246 with vllm_runner(
247 model,
248 runner="generate",
249 max_model_len=4096,
250 max_num_seqs=2,
251 dtype=dtype,
252 limit_mm_per_prompt=IMAGE_ONLY_LIMIT_MM_PER_PROMPT,
253 compilation_config=_encoder_cudagraph_config(max_vision_items=2),
254 ) as vllm_model:
255 outputs = vllm_model.generate_greedy(prompts, max_tokens, images=images)
256
257 assert len(outputs) == 2
258 for output_ids, output_text in outputs:
259 assert len(output_ids) > 0
260 assert len(output_text) > 0
261 assert isinstance(output_text, str)