@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 158 lines 5.04 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """Custom input builders for edge-cases in different models."""
4
5 from collections.abc import Callable
6
7 from vllm.assets.image import ImageAsset
8 from vllm.multimodal.image import rescale_image_size
9 from vllm.multimodal.video import (
10 rescale_video_size,
11 resize_video,
12 sample_frames_from_video,
13 )
14
15 from .....conftest import IMAGE_ASSETS, VIDEO_ASSETS
16 from .builders import build_multi_image_inputs, build_single_image_inputs
17 from .types import ImageSizeWrapper, PromptWithMultiModalInput, SizeType
18
19
20 def multi_image_multi_aspect_ratio_inputs(formatter: Callable[[str], str]):
21 """Builds inputs for multi-image (varied sizes/aspect ratio) testing.
22
23 Args:
24 formatter: model-specific prompt formatter.
25
26 """
27 stop_sign = IMAGE_ASSETS[0].pil_image
28 cherry_blossom = IMAGE_ASSETS[1].pil_image
29
30 # Apply the selected formatter to the base prompts
31 img_prompts = [
32 "<image><image>\nDescribe 2 images.",
33 "<image><image>\nDescribe 2 images.",
34 "<image><image><image><image>\nDescribe 4 images.",
35 "<image>\nWhat is the season?",
36 ]
37 formatted_prompts = [formatter(prompt) for prompt in img_prompts]
38 aspect_ratio_images = [
39 [stop_sign, cherry_blossom],
40 # Images with different sizes and aspect-ratios
41 [
42 rescale_image_size(stop_sign, 0.1),
43 stop_sign,
44 ],
45 [
46 stop_sign,
47 rescale_image_size(stop_sign, 0.25),
48 cherry_blossom.resize((183, 488)),
49 cherry_blossom.resize((488, 183)),
50 ],
51 cherry_blossom,
52 ]
53
54 return [
55 PromptWithMultiModalInput(
56 prompts=formatted_prompts,
57 image_data=aspect_ratio_images,
58 )
59 ]
60
61
62 def multi_video_multi_aspect_ratio_inputs(
63 formatter: Callable[[str], str], num_frames: int = 16
64 ):
65 """Builds inputs for multi-video (varied sizes/aspect ratio) testing.
66
67 Args:
68 formatter: model-specific prompt formatter.
69
70 """
71 video = sample_frames_from_video(VIDEO_ASSETS[0].np_ndarrays, num_frames)
72 # Apply the selected formatter to the base prompts
73 video_prompts = [
74 "<video><video>\nDescribe 2 videos.",
75 "<video><video>\nDescribe 2 videos.",
76 "<video><video><video><video>\nDescribe 4 videos.",
77 "<video>\nWhy is this video funny?",
78 ]
79 formatted_prompts = [formatter(prompt) for prompt in video_prompts]
80 aspect_ratio_videos = [
81 [video, video],
82 # Videos with different sizes and aspect-ratios
83 [
84 rescale_video_size(video, 0.1),
85 video,
86 ],
87 [
88 video,
89 rescale_video_size(video, 0.25),
90 resize_video(video, (183, 488)),
91 resize_video(video, (488, 183)),
92 ],
93 video,
94 ]
95
96 return [
97 PromptWithMultiModalInput(
98 prompts=formatted_prompts,
99 video_data=aspect_ratio_videos,
100 )
101 ]
102
103
104 def different_patch_input_cases_internvl():
105 images = [asset.pil_image.resize((896, 896)) for asset in IMAGE_ASSETS]
106 formatter = (
107 lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n" # noqa: E501
108 )
109 single_img_prompts = [
110 "<image>\nWhat's the content in the center of the image?",
111 "<image>\nWhat is the season?",
112 ]
113 multi_img_prompts = [
114 "Image-1: <image>\nImage-2: <image>\nDescribe the two images in detail.\n", # noqa: E501
115 ]
116 formatted_sprompts = [formatter(prompt) for prompt in single_img_prompts]
117 formatted_mprompts = [formatter(prompt) for prompt in multi_img_prompts]
118
119 wrapped_sf = ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=[0.5, 1.0])
120 return [
121 build_single_image_inputs(images, formatted_sprompts, wrapped_sf),
122 build_multi_image_inputs([images], formatted_mprompts, wrapped_sf),
123 ]
124
125
126 def windows_attention_image_qwen2_5_vl():
127 # image from regression issue: https://github.com/vllm-project/vllm/issues/15122 # noqa: E501
128 image = ImageAsset("hato").pil_image
129
130 question = "Describe the image."
131 img_prompt = "<|vision_start|><|image_pad|><|vision_end|>"
132 prompt = (
133 f"<|im_start|>User\n{img_prompt}{question}<|im_end|>\n<|im_start|>assistant\n"
134 )
135
136 wrapped_sf = ImageSizeWrapper(type=SizeType.SIZE_FACTOR, data=[0.5])
137 return build_single_image_inputs([image], [prompt], wrapped_sf)
138
139
140 def video_with_metadata_glm4_1v():
141 video_array = VIDEO_ASSETS[0].np_ndarrays
142 metadata = VIDEO_ASSETS[0].metadata
143 question = "Describe the video."
144 video_prompt = "<|begin_of_video|><|video|><|end_of_video|>"
145 formatted_prompt = f"[gMASK]<|user|>\n{video_prompt}{question}<|assistant|>\n"
146
147 scales = [0.1, 0.2, 0.25]
148 video_input = [
149 [(rescale_video_size(video_array, scale), metadata)] for scale in scales
150 ]
151 prompts = [formatted_prompt] * len(video_input)
152
153 return [
154 PromptWithMultiModalInput(
155 prompts=prompts,
156 video_data=video_input,
157 )
158 ]