| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Helpers for building inputs that can be leveraged for different test types.""" |
| 4 | |
| 5 | from collections.abc import Callable, Iterable |
| 6 | from pathlib import PosixPath |
| 7 | from typing import Any |
| 8 | |
| 9 | import numpy.typing as npt |
| 10 | import torch |
| 11 | |
| 12 | from vllm.multimodal.audio import AudioResampler |
| 13 | from vllm.multimodal.image import rescale_image_size |
| 14 | from vllm.multimodal.video import ( |
| 15 | rescale_video_size, |
| 16 | resize_video, |
| 17 | sample_frames_from_video, |
| 18 | ) |
| 19 | |
| 20 | from .....conftest import AudioTestAssets, ImageTestAssets, VideoTestAssets |
| 21 | from .types import ( |
| 22 | SINGLE_AUDIO_BASE_PROMPT, |
| 23 | SINGLE_IMAGE_BASE_PROMPTS, |
| 24 | TEST_AUDIO_PLACEHOLDER, |
| 25 | TEST_IMG_PLACEHOLDER, |
| 26 | TEST_VIDEO_PLACEHOLDER, |
| 27 | VIDEO_BASE_PROMPT, |
| 28 | ImageSizeWrapper, |
| 29 | PromptWithMultiModalInput, |
| 30 | SizeType, |
| 31 | VLMTestInfo, |
| 32 | ) |
| 33 | |
| 34 | |
| 35 | def replace_test_placeholder( |
| 36 | prompt: str, mm_idx_to_prompt: Callable[[int], str], test_placeholder: str |
| 37 | ) -> str: |
| 38 | """Given a prompt, replaces each test placeholder with the |
| 39 | model-specific tag. |
| 40 | """ |
| 41 | prompt_segments = prompt.split(test_placeholder) |
| 42 | img_prompt = prompt_segments[0] |
| 43 | for placeholder_idx, next_seg in enumerate(prompt_segments[1:], start=1): |
| 44 | img_prompt += mm_idx_to_prompt(placeholder_idx) |
| 45 | img_prompt += next_seg |
| 46 | return img_prompt |
| 47 | |
| 48 | |
| 49 | def get_model_prompts( |
| 50 | base_prompts: Iterable[str], |
| 51 | img_idx_to_prompt: Callable[[int], str] | None, |
| 52 | video_idx_to_prompt: Callable[[int], str] | None, |
| 53 | audio_idx_to_prompt: Callable[[int], str] | None, |
| 54 | prompt_formatter: Callable[[str], str], |
| 55 | ) -> list[str]: |
| 56 | """Given a model-agnostic base prompt and test configuration for a model(s) |
| 57 | to be tested, update the media placeholders and apply the prompt formatting |
| 58 | to get the test prompt string for this model. |
| 59 | |
| 60 | Example for phi3v, given the base_prompt: "<image>What is the season?" |
| 61 | 1. Replace img placeholder(s) |
| 62 | -> "<|image_1|>\nWhat is the season?" |
| 63 | 2. Apply prompt formatter: |
| 64 | -> <|user|>\n<|image_1|>\nWhat is the season?<|end|>\n<|assistant|>\n |
| 65 | """ |
| 66 | assert isinstance(base_prompts, (list, tuple)) |
| 67 | model_prompts = [] |
| 68 | for base_prompt in base_prompts: |
| 69 | # Replace the multimodal placeholders in the base prompt with |
| 70 | # the correct ones for the model that we are testing |
| 71 | if img_idx_to_prompt: |
| 72 | base_prompt = replace_test_placeholder( |
| 73 | base_prompt, img_idx_to_prompt, TEST_IMG_PLACEHOLDER |
| 74 | ) |
| 75 | |
| 76 | if video_idx_to_prompt: |
| 77 | base_prompt = replace_test_placeholder( |
| 78 | base_prompt, video_idx_to_prompt, TEST_VIDEO_PLACEHOLDER |
| 79 | ) |
| 80 | |
| 81 | if audio_idx_to_prompt: |
| 82 | base_prompt = replace_test_placeholder( |
| 83 | base_prompt, audio_idx_to_prompt, TEST_AUDIO_PLACEHOLDER |
| 84 | ) |
| 85 | |
| 86 | # Apply the prompt formatter to wrap the base prompt with |
| 87 | # the correct media placeholders to get the model test prompt |
| 88 | model_prompt = prompt_formatter(base_prompt) |
| 89 | model_prompts.append(model_prompt) |
| 90 | return model_prompts |
| 91 | |
| 92 | |
| 93 | def build_single_image_inputs_from_test_info( |
| 94 | test_info: VLMTestInfo, |
| 95 | image_assets: ImageTestAssets, |
| 96 | size_wrapper: ImageSizeWrapper, |
| 97 | tmp_path: PosixPath | None = None, |
| 98 | ) -> list[PromptWithMultiModalInput]: |
| 99 | if test_info.prompt_formatter is None: |
| 100 | raise ValueError("Prompt formatter must be set to build single image inputs") |
| 101 | |
| 102 | model_prompts = get_model_prompts( |
| 103 | test_info.single_image_prompts, |
| 104 | test_info.img_idx_to_prompt, |
| 105 | test_info.video_idx_to_prompt, |
| 106 | test_info.audio_idx_to_prompt, |
| 107 | test_info.prompt_formatter, |
| 108 | ) |
| 109 | |
| 110 | # For models that require a local path / URL encoded in the image; export |
| 111 | # assets and encode into tmp_path for this test. This should be avoided |
| 112 | # where possible (currently needed for Qwen-VL). |
| 113 | if test_info.prompt_path_encoder is not None: |
| 114 | if tmp_path is None: |
| 115 | raise ValueError("Prompt path encoder requires setting local path") |
| 116 | model_prompts = [ |
| 117 | test_info.prompt_path_encoder(tmp_path, prompt, [asset]) |
| 118 | for prompt, asset in zip(model_prompts, image_assets) |
| 119 | ] |
| 120 | |
| 121 | images = [asset.pil_image for asset in image_assets] |
| 122 | assert len(images) == len(model_prompts) |
| 123 | return build_single_image_inputs(images, model_prompts, size_wrapper) |
| 124 | |
| 125 | |
| 126 | def build_single_image_inputs( |
| 127 | images, model_prompts, size_wrapper: ImageSizeWrapper |
| 128 | ) -> list[PromptWithMultiModalInput]: |
| 129 | # For every image / prompt pair, get a pair containing two lists of |
| 130 | # length size_factors, where the first contains duplicates of the model |
| 131 | # prompt [str], and the second contains copies of the image after being |
| 132 | # scaled by one of the size factors. |
| 133 | # |
| 134 | # NOTE: rescaling preserves the image aspect ratio. |
| 135 | return [ |
| 136 | PromptWithMultiModalInput( |
| 137 | prompts=[prompt for _ in size_wrapper.data], |
| 138 | image_data=[ |
| 139 | apply_image_size_scaling(image, size, size_wrapper.type) |
| 140 | for size in size_wrapper.data |
| 141 | ], |
| 142 | ) |
| 143 | for image, prompt in zip(images, model_prompts) |
| 144 | ] |
| 145 | |
| 146 | |
| 147 | def build_multi_image_inputs_from_test_info( |
| 148 | test_info: VLMTestInfo, |
| 149 | image_assets: ImageTestAssets, |
| 150 | size_wrapper: ImageSizeWrapper, |
| 151 | tmp_path: PosixPath | None = None, |
| 152 | ) -> list[PromptWithMultiModalInput]: |
| 153 | if test_info.prompt_formatter is None: |
| 154 | raise ValueError("Prompt formatter must be set to build multi image inputs") |
| 155 | |
| 156 | model_prompts = get_model_prompts( |
| 157 | [test_info.multi_image_prompt], |
| 158 | test_info.img_idx_to_prompt, |
| 159 | test_info.video_idx_to_prompt, |
| 160 | test_info.audio_idx_to_prompt, |
| 161 | test_info.prompt_formatter, |
| 162 | ) |
| 163 | |
| 164 | if test_info.prompt_path_encoder is not None: |
| 165 | if tmp_path is None: |
| 166 | raise ValueError("Prompt path encoder requires setting local path") |
| 167 | model_prompts = [ |
| 168 | test_info.prompt_path_encoder(tmp_path, model_prompt, image_assets) |
| 169 | for model_prompt in model_prompts |
| 170 | ] |
| 171 | |
| 172 | images = [asset.pil_image for asset in image_assets] |
| 173 | |
| 174 | # Currently, we only have one multi-image list & one multi-image prompt |
| 175 | return build_multi_image_inputs( |
| 176 | image_lists=[images], |
| 177 | model_prompts=model_prompts, |
| 178 | size_wrapper=size_wrapper, |
| 179 | ) |
| 180 | |
| 181 | |
| 182 | def build_multi_image_inputs( |
| 183 | image_lists, model_prompts, size_wrapper: ImageSizeWrapper |
| 184 | ) -> list[PromptWithMultiModalInput]: |
| 185 | return [ |
| 186 | PromptWithMultiModalInput( |
| 187 | prompts=[prompt for _ in size_wrapper.data], |
| 188 | image_data=[ |
| 189 | [ |
| 190 | apply_image_size_scaling(image, size, size_wrapper.type) |
| 191 | for image in images |
| 192 | ] |
| 193 | for size in size_wrapper.data |
| 194 | ], |
| 195 | ) |
| 196 | for images, prompt in zip(image_lists, model_prompts) |
| 197 | ] |
| 198 | |
| 199 | |
| 200 | def build_embedding_inputs_from_test_info( |
| 201 | test_info: VLMTestInfo, |
| 202 | image_assets: ImageTestAssets, |
| 203 | size_wrapper: ImageSizeWrapper, |
| 204 | ): |
| 205 | # These conditions will always be true if invoked through filtering, |
| 206 | # but we still check them in case this is ever called directly |
| 207 | if test_info.prompt_formatter is None: |
| 208 | raise ValueError("Prompt formatter must be set to build image embedding inputs") |
| 209 | if size_wrapper.type != SizeType.SIZE_FACTOR or not all( |
| 210 | factor == 1.0 for factor in size_wrapper.data |
| 211 | ): |
| 212 | raise ValueError("Embedding tests require constant (1.0) size factors") |
| 213 | if test_info.convert_assets_to_embeddings is None: |
| 214 | raise ValueError("No conversion func for getting embeddings found") |
| 215 | |
| 216 | model_prompts = get_model_prompts( |
| 217 | SINGLE_IMAGE_BASE_PROMPTS, |
| 218 | test_info.img_idx_to_prompt, |
| 219 | test_info.video_idx_to_prompt, |
| 220 | test_info.audio_idx_to_prompt, |
| 221 | test_info.prompt_formatter, |
| 222 | ) |
| 223 | |
| 224 | images = [asset.pil_image for asset in image_assets] |
| 225 | embeds = test_info.convert_assets_to_embeddings(image_assets) |
| 226 | if test_info.dtype != "auto": |
| 227 | dtype = getattr(torch, test_info.dtype) # type: ignore |
| 228 | embeds = [e.to(dtype=dtype) for e in embeds] |
| 229 | assert len(images) == len(model_prompts) |
| 230 | |
| 231 | inputs = build_single_image_inputs(images, model_prompts, size_wrapper) |
| 232 | vllm_embeddings = build_single_image_inputs(embeds, model_prompts, size_wrapper) |
| 233 | return inputs, vllm_embeddings |
| 234 | |
| 235 | |
| 236 | def build_video_inputs_from_test_info( |
| 237 | test_info: VLMTestInfo, |
| 238 | video_assets: VideoTestAssets, |
| 239 | size_wrapper: ImageSizeWrapper, |
| 240 | num_frames: int, |
| 241 | needs_video_metadata: bool, |
| 242 | ) -> list[PromptWithMultiModalInput]: |
| 243 | if test_info.prompt_formatter is None: |
| 244 | raise ValueError("Prompt formatter must be set to build video inputs") |
| 245 | model_prompts = get_model_prompts( |
| 246 | [VIDEO_BASE_PROMPT], |
| 247 | test_info.img_idx_to_prompt, |
| 248 | test_info.video_idx_to_prompt, |
| 249 | test_info.audio_idx_to_prompt, |
| 250 | test_info.prompt_formatter, |
| 251 | ) |
| 252 | |
| 253 | sampled_vids = [ |
| 254 | sample_frames_with_video_metadata( |
| 255 | (asset.np_ndarrays, asset.metadata), |
| 256 | num_frames, |
| 257 | ) |
| 258 | for asset in video_assets |
| 259 | ] |
| 260 | |
| 261 | video_scaler = ( |
| 262 | resize_video if size_wrapper.type == SizeType.FIXED_SIZE else rescale_video_size |
| 263 | ) |
| 264 | |
| 265 | return [ |
| 266 | PromptWithMultiModalInput( |
| 267 | prompts=[prompt for _ in size_wrapper.data], |
| 268 | video_data=[ |
| 269 | ( |
| 270 | video_scaler(video, size) |
| 271 | if not needs_video_metadata |
| 272 | else (video_scaler(video, size), meta) |
| 273 | ) |
| 274 | for size in size_wrapper.data |
| 275 | ], |
| 276 | ) |
| 277 | for (video, meta), prompt in zip(sampled_vids, model_prompts) |
| 278 | ] |
| 279 | |
| 280 | |
| 281 | def sample_frames_with_video_metadata( |
| 282 | video_with_meta: tuple[npt.NDArray, dict[str, Any]], |
| 283 | num_frames: int, |
| 284 | ) -> tuple[npt.NDArray, dict[str, Any]]: |
| 285 | video, meta = video_with_meta |
| 286 | video = sample_frames_from_video(video, num_frames) |
| 287 | |
| 288 | meta["do_sample_frames"] = meta["total_num_frames"] == num_frames |
| 289 | meta["total_num_frames"] = num_frames |
| 290 | meta["fps"] = meta["duration"] / num_frames |
| 291 | meta["frames_indices"] = list(range(num_frames)) |
| 292 | return video, meta |
| 293 | |
| 294 | |
| 295 | def apply_image_size_scaling(image, size: float | tuple[int, int], size_type: SizeType): |
| 296 | """Applies a size scaler to one image; this can be an image size factor, |
| 297 | which scales the image while maintaining the aspect ratio""" |
| 298 | # Special case for embeddings; if it's a tensor, it's only valid if we |
| 299 | # are considering size factors at constant scale, i.e., we just clone |
| 300 | # the tensor |
| 301 | if isinstance(image, torch.Tensor): |
| 302 | assert size_type == SizeType.SIZE_FACTOR and size == 1 |
| 303 | return image |
| 304 | if size_type == SizeType.SIZE_FACTOR: |
| 305 | # We have a list of image size factors |
| 306 | return rescale_image_size(image, size) |
| 307 | elif size_type == SizeType.FIXED_SIZE: |
| 308 | # We have a list of fixed sizes |
| 309 | return image.resize(size) |
| 310 | raise ValueError("ImageSizeWrapper type must be FIXED_SIZE or SIZE_FACTOR") |
| 311 | |
| 312 | |
| 313 | def build_audio_inputs_from_test_info( |
| 314 | test_info: VLMTestInfo, |
| 315 | audio_assets: AudioTestAssets, |
| 316 | ) -> list[PromptWithMultiModalInput]: |
| 317 | if test_info.prompt_formatter is None: |
| 318 | raise ValueError("Prompt formatter must be set to build audio inputs") |
| 319 | model_prompts = get_model_prompts( |
| 320 | SINGLE_AUDIO_BASE_PROMPT, |
| 321 | test_info.img_idx_to_prompt, |
| 322 | test_info.video_idx_to_prompt, |
| 323 | test_info.audio_idx_to_prompt, |
| 324 | test_info.prompt_formatter, |
| 325 | ) |
| 326 | resampler = AudioResampler(target_sr=16000) |
| 327 | audios = [asset.audio_and_sample_rate for asset in audio_assets] |
| 328 | resampled_audios = [ |
| 329 | ( |
| 330 | resampler.resample( |
| 331 | audio, |
| 332 | orig_sr=sr, |
| 333 | ), |
| 334 | int(resampler.target_sr), |
| 335 | ) |
| 336 | for audio, sr in audios |
| 337 | ] |
| 338 | |
| 339 | return [ |
| 340 | PromptWithMultiModalInput( |
| 341 | prompts=model_prompts, |
| 342 | audio_data=resampled_audios, |
| 343 | ) |
| 344 | ] |