| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import json |
| 5 | from collections.abc import Set as AbstractSet |
| 6 | from functools import partial |
| 7 | |
| 8 | import numpy as np |
| 9 | import pytest |
| 10 | import torch |
| 11 | from PIL import Image |
| 12 | |
| 13 | from tests.models.utils import build_model_context |
| 14 | from vllm.config import ModelConfig |
| 15 | from vllm.config.multimodal import ( |
| 16 | AudioDummyOptions, |
| 17 | BaseDummyOptions, |
| 18 | ImageDummyOptions, |
| 19 | MultiModalDummyOptions, |
| 20 | VideoDummyOptions, |
| 21 | ) |
| 22 | from vllm.distributed.ec_transfer.ec_connector.utils import ( |
| 23 | PlaceholderMetadataResolver, |
| 24 | collect_ec_item_metadata, |
| 25 | ) |
| 26 | from vllm.entrypoints.chat_utils import _get_embeds_data |
| 27 | from vllm.inputs import MultiModalDataDict, MultiModalInput |
| 28 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 29 | from vllm.multimodal.cache import MultiModalProcessorOnlyCache |
| 30 | from vllm.multimodal.inputs import MultiModalFeatureSpec, batched_tensors_equal |
| 31 | from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext |
| 32 | from vllm.platforms import current_platform |
| 33 | from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config |
| 34 | from vllm.utils.mistral import is_mistral_tokenizer |
| 35 | |
| 36 | from ....multimodal.utils import random_audio, random_image, random_video |
| 37 | from ...registry import ( |
| 38 | _MULTIMODAL_EXAMPLE_MODELS, |
| 39 | _TRANSFORMERS_BACKEND_MODELS, |
| 40 | HF_EXAMPLE_MODELS, |
| 41 | ) |
| 42 | |
| 43 | |
| 44 | def add_video_metadata(mm_data: MultiModalDataDict) -> MultiModalDataDict: |
| 45 | """Add metadata to video mm_data.""" |
| 46 | |
| 47 | def create_metadata(frames: np.ndarray): |
| 48 | num_frames = len(frames) |
| 49 | return { |
| 50 | "total_num_frames": num_frames, |
| 51 | "fps": 2.0, |
| 52 | "duration": num_frames / 2.0, |
| 53 | "video_backend": "opencv", |
| 54 | "frames_indices": list(range(num_frames)), |
| 55 | "do_sample_frames": True, |
| 56 | } |
| 57 | |
| 58 | # Ensure video metadata is included |
| 59 | if "video" in mm_data: |
| 60 | video = mm_data["video"] |
| 61 | if isinstance(video, list): |
| 62 | # multiple videos |
| 63 | mm_data["video"] = [(vid, create_metadata(vid)) for vid in video] |
| 64 | else: |
| 65 | # single video |
| 66 | mm_data["video"] = (video, create_metadata(video)) |
| 67 | return mm_data |
| 68 | |
| 69 | |
| 70 | def glmasr_patch_mm_data(mm_data: MultiModalDataDict) -> MultiModalDataDict: |
| 71 | """Patch the multimodal data for GLM-ASR model. |
| 72 | GLM-ASR requires text and audio to match 1:1, so we limit audio to 1. |
| 73 | """ |
| 74 | if "audio" in mm_data: |
| 75 | audio = mm_data["audio"] |
| 76 | if isinstance(audio, list) and len(audio) > 1: |
| 77 | # Limit to single audio to match text requirement |
| 78 | mm_data["audio"] = [audio[0]] |
| 79 | return mm_data |
| 80 | |
| 81 | |
| 82 | _IGNORE_MM_KEYS = { |
| 83 | # Dithering causes minor divergence |
| 84 | "cohere_asr": {"input_features"}, |
| 85 | # In Ultravox, the audio_features can be different depending on padding |
| 86 | # The slight difference should not be a problem though, since |
| 87 | # attention_mask lets us ignore the difference. |
| 88 | "ultravox": {"audio_features"}, |
| 89 | } |
| 90 | |
| 91 | MM_DATA_PATCHES = { |
| 92 | "glmasr": glmasr_patch_mm_data, |
| 93 | } |
| 94 | |
| 95 | _XPU_EXCLUDED_MODEL_IDS = { |
| 96 | "baidu/Unlimited-OCR", |
| 97 | "mistralai/Mistral-Large-3-675B-Instruct-2512-NVFP4", |
| 98 | "moonshotai/Kimi-K3", |
| 99 | "Qwen/Qwen2.5-Omni-7B-AWQ", |
| 100 | "thinkingmachines/Inkling-NVFP4", |
| 101 | } |
| 102 | |
| 103 | _CPU_EXCLUDED_MODEL_IDS = { |
| 104 | # DeepSeek-V4 vision variant is only supported on NVIDIA GPUs for now. |
| 105 | "deepseek-ai/DeepSeek-V4-Flash-Vision-Exp", |
| 106 | } |
| 107 | |
| 108 | |
| 109 | def _iter_model_ids_to_test(model_arch_list: AbstractSet[str]): |
| 110 | for model_arch in model_arch_list: |
| 111 | model_info = HF_EXAMPLE_MODELS.get_hf_info(model_arch) |
| 112 | yield model_info.default |
| 113 | |
| 114 | for extra_type, extra_model_id in model_info.extras.items(): |
| 115 | if "fp" in extra_type: |
| 116 | continue # Redundant to test quantized models |
| 117 | |
| 118 | yield extra_model_id |
| 119 | |
| 120 | |
| 121 | def _get_model_ids_to_test(model_arch_list: AbstractSet[str]): |
| 122 | model_ids = list(_iter_model_ids_to_test(model_arch_list)) |
| 123 | |
| 124 | if current_platform.is_xpu(): |
| 125 | for excluded_model_id in _XPU_EXCLUDED_MODEL_IDS: |
| 126 | while excluded_model_id in model_ids: |
| 127 | model_ids.remove(excluded_model_id) |
| 128 | |
| 129 | if current_platform.is_cpu(): |
| 130 | for excluded_model_id in _CPU_EXCLUDED_MODEL_IDS: |
| 131 | while excluded_model_id in model_ids: |
| 132 | model_ids.remove(excluded_model_id) |
| 133 | |
| 134 | return model_ids |
| 135 | |
| 136 | |
| 137 | def get_model_ids_to_test(): |
| 138 | transformers_arch_ids = { |
| 139 | model_id |
| 140 | for info in _TRANSFORMERS_BACKEND_MODELS.values() |
| 141 | for model_id in (info.default, *info.extras.values()) |
| 142 | } |
| 143 | vllm_only_archs = { |
| 144 | arch |
| 145 | for arch, info in _MULTIMODAL_EXAMPLE_MODELS.items() |
| 146 | if not any( |
| 147 | model_id in transformers_arch_ids |
| 148 | for model_id in (info.default, *info.extras.values()) |
| 149 | ) |
| 150 | } |
| 151 | |
| 152 | return _get_model_ids_to_test(vllm_only_archs) |
| 153 | |
| 154 | |
| 155 | def get_transformers_backend_model_ids_to_test(): |
| 156 | return sorted( |
| 157 | { |
| 158 | model_id |
| 159 | for arch, info in _TRANSFORMERS_BACKEND_MODELS.items() |
| 160 | if "MultiModal" in arch |
| 161 | for model_id in (info.default, *info.extras.values()) |
| 162 | } |
| 163 | ) |
| 164 | |
| 165 | |
| 166 | def get_token_prompt( |
| 167 | processor: BaseMultiModalProcessor, |
| 168 | mm_data: MultiModalDataDict, |
| 169 | ) -> list[int]: |
| 170 | tokenizer: TokenizerLike = processor.info.get_tokenizer() |
| 171 | model_config = processor.info.ctx.model_config |
| 172 | |
| 173 | if processor.info.data_parser.video_needs_metadata: |
| 174 | mm_data = add_video_metadata(mm_data) |
| 175 | |
| 176 | model_type = model_config.hf_config.model_type |
| 177 | if model_type in MM_DATA_PATCHES: |
| 178 | mm_data = MM_DATA_PATCHES[model_type](mm_data) |
| 179 | |
| 180 | parsed_data = processor.info.parse_mm_data(mm_data) |
| 181 | mm_counts = {k: len(vs) for k, vs in parsed_data.items()} |
| 182 | |
| 183 | if is_mistral_tokenizer(tokenizer): |
| 184 | inputs = processor.get_dummy_inputs( |
| 185 | model_config.max_model_len, |
| 186 | mm_counts, |
| 187 | mm_options=MultiModalDummyOptions(), |
| 188 | # Assume all Mistral models define this extra argument |
| 189 | mm_data=mm_data, # type: ignore[call-arg] |
| 190 | ) |
| 191 | else: |
| 192 | inputs = processor.get_dummy_inputs( |
| 193 | model_config.max_model_len, |
| 194 | mm_counts, |
| 195 | mm_options=MultiModalDummyOptions(), |
| 196 | ) |
| 197 | |
| 198 | if not isinstance(inputs.prompt, list): |
| 199 | raise TypeError(type(inputs.prompt)) |
| 200 | |
| 201 | return inputs.prompt |
| 202 | |
| 203 | |
| 204 | @pytest.mark.parametrize( |
| 205 | "model_id,durations", |
| 206 | [ |
| 207 | ("Qwen/Qwen2-Audio-7B-Instruct", [1.0, 2.3]), |
| 208 | ("nvidia/audio-flamingo-3-hf", [31.0]), |
| 209 | ("fixie-ai/ultravox-v0_5-llama-3_2-1b", [1.0, 31.0]), |
| 210 | ("Qwen/Qwen2.5-Omni-3B", [1.0, 2.3]), |
| 211 | ("Qwen/Qwen3-Omni-30B-A3B-Instruct", [1.0, 2.3]), |
| 212 | ], |
| 213 | ) |
| 214 | def test_audio_metadata_only_roundtrip(model_id, durations, monkeypatch): |
| 215 | """EC metadata preserves the raw-audio prompt without running the HF processor.""" |
| 216 | ctx = build_model_context(model_id, limit_mm_per_prompt={"audio": len(durations)}) |
| 217 | mm_config = ctx.model_config.multimodal_config |
| 218 | mm_config.enable_mm_embeds = True |
| 219 | mm_config.allow_missing_mm_embeddings = True |
| 220 | processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 221 | sample_rate = processor.info.get_feature_extractor().sampling_rate |
| 222 | audios = [ |
| 223 | np.zeros(int(duration * sample_rate), dtype=np.float32) |
| 224 | for duration in durations |
| 225 | ] |
| 226 | prompt = processor.dummy_inputs.get_dummy_text({"audio": len(audios)}) |
| 227 | raw = processor( |
| 228 | prompt, |
| 229 | mm_items=processor.info.parse_mm_data({"audio": audios}), |
| 230 | hf_processor_mm_kwargs={}, |
| 231 | ) |
| 232 | features = [ |
| 233 | MultiModalFeatureSpec( |
| 234 | data=item, |
| 235 | modality="audio", |
| 236 | identifier=str(index), |
| 237 | mm_position=position, |
| 238 | ) |
| 239 | for index, (item, position) in enumerate( |
| 240 | zip(raw["mm_kwargs"]["audio"], raw["mm_placeholders"]["audio"]) |
| 241 | ) |
| 242 | ] |
| 243 | published = collect_ec_item_metadata( |
| 244 | features, PlaceholderMetadataResolver(ctx.model_config) |
| 245 | ) |
| 246 | metadata = json.loads( |
| 247 | json.dumps([entry["metadata"] for entry in published.values()]) |
| 248 | ) |
| 249 | assert len(metadata) == len(audios) |
| 250 | assert all(metadata) |
| 251 | audio_data = _get_embeds_data("audio", metadata, processor) |
| 252 | |
| 253 | def unexpected_preprocessing(*args, **kwargs): |
| 254 | pytest.fail("The EC consumer must not preprocess audio") |
| 255 | |
| 256 | monkeypatch.setattr(processor, "_call_hf_processor", unexpected_preprocessing) |
| 257 | remote = processor( |
| 258 | prompt, |
| 259 | mm_items=processor.info.parse_mm_data({"audio": audio_data}), |
| 260 | hf_processor_mm_kwargs={}, |
| 261 | ) |
| 262 | assert remote["prompt_token_ids"] == raw["prompt_token_ids"] |
| 263 | assert remote["mm_placeholders"] == raw["mm_placeholders"] |
| 264 | # Omni also consumes these lengths when constructing M-RoPE positions. |
| 265 | if "audio_feature_lengths" in audio_data: |
| 266 | assert torch.equal( |
| 267 | remote["mm_kwargs"].get_data()["audio_feature_lengths"], |
| 268 | raw["mm_kwargs"].get_data()["audio_feature_lengths"], |
| 269 | ) |
| 270 | |
| 271 | mm_config.allow_missing_mm_embeddings = False |
| 272 | ordinary_processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) |
| 273 | with pytest.raises(ValueError, match="should contain the fields"): |
| 274 | ordinary_processor.info.parse_mm_data({"audio": audio_data}) |
| 275 | |
| 276 | |
| 277 | def random_vision_chunk( |
| 278 | rng: np.random.RandomState, |
| 279 | min_wh: int, |
| 280 | max_wh: int, |
| 281 | min_frames: int, |
| 282 | max_frames: int, |
| 283 | ) -> dict: |
| 284 | num_frames = rng.randint(min_frames, max_frames + 1) |
| 285 | if num_frames == 1: |
| 286 | # Single image chunk |
| 287 | wh = rng.randint(min_wh, max_wh + 1) |
| 288 | image = random_image(rng, wh, wh + 1) |
| 289 | return {"type": "image", "image": image} |
| 290 | frames = [] |
| 291 | for _ in range(num_frames): |
| 292 | wh = rng.randint(min_wh, max_wh + 1) |
| 293 | frame = rng.randint(0, 256, size=(wh, wh, 3), dtype=np.uint8) |
| 294 | frames.append(frame) |
| 295 | video_array = np.stack(frames, axis=0) |
| 296 | return {"type": "video_chunk", "video_chunk": video_array} |
| 297 | |
| 298 | |
| 299 | def _test_processing_correctness( |
| 300 | model_id_or_arch: str, |
| 301 | hit_rate: float, |
| 302 | num_batches: int, |
| 303 | simplify_rate: float, |
| 304 | model_impl: str = "auto", |
| 305 | ): |
| 306 | if model_id_or_arch in HF_EXAMPLE_MODELS.get_supported_archs(): |
| 307 | # Use model architecture to get the default model id |
| 308 | model_info = HF_EXAMPLE_MODELS.get_hf_info(model_id_or_arch) |
| 309 | model_id = model_info.default |
| 310 | else: |
| 311 | model_info = HF_EXAMPLE_MODELS.find_hf_info(model_id_or_arch) |
| 312 | model_id = model_id_or_arch |
| 313 | model_info.check_available_online(on_fail="skip") |
| 314 | model_info.check_transformers_version( |
| 315 | on_fail="skip", |
| 316 | check_max_version=False, |
| 317 | check_version_reason="vllm", |
| 318 | ) |
| 319 | |
| 320 | model_config = ModelConfig( |
| 321 | model_id, |
| 322 | tokenizer=model_info.tokenizer or model_id, |
| 323 | tokenizer_mode=model_info.tokenizer_mode, |
| 324 | revision=model_info.revision, |
| 325 | trust_remote_code=model_info.trust_remote_code, |
| 326 | hf_overrides=model_info.hf_overrides, |
| 327 | skip_tokenizer_init=model_info.require_embed_inputs, |
| 328 | enable_prompt_embeds=model_info.require_embed_inputs, |
| 329 | enable_mm_embeds=model_info.require_embed_inputs, |
| 330 | enforce_eager=model_info.enforce_eager, |
| 331 | dtype=model_info.dtype, |
| 332 | model_impl=model_impl, |
| 333 | ) |
| 334 | # Ensure that the cache can fit all of the data |
| 335 | # (set after because ModelConfig would set it to 0 for encoder-decoder models) |
| 336 | model_config.multimodal_config.mm_processor_cache_gb = 2048 |
| 337 | |
| 338 | model_cls = MULTIMODAL_REGISTRY._get_model_cls(model_config) |
| 339 | factories = model_cls._processor_factory |
| 340 | ctx = InputProcessingContext( |
| 341 | model_config, |
| 342 | tokenizer=cached_tokenizer_from_config(model_config), |
| 343 | ) |
| 344 | cache = MultiModalProcessorOnlyCache(model_config) |
| 345 | |
| 346 | processing_info = factories.info(ctx) |
| 347 | supported_mm_limits = processing_info.get_supported_mm_limits() |
| 348 | # Keep integer limits for local data generation |
| 349 | limit_mm_per_prompt_ints = { |
| 350 | modality: 3 if limit is None else limit |
| 351 | for modality, limit in supported_mm_limits.items() |
| 352 | } |
| 353 | |
| 354 | def _to_dummy_options(modality: str, count: int) -> BaseDummyOptions: |
| 355 | if modality == "video": |
| 356 | return VideoDummyOptions(count=count) |
| 357 | if modality == "image": |
| 358 | return ImageDummyOptions(count=count) |
| 359 | if modality == "audio": |
| 360 | return AudioDummyOptions(count=count) |
| 361 | return BaseDummyOptions(count=count) |
| 362 | |
| 363 | # Assign normalized DummyOptions to the model config |
| 364 | model_config.get_multimodal_config().limit_per_prompt = MultiModalDummyOptions( |
| 365 | { |
| 366 | modality: _to_dummy_options(modality, count) |
| 367 | for modality, count in limit_mm_per_prompt_ints.items() |
| 368 | } |
| 369 | ) |
| 370 | |
| 371 | processor = factories.build_processor(ctx) |
| 372 | |
| 373 | rng = np.random.RandomState(0) |
| 374 | |
| 375 | # GLM-ASR requires a minimum audio length of 70ms |
| 376 | min_audio_len = 512 if model_config.hf_config.model_type != "glmasr" else 1120 |
| 377 | input_to_hit = { |
| 378 | "image": Image.new("RGB", size=(128, 128)), |
| 379 | "video": np.zeros((4, 128, 128, 3), dtype=np.uint8), |
| 380 | "audio": (np.zeros((min_audio_len,)), 16000), |
| 381 | "vision_chunk": {"type": "image", "image": Image.new("RGB", size=(128, 128))}, |
| 382 | } |
| 383 | input_factory = { |
| 384 | "image": partial(random_image, rng, min_wh=128, max_wh=256), |
| 385 | "video": partial( |
| 386 | random_video, rng, min_frames=2, max_frames=16, min_wh=128, max_wh=256 |
| 387 | ), |
| 388 | "audio": partial( |
| 389 | random_audio, |
| 390 | rng, |
| 391 | min_len=min_audio_len, |
| 392 | max_len=min_audio_len + 512, |
| 393 | sr=16000, |
| 394 | ), |
| 395 | "vision_chunk": partial( |
| 396 | random_vision_chunk, rng, min_wh=128, max_wh=256, min_frames=1, max_frames=1 |
| 397 | ), |
| 398 | } |
| 399 | |
| 400 | for batch_idx in range(num_batches): |
| 401 | mm_data = { |
| 402 | k: [ |
| 403 | (input_to_hit[k] if rng.rand() < hit_rate else input_factory[k]()) |
| 404 | for _ in range(rng.randint(limit + 1)) |
| 405 | ] |
| 406 | for k, limit in limit_mm_per_prompt_ints.items() |
| 407 | } |
| 408 | |
| 409 | # Drop unnecessary keys and test single -> multi conversion |
| 410 | if rng.rand() < simplify_rate: |
| 411 | for k in list(mm_data.keys()): |
| 412 | if not mm_data[k]: |
| 413 | del mm_data[k] |
| 414 | elif len(mm_data[k]) == 1: |
| 415 | mm_data[k] = mm_data[k][0] |
| 416 | |
| 417 | _test_processing_correctness_one( |
| 418 | model_config, |
| 419 | mm_data, |
| 420 | processor, |
| 421 | batch_idx, |
| 422 | hit_rate, |
| 423 | num_batches, |
| 424 | simplify_rate, |
| 425 | cache=cache, |
| 426 | ) |
| 427 | |
| 428 | |
| 429 | def _test_processing_correctness_one( |
| 430 | model_config: ModelConfig, |
| 431 | mm_data: MultiModalDataDict, |
| 432 | processor: BaseMultiModalProcessor, |
| 433 | batch_idx: int, |
| 434 | hit_rate: float, |
| 435 | num_batches: int, |
| 436 | simplify_rate: float, |
| 437 | cache: MultiModalProcessorOnlyCache, |
| 438 | ): |
| 439 | model_type = model_config.hf_config.model_type |
| 440 | |
| 441 | token_prompt = get_token_prompt(processor, mm_data) |
| 442 | mm_items = processor.info.parse_mm_data(mm_data) |
| 443 | ignore_mm_keys = _IGNORE_MM_KEYS.get(model_type, set[str]()) |
| 444 | |
| 445 | baseline_tokenized_result = processor( |
| 446 | token_prompt, |
| 447 | mm_items=mm_items, |
| 448 | hf_processor_mm_kwargs={}, |
| 449 | ) |
| 450 | |
| 451 | cached_tokenized_result = processor( |
| 452 | token_prompt, |
| 453 | mm_items=mm_items, |
| 454 | hf_processor_mm_kwargs={}, |
| 455 | cache=cache, |
| 456 | ) |
| 457 | |
| 458 | _assert_inputs_equal( |
| 459 | baseline_tokenized_result, |
| 460 | cached_tokenized_result, |
| 461 | ignore_mm_keys=ignore_mm_keys, |
| 462 | msg=( |
| 463 | f"Failed ({batch_idx=}, {hit_rate=}, " |
| 464 | f"{num_batches=}, {simplify_rate=}, " |
| 465 | f"{token_prompt=}, {mm_data=})" |
| 466 | ), |
| 467 | ) |
| 468 | |
| 469 | |
| 470 | @pytest.mark.parametrize("model_id", get_model_ids_to_test()) |
| 471 | @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) |
| 472 | @pytest.mark.parametrize("num_batches", [32]) |
| 473 | @pytest.mark.parametrize("simplify_rate", [1.0]) |
| 474 | def test_processing_correctness( |
| 475 | model_id: str, |
| 476 | hit_rate: float, |
| 477 | num_batches: int, |
| 478 | simplify_rate: float, |
| 479 | ): |
| 480 | if model_id == "openvla/openvla-7b": |
| 481 | pytest.skip( |
| 482 | "OpenVLA uses a custom vLLM processor because its HF remote " |
| 483 | "processor is incompatible with current Transformers." |
| 484 | ) |
| 485 | if model_id == "mistralai/Voxtral-Mini-4B-Realtime-2602": |
| 486 | pytest.skip( |
| 487 | "Voxtral Realtime doesn't make use of any place-holder " |
| 488 | "tokens and hence cannot pass the processing " |
| 489 | "correctness test as is. Let's revisit adapting this " |
| 490 | "test once more realtime models exist." |
| 491 | ) |
| 492 | if model_id.startswith("OpenMOSS-Team/MOSS-Audio-"): |
| 493 | pytest.skip( |
| 494 | "MOSS-Audio uses a custom processor that dynamically expands " |
| 495 | "audio placeholders from processed audio lengths. Its vLLM " |
| 496 | "processor paths are covered by test_moss_audio.py." |
| 497 | ) |
| 498 | if model_id == "lmms-lab-encoder/LLaVA-OneVision-2-8B-Instruct": |
| 499 | pytest.skip( |
| 500 | "LLaVA-OneVision-2 video processing routes frames through custom " |
| 501 | "video backends (qwen_vl_utils / codec) that require real encoded " |
| 502 | "video bytes and metadata. The synthetic numpy-array videos used by " |
| 503 | "this test yield empty video features, so the generic correctness " |
| 504 | "check cannot exercise the video path. Image processing is covered " |
| 505 | "by registration/inference tests." |
| 506 | ) |
| 507 | |
| 508 | _test_processing_correctness( |
| 509 | model_id, |
| 510 | hit_rate=hit_rate, |
| 511 | num_batches=num_batches, |
| 512 | simplify_rate=simplify_rate, |
| 513 | ) |
| 514 | |
| 515 | |
| 516 | @pytest.mark.parametrize("model_id", get_transformers_backend_model_ids_to_test()) |
| 517 | @pytest.mark.parametrize("hit_rate", [0.3, 0.5, 1.0]) |
| 518 | @pytest.mark.parametrize("num_batches", [32]) |
| 519 | @pytest.mark.parametrize("simplify_rate", [1.0]) |
| 520 | def test_processing_correctness_transformers( |
| 521 | model_id: str, |
| 522 | hit_rate: float, |
| 523 | num_batches: int, |
| 524 | simplify_rate: float, |
| 525 | ): |
| 526 | _test_processing_correctness( |
| 527 | model_id, |
| 528 | hit_rate=hit_rate, |
| 529 | num_batches=num_batches, |
| 530 | simplify_rate=simplify_rate, |
| 531 | model_impl="transformers", |
| 532 | ) |
| 533 | |
| 534 | |
| 535 | def _assert_inputs_equal( |
| 536 | a: MultiModalInput, |
| 537 | b: MultiModalInput, |
| 538 | *, |
| 539 | ignore_mm_keys: set[str] | None = None, |
| 540 | msg: str = "", |
| 541 | ): |
| 542 | if ignore_mm_keys is None: |
| 543 | ignore_mm_keys = set() |
| 544 | |
| 545 | ignore_prompt_keys = ("prompt", "mm_kwargs") |
| 546 | a_rest = {k: v for k, v in a.items() if k not in ignore_prompt_keys} |
| 547 | b_rest = {k: v for k, v in b.items() if k not in ignore_prompt_keys} |
| 548 | |
| 549 | assert a_rest == b_rest, msg |
| 550 | |
| 551 | a_data = a["mm_kwargs"].get_data() |
| 552 | b_data = b["mm_kwargs"].get_data() |
| 553 | |
| 554 | for key in ignore_mm_keys: |
| 555 | a_data.pop(key, None) |
| 556 | b_data.pop(key, None) |
| 557 | |
| 558 | assert batched_tensors_equal(a_data, b_data), msg |