| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from vllm.assets.video import VideoAsset |
| 7 | from vllm.config import ModelConfig, VllmConfig |
| 8 | from vllm.multimodal.utils import encode_video_url |
| 9 | from vllm.renderers.hf import HfRenderer |
| 10 | from vllm.renderers.params import ChatParams |
| 11 | from vllm.tokenizers.registry import cached_tokenizer_from_config |
| 12 | |
| 13 | pytestmark = pytest.mark.cpu_test |
| 14 | |
| 15 | video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays) |
| 16 | |
| 17 | DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}} |
| 18 | DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}} |
| 19 | |
| 20 | |
| 21 | def _build_video_renderer() -> HfRenderer: |
| 22 | model_config = ModelConfig( |
| 23 | model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf", |
| 24 | max_model_len=4096, |
| 25 | mm_processor_cache_gb=4.0, |
| 26 | ) |
| 27 | |
| 28 | return HfRenderer( |
| 29 | VllmConfig(model_config=model_config), |
| 30 | cached_tokenizer_from_config(model_config), |
| 31 | ) |
| 32 | |
| 33 | |
| 34 | def _get_message_hash( |
| 35 | renderer: HfRenderer, |
| 36 | messages: list[dict], |
| 37 | media_io_kwargs: dict | None = None, |
| 38 | mm_processor_kwargs: dict | None = None, |
| 39 | ) -> str: |
| 40 | media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS |
| 41 | mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS |
| 42 | _, inputs = renderer.render_chat( |
| 43 | [messages], |
| 44 | ChatParams( |
| 45 | media_io_kwargs=media_io_kwargs, |
| 46 | mm_processor_kwargs=mm_processor_kwargs, |
| 47 | ), |
| 48 | prompt_extras={"mm_processor_kwargs": mm_processor_kwargs}, |
| 49 | ) |
| 50 | |
| 51 | return inputs[0]["mm_hashes"]["video"][0] |
| 52 | |
| 53 | |
| 54 | def _video_chat_messages(client_uuid: str): |
| 55 | return [ |
| 56 | { |
| 57 | "role": "user", |
| 58 | "content": [ |
| 59 | { |
| 60 | "type": "video_url", |
| 61 | "video_url": {"url": video_data_url}, |
| 62 | "uuid": client_uuid, |
| 63 | }, |
| 64 | {"type": "text", "text": "Describe the video."}, |
| 65 | ], |
| 66 | } |
| 67 | ] |
| 68 | |
| 69 | |
| 70 | def test_mm_hash_includes_uuid_and_processing_kwargs(): |
| 71 | renderer = _build_video_renderer() |
| 72 | messages = _video_chat_messages(video_data_url) |
| 73 | |
| 74 | base_hash = _get_message_hash(renderer, messages) |
| 75 | media_io_hash = _get_message_hash( |
| 76 | renderer, |
| 77 | messages, |
| 78 | media_io_kwargs={"video": {"num_frames": 4}}, |
| 79 | ) |
| 80 | uuid_hash = _get_message_hash( |
| 81 | renderer, |
| 82 | _video_chat_messages("different-video-uuid"), |
| 83 | ) |
| 84 | mm_processor_hash = _get_message_hash( |
| 85 | renderer, |
| 86 | messages, |
| 87 | mm_processor_kwargs={"videos_kwargs": {"do_resize": False}}, |
| 88 | ) |
| 89 | |
| 90 | assert base_hash != media_io_hash |
| 91 | assert base_hash != uuid_hash |
| 92 | assert base_hash != mm_processor_hash |