| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from collections.abc import Mapping |
| 4 | from dataclasses import dataclass, field |
| 5 | |
| 6 | from vllm.config.multimodal import MMHasherAlgorithm |
| 7 | from vllm.inputs import MultiModalHashes |
| 8 | |
| 9 | from ..cache import BaseMultiModalProcessorCache |
| 10 | from ..hasher import MultiModalHasher |
| 11 | from ..parse import MultiModalDataItems, MultiModalUUIDItems |
| 12 | |
| 13 | _HF_MODALITY_PROCESSOR_KWARGS = { |
| 14 | "image": "images_kwargs", |
| 15 | "video": "videos_kwargs", |
| 16 | "audio": "audio_kwargs", |
| 17 | } |
| 18 | |
| 19 | |
| 20 | @dataclass |
| 21 | class ProcessorInputs: |
| 22 | """Represents the keyword arguments to |
| 23 | [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][]. |
| 24 | """ |
| 25 | |
| 26 | prompt: list[int] |
| 27 | mm_data_items: MultiModalDataItems |
| 28 | mm_uuid_items: MultiModalUUIDItems | None = None |
| 29 | hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) |
| 30 | media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict) |
| 31 | |
| 32 | cache: BaseMultiModalProcessorCache | None = None |
| 33 | |
| 34 | def get_mm_hashes( |
| 35 | self, |
| 36 | model_id: str, |
| 37 | hash_algorithm: MMHasherAlgorithm, |
| 38 | ) -> MultiModalHashes: |
| 39 | mm_data_items = self.mm_data_items |
| 40 | mm_uuid_items = self.mm_uuid_items or {} |
| 41 | |
| 42 | mm_hashes = dict[str, list[str]]() |
| 43 | hasher = MultiModalHasher |
| 44 | |
| 45 | for modality, data_items in mm_data_items.items(): |
| 46 | # Each cache entry represents one item from one modality. Keep |
| 47 | # modality-scoped options out of unrelated entries, while |
| 48 | # retaining flat processor kwargs because they are shared by HF. |
| 49 | if modality in _HF_MODALITY_PROCESSOR_KWARGS: |
| 50 | modality_media_io_kwargs = self.media_io_kwargs.get(modality, {}) |
| 51 | media_io_kwargs = ( |
| 52 | {modality: modality_media_io_kwargs} |
| 53 | if modality_media_io_kwargs |
| 54 | else {} |
| 55 | ) |
| 56 | scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] |
| 57 | mm_processor_kwargs = { |
| 58 | key: value |
| 59 | for key, value in self.hf_processor_mm_kwargs.items() |
| 60 | if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() |
| 61 | or key == scoped_key |
| 62 | } |
| 63 | else: |
| 64 | # Unified modalities such as ``vision_chunk`` can contain |
| 65 | # items originating from multiple media modalities. There is |
| 66 | # no per-item origin metadata here, so preserve all factors. |
| 67 | media_io_kwargs = dict(self.media_io_kwargs) |
| 68 | mm_processor_kwargs = dict(self.hf_processor_mm_kwargs) |
| 69 | |
| 70 | hash_factors = { |
| 71 | "media_io_kwargs": media_io_kwargs, |
| 72 | "mm_processor_kwargs": mm_processor_kwargs, |
| 73 | } |
| 74 | hash_factors = {key: value for key, value in hash_factors.items() if value} |
| 75 | has_hash_factors = bool(hash_factors) |
| 76 | |
| 77 | uuid_items = ( |
| 78 | mm_uuid_items[modality] |
| 79 | if modality in mm_uuid_items |
| 80 | else ([None] * len(data_items)) |
| 81 | ) |
| 82 | |
| 83 | # For None entries, compute a hash; otherwise, use provided ID. |
| 84 | hashes: list[str] = [] |
| 85 | for i, item in enumerate(data_items.get_all_items_for_hash()): |
| 86 | uuid_item = uuid_items[i] |
| 87 | |
| 88 | # NOTE: Even if a uuid_item is provided, model output depends |
| 89 | # on the current modality's hash factors, so they are taken |
| 90 | # into account. |
| 91 | if uuid_item is None or has_hash_factors: |
| 92 | # NOTE: use provided hash string to hash with kwargs |
| 93 | # if available for better performance. |
| 94 | item = uuid_item if uuid_item is not None else item |
| 95 | hashes.append( |
| 96 | hasher.hash_kwargs( |
| 97 | hash_algorithm, |
| 98 | model_id=model_id, |
| 99 | **{modality: item}, |
| 100 | **hash_factors, |
| 101 | ) |
| 102 | ) |
| 103 | else: |
| 104 | # If there are no extra kwargs, use the client-provided UUID. |
| 105 | hashes.append(uuid_item) |
| 106 | |
| 107 | mm_hashes[modality] = hashes |
| 108 | |
| 109 | return mm_hashes |