| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from abc import ABC, abstractmethod |
| 4 | from collections import defaultdict, deque |
| 5 | from collections.abc import Callable, Generator, ItemsView, Iterable, Mapping, Sequence |
| 6 | from dataclasses import dataclass, field, replace |
| 7 | from enum import Enum |
| 8 | from functools import lru_cache |
| 9 | from typing import ( |
| 10 | TYPE_CHECKING, |
| 11 | ClassVar, |
| 12 | Generic, |
| 13 | NamedTuple, |
| 14 | Protocol, |
| 15 | TypeAlias, |
| 16 | ) |
| 17 | |
| 18 | import torch |
| 19 | from typing_extensions import TypeVar, assert_never |
| 20 | |
| 21 | from vllm.config import SchedulerConfig |
| 22 | from vllm.inputs import ( |
| 23 | MultiModalEncDecInput, |
| 24 | MultiModalHashes, |
| 25 | MultiModalInput, |
| 26 | mm_enc_dec_input, |
| 27 | mm_input, |
| 28 | ) |
| 29 | from vllm.logger import init_logger |
| 30 | from vllm.tokenizers import TokenizerLike |
| 31 | from vllm.utils.collection_utils import flatten_2d_lists, full_groupby |
| 32 | |
| 33 | from ..inputs import ( |
| 34 | MultiModalFieldConfig, |
| 35 | MultiModalKwargsItem, |
| 36 | MultiModalKwargsItems, |
| 37 | MultiModalKwargsOptionalItems, |
| 38 | PlaceholderRange, |
| 39 | ) |
| 40 | from ..parse import MultiModalDataItems, MultiModalUUIDItems |
| 41 | from .context import BaseProcessingInfo, TimingContext |
| 42 | from .dummy_inputs import BaseDummyInputsBuilder, MultiModalDummyOptions |
| 43 | from .inputs import ProcessorInputs |
| 44 | |
| 45 | if TYPE_CHECKING: |
| 46 | from transformers.feature_extraction_utils import BatchFeature |
| 47 | |
| 48 | from ..cache import BaseMultiModalProcessorCache |
| 49 | else: |
| 50 | BatchFeature = object |
| 51 | |
| 52 | BaseMultiModalProcessorCache = object |
| 53 | |
| 54 | logger = init_logger(__name__) |
| 55 | |
| 56 | |
| 57 | @lru_cache(maxsize=2048) |
| 58 | def cached_encode( |
| 59 | tokenizer: TokenizerLike, |
| 60 | text: str, |
| 61 | *, |
| 62 | truncation: bool | None = None, |
| 63 | add_special_tokens: bool = True, |
| 64 | ) -> list[int]: |
| 65 | """Encode text while caching repeated tokenizer calls.""" |
| 66 | return tokenizer.encode( |
| 67 | text, |
| 68 | truncation=truncation, |
| 69 | add_special_tokens=add_special_tokens, |
| 70 | ) |
| 71 | |
| 72 | |
| 73 | class _GetMatchIndex(Protocol): |
| 74 | def __call__( |
| 75 | self, |
| 76 | prompt: list[int], |
| 77 | start_idx: int = 0, |
| 78 | ) -> int | None: ... |
| 79 | |
| 80 | |
| 81 | class _GetTextMatchIndex(Protocol): |
| 82 | def __call__( |
| 83 | self, |
| 84 | tokenizer: TokenizerLike, |
| 85 | prompt: str, |
| 86 | start_idx: int = 0, |
| 87 | ) -> int | None: ... |
| 88 | |
| 89 | |
| 90 | @dataclass |
| 91 | class PromptIndex: |
| 92 | """Resolves to an index in the prompt.""" |
| 93 | |
| 94 | get_match_index: _GetMatchIndex |
| 95 | |
| 96 | get_text_match_index: _GetTextMatchIndex |
| 97 | """ |
| 98 | Resolves the index against a decoded text prompt. |
| 99 | |
| 100 | Used only by the private text-based fallback of |
| 101 | [`BaseMultiModalProcessor._apply_prompt_updates`][vllm.multimodal.processing.BaseMultiModalProcessor]. |
| 102 | """ |
| 103 | |
| 104 | |
| 105 | class PromptIndexTargets: |
| 106 | @staticmethod |
| 107 | def start() -> PromptIndex: |
| 108 | """Resolves to the start of the prompt (before the first token). |
| 109 | |
| 110 | This results in a match even if the prompt is empty. |
| 111 | """ |
| 112 | return PromptIndex( |
| 113 | lambda prompt, start_idx=0: 0, |
| 114 | lambda tokenizer, prompt, start_idx=0: 0, |
| 115 | ) |
| 116 | |
| 117 | @staticmethod |
| 118 | def prefix(seq: list[int]) -> PromptIndex: |
| 119 | """Resolves to a location in the prompt after the given prefix.""" |
| 120 | |
| 121 | def get_match_index( |
| 122 | prompt: list[int], |
| 123 | start_idx: int = 0, |
| 124 | ) -> int | None: |
| 125 | if start_idx != 0: |
| 126 | return None |
| 127 | |
| 128 | match_idx = len(seq) |
| 129 | return match_idx if prompt[:match_idx] == seq else None |
| 130 | |
| 131 | def get_text_match_index( |
| 132 | tokenizer: TokenizerLike, |
| 133 | prompt: str, |
| 134 | start_idx: int = 0, |
| 135 | ) -> int | None: |
| 136 | if start_idx != 0: |
| 137 | return None |
| 138 | |
| 139 | prefix = tokenizer.decode(seq) |
| 140 | match_idx = len(prefix) |
| 141 | return match_idx if prompt[:match_idx] == prefix else None |
| 142 | |
| 143 | return PromptIndex(get_match_index, get_text_match_index) |
| 144 | |
| 145 | @staticmethod |
| 146 | def end() -> PromptIndex: |
| 147 | """Resolves to the end of the prompt (after the last token). |
| 148 | |
| 149 | This results in a match even if the prompt is empty. |
| 150 | """ |
| 151 | return PromptIndex( |
| 152 | lambda prompt, start_idx=0: len(prompt), |
| 153 | lambda tokenizer, prompt, start_idx=0: len(prompt), |
| 154 | ) |
| 155 | |
| 156 | |
| 157 | UpdateTarget: TypeAlias = list[int] | PromptIndex |
| 158 | """ |
| 159 | The token sequence to update. |
| 160 | """ |
| 161 | |
| 162 | PromptUpdateTarget: TypeAlias = Callable[[int], UpdateTarget] | UpdateTarget |
| 163 | """ |
| 164 | Given the index of the processed item within |
| 165 | [`modality`][vllm.multimodal.processing.PromptUpdate.modality], |
| 166 | output the corresponding token sequence. |
| 167 | |
| 168 | For convenience, you can directly pass in the token sequence |
| 169 | instead of a function if it does not depend on the input. |
| 170 | """ |
| 171 | |
| 172 | |
| 173 | @dataclass |
| 174 | class PromptUpdateDetails: |
| 175 | """Details about the token sequence that is part of the update.""" |
| 176 | |
| 177 | full: list[int] |
| 178 | """The full content.""" |
| 179 | |
| 180 | is_embed: Callable[[list[int]], torch.Tensor] | None = None |
| 181 | """ |
| 182 | Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full], |
| 183 | return a boolean mask of shape `(len(full),)` indicating which positions |
| 184 | of `full` to assign embeddings to. |
| 185 | |
| 186 | `None` (default) means to assign embeddings to all positions of `full`. |
| 187 | |
| 188 | The embeddings are obtained by calling |
| 189 | [`SupportsMultiModal.embed_multimodal`][vllm.model_executor.models.interfaces.SupportsMultiModal.embed_multimodal]. |
| 190 | """ |
| 191 | |
| 192 | @staticmethod |
| 193 | def from_seq(seq: list[int]) -> "PromptUpdateDetails": |
| 194 | return PromptUpdateDetails(full=seq) |
| 195 | |
| 196 | @staticmethod |
| 197 | def select_token_id( |
| 198 | seq: list[int], |
| 199 | embed_token_id: int, |
| 200 | ) -> "PromptUpdateDetails": |
| 201 | def is_embed(full: list[int]) -> torch.Tensor: |
| 202 | return torch.tensor(full) == embed_token_id |
| 203 | |
| 204 | return PromptUpdateDetails(full=seq, is_embed=is_embed) |
| 205 | |
| 206 | @staticmethod |
| 207 | def select_token_ids( |
| 208 | seq: list[int], |
| 209 | embed_token_ids: list[int], |
| 210 | ) -> "PromptUpdateDetails": |
| 211 | def is_embed(full: list[int]) -> torch.Tensor: |
| 212 | return torch.isin( |
| 213 | torch.tensor(full), |
| 214 | torch.tensor(embed_token_ids), |
| 215 | ) |
| 216 | |
| 217 | return PromptUpdateDetails(full=seq, is_embed=is_embed) |
| 218 | |
| 219 | |
| 220 | PromptUpdateInfo: TypeAlias = list[int] | PromptUpdateDetails |
| 221 | """ |
| 222 | The token sequence that is part of the update. |
| 223 | |
| 224 | If only part of the content corresponds to feature placeholders, you can |
| 225 | use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to |
| 226 | specify which part. |
| 227 | """ |
| 228 | |
| 229 | PromptUpdateContent: TypeAlias = Callable[[int], PromptUpdateInfo] | PromptUpdateInfo |
| 230 | """ |
| 231 | Given the index of the processed item within |
| 232 | [`modality`][vllm.multimodal.processing.PromptUpdate.modality], |
| 233 | output the corresponding token sequence. |
| 234 | |
| 235 | For convenience, you can directly pass in the token sequence |
| 236 | instead of a function if it does not depend on the input. |
| 237 | """ |
| 238 | |
| 239 | |
| 240 | class UpdateMode(str, Enum): |
| 241 | INSERT = "insert" |
| 242 | REPLACE = "replace" |
| 243 | |
| 244 | |
| 245 | @dataclass |
| 246 | class PromptUpdate(ABC): |
| 247 | """Defines how to update a prompt with placeholder tokens. |
| 248 | |
| 249 | Note: |
| 250 | The target and content are token sequences. When converting text |
| 251 | to token sequences, remember to encode it with |
| 252 | `add_special_tokens=False` to match how the prompt itself is |
| 253 | tokenized. Otherwise, the updated prompt may contain duplicated |
| 254 | special tokens or fail to match the target. |
| 255 | |
| 256 | """ |
| 257 | |
| 258 | modality: str |
| 259 | """The modality for which the update is made.""" |
| 260 | |
| 261 | target: PromptUpdateTarget |
| 262 | """The token sequence to update.""" |
| 263 | |
| 264 | @property |
| 265 | @abstractmethod |
| 266 | def content(self) -> PromptUpdateContent: |
| 267 | """The placeholder tokens that are part of the update.""" |
| 268 | raise NotImplementedError |
| 269 | |
| 270 | @property |
| 271 | @abstractmethod |
| 272 | def mode(self) -> UpdateMode: |
| 273 | """Defines how to update the prompt.""" |
| 274 | raise NotImplementedError |
| 275 | |
| 276 | def _resolve_target(self, item_idx: int) -> UpdateTarget: |
| 277 | target = self.target |
| 278 | if callable(target): |
| 279 | target = target(item_idx) |
| 280 | |
| 281 | return target |
| 282 | |
| 283 | def _resolve_content(self, item_idx: int) -> PromptUpdateDetails: |
| 284 | content = self.content |
| 285 | if callable(content): |
| 286 | content = content(item_idx) |
| 287 | |
| 288 | if not isinstance(content, PromptUpdateDetails): |
| 289 | content = PromptUpdateDetails.from_seq(content) |
| 290 | |
| 291 | return content |
| 292 | |
| 293 | def resolve(self, item_idx: int) -> "ResolvedPromptUpdate": |
| 294 | """Given the index of the processed item within |
| 295 | [`modality`][vllm.multimodal.processing.PromptUpdate.modality], |
| 296 | output a copy of this object with its lazy attributes resolved. |
| 297 | """ |
| 298 | return ResolvedPromptUpdate( |
| 299 | modality=self.modality, |
| 300 | item_idx=item_idx, |
| 301 | mode=self.mode, |
| 302 | target=self._resolve_target(item_idx), |
| 303 | content=self._resolve_content(item_idx), |
| 304 | ) |
| 305 | |
| 306 | |
| 307 | @dataclass |
| 308 | class PromptInsertion(PromptUpdate): |
| 309 | """Defines how to insert placeholder tokens into a prompt. |
| 310 | |
| 311 | Example: |
| 312 | For each image, insert a number of `<image>` feature placeholders |
| 313 | equal to the feature size of the vision encoder after the `<s>` token: |
| 314 | |
| 315 | ```python |
| 316 | PromptInsertion( |
| 317 | modality="image", |
| 318 | target=[bos_token_id], |
| 319 | insertion=[image_token_id] * image_feature_size, |
| 320 | ) |
| 321 | ``` |
| 322 | |
| 323 | Insert these tokens at the start of the prompt: |
| 324 | |
| 325 | ```python |
| 326 | PromptInsertion( |
| 327 | modality="image", |
| 328 | target=PromptIndexTargets.start(), |
| 329 | insertion=[image_token_id] * image_feature_size, |
| 330 | ) |
| 331 | ``` |
| 332 | |
| 333 | Insert these tokens after a prefix `Images:`: |
| 334 | |
| 335 | ```python |
| 336 | PromptInsertion( |
| 337 | modality="image", |
| 338 | target=PromptIndexTargets.prefix(images_prefix_token_ids), |
| 339 | insertion=[image_token_id] * image_feature_size, |
| 340 | ) |
| 341 | ``` |
| 342 | |
| 343 | Insert these tokens at the end of the prompt: |
| 344 | |
| 345 | ```python |
| 346 | PromptInsertion( |
| 347 | modality="image", |
| 348 | target=PromptIndexTargets.end(), |
| 349 | insertion=[image_token_id] * image_feature_size, |
| 350 | ) |
| 351 | ``` |
| 352 | |
| 353 | """ |
| 354 | |
| 355 | insertion: PromptUpdateContent = field(repr=False) |
| 356 | """ |
| 357 | Given the index of the processed item within |
| 358 | [`modality`][vllm.multimodal.processing.PromptUpdate.modality], |
| 359 | output the token sequence to insert right after |
| 360 | [`target`][vllm.multimodal.processing.PromptUpdate.target]. |
| 361 | |
| 362 | For convenience, you can directly pass in the token sequence |
| 363 | instead of a function if it does not depend on the input. |
| 364 | """ |
| 365 | |
| 366 | @property |
| 367 | def content(self) -> PromptUpdateContent: |
| 368 | return self.insertion |
| 369 | |
| 370 | @property |
| 371 | def mode(self) -> UpdateMode: |
| 372 | return UpdateMode.INSERT |
| 373 | |
| 374 | |
| 375 | @dataclass |
| 376 | class PromptReplacement(PromptUpdate): |
| 377 | """Defines how to replace portions of an input prompt with placeholder tokens. |
| 378 | |
| 379 | Example: |
| 380 | For each image, replace one `<image>` input placeholder in the prompt |
| 381 | with a number of `<image>` feature placeholders |
| 382 | equal to the feature size of the vision encoder: |
| 383 | |
| 384 | ```python |
| 385 | PromptReplacement( |
| 386 | modality="image", |
| 387 | target=[image_token_id], |
| 388 | replacement=[image_token_id] * image_feature_size, |
| 389 | ) |
| 390 | ``` |
| 391 | |
| 392 | As above, but further pad the feature placeholders with `<image_bos>` |
| 393 | and `<image_eos>`, which are not supposed to be passed to the vision |
| 394 | encoder: |
| 395 | |
| 396 | ```python |
| 397 | PromptReplacement( |
| 398 | modality="image", |
| 399 | target=[image_token_id], |
| 400 | replacement=PromptUpdateDetails( |
| 401 | full=( |
| 402 | [image_bos_id] + [image_token_id] * image_feature_size + [image_eos_id] |
| 403 | ), |
| 404 | is_embed=lambda full: torch.tensor(full) == image_token_id, |
| 405 | ), |
| 406 | ) |
| 407 | ``` |
| 408 | |
| 409 | """ |
| 410 | |
| 411 | replacement: PromptUpdateContent = field(repr=False) |
| 412 | """ |
| 413 | Given the index of the processed item within |
| 414 | [`modality`][vllm.multimodal.processing.PromptUpdate.modality], |
| 415 | output the token sequence to replace |
| 416 | [`target`][vllm.multimodal.processing.PromptUpdate.target]. |
| 417 | |
| 418 | For convenience, you can directly pass in the token sequence |
| 419 | instead of a function if it does not depend on the input. |
| 420 | """ |
| 421 | |
| 422 | @property |
| 423 | def content(self) -> PromptUpdateContent: |
| 424 | return self.replacement |
| 425 | |
| 426 | @property |
| 427 | def mode(self) -> UpdateMode: |
| 428 | return UpdateMode.REPLACE |
| 429 | |
| 430 | |
| 431 | class _HasModalityAttr(Protocol): |
| 432 | modality: str |
| 433 | |
| 434 | |
| 435 | class _HasModalityProp(Protocol): |
| 436 | @property |
| 437 | def modality(self) -> str: ... |
| 438 | |
| 439 | |
| 440 | _M = TypeVar("_M", bound=_HasModalityAttr | _HasModalityProp) |
| 441 | |
| 442 | |
| 443 | def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]: |
| 444 | """Convenience function to apply |
| 445 | [`full_groupby`][vllm.utils.collection_utils.full_groupby] |
| 446 | based on modality. |
| 447 | """ |
| 448 | return full_groupby(values, key=lambda x: x.modality) |
| 449 | |
| 450 | |
| 451 | class PromptTargetMatch(NamedTuple): |
| 452 | start_idx: int |
| 453 | end_idx: int |
| 454 | |
| 455 | |
| 456 | @dataclass(frozen=True) |
| 457 | class ResolvedPromptUpdate: |
| 458 | """A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] with its |
| 459 | lazy attributes resolved. |
| 460 | """ |
| 461 | |
| 462 | modality: str |
| 463 | """The modality for which the update is made.""" |
| 464 | |
| 465 | item_idx: int |
| 466 | """The index within `modality` of the item this update pertains to.""" |
| 467 | |
| 468 | mode: UpdateMode |
| 469 | """Defines how to update the prompt.""" |
| 470 | |
| 471 | target: UpdateTarget |
| 472 | """The token sequence to update.""" |
| 473 | |
| 474 | content: PromptUpdateDetails = field(repr=False) |
| 475 | """The placeholder tokens that are part of the update.""" |
| 476 | |
| 477 | def iter_token_matches( |
| 478 | self, |
| 479 | prompt: list[int], |
| 480 | *, |
| 481 | start_idx: int = 0, |
| 482 | ) -> Generator[PromptTargetMatch]: |
| 483 | """Yield each instance of `self.target` found in `prompt`.""" |
| 484 | target = self.target |
| 485 | |
| 486 | if isinstance(target, PromptIndex): |
| 487 | match_idx = target.get_match_index(prompt, start_idx) |
| 488 | if match_idx is not None: |
| 489 | yield PromptTargetMatch(match_idx, match_idx) |
| 490 | |
| 491 | return |
| 492 | |
| 493 | for match in iter_token_matches(prompt, target, start_idx=start_idx): |
| 494 | yield PromptTargetMatch(match.start_idx, match.end_idx) |
| 495 | |
| 496 | def with_target(self, target: UpdateTarget): |
| 497 | return replace(self, target=target) |
| 498 | |
| 499 | def with_content(self, content: PromptUpdateInfo): |
| 500 | if not isinstance(content, PromptUpdateDetails): |
| 501 | content = PromptUpdateDetails.from_seq(content) |
| 502 | |
| 503 | return replace(self, content=content) |
| 504 | |
| 505 | |
| 506 | class _TokenMatch(NamedTuple): |
| 507 | start_idx: int |
| 508 | end_idx: int |
| 509 | |
| 510 | |
| 511 | def iter_token_matches( |
| 512 | token_ids: list[int], |
| 513 | match_ids: list[int], |
| 514 | *, |
| 515 | start_idx: int = 0, |
| 516 | ) -> Generator[_TokenMatch]: |
| 517 | """Yield each occurrence of `match_ids` in `token_ids`. |
| 518 | |
| 519 | Note that empty matches are ignored. |
| 520 | """ |
| 521 | if start_idx < 0: |
| 522 | raise ValueError("start_idx must be non-negative") |
| 523 | |
| 524 | prompt_len = len(token_ids) |
| 525 | match_len = len(match_ids) |
| 526 | |
| 527 | if match_len == 0: |
| 528 | return |
| 529 | |
| 530 | first_id = match_ids[0] |
| 531 | last_start_idx = prompt_len - match_len |
| 532 | |
| 533 | while start_idx <= last_start_idx: |
| 534 | # Fast-forward to the next candidate position using a C-level scan |
| 535 | try: |
| 536 | start_idx = token_ids.index(first_id, start_idx, last_start_idx + 1) |
| 537 | except ValueError: |
| 538 | return |
| 539 | |
| 540 | end_idx = start_idx + match_len |
| 541 | |
| 542 | if token_ids[start_idx:end_idx] == match_ids: |
| 543 | yield _TokenMatch(start_idx=start_idx, end_idx=end_idx) |
| 544 | |
| 545 | # Exclude overlapping matches |
| 546 | start_idx = end_idx |
| 547 | else: |
| 548 | start_idx += 1 |
| 549 | |
| 550 | |
| 551 | def replace_token_matches( |
| 552 | token_ids: list[int], |
| 553 | match_ids: list[int], |
| 554 | new_ids: list[int], |
| 555 | ) -> list[int]: |
| 556 | """Replace each occurrence of `match_ids` in `token_ids` |
| 557 | with `new_ids`. |
| 558 | |
| 559 | Note that empty matches are ignored. |
| 560 | """ |
| 561 | out_seqs = list[list[int]]() |
| 562 | prev_end_idx = 0 |
| 563 | |
| 564 | for match in iter_token_matches(token_ids, match_ids): |
| 565 | start_idx = match.start_idx |
| 566 | end_idx = match.end_idx |
| 567 | |
| 568 | out_seqs.append(token_ids[prev_end_idx:start_idx]) |
| 569 | out_seqs.append(new_ids) |
| 570 | prev_end_idx = end_idx |
| 571 | |
| 572 | out_seqs.append(token_ids[prev_end_idx:]) |
| 573 | |
| 574 | return flatten_2d_lists(out_seqs) |
| 575 | |
| 576 | |
| 577 | @dataclass |
| 578 | class PlaceholderFeaturesInfo: |
| 579 | modality: str |
| 580 | item_idx: int |
| 581 | start_idx: int |
| 582 | tokens: list[int] |
| 583 | is_embed: torch.Tensor | None |
| 584 | |
| 585 | @property |
| 586 | def length(self) -> int: |
| 587 | return len(self.tokens) |
| 588 | |
| 589 | def to_range(self) -> PlaceholderRange: |
| 590 | # TODO: Is it worth it to optimize this by stripping the |
| 591 | # leading and ending positions where `is_embed=False`? |
| 592 | return PlaceholderRange( |
| 593 | offset=self.start_idx, |
| 594 | length=self.length, |
| 595 | is_embed=self.is_embed, |
| 596 | ) |
| 597 | |
| 598 | |
| 599 | class _MatchedUpdate(NamedTuple): |
| 600 | """A resolved update selected for a match in the original prompt.""" |
| 601 | |
| 602 | priority: int |
| 603 | """The original item order used to preserve match tie-breaking.""" |
| 604 | |
| 605 | update: ResolvedPromptUpdate |
| 606 | """The selected update for the multimodal item.""" |
| 607 | |
| 608 | update_idx: int |
| 609 | """The selected update's index within the item's alternatives.""" |
| 610 | |
| 611 | match: PromptTargetMatch |
| 612 | """The target range in the original prompt.""" |
| 613 | |
| 614 | |
| 615 | _UpdateQueue: TypeAlias = deque[tuple[int, Sequence[ResolvedPromptUpdate]]] |
| 616 | """ |
| 617 | Items with the same ordered match rules, stored as `(priority, alternatives)`. |
| 618 | """ |
| 619 | |
| 620 | _QueueMatch: TypeAlias = tuple[_UpdateQueue, PromptTargetMatch, int] |
| 621 | """A queue together with its next match and selected alternative index.""" |
| 622 | |
| 623 | |
| 624 | def _target_key(target: UpdateTarget) -> tuple[str, object]: |
| 625 | """Return a hashable key that preserves target matching semantics.""" |
| 626 | if isinstance(target, PromptIndex): |
| 627 | return ("index", id(target)) |
| 628 | |
| 629 | return ("tokens", tuple(target)) |
| 630 | |
| 631 | |
| 632 | def _compile_prompt_update_queues( |
| 633 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 634 | ) -> dict[ |
| 635 | tuple[tuple[UpdateMode, tuple[str, object]], ...], |
| 636 | _UpdateQueue, |
| 637 | ]: |
| 638 | """Group items with identical match rules into ordered queues.""" |
| 639 | queues_by_signature = dict[ |
| 640 | tuple[tuple[UpdateMode, tuple[str, object]], ...], |
| 641 | _UpdateQueue, |
| 642 | ]() |
| 643 | priority = 0 |
| 644 | |
| 645 | for modality_updates in mm_prompt_updates.values(): |
| 646 | for updates in modality_updates: |
| 647 | signature = tuple( |
| 648 | (update.mode, _target_key(update.target)) for update in updates |
| 649 | ) |
| 650 | queues_by_signature.setdefault(signature, deque()).append( |
| 651 | (priority, updates) |
| 652 | ) |
| 653 | priority += 1 |
| 654 | |
| 655 | return queues_by_signature |
| 656 | |
| 657 | |
| 658 | _IterMatches: TypeAlias = Callable[ |
| 659 | [ResolvedPromptUpdate, int], |
| 660 | Generator[PromptTargetMatch, None, None], |
| 661 | ] |
| 662 | """Yields each match of an update's target, given the search start index.""" |
| 663 | |
| 664 | |
| 665 | def _find_queue_match( |
| 666 | queue: _UpdateQueue, |
| 667 | iter_matches: _IterMatches, |
| 668 | *, |
| 669 | start_idx: int, |
| 670 | mode: UpdateMode | None = None, |
| 671 | ) -> tuple[PromptTargetMatch, int] | None: |
| 672 | """Find the first matching alternative for the next queued item.""" |
| 673 | _, updates = queue[0] |
| 674 | for update_idx, update in enumerate(updates): |
| 675 | if mode is not None and update.mode != mode: |
| 676 | continue |
| 677 | |
| 678 | match = next(iter_matches(update, start_idx), None) |
| 679 | if match is not None: |
| 680 | return match, update_idx |
| 681 | |
| 682 | return None |
| 683 | |
| 684 | |
| 685 | def _next_priority(queue: _UpdateQueue) -> int: |
| 686 | """Return the original priority of the next queued item.""" |
| 687 | priority, _ = queue[0] |
| 688 | return priority |
| 689 | |
| 690 | |
| 691 | def _plan_prompt_updates_with( |
| 692 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 693 | iter_matches: _IterMatches, |
| 694 | ) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]: |
| 695 | """Plan non-overlapping prompt updates before rendering the output.""" |
| 696 | queues = list(_compile_prompt_update_queues(mm_prompt_updates).values()) |
| 697 | result: MultiModalPromptUpdatesApplyResult = { |
| 698 | modality: [None] * len(items) for modality, items in mm_prompt_updates.items() |
| 699 | } |
| 700 | planned_updates = list[_MatchedUpdate]() |
| 701 | prev_end_idx = 0 |
| 702 | |
| 703 | while queues: |
| 704 | first_matches = list[_QueueMatch]() |
| 705 | for queue in queues: |
| 706 | queue_match = _find_queue_match( |
| 707 | queue, |
| 708 | iter_matches, |
| 709 | start_idx=prev_end_idx, |
| 710 | ) |
| 711 | if queue_match is not None: |
| 712 | prompt_match, update_idx = queue_match |
| 713 | first_matches.append((queue, prompt_match, update_idx)) |
| 714 | |
| 715 | if not first_matches: |
| 716 | break |
| 717 | |
| 718 | mode_queue, _, mode_update_idx = min( |
| 719 | first_matches, |
| 720 | key=lambda item: _next_priority(item[0]), |
| 721 | ) |
| 722 | _, mode_updates = mode_queue[0] |
| 723 | mode = mode_updates[mode_update_idx].mode |
| 724 | |
| 725 | mode_matches = list[_QueueMatch]() |
| 726 | for queue, prompt_match, first_update_idx in first_matches: |
| 727 | if queue[0][1][first_update_idx].mode == mode: |
| 728 | mode_matches.append((queue, prompt_match, first_update_idx)) |
| 729 | continue |
| 730 | |
| 731 | queue_match = _find_queue_match( |
| 732 | queue, |
| 733 | iter_matches, |
| 734 | start_idx=prev_end_idx, |
| 735 | mode=mode, |
| 736 | ) |
| 737 | if queue_match is not None: |
| 738 | prompt_match, update_idx = queue_match |
| 739 | mode_matches.append((queue, prompt_match, update_idx)) |
| 740 | |
| 741 | updates_to_apply = list[_MatchedUpdate]() |
| 742 | non_empty_replacements = list[_QueueMatch]() |
| 743 | for queue, match, update_idx in mode_matches: |
| 744 | if mode == UpdateMode.REPLACE and match.start_idx != match.end_idx: |
| 745 | non_empty_replacements.append((queue, match, update_idx)) |
| 746 | else: |
| 747 | while queue: |
| 748 | priority, updates = queue.popleft() |
| 749 | updates_to_apply.append( |
| 750 | _MatchedUpdate( |
| 751 | priority=priority, |
| 752 | update=updates[update_idx], |
| 753 | update_idx=update_idx, |
| 754 | match=match, |
| 755 | ) |
| 756 | ) |
| 757 | |
| 758 | if non_empty_replacements: |
| 759 | queue, match, update_idx = min( |
| 760 | non_empty_replacements, |
| 761 | key=lambda item: (item[1], _next_priority(item[0])), |
| 762 | ) |
| 763 | priority, updates = queue.popleft() |
| 764 | updates_to_apply.append( |
| 765 | _MatchedUpdate( |
| 766 | priority=priority, |
| 767 | update=updates[update_idx], |
| 768 | update_idx=update_idx, |
| 769 | match=match, |
| 770 | ) |
| 771 | ) |
| 772 | |
| 773 | updates_to_apply.sort(key=lambda item: (item.match, item.priority)) |
| 774 | for matched_update in updates_to_apply: |
| 775 | update = matched_update.update |
| 776 | result[update.modality][update.item_idx] = matched_update.update_idx |
| 777 | prev_end_idx = matched_update.match.end_idx |
| 778 | planned_updates.extend(updates_to_apply) |
| 779 | queues = [queue for queue in queues if queue] |
| 780 | |
| 781 | return planned_updates, result |
| 782 | |
| 783 | |
| 784 | def _plan_prompt_updates( |
| 785 | prompt: list[int], |
| 786 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 787 | ) -> tuple[list[_MatchedUpdate], "MultiModalPromptUpdatesApplyResult"]: |
| 788 | """Plan non-overlapping prompt updates before rendering the output.""" |
| 789 | return _plan_prompt_updates_with( |
| 790 | mm_prompt_updates, |
| 791 | lambda update, start_idx: update.iter_token_matches( |
| 792 | prompt, start_idx=start_idx |
| 793 | ), |
| 794 | ) |
| 795 | |
| 796 | |
| 797 | def _all_items_found( |
| 798 | mm_item_counts: dict[str, int], |
| 799 | mm_found_counts: dict[str, int], |
| 800 | ) -> bool: |
| 801 | return all( |
| 802 | item_idx >= mm_item_counts[modality] |
| 803 | for modality, item_idx in mm_found_counts.items() |
| 804 | ) |
| 805 | |
| 806 | |
| 807 | def _apply_matches( |
| 808 | prompt: list[int], |
| 809 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 810 | ) -> tuple[list[list[int]], "MultiModalPromptUpdatesApplyResult"]: |
| 811 | out_seqs = list[list[int]]() |
| 812 | matched_updates, result = _plan_prompt_updates(prompt, mm_prompt_updates) |
| 813 | |
| 814 | prev_end_idx = 0 |
| 815 | for matched_update in matched_updates: |
| 816 | update = matched_update.update |
| 817 | match = matched_update.match |
| 818 | |
| 819 | if update.mode == UpdateMode.INSERT: |
| 820 | end_idx_to_insert = match.end_idx |
| 821 | elif update.mode == UpdateMode.REPLACE: |
| 822 | end_idx_to_insert = match.start_idx |
| 823 | else: |
| 824 | assert_never(update.mode) |
| 825 | |
| 826 | out_seqs.append(prompt[prev_end_idx:end_idx_to_insert]) |
| 827 | out_seqs.append(update.content.full) |
| 828 | prev_end_idx = match.end_idx |
| 829 | |
| 830 | out_seqs.append(prompt[prev_end_idx:]) |
| 831 | |
| 832 | return out_seqs, result |
| 833 | |
| 834 | |
| 835 | def apply_token_matches( |
| 836 | prompt: list[int], |
| 837 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 838 | ) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]: |
| 839 | """Apply the updates in `mm_prompt_updates` to `prompt`. |
| 840 | |
| 841 | Matches are exclusive even when multiple modalities share |
| 842 | the same placeholder tokens. In that case, the modality that |
| 843 | appears earlier in `mm_prompt_updates` takes priority. |
| 844 | """ |
| 845 | token_id_seqs, result = _apply_matches(prompt, mm_prompt_updates) |
| 846 | |
| 847 | return flatten_2d_lists(token_id_seqs), result |
| 848 | |
| 849 | |
| 850 | def _apply_token_matches_with_placeholders( |
| 851 | token_ids: list[int], |
| 852 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 853 | ) -> tuple[ |
| 854 | list[int], |
| 855 | "MultiModalPromptUpdatesApplyResult", |
| 856 | Mapping[str, list[PlaceholderFeaturesInfo]], |
| 857 | ]: |
| 858 | matched_updates, result = _plan_prompt_updates(token_ids, mm_prompt_updates) |
| 859 | placeholders: dict[str, list[PlaceholderFeaturesInfo]] = { |
| 860 | modality: [] for modality in mm_prompt_updates |
| 861 | } |
| 862 | |
| 863 | new_token_ids = list[int]() |
| 864 | prev_end_idx = 0 |
| 865 | for matched_update in matched_updates: |
| 866 | update = matched_update.update |
| 867 | match = matched_update.match |
| 868 | |
| 869 | if update.mode == UpdateMode.INSERT: |
| 870 | end_idx_to_insert = match.end_idx |
| 871 | elif update.mode == UpdateMode.REPLACE: |
| 872 | end_idx_to_insert = match.start_idx |
| 873 | else: |
| 874 | assert_never(update.mode) |
| 875 | |
| 876 | new_token_ids.extend(token_ids[prev_end_idx:end_idx_to_insert]) |
| 877 | start_idx = len(new_token_ids) |
| 878 | |
| 879 | tokens = update.content.full |
| 880 | if tokens: |
| 881 | content_is_embed = update.content.is_embed |
| 882 | if content_is_embed is not None: |
| 883 | content_is_embed = content_is_embed(tokens) |
| 884 | |
| 885 | placeholders[update.modality].append( |
| 886 | PlaceholderFeaturesInfo( |
| 887 | modality=update.modality, |
| 888 | item_idx=update.item_idx, |
| 889 | start_idx=start_idx, |
| 890 | tokens=tokens, |
| 891 | is_embed=content_is_embed, |
| 892 | ) |
| 893 | ) |
| 894 | new_token_ids.extend(tokens) |
| 895 | |
| 896 | prev_end_idx = match.end_idx |
| 897 | |
| 898 | new_token_ids.extend(token_ids[prev_end_idx:]) |
| 899 | return new_token_ids, result, placeholders |
| 900 | |
| 901 | |
| 902 | def _iter_placeholders( |
| 903 | prompt: list[int], |
| 904 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 905 | ) -> Iterable[PlaceholderFeaturesInfo]: |
| 906 | """Yield each set of placeholder tokens found in `prompt`. |
| 907 | |
| 908 | Matches are exclusive even when multiple modalities share |
| 909 | the same placeholder tokens. In that case, the modality that |
| 910 | appears earlier in `mm_prompt_updates` takes priority. |
| 911 | |
| 912 | Note that empty matches are ignored. |
| 913 | """ |
| 914 | mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()} |
| 915 | item_idx_by_modality = {modality: 0 for modality in mm_prompt_updates} |
| 916 | |
| 917 | if _all_items_found(mm_item_counts, item_idx_by_modality): |
| 918 | return |
| 919 | |
| 920 | prompt_len = len(prompt) |
| 921 | start_idx = 0 |
| 922 | |
| 923 | # The current (unfound) item's updates for each modality; rebuilt |
| 924 | # whenever an item is found. Items are only resolved once the scan |
| 925 | # reaches them. |
| 926 | candidates: list[tuple[str, ResolvedPromptUpdate, list[int]]] | None = None |
| 927 | |
| 928 | while start_idx < prompt_len: |
| 929 | if candidates is None: |
| 930 | candidates = [ |
| 931 | (modality, update, update.content.full) |
| 932 | for modality, modality_updates in mm_prompt_updates.items() |
| 933 | if item_idx_by_modality[modality] < mm_item_counts.get(modality, 0) |
| 934 | for update in modality_updates[item_idx_by_modality[modality]] |
| 935 | ] |
| 936 | if not candidates: |
| 937 | return |
| 938 | |
| 939 | found = False |
| 940 | first_token = prompt[start_idx] |
| 941 | |
| 942 | for modality, update, content_tokens_full in candidates: |
| 943 | content_len_full = len(content_tokens_full) |
| 944 | end_idx_full = start_idx + content_len_full |
| 945 | |
| 946 | if content_len_full == 0 or end_idx_full > prompt_len: |
| 947 | continue |
| 948 | |
| 949 | # Check the first token before comparing the full slice |
| 950 | if ( |
| 951 | first_token == content_tokens_full[0] |
| 952 | and prompt[start_idx:end_idx_full] == content_tokens_full |
| 953 | ): |
| 954 | content = update.content |
| 955 | content_is_embed = content.is_embed |
| 956 | if content_is_embed is not None: |
| 957 | content_is_embed = content_is_embed(content.full) |
| 958 | |
| 959 | yield PlaceholderFeaturesInfo( |
| 960 | modality=modality, |
| 961 | item_idx=item_idx_by_modality[modality], |
| 962 | start_idx=start_idx, |
| 963 | tokens=content_tokens_full, |
| 964 | is_embed=content_is_embed, |
| 965 | ) |
| 966 | |
| 967 | # Exclude overlapping matches |
| 968 | start_idx = end_idx_full |
| 969 | item_idx_by_modality[modality] += 1 |
| 970 | if _all_items_found(mm_item_counts, item_idx_by_modality): |
| 971 | return |
| 972 | |
| 973 | candidates = None |
| 974 | found = True |
| 975 | break |
| 976 | |
| 977 | if not found: |
| 978 | start_idx += 1 |
| 979 | |
| 980 | |
| 981 | def find_mm_placeholders( |
| 982 | prompt: list[int], |
| 983 | mm_prompt_updates: "MultiModalPromptUpdates", |
| 984 | ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: |
| 985 | it = _iter_placeholders(prompt, mm_prompt_updates) |
| 986 | return dict(full_groupby_modality(it)) |
| 987 | |
| 988 | |
| 989 | MultiModalIsCached = dict[str, list[bool]] |
| 990 | """ |
| 991 | A collection of the `is_cached` flag for each item, with a similar structure as |
| 992 | [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems]. |
| 993 | """ |
| 994 | |
| 995 | MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]] |
| 996 | """ |
| 997 | A collection of prompt updates with a similar structure as |
| 998 | [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems]. |
| 999 | """ |
| 1000 | |
| 1001 | MultiModalPromptUpdatesApplyResult = Mapping[str, list[int | None]] |
| 1002 | """ |
| 1003 | For an item `MultiModalPromptUpdates[k][i]`, |
| 1004 | `MultiModalPromptUpdatesApplyResult[k][i]` represents the index of the |
| 1005 | `ResolvedPromptUpdate` instance that has been applied, or `None` if none of the |
| 1006 | `ResolvedPromptUpdate` instances have been applied. |
| 1007 | """ |
| 1008 | |
| 1009 | _I = TypeVar("_I", bound=BaseProcessingInfo) |
| 1010 | |
| 1011 | |
| 1012 | class HFMultiModalInputs(NamedTuple): |
| 1013 | hf_data: dict[str, object] |
| 1014 | """The data inputs to the HF processor.""" |
| 1015 | |
| 1016 | hf_kwargs: Mapping[str, object] |
| 1017 | """The non-data inputs to the HF processor.""" |
| 1018 | |
| 1019 | passthrough_data: dict[str, object] |
| 1020 | """The inputs to assign to the output of the HF processor, |
| 1021 | instead of passing them to the HF processor.""" |
| 1022 | |
| 1023 | |
| 1024 | class MultiModalProcessingResult(NamedTuple): |
| 1025 | prompt_ids: list[int] |
| 1026 | kwargs: MultiModalKwargsOptionalItems |
| 1027 | hashes: MultiModalHashes |
| 1028 | prompt_updates: MultiModalPromptUpdates |
| 1029 | |
| 1030 | |
| 1031 | class BaseMultiModalProcessor(ABC, Generic[_I]): |
| 1032 | """Abstract base class to process multi-modal inputs to be used in vLLM. |
| 1033 | |
| 1034 | Not to be confused with `transformers.ProcessorMixin`. |
| 1035 | """ |
| 1036 | |
| 1037 | requires_tokenizer: ClassVar[bool] = True |
| 1038 | |
| 1039 | def __init__( |
| 1040 | self, |
| 1041 | info: _I, |
| 1042 | dummy_inputs: "BaseDummyInputsBuilder[_I]", |
| 1043 | ) -> None: |
| 1044 | super().__init__() |
| 1045 | |
| 1046 | if self.requires_tokenizer and info.ctx.tokenizer is None: |
| 1047 | raise ValueError( |
| 1048 | f"{type(self).__name__} requires a tokenizer and cannot be " |
| 1049 | "initialized when `skip_tokenizer_init=True`. Disable " |
| 1050 | "`--skip-tokenizer-init`." |
| 1051 | ) |
| 1052 | |
| 1053 | self.info = info |
| 1054 | self.dummy_inputs = dummy_inputs |
| 1055 | |
| 1056 | self.data_parser = self.info.get_data_parser() |
| 1057 | |
| 1058 | def get_dummy_inputs( |
| 1059 | self, |
| 1060 | seq_len: int, |
| 1061 | mm_counts: Mapping[str, int], |
| 1062 | mm_options: MultiModalDummyOptions, |
| 1063 | ) -> ProcessorInputs: |
| 1064 | """Build the input which, after processing, results in |
| 1065 | the maximum possible number of placeholder tokens. |
| 1066 | |
| 1067 | Args: |
| 1068 | seq_len: Sequence length |
| 1069 | mm_counts: Count of items per modality |
| 1070 | mm_options: Configurable options per modality (optional) |
| 1071 | |
| 1072 | """ |
| 1073 | builder = self.dummy_inputs |
| 1074 | dummy_text = builder.get_dummy_text(mm_counts) |
| 1075 | dummy_mm_data = builder.get_dummy_mm_data(seq_len, mm_counts, mm_options) |
| 1076 | dummy_mm_items = self.info.parse_mm_data(dummy_mm_data, validate=False) |
| 1077 | |
| 1078 | tokenizer = self.info.ctx.tokenizer |
| 1079 | dummy_prompt: list[int] |
| 1080 | if tokenizer is None: |
| 1081 | # Tokenizer-less models (e.g. `skip_tokenizer_init=True`) only |
| 1082 | # accept embeddings and have an empty dummy text, so there are no |
| 1083 | # prompt tokens. |
| 1084 | dummy_prompt = [] |
| 1085 | else: |
| 1086 | dummy_prompt = cached_encode(tokenizer, dummy_text, truncation=False) |
| 1087 | |
| 1088 | return ProcessorInputs( |
| 1089 | prompt=dummy_prompt, |
| 1090 | mm_data_items=dummy_mm_items, |
| 1091 | ) |
| 1092 | |
| 1093 | def get_dummy_mm_inputs( |
| 1094 | self, |
| 1095 | mm_counts: Mapping[str, int], |
| 1096 | *, |
| 1097 | cache: BaseMultiModalProcessorCache | None = None, |
| 1098 | scheduler_config: "SchedulerConfig | None" = None, |
| 1099 | ) -> MultiModalInput: |
| 1100 | """Create dummy data for profiling the memory usage of a model.""" |
| 1101 | model_config = self.info.ctx.model_config |
| 1102 | seq_len = model_config.max_model_len |
| 1103 | if scheduler_config is not None and scheduler_config.enable_chunked_prefill: |
| 1104 | seq_len = min(seq_len, scheduler_config.max_num_batched_tokens) |
| 1105 | |
| 1106 | mm_config = model_config.get_multimodal_config() |
| 1107 | |
| 1108 | processor_inputs = self.get_dummy_inputs( |
| 1109 | seq_len=seq_len, |
| 1110 | mm_counts=mm_counts, |
| 1111 | mm_options=mm_config.limit_per_prompt, |
| 1112 | ) |
| 1113 | processor_inputs.cache = cache |
| 1114 | |
| 1115 | mm_inputs = self.apply( |
| 1116 | processor_inputs, |
| 1117 | timing_ctx=TimingContext(enabled=False), |
| 1118 | ) |
| 1119 | |
| 1120 | prompt_token_ids = mm_inputs["prompt_token_ids"] |
| 1121 | total_len = len(prompt_token_ids) |
| 1122 | if total_len < seq_len: |
| 1123 | prompt_token_ids.extend([0] * (seq_len - total_len)) |
| 1124 | |
| 1125 | return mm_inputs |
| 1126 | |
| 1127 | def __call__( |
| 1128 | self, |
| 1129 | prompt: str | list[int], |
| 1130 | mm_items: MultiModalDataItems, |
| 1131 | mm_uuid_items: MultiModalUUIDItems | None = None, |
| 1132 | hf_processor_mm_kwargs: Mapping[str, object] | None = None, |
| 1133 | cache: BaseMultiModalProcessorCache | None = None, |
| 1134 | ) -> MultiModalInput: |
| 1135 | if isinstance(prompt, str): |
| 1136 | tokenizer = self.info.get_tokenizer() |
| 1137 | prompt = tokenizer.encode( |
| 1138 | prompt, |
| 1139 | **self.info.default_tok_params.get_encode_kwargs(), |
| 1140 | ) |
| 1141 | |
| 1142 | processor_inputs = ProcessorInputs( |
| 1143 | prompt, |
| 1144 | mm_items, |
| 1145 | mm_uuid_items, |
| 1146 | hf_processor_mm_kwargs=hf_processor_mm_kwargs or {}, |
| 1147 | cache=cache, |
| 1148 | ) |
| 1149 | |
| 1150 | return self.apply(processor_inputs, TimingContext(enabled=False)) |
| 1151 | |
| 1152 | @abstractmethod |
| 1153 | def _get_mm_fields_config( |
| 1154 | self, |
| 1155 | hf_inputs: BatchFeature, |
| 1156 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1157 | ) -> Mapping[str, MultiModalFieldConfig]: |
| 1158 | """Given the HF-processed data, output the metadata of each field.""" |
| 1159 | raise NotImplementedError |
| 1160 | |
| 1161 | @abstractmethod |
| 1162 | def _get_prompt_updates( |
| 1163 | self, |
| 1164 | mm_items: MultiModalDataItems, |
| 1165 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1166 | out_mm_kwargs: MultiModalKwargsItems, |
| 1167 | ) -> Sequence[PromptUpdate]: |
| 1168 | """Given the original multi-modal items for this modality |
| 1169 | and HF-processed data, output the updates to perform. |
| 1170 | |
| 1171 | The information returned by this method is used to update token inputs |
| 1172 | which bypass the HF processor. It is also used to update the output of |
| 1173 | HF processor if the HF process does not apply prompt updates to text |
| 1174 | inputs. |
| 1175 | |
| 1176 | Moreover, this information is critical to determine the token positions |
| 1177 | in order to construct |
| 1178 | [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange] |
| 1179 | for each multi-modal item. |
| 1180 | """ |
| 1181 | raise NotImplementedError |
| 1182 | |
| 1183 | def _bind_and_group_updates( |
| 1184 | self, |
| 1185 | prompt_updates: Sequence[PromptUpdate], |
| 1186 | mm_item_counts: Mapping[str, int], |
| 1187 | ) -> MultiModalPromptUpdates: |
| 1188 | return { |
| 1189 | modality: [ |
| 1190 | [update.resolve(item_idx) for update in updates] |
| 1191 | for item_idx in range(mm_item_counts.get(modality, 0)) |
| 1192 | ] |
| 1193 | for modality, updates in full_groupby_modality(prompt_updates) |
| 1194 | } |
| 1195 | |
| 1196 | def _get_mm_prompt_updates( |
| 1197 | self, |
| 1198 | mm_items: MultiModalDataItems, |
| 1199 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1200 | out_mm_kwargs: MultiModalKwargsItems, |
| 1201 | ) -> MultiModalPromptUpdates: |
| 1202 | unbound_prompt_updates = self._get_prompt_updates( |
| 1203 | mm_items=mm_items, |
| 1204 | hf_processor_mm_kwargs=hf_processor_mm_kwargs, |
| 1205 | out_mm_kwargs=out_mm_kwargs, |
| 1206 | ) |
| 1207 | |
| 1208 | mm_prompt_updates = self._bind_and_group_updates( |
| 1209 | unbound_prompt_updates, |
| 1210 | mm_items.get_all_counts(), |
| 1211 | ) |
| 1212 | |
| 1213 | return mm_prompt_updates |
| 1214 | |
| 1215 | def _find_mm_placeholders( |
| 1216 | self, |
| 1217 | new_token_ids: list[int], |
| 1218 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1219 | ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: |
| 1220 | return find_mm_placeholders(new_token_ids, mm_prompt_updates) |
| 1221 | |
| 1222 | def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str | None: |
| 1223 | """Get the text to pass to the HF processor alongside the multi-modal data. |
| 1224 | |
| 1225 | By default, no text is passed. If the HF processor requires that |
| 1226 | text and multi-modal items correspond to each other, you should |
| 1227 | override this method to return dummy text generated by |
| 1228 | [`DummyInputsBuilder`][vllm.multimodal.processing.BaseDummyInputsBuilder]. |
| 1229 | """ |
| 1230 | return None |
| 1231 | |
| 1232 | def _get_hf_mm_inputs( |
| 1233 | self, |
| 1234 | mm_items: MultiModalDataItems, |
| 1235 | hf_kwargs: Mapping[str, object], |
| 1236 | ) -> HFMultiModalInputs: |
| 1237 | """Extract the multi-modal data and corresponding keyword arguments |
| 1238 | for HF processing. |
| 1239 | |
| 1240 | If the HF processor expects the multi-modal data under different keys |
| 1241 | than those provided by the multi-modal items, or requires additional |
| 1242 | keyword arguments, you should override this method. |
| 1243 | """ |
| 1244 | hf_data = dict[str, object]() |
| 1245 | passthrough_data = dict[str, object]() |
| 1246 | |
| 1247 | for items in mm_items.values(): |
| 1248 | if not items: |
| 1249 | continue |
| 1250 | |
| 1251 | hf_data.update(items.get_processor_data()) |
| 1252 | passthrough_data.update(items.get_passthrough_data()) |
| 1253 | |
| 1254 | if hf_data: |
| 1255 | prompt_text = self._get_hf_mm_text(mm_items.get_all_counts()) |
| 1256 | if prompt_text is not None: |
| 1257 | hf_data = dict(text=prompt_text, **hf_data) |
| 1258 | |
| 1259 | # vLLM needs the untruncated sequence to keep placeholder |
| 1260 | # tokens aligned. Note that the text inputs are just dummy |
| 1261 | # text, not the original prompt. The original prompt is |
| 1262 | # already tokenized by the renderer. |
| 1263 | hf_kwargs = {"truncation": False, **hf_kwargs} |
| 1264 | |
| 1265 | # HF processors accept (images, text, videos, audio) |
| 1266 | # so we need to remap vLLM's "audios" -> HF "audio" |
| 1267 | if "audios" in hf_data: |
| 1268 | hf_data["audio"] = hf_data.pop("audios") |
| 1269 | |
| 1270 | return HFMultiModalInputs(hf_data, hf_kwargs, passthrough_data) |
| 1271 | |
| 1272 | def _postprocess_hf_mm_data( |
| 1273 | self, |
| 1274 | mm_inputs: Mapping[str, object], |
| 1275 | hf_processor_mm_kwargs: Mapping[str, object], |
| 1276 | processed_data: BatchFeature, |
| 1277 | ) -> BatchFeature: |
| 1278 | """Post-process the combined processor and passthrough data. |
| 1279 | |
| 1280 | By default, the output is returned as-is. If you need to modify the |
| 1281 | output of the HF processor before it is converted into multi-modal |
| 1282 | keyword arguments, you should override this method. |
| 1283 | """ |
| 1284 | return processed_data |
| 1285 | |
| 1286 | def _call_hf_processor( |
| 1287 | self, |
| 1288 | hf_data: Mapping[str, object], |
| 1289 | hf_kwargs: Mapping[str, object], |
| 1290 | ) -> BatchFeature: |
| 1291 | """Call the HF processor on the normalized multi-modal inputs.""" |
| 1292 | return self.info.ctx.call_hf_processor( |
| 1293 | self.info.get_hf_processor(**hf_kwargs), |
| 1294 | hf_data, |
| 1295 | hf_kwargs, |
| 1296 | ) |
| 1297 | |
| 1298 | def _finalize_hf_mm_data( |
| 1299 | self, |
| 1300 | hf_data: Mapping[str, object], |
| 1301 | hf_kwargs: Mapping[str, object], |
| 1302 | passthrough_data: Mapping[str, object], |
| 1303 | processed_data: BatchFeature | None = None, |
| 1304 | ) -> BatchFeature: |
| 1305 | """Merge passthrough fields and apply model-specific postprocessing.""" |
| 1306 | if processed_data is None: |
| 1307 | from transformers.feature_extraction_utils import BatchFeature |
| 1308 | |
| 1309 | processed_data = BatchFeature() |
| 1310 | |
| 1311 | processed_data.update(passthrough_data) |
| 1312 | return self._postprocess_hf_mm_data(hf_data, hf_kwargs, processed_data) |
| 1313 | |
| 1314 | def _apply_hf_processor_main( |
| 1315 | self, |
| 1316 | mm_items: MultiModalDataItems, |
| 1317 | hf_kwargs: Mapping[str, object], |
| 1318 | ) -> BatchFeature: |
| 1319 | """Apply the HF processor on the multi-modal data.""" |
| 1320 | hf_data, hf_kwargs, passthrough_data = self._get_hf_mm_inputs( |
| 1321 | mm_items, hf_kwargs |
| 1322 | ) |
| 1323 | |
| 1324 | if hf_data: |
| 1325 | processed_data = self._call_hf_processor(hf_data, hf_kwargs) |
| 1326 | else: |
| 1327 | processed_data = None |
| 1328 | |
| 1329 | return self._finalize_hf_mm_data( |
| 1330 | hf_data, hf_kwargs, passthrough_data, processed_data |
| 1331 | ) |
| 1332 | |
| 1333 | def _postprocess_prompt(self, prompt: list[int]) -> list[int]: |
| 1334 | """Post-process the prompt token IDs before locating or applying |
| 1335 | multi-modal prompt updates. |
| 1336 | |
| 1337 | By default, the prompt is returned as-is. If the HF processor (or |
| 1338 | chat template) applies additional transformations to the prompt |
| 1339 | that are not reflected in its multi-modal outputs, you should |
| 1340 | override this method to replicate them. |
| 1341 | """ |
| 1342 | return prompt |
| 1343 | |
| 1344 | def _get_cache_missing_items( |
| 1345 | self, |
| 1346 | cache: BaseMultiModalProcessorCache, |
| 1347 | mm_data_items: MultiModalDataItems, |
| 1348 | mm_hashes: MultiModalHashes, |
| 1349 | ) -> tuple[MultiModalIsCached, MultiModalDataItems]: |
| 1350 | mm_is_cached = { |
| 1351 | modality: cache.is_cached(hashes) for modality, hashes in mm_hashes.items() |
| 1352 | } |
| 1353 | |
| 1354 | mm_missing_idxs = { |
| 1355 | modality: [ |
| 1356 | idx |
| 1357 | for idx, item_is_cached in enumerate(items_is_cached) |
| 1358 | if not item_is_cached |
| 1359 | ] |
| 1360 | for modality, items_is_cached in mm_is_cached.items() |
| 1361 | } |
| 1362 | |
| 1363 | mm_missing_data = {} |
| 1364 | for modality, idxs in mm_missing_idxs.items(): |
| 1365 | if not idxs: |
| 1366 | continue |
| 1367 | |
| 1368 | missing_modality_data = [] |
| 1369 | for idx in idxs: |
| 1370 | data = mm_data_items[modality][idx] |
| 1371 | if data is None: |
| 1372 | raise ValueError( |
| 1373 | f"Cache miss for {modality} at index {idx} " |
| 1374 | f"but data is not provided." |
| 1375 | ) |
| 1376 | else: |
| 1377 | missing_modality_data.append(data) |
| 1378 | |
| 1379 | mm_missing_data[modality] = missing_modality_data |
| 1380 | |
| 1381 | mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False) |
| 1382 | |
| 1383 | return mm_is_cached, mm_missing_items |
| 1384 | |
| 1385 | def _recompute_cached_prompt_update( |
| 1386 | self, |
| 1387 | cached_update: ResolvedPromptUpdate, |
| 1388 | new_item_idx: int, |
| 1389 | ) -> ResolvedPromptUpdate: |
| 1390 | """Override this if other attributes of `ResolvedPromptUpdate` |
| 1391 | also need to be recomputed after retrieving from the cache. |
| 1392 | """ |
| 1393 | return replace(cached_update, item_idx=new_item_idx) |
| 1394 | |
| 1395 | def _merge_mm_kwargs( |
| 1396 | self, |
| 1397 | cache: BaseMultiModalProcessorCache, |
| 1398 | mm_hashes: MultiModalHashes, |
| 1399 | mm_is_cached: MultiModalIsCached, |
| 1400 | mm_missing_kwargs: MultiModalKwargsItems, |
| 1401 | mm_missing_prompt_updates: MultiModalPromptUpdates, |
| 1402 | ) -> tuple[MultiModalKwargsOptionalItems, MultiModalPromptUpdates]: |
| 1403 | # Need to touch all mm hashes before update to avoid hash in updated |
| 1404 | # list evict during update |
| 1405 | for hashes in mm_hashes.values(): |
| 1406 | for item_hash in hashes: |
| 1407 | cache.touch_sender_cache_item(item_hash) |
| 1408 | |
| 1409 | mm_missing_next_idx = defaultdict[str, int](lambda: 0) |
| 1410 | |
| 1411 | merged_kwargs = defaultdict[str, list[MultiModalKwargsItem | None]](list) |
| 1412 | merged_prompt_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]]( |
| 1413 | list |
| 1414 | ) |
| 1415 | for modality, hashes in mm_hashes.items(): |
| 1416 | missing_kwargs = mm_missing_kwargs.get(modality, []) |
| 1417 | missing_prompt_updates = mm_missing_prompt_updates.get(modality, []) |
| 1418 | |
| 1419 | for item_idx, item_hash in enumerate(hashes): |
| 1420 | if not mm_is_cached[modality][item_idx]: |
| 1421 | missing_next_idx = mm_missing_next_idx[modality] |
| 1422 | missing_kwargs_item = missing_kwargs[missing_next_idx] |
| 1423 | missing_updates_item = missing_prompt_updates[missing_next_idx] |
| 1424 | |
| 1425 | mm_missing_next_idx[modality] += 1 |
| 1426 | |
| 1427 | item = missing_kwargs_item, missing_updates_item |
| 1428 | else: |
| 1429 | item = None |
| 1430 | |
| 1431 | kwargs, updates = cache.get_and_update_item(item, item_hash) |
| 1432 | |
| 1433 | merged_kwargs[modality].append(kwargs) |
| 1434 | merged_prompt_updates[modality].append( |
| 1435 | [ |
| 1436 | self._recompute_cached_prompt_update(update, item_idx) |
| 1437 | for update in updates |
| 1438 | ] |
| 1439 | ) |
| 1440 | |
| 1441 | mm_kwargs = MultiModalKwargsItems(merged_kwargs) |
| 1442 | mm_prompt_updates = dict(merged_prompt_updates) |
| 1443 | |
| 1444 | return mm_kwargs, mm_prompt_updates |
| 1445 | |
| 1446 | def _apply_hf_processor( |
| 1447 | self, |
| 1448 | inputs: ProcessorInputs, |
| 1449 | timing_ctx: TimingContext, |
| 1450 | ) -> MultiModalProcessingResult: |
| 1451 | with timing_ctx.record("apply_hf_processor"): |
| 1452 | mm_processed_data = self._apply_hf_processor_main( |
| 1453 | mm_items=inputs.mm_data_items, |
| 1454 | hf_kwargs=inputs.hf_processor_mm_kwargs, |
| 1455 | ) |
| 1456 | |
| 1457 | mm_kwargs = MultiModalKwargsItems.from_hf_inputs( |
| 1458 | mm_processed_data, |
| 1459 | self._get_mm_fields_config( |
| 1460 | mm_processed_data, inputs.hf_processor_mm_kwargs |
| 1461 | ), |
| 1462 | ) |
| 1463 | |
| 1464 | # Use overrides if provided; fallback to data-dependent hashing. |
| 1465 | with timing_ctx.record("get_mm_hashes"): |
| 1466 | mm_hashes = inputs.get_mm_hashes( |
| 1467 | self.info.model_id, |
| 1468 | self.info.ctx.get_mm_config().mm_hasher_algorithm, |
| 1469 | ) |
| 1470 | |
| 1471 | mm_prompt_updates = self._get_mm_prompt_updates( |
| 1472 | inputs.mm_data_items, |
| 1473 | inputs.hf_processor_mm_kwargs, |
| 1474 | mm_kwargs, |
| 1475 | ) |
| 1476 | |
| 1477 | return MultiModalProcessingResult( |
| 1478 | prompt_ids=self._postprocess_prompt(inputs.prompt), |
| 1479 | kwargs=mm_kwargs, |
| 1480 | hashes=mm_hashes, |
| 1481 | prompt_updates=mm_prompt_updates, |
| 1482 | ) |
| 1483 | |
| 1484 | def _cached_apply_hf_processor( |
| 1485 | self, |
| 1486 | inputs: ProcessorInputs, |
| 1487 | timing_ctx: TimingContext, |
| 1488 | ) -> MultiModalProcessingResult: |
| 1489 | """Apply the HF processor on the full prompt text, |
| 1490 | caching the results and reusing cached results. |
| 1491 | """ |
| 1492 | cache = inputs.cache |
| 1493 | has_passthrough_data = any( |
| 1494 | len(items.get_passthrough_data()) > 0 |
| 1495 | for items in inputs.mm_data_items.values() |
| 1496 | ) |
| 1497 | |
| 1498 | if cache is None or has_passthrough_data: |
| 1499 | return self._apply_hf_processor(inputs, timing_ctx) |
| 1500 | |
| 1501 | with timing_ctx.record("get_mm_hashes"): |
| 1502 | mm_hashes = inputs.get_mm_hashes( |
| 1503 | self.info.model_id, |
| 1504 | self.info.ctx.get_mm_config().mm_hasher_algorithm, |
| 1505 | ) |
| 1506 | |
| 1507 | with timing_ctx.record("get_cache_missing_items"): |
| 1508 | mm_is_cached, mm_missing_data_items = self._get_cache_missing_items( |
| 1509 | cache=cache, |
| 1510 | mm_data_items=inputs.mm_data_items, |
| 1511 | mm_hashes=mm_hashes, |
| 1512 | ) |
| 1513 | |
| 1514 | # NOTE: The prompt does not correspond to `mm_missing_data_items`, |
| 1515 | # so we can't apply prompt updates until the new multimodal |
| 1516 | # items are combined with the cached multimodal items |
| 1517 | with timing_ctx.record("apply_hf_processor"): |
| 1518 | mm_missing_processed_data = self._apply_hf_processor_main( |
| 1519 | mm_items=mm_missing_data_items, |
| 1520 | hf_kwargs=inputs.hf_processor_mm_kwargs, |
| 1521 | ) |
| 1522 | |
| 1523 | mm_missing_kwargs = MultiModalKwargsItems.from_hf_inputs( |
| 1524 | mm_missing_processed_data, |
| 1525 | self._get_mm_fields_config( |
| 1526 | mm_missing_processed_data, inputs.hf_processor_mm_kwargs |
| 1527 | ), |
| 1528 | ) |
| 1529 | |
| 1530 | mm_missing_prompt_updates = self._get_mm_prompt_updates( |
| 1531 | mm_missing_data_items, |
| 1532 | inputs.hf_processor_mm_kwargs, |
| 1533 | mm_missing_kwargs, |
| 1534 | ) |
| 1535 | |
| 1536 | with timing_ctx.record("merge_mm_kwargs"): |
| 1537 | try: |
| 1538 | mm_kwargs, mm_prompt_updates = self._merge_mm_kwargs( |
| 1539 | cache, |
| 1540 | mm_hashes=mm_hashes, |
| 1541 | mm_is_cached=mm_is_cached, |
| 1542 | mm_missing_kwargs=mm_missing_kwargs, |
| 1543 | mm_missing_prompt_updates=mm_missing_prompt_updates, |
| 1544 | ) |
| 1545 | finally: |
| 1546 | cache.release_sender_touches() |
| 1547 | |
| 1548 | return MultiModalProcessingResult( |
| 1549 | prompt_ids=self._postprocess_prompt(inputs.prompt), |
| 1550 | kwargs=mm_kwargs, |
| 1551 | hashes=mm_hashes, |
| 1552 | prompt_updates=mm_prompt_updates, |
| 1553 | ) |
| 1554 | |
| 1555 | def _apply_token_matches( |
| 1556 | self, |
| 1557 | prompt: list[int], |
| 1558 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1559 | ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: |
| 1560 | return apply_token_matches(prompt, mm_prompt_updates) |
| 1561 | |
| 1562 | def _apply_token_matches_with_placeholders( |
| 1563 | self, |
| 1564 | token_ids: list[int], |
| 1565 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1566 | ) -> tuple[ |
| 1567 | list[int], |
| 1568 | MultiModalPromptUpdatesApplyResult, |
| 1569 | Mapping[str, list[PlaceholderFeaturesInfo]], |
| 1570 | ]: |
| 1571 | return _apply_token_matches_with_placeholders( |
| 1572 | token_ids, |
| 1573 | mm_prompt_updates, |
| 1574 | ) |
| 1575 | |
| 1576 | def _iter_text_matches( |
| 1577 | self, |
| 1578 | update: ResolvedPromptUpdate, |
| 1579 | prompt_text: str, |
| 1580 | tokenizer: TokenizerLike, |
| 1581 | *, |
| 1582 | start_idx: int = 0, |
| 1583 | ) -> Generator[PromptTargetMatch, None, None]: |
| 1584 | """Yield each instance of `update.target` found in `prompt_text`.""" |
| 1585 | target = update.target |
| 1586 | |
| 1587 | if isinstance(target, PromptIndex): |
| 1588 | match_idx = target.get_text_match_index(tokenizer, prompt_text, start_idx) |
| 1589 | if match_idx is not None: |
| 1590 | yield PromptTargetMatch(match_idx, match_idx) |
| 1591 | |
| 1592 | return |
| 1593 | |
| 1594 | target_text = tokenizer.decode(target) |
| 1595 | if not target_text: |
| 1596 | return |
| 1597 | |
| 1598 | idx = prompt_text.find(target_text, start_idx) |
| 1599 | while idx != -1: |
| 1600 | end_idx = idx + len(target_text) |
| 1601 | yield PromptTargetMatch(idx, end_idx) |
| 1602 | idx = prompt_text.find(target_text, end_idx) |
| 1603 | |
| 1604 | def _apply_prompt_updates_via_text( |
| 1605 | self, |
| 1606 | token_ids: list[int], |
| 1607 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1608 | *, |
| 1609 | encode_segments_separately: bool = False, |
| 1610 | ) -> tuple[list[int], MultiModalPromptUpdatesApplyResult]: |
| 1611 | """Apply multi-modal prompt updates in text space, then encode the |
| 1612 | result. |
| 1613 | |
| 1614 | A non-special-token target may be tokenized differently inside the |
| 1615 | prompt because the tokens may go across the boundaries of the target |
| 1616 | (e.g. when searching for "foo" in "food", if "food" itself makes up |
| 1617 | a token, then the token ID of "foo" will not appear at all). Since |
| 1618 | it is inefficient to search for all possible tokenizations of the |
| 1619 | target in the prompt, we instead perform text-based updates on the |
| 1620 | decoded token IDs, then encode them back. |
| 1621 | """ |
| 1622 | tokenizer = self.info.get_tokenizer() |
| 1623 | prompt_text = tokenizer.decode(token_ids) |
| 1624 | |
| 1625 | matched_updates, result = _plan_prompt_updates_with( |
| 1626 | mm_prompt_updates, |
| 1627 | lambda update, start_idx: self._iter_text_matches( |
| 1628 | update, prompt_text, tokenizer, start_idx=start_idx |
| 1629 | ), |
| 1630 | ) |
| 1631 | |
| 1632 | out_texts = list[str]() |
| 1633 | prev_end_idx = 0 |
| 1634 | for matched_update in matched_updates: |
| 1635 | update = matched_update.update |
| 1636 | match = matched_update.match |
| 1637 | |
| 1638 | if update.mode == UpdateMode.INSERT: |
| 1639 | end_idx_to_insert = match.end_idx |
| 1640 | elif update.mode == UpdateMode.REPLACE: |
| 1641 | end_idx_to_insert = match.start_idx |
| 1642 | else: |
| 1643 | assert_never(update.mode) |
| 1644 | |
| 1645 | out_texts.append(prompt_text[prev_end_idx:end_idx_to_insert]) |
| 1646 | out_texts.append(tokenizer.decode(update.content.full)) |
| 1647 | prev_end_idx = match.end_idx |
| 1648 | |
| 1649 | out_texts.append(prompt_text[prev_end_idx:]) |
| 1650 | |
| 1651 | if encode_segments_separately: |
| 1652 | # Encode each segment separately instead of joining into one |
| 1653 | # text and encoding it in a single pass. Joining first would |
| 1654 | # let BPE merge tokens across a segment boundary, silently |
| 1655 | # changing how a non-special-token placeholder is tokenized. |
| 1656 | new_token_ids = flatten_2d_lists( |
| 1657 | [tokenizer.encode(text, add_special_tokens=False) for text in out_texts] |
| 1658 | ) |
| 1659 | else: |
| 1660 | new_token_ids = tokenizer.encode( |
| 1661 | "".join(out_texts), add_special_tokens=False |
| 1662 | ) |
| 1663 | |
| 1664 | return new_token_ids, result |
| 1665 | |
| 1666 | def _matched_updates_from_result( |
| 1667 | self, |
| 1668 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1669 | match_result: MultiModalPromptUpdatesApplyResult, |
| 1670 | ) -> dict[str, list[Sequence[ResolvedPromptUpdate]]]: |
| 1671 | matched_updates = defaultdict[str, list[Sequence[ResolvedPromptUpdate]]](list) |
| 1672 | for modality, update_idxs in match_result.items(): |
| 1673 | for item_idx, update_idx in enumerate(update_idxs): |
| 1674 | assert update_idx is not None, ( |
| 1675 | "Failed to apply prompt replacement for " |
| 1676 | f"mm_items[{modality!r}][{item_idx}]" |
| 1677 | ) |
| 1678 | |
| 1679 | matched_updates[modality].append( |
| 1680 | [mm_prompt_updates[modality][item_idx][update_idx]] |
| 1681 | ) |
| 1682 | |
| 1683 | return dict(matched_updates) |
| 1684 | |
| 1685 | def _apply_prompt_updates( |
| 1686 | self, |
| 1687 | token_ids: list[int], |
| 1688 | mm_prompt_updates: MultiModalPromptUpdates, |
| 1689 | ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: |
| 1690 | """Apply multi-modal prompt updates to token IDs.""" |
| 1691 | new_token_ids, match_result, placeholders = ( |
| 1692 | self._apply_token_matches_with_placeholders( |
| 1693 | token_ids, |
| 1694 | mm_prompt_updates, |
| 1695 | ) |
| 1696 | ) |
| 1697 | |
| 1698 | if all( |
| 1699 | all(update_idx is not None for update_idx in update_idxs) |
| 1700 | for update_idxs in match_result.values() |
| 1701 | ): |
| 1702 | placeholders = { |
| 1703 | modality: modality_placeholders |
| 1704 | for modality, modality_placeholders in placeholders.items() |
| 1705 | if modality_placeholders |
| 1706 | } |
| 1707 | return new_token_ids, placeholders |
| 1708 | |
| 1709 | # A non-special-token target may be tokenized differently inside |
| 1710 | # the prompt, so fall back to performing the updates on the |
| 1711 | # decoded text, then encoding the result back. |
| 1712 | new_token_ids, match_result = self._apply_prompt_updates_via_text( |
| 1713 | token_ids, |
| 1714 | mm_prompt_updates, |
| 1715 | ) |
| 1716 | |
| 1717 | placeholders = self._find_mm_placeholders( |
| 1718 | new_token_ids, |
| 1719 | self._matched_updates_from_result(mm_prompt_updates, match_result), |
| 1720 | ) |
| 1721 | |
| 1722 | return new_token_ids, placeholders |
| 1723 | |
| 1724 | def _validate_mm_kwargs( |
| 1725 | self, |
| 1726 | mm_kwargs: MultiModalKwargsOptionalItems, |
| 1727 | mm_item_counts: Mapping[str, int], |
| 1728 | ) -> None: |
| 1729 | for modality, item_count in mm_item_counts.items(): |
| 1730 | items = mm_kwargs.get(modality, []) |
| 1731 | |
| 1732 | if len(items) != item_count: |
| 1733 | raise RuntimeError( |
| 1734 | f"Expected there to be {item_count} {modality} items in " |
| 1735 | f"keyword arguments corresponding to {item_count} " |
| 1736 | f"{modality} data items, but only found {len(items)}! " |
| 1737 | "There is likely a problem with your " |
| 1738 | "implementation of merged multi-modal processor for this " |
| 1739 | "model (usually arising from an inconsistency between " |
| 1740 | "`_apply_hf_processor_main` and `_get_mm_fields_config`)." |
| 1741 | ) |
| 1742 | |
| 1743 | def _validate_mm_updates( |
| 1744 | self, |
| 1745 | mm_updates: MultiModalPromptUpdates, |
| 1746 | mm_item_counts: Mapping[str, int], |
| 1747 | ) -> None: |
| 1748 | for modality, item_count in mm_item_counts.items(): |
| 1749 | placeholders = mm_updates.get(modality, []) |
| 1750 | |
| 1751 | if len(placeholders) != item_count: |
| 1752 | raise RuntimeError( |
| 1753 | f"Expected there to be {item_count} prompt updates " |
| 1754 | f"corresponding to {item_count} {modality} items, but " |
| 1755 | f"instead found {len(placeholders)} prompt updates! " |
| 1756 | "This is likely because you forgot to include input " |
| 1757 | "placeholder tokens (e.g., `<image>`, `<|image_pad|>`) " |
| 1758 | "in the prompt. If the model has a chat template, make " |
| 1759 | "sure you have applied it before calling `LLM.generate`." |
| 1760 | ) |
| 1761 | |
| 1762 | def _validate_mm_placeholders( |
| 1763 | self, |
| 1764 | mm_placeholders: Mapping[str, list[PlaceholderFeaturesInfo]], |
| 1765 | mm_item_counts: Mapping[str, int], |
| 1766 | ) -> None: |
| 1767 | for modality, item_count in mm_item_counts.items(): |
| 1768 | placeholders = mm_placeholders.get(modality, []) |
| 1769 | |
| 1770 | if len(placeholders) != item_count: |
| 1771 | raise RuntimeError( |
| 1772 | f"Expected there to be {item_count} prompt placeholders " |
| 1773 | f"corresponding to {item_count} {modality} items, but " |
| 1774 | f"instead found {len(placeholders)} prompt placeholders! " |
| 1775 | "Make sure the implementation of `_apply_hf_processor_main` " |
| 1776 | "and `_get_mm_fields_config` are consistent with each other." |
| 1777 | ) |
| 1778 | |
| 1779 | def _maybe_apply_prompt_updates( |
| 1780 | self, |
| 1781 | mm_items: MultiModalDataItems, |
| 1782 | mm_res: MultiModalProcessingResult, |
| 1783 | ) -> tuple[list[int], Mapping[str, list[PlaceholderFeaturesInfo]]]: |
| 1784 | mm_item_counts = mm_items.get_all_counts() |
| 1785 | self._validate_mm_kwargs(mm_res.kwargs, mm_item_counts) |
| 1786 | self._validate_mm_updates(mm_res.prompt_updates, mm_item_counts) |
| 1787 | |
| 1788 | prompt_ids, mm_placeholders = self._apply_prompt_updates( |
| 1789 | mm_res.prompt_ids, |
| 1790 | mm_res.prompt_updates, |
| 1791 | ) |
| 1792 | |
| 1793 | self._validate_mm_placeholders(mm_placeholders, mm_item_counts) |
| 1794 | |
| 1795 | return prompt_ids, mm_placeholders |
| 1796 | |
| 1797 | def apply( |
| 1798 | self, |
| 1799 | inputs: ProcessorInputs, |
| 1800 | timing_ctx: TimingContext, |
| 1801 | ) -> MultiModalInput: |
| 1802 | """Process multi-modal inputs to be used in vLLM. |
| 1803 | |
| 1804 | The main steps are: |
| 1805 | |
| 1806 | 1. Perform MM-specific post-processing to the token inputs. |
| 1807 | 2. Apply HF Processor on multi-modal data together with optional |
| 1808 | dummy text, outputting processed multi-modal data tensors. |
| 1809 | 3. Find and update sequences in the token IDs with placeholder tokens. |
| 1810 | The number of placeholder tokens equals the feature size of the |
| 1811 | multi-modal data outputted by the multi-modal encoder. |
| 1812 | 4. Extract information about the placeholder tokens from the |
| 1813 | processed token IDs. |
| 1814 | """ |
| 1815 | mm_res = self._cached_apply_hf_processor(inputs, timing_ctx) |
| 1816 | |
| 1817 | with timing_ctx.record("apply_prompt_updates"): |
| 1818 | prompt_ids, mm_placeholders = self._maybe_apply_prompt_updates( |
| 1819 | mm_items=inputs.mm_data_items, |
| 1820 | mm_res=mm_res, |
| 1821 | ) |
| 1822 | |
| 1823 | mm_placeholder_ranges = { |
| 1824 | modality: [item.to_range() for item in placeholders] |
| 1825 | for modality, placeholders in mm_placeholders.items() |
| 1826 | } |
| 1827 | |
| 1828 | return mm_input( |
| 1829 | prompt_token_ids=prompt_ids, |
| 1830 | mm_kwargs=mm_res.kwargs, |
| 1831 | mm_hashes=mm_res.hashes, |
| 1832 | mm_placeholders=mm_placeholder_ranges, |
| 1833 | ) |
| 1834 | |
| 1835 | |
| 1836 | class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]): |
| 1837 | skip_decoder_start_token: bool = False |
| 1838 | |
| 1839 | @abstractmethod |
| 1840 | def create_encoder_prompt( |
| 1841 | self, |
| 1842 | prompt: list[int], |
| 1843 | mm_items: MultiModalDataItems, |
| 1844 | ) -> list[int]: |
| 1845 | """Create input prompt for the encoder. HF processor will be applied on |
| 1846 | this prompt during profiling and generation. |
| 1847 | """ |
| 1848 | raise NotImplementedError |
| 1849 | |
| 1850 | def create_decoder_prompt( |
| 1851 | self, |
| 1852 | prompt: str | list[int], |
| 1853 | mm_items: MultiModalDataItems, |
| 1854 | ) -> str | list[int]: |
| 1855 | """Create input prompt for the decoder.""" |
| 1856 | return prompt |
| 1857 | |
| 1858 | def _get_enc_dec_inputs( |
| 1859 | self, |
| 1860 | prompt: list[int], |
| 1861 | mm_items: MultiModalDataItems, |
| 1862 | encoder_inputs: MultiModalInput, |
| 1863 | ): |
| 1864 | tokenizer = self.info.get_tokenizer() |
| 1865 | decoder_prompt_raw = self.create_decoder_prompt(prompt, mm_items) |
| 1866 | if isinstance(decoder_prompt_raw, str): |
| 1867 | decoder_prompt_text = decoder_prompt_raw |
| 1868 | decoder_prompt_ids = tokenizer.encode( |
| 1869 | decoder_prompt_raw, add_special_tokens=False |
| 1870 | ) |
| 1871 | else: |
| 1872 | decoder_prompt_text = None |
| 1873 | decoder_prompt_ids = decoder_prompt_raw |
| 1874 | |
| 1875 | return mm_enc_dec_input( |
| 1876 | encoder_inputs, |
| 1877 | decoder_prompt_ids, |
| 1878 | decoder_prompt=decoder_prompt_text, |
| 1879 | ) |
| 1880 | |
| 1881 | def apply( |
| 1882 | self, |
| 1883 | inputs: ProcessorInputs, |
| 1884 | timing_ctx: TimingContext, |
| 1885 | ) -> MultiModalEncDecInput: |
| 1886 | """Process multi-modal inputs to be used in vLLM. |
| 1887 | The main processing steps are modified to fit encoder-decoder model: |
| 1888 | 1. Create encoder prompt from input prompt text. |
| 1889 | 2. Apply the HF processor on encoder prompt. |
| 1890 | 3. Copy the input prompt text as decoder prompt inputs. |
| 1891 | """ |
| 1892 | encoder_prompt = self.create_encoder_prompt( |
| 1893 | inputs.prompt, |
| 1894 | inputs.mm_data_items, |
| 1895 | ) |
| 1896 | encoder_processor_inputs = ProcessorInputs( |
| 1897 | encoder_prompt, |
| 1898 | inputs.mm_data_items, |
| 1899 | inputs.mm_uuid_items, |
| 1900 | hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs, |
| 1901 | media_io_kwargs=inputs.media_io_kwargs, |
| 1902 | cache=inputs.cache, |
| 1903 | ) |
| 1904 | |
| 1905 | encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx) |
| 1906 | |
| 1907 | return self._get_enc_dec_inputs( |
| 1908 | prompt=inputs.prompt, |
| 1909 | mm_items=inputs.mm_data_items, |
| 1910 | encoder_inputs=encoder_inputs, |
| 1911 | ) |