@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f1579b229d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 21:00 UTC

[VLM] Generalized prompt updates for multi-modal processor (#13964)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Feb 28, 2025 at 01:44 UTC f1579b229de7b21a9e4aec34be2fab29b2c84675
29 files changed +635 -492
docs/source/contributing/model/multimodal.md
+13 -13
@@ -720,13 +720,13 @@ def _get_mm_fields_config(
720
721 :::::
722
723 -### Prompt replacements
723 +### Prompt updates
724
725 -Override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` to
726 -return a list of {class}`~vllm.multimodal.processing.PromptReplacement` instances.
725 +Override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates` to
726 +return a list of {class}`~vllm.multimodal.processing.PromptUpdate` instances.
727
728 -Each {class}`~vllm.multimodal.processing.PromptReplacement` instance specifies a find-and-replace
729 -operation performed by the HF processor.
728 +Each {class}`~vllm.multimodal.processing.PromptUpdate` instance specifies an update operation
729 +(e.g.: insertion, replacement) performed by the HF processor.
730
731 ::::{tab-set}
732 :::{tab-item} Basic example: LLaVA
@@ -743,15 +743,15 @@ for sample in text:
743 ```
744
745 It simply repeats each input `image_token` a number of times equal to the number of placeholder feature tokens (`num_image_tokens`).
746 -Based on this, we override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements` as follows:
746 +Based on this, we override {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates` as follows:
747
748 ```python
749 -def _get_prompt_replacements(
749 +def _get_prompt_updates(
750 self,
751 mm_items: MultiModalDataItems,
752 hf_processor_mm_kwargs: Mapping[str, object],
753 out_mm_kwargs: MultiModalKwargs,
754 -) -> list[PromptReplacement]:
754 +) -> Sequence[PromptUpdate]:
755 hf_config = self.info.get_hf_config()
756 image_token_id = hf_config.image_token_index
757
@@ -859,7 +859,7 @@ prompt_tokens, prompts_length = _tokenize_prompts_with_image_and_batch(
859 )
860 ```
861
862 -To accommodate this, instead of a string you can return an instance of `PromptReplacementDetails`
862 +To accommodate this, instead of a string you can return an instance of `PromptUpdateDetails`
863 with different `full` and `feature` attributes:
864
865 ```python
@@ -878,7 +878,7 @@ def get_replacement_fuyu(item_idx: int):
878 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
879 [_NEWLINE_TOKEN_ID]) * nrows
880
881 - return PromptReplacementDetails(
881 + return PromptUpdateDetails(
882 full=image_tokens + [bos_token_id],
883 features=image_tokens,
884 )
@@ -888,12 +888,12 @@ Finally, noticing that the HF processor removes the `|ENDOFTEXT|` token from the
888 we can search for it to conduct the replacement at the start of the string:
889
890 ```python
891 -def _get_prompt_replacements(
891 +def _get_prompt_updates(
892 self,
893 mm_items: MultiModalDataItems,
894 hf_processor_mm_kwargs: Mapping[str, object],
895 out_mm_kwargs: MultiModalKwargs,
896 -) -> list[PromptReplacement]:
896 +) -> Sequence[PromptUpdate]:
897 hf_config = self.info.get_hf_config()
898 bos_token_id = hf_config.bos_token_id
899 assert isinstance(bos_token_id, int)
@@ -913,7 +913,7 @@ def _get_prompt_replacements(
913 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
914 [_NEWLINE_TOKEN_ID]) * nrows
915
916 - return PromptReplacementDetails(
916 + return PromptUpdateDetails(
917 full=image_tokens + [bos_token_id],
918 features=image_tokens,
919 )
docs/source/design/mm_processing.md
+14 -9
@@ -6,11 +6,16 @@ To enable various optimizations in vLLM such as [chunked prefill](#chunked-prefi
6
7 Here are the main features of {class}`~vllm.multimodal.processing.BaseMultiModalProcessor`:
8
9 -## Prompt Replacement Detection
9 +## Prompt Update Detection
10
11 -One of the main responsibilies of HF processor is to replace input placeholder tokens (e.g. `<image>` for a single image) with feature placeholder tokens (e.g. `<image><image>...<image>`, the number of which equals to the feature size). The information about which tokens have been replaced is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
11 +One of the main responsibilies of HF processor is to update the prompt with placeholder tokens. For example:
12
13 -In vLLM, this information is specified using {class}`~vllm.multimodal.processing.PromptReplacement` in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`. Given this specification, we can automatically detect whether HF has replaced the input placeholder tokens by checking whether the feature placeholder tokens exist in the prompt.
13 +- Insert feature placeholder tokens (e.g. `<image><image>...<image>`, the number of which equals to the feature size) at the start of the string.
14 +- Replace existing input placeholder tokens (e.g. `<image>` for a single image) with feature placeholder tokens (e.g. `<image><image>...<image>`, the number of which equals to the feature size).
15 +
16 +The information about which tokens have been updated is key to finding the correspondence between placeholder feature tokens and multi-modal inputs.
17 +
18 +In vLLM, this information is specified using {class}`~vllm.multimodal.processing.PromptUpdate` in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates`. We can automatically detect whether HF has updated the prompt by checking the existence of the updated tokens.
19
20 ## Tokenized Prompt Inputs
21
@@ -22,7 +27,7 @@ Consider that HF processors follow these main steps:
27
28 1. Tokenize the text
29 2. Process multi-modal inputs
25 -3. Perform prompt replacement
30 +3. Perform prompt updates
31
32 And we require that:
33
@@ -44,16 +49,16 @@ Moreover, since the tokenized text has not passed through the HF processor, we h
49
50 We work around the first issue by requiring each model to define how to generate dummy text based on the number of multi-modal inputs, via {meth}`~vllm.multimodal.profiling.BaseDummyInputsBuilder.get_dummy_processor_inputs`. This lets us generate dummy text corresponding to the multi-modal inputs and input them together to obtain the processed multi-modal data.
51
47 -(mm-automatic-prompt-replacement)=
52 +(mm-automatic-prompt-updating)=
53
49 -### Automatic prompt replacement
54 +### Automatic prompt updating
55
56 We address the second issue by implementing model-agnostic code in
52 -{meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_replacements` to automatically replace input placeholder tokens with feature placeholder tokens based on the specification outputted by {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_replacements`.
57 +{meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_prompt_updates` to automatically update the prompt with feature placeholder tokens based on the specification outputted by {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._get_prompt_updates`.
58
59 ### Summary
60
56 -With the help of dummy text and automatic prompt replacement, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main`.
61 +With the help of dummy text and automatic prompt updating, our multi-modal processor can finally accept both text and token prompts with multi-modal data. The detailed logic is shown in {meth}`~vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_main`.
62
63 ## Processor Output Caching
64
@@ -61,4 +66,4 @@ Some HF processors, such as the one for Qwen2-VL, are [very slow](gh-issue:9238)
66
67 When new data is passed in, we first check which items are in the cache, and which ones are missing. The missing items are passed into the HF processor in a single batch and cached, before being merged with the existing items in the cache.
68
64 -Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#mm-dummy-text) to avoid HF errors. Since this skips HF's prompt replacement code, we apply [automatic prompt replacement](#mm-automatic-prompt-replacement) afterwards to keep the output tokens and multi-modal data consistent with each other.
69 +Since we only process the missing multi-modal data items, the number of input placeholder tokens no longer corresponds to the number of the multi-modal inputs, so they can't be passed alongside the text prompt to HF processor. Therefore, we process the text and multi-modal inputs separately, using [dummy text](#mm-dummy-text) to avoid HF errors. Since this skips HF's prompt updating code, we apply [automatic prompt updating](#mm-automatic-prompt-updating) afterwards to keep the output tokens and multi-modal data consistent with each other.
tests/multimodal/test_processing.py
+120 -90
@@ -14,12 +14,12 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
14 # yapf conflicts with isort for this block
15 # yapf: disable
16 from vllm.multimodal.processing import (PlaceholderFeaturesInfo,
17 - PromptReplacement,
17 + PromptInsertion, PromptReplacement,
18 + apply_text_matches,
19 + apply_token_matches,
20 find_mm_placeholders,
21 find_text_matches, find_token_matches,
20 - iter_token_matches,
21 - replace_text_matches,
22 - replace_token_matches)
22 + iter_token_matches)
23 # yapf: enable
24 from vllm.multimodal.profiling import MultiModalProfiler
25 from vllm.transformers_utils.tokenizer import (AnyTokenizer,
@@ -102,7 +102,7 @@ def test_iter_token_matches(token_ids, match_ids, expected):
102 {
103 "pattern_1": [],
104 "pattern_2": [],
105 - }
105 + },
106 ),
107 (
108 [32000, 32000, 32000, 32000],
@@ -147,16 +147,22 @@ def test_iter_token_matches(token_ids, match_ids, expected):
147 ),
148 ],
149 )
150 +@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
151 # yapf: enable
151 -def test_find_token_matches(prompt, target_by_key, expected_by_key):
152 +def test_find_token_matches(
153 + prompt,
154 + target_by_key,
155 + expected_by_key,
156 + update_type,
157 +):
158 # Should not be used since there is nothing to convert to token IDs
159 mock_tokenizer = cast(AnyTokenizer, object())
160
155 - prompt_repls = [
156 - PromptReplacement(key, target, []).bind(mock_tokenizer)
161 + prompt_updates = [
162 + update_type(key, target, []).bind(mock_tokenizer)
163 for key, target in target_by_key.items()
164 ]
159 - result = find_token_matches(prompt, prompt_repls)
165 + result = find_token_matches(prompt, prompt_updates)
166
167 # Only displayed on error
168 print("result:", result)
@@ -254,16 +260,22 @@ def test_find_token_matches(prompt, target_by_key, expected_by_key):
260 ),
261 ],
262 )
263 +@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
264 # yapf: enable
258 -def test_find_text_matches(prompt, target_by_key, expected_by_key):
265 +def test_find_text_matches(
266 + prompt,
267 + target_by_key,
268 + expected_by_key,
269 + update_type,
270 +):
271 # Should not be used since there is nothing to convert to text
272 mock_tokenizer = cast(AnyTokenizer, object())
273
262 - prompt_repls = [
263 - PromptReplacement(key, target, []).bind(mock_tokenizer)
274 + prompt_updates = [
275 + update_type(key, target, []).bind(mock_tokenizer)
276 for key, target in target_by_key.items()
277 ]
266 - result = find_text_matches(prompt, prompt_repls)
278 + result = find_text_matches(prompt, prompt_updates)
279
280 # Only displayed on error
281 print("result:", result)
@@ -281,7 +293,7 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
293
294 # yapf: disable
295 @pytest.mark.parametrize(
284 - ("prompt", "target_by_key", "repl_by_key"),
296 + ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
297 [
298 (
299 "Image:<image>Image:<image><image>!",
@@ -300,58 +312,66 @@ def test_find_text_matches(prompt, target_by_key, expected_by_key):
312 # Test dynamic replacement (beyond the form of `unit * count`)
313 "pattern_3": "?!?",
314 },
315 + {
316 + PromptInsertion: {
317 + 0: "Image:<image>Image:<image><image>!",
318 + 1: "Image:<image><image><image>Image:<image><image>!?!?",
319 + 2: "Image:<image><image><image><image><image>Image:<image><image>!?!??!?", # noqa: E501
320 + },
321 + PromptReplacement: {
322 + 0: "Image:<image>Image:<image><image>!",
323 + 1: "<image><image>Image:<image><image>?!?",
324 + 2: "<image><image><image><image><image>?!?",
325 + },
326 + },
327 ),
328 ]
329 )
306 -@pytest.mark.parametrize(
307 - ("mm_count", "expected"),
308 - [
309 - (0, "Image:<image>Image:<image><image>!"),
310 - (1, "<image><image>Image:<image><image>?!?"),
311 - (2, "<image><image><image><image><image>?!?"),
312 - ]
313 -)
330 # yapf: enable
315 -def test_find_replace_text(
331 +def test_find_update_text(
332 prompt,
333 target_by_key,
334 repl_by_key,
319 - mm_count,
320 - expected,
335 + expected_by_update_type_mm_count,
336 ):
337 # Should not be used since there is nothing to convert to text
338 mock_tokenizer = cast(AnyTokenizer, object())
339
325 - mm_prompt_repls = {
326 - key: [
327 - PromptReplacement(key, target,
328 - repl_by_key[key]).bind(mock_tokenizer)
329 - ]
330 - for key, target in target_by_key.items()
331 - }
332 - mm_matches = {
333 - key: find_text_matches(prompt, prompt_repls)
334 - for key, prompt_repls in mm_prompt_repls.items()
335 - }
336 -
337 - result = replace_text_matches(
338 - prompt,
339 - mm_matches,
340 - {key: mm_count
341 - for key in repl_by_key},
342 - )
343 -
344 - # Only displayed on error
345 - print("mm_matches:", mm_matches)
346 - print("result:", result)
347 -
348 - # Manually constructed results
349 - assert result == expected
340 + for (
341 + update_type,
342 + expected_by_mm_count,
343 + ) in expected_by_update_type_mm_count.items():
344 + mm_prompt_updates = {
345 + key:
346 + [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
347 + for key, target in target_by_key.items()
348 + }
349 + mm_matches = {
350 + key: find_text_matches(prompt, updates)
351 + for key, updates in mm_prompt_updates.items()
352 + }
353 +
354 + for mm_count, expected in expected_by_mm_count.items():
355 + result = apply_text_matches(
356 + prompt,
357 + mm_matches,
358 + {key: mm_count
359 + for key in repl_by_key},
360 + )
361 +
362 + # Only displayed on error
363 + print("update_type:", update_type)
364 + print("mm_count:", mm_count)
365 + print("mm_matches:", mm_matches)
366 + print("result:", result)
367 +
368 + # Manually constructed results
369 + assert result == expected
370
371
372 # yapf: disable
373 @pytest.mark.parametrize(
354 - ("prompt", "target_by_key", "repl_by_key"),
374 + ("prompt", "target_by_key", "repl_by_key", "expected_by_update_type_mm_count"), # noqa: E501
375 [
376 # Tokenized test cases of `test_find_replace_text`
377 # using the vocab of llava-hf/llava-v1.6-mistral-7b-hf
@@ -372,53 +392,61 @@ def test_find_replace_text(
392 # Test dynamic replacement (beyond the form of `unit * count`)
393 "pattern_3": [1550, 918, 1550],
394 },
395 + {
396 + PromptInsertion: {
397 + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
398 + 1: [1, 9833, 28747, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550], # noqa: E501
399 + 2: [1, 9833, 28747, 32000, 32000, 32000, 32000, 32000, 9833, 28747, 32000, 32000, 918, 1550, 918, 1550, 1550, 918, 1550], # noqa: E501
400 + },
401 + PromptReplacement: {
402 + 0: [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
403 + 1: [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550], # noqa: E501
404 + 2: [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
405 + },
406 + },
407 ),
408 ]
409 )
378 -@pytest.mark.parametrize(
379 - ("mm_count", "expected"),
380 - [
381 - (0, [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918]),
382 - (1, [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550]),
383 - (2, [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550]),
384 - ]
385 -)
410 # yapf: enable
387 -def test_find_replace_tokens(
411 +def test_find_update_tokens(
412 prompt,
413 target_by_key,
414 repl_by_key,
391 - mm_count,
392 - expected,
415 + expected_by_update_type_mm_count,
416 ):
417 # Should not be used since there is nothing to convert to tokens
418 mock_tokenizer = cast(AnyTokenizer, object())
419
397 - mm_prompt_repls = {
398 - key: [
399 - PromptReplacement(key, target,
400 - repl_by_key[key]).bind(mock_tokenizer)
401 - ]
402 - for key, target in target_by_key.items()
403 - }
404 - mm_matches = {
405 - key: find_token_matches(prompt, prompt_repls)
406 - for key, prompt_repls in mm_prompt_repls.items()
407 - }
408 -
409 - result = replace_token_matches(
410 - prompt,
411 - mm_matches,
412 - {key: mm_count
413 - for key in repl_by_key},
414 - )
415 -
416 - # Only displayed on error
417 - print("mm_matches:", mm_matches)
418 - print("result:", result)
419 -
420 - # Manually constructed results
421 - assert result == expected
420 + for (
421 + update_type,
422 + expected_by_mm_count,
423 + ) in expected_by_update_type_mm_count.items():
424 + mm_prompt_updates = {
425 + key:
426 + [update_type(key, target, repl_by_key[key]).bind(mock_tokenizer)]
427 + for key, target in target_by_key.items()
428 + }
429 + mm_matches = {
430 + key: find_token_matches(prompt, updates)
431 + for key, updates in mm_prompt_updates.items()
432 + }
433 +
434 + for mm_count, expected in expected_by_mm_count.items():
435 + result = apply_token_matches(
436 + prompt,
437 + mm_matches,
438 + {key: mm_count
439 + for key in repl_by_key},
440 + )
441 +
442 + # Only displayed on error
443 + print("update_type:", update_type)
444 + print("mm_count:", mm_count)
445 + print("mm_matches:", mm_matches)
446 + print("result:", result)
447 +
448 + # Manually constructed results
449 + assert result == expected
450
451
452 # yapf: disable
@@ -524,22 +552,24 @@ def test_find_replace_tokens(
552 ),
553 ]
554 )
555 +@pytest.mark.parametrize("update_type", [PromptInsertion, PromptReplacement])
556 # yapf: enable
557 def test_find_mm_placeholders(
558 repl_by_key,
559 prompt,
560 expected,
561 + update_type,
562 ):
563 # Should not be used since there is nothing to convert to tokens
564 mock_tokenizer = cast(AnyTokenizer, object())
565
536 - mm_prompt_repls = {
537 - key: [PromptReplacement(key, [], repl).bind(mock_tokenizer)]
566 + mm_prompt_updates = {
567 + key: [update_type(key, [], repl).bind(mock_tokenizer)]
568 for key, repl in repl_by_key.items()
569 }
570
571 result = find_mm_placeholders(
542 - mm_prompt_repls,
572 + mm_prompt_updates,
573 prompt,
574 # Effectively match all occurrences in the prompt
575 {key: 3
vllm/model_executor/models/aria.py
+6 -6
@@ -1,7 +1,6 @@
1 # SPDX-License-Identifier: Apache-2.0
2 -
3 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
4 - Union)
2 +from collections.abc import Iterable, Mapping, Sequence
3 +from typing import List, Optional, Set, Tuple, TypedDict, Union
4
5 import torch
6 import torch.nn as nn
@@ -26,7 +25,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
25 NestedTensors)
26 from vllm.multimodal.parse import MultiModalDataItems
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 - BaseProcessingInfo, PromptReplacement)
28 + BaseProcessingInfo, PromptReplacement,
29 + PromptUpdate)
30 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 from vllm.sequence import IntermediateTensors
32
@@ -457,12 +457,12 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor[AriaProcessingInfo]):
457 pixel_mask=MultiModalFieldConfig.batched("image"),
458 )
459
460 - def _get_prompt_replacements(
460 + def _get_prompt_updates(
461 self,
462 mm_items: MultiModalDataItems,
463 hf_processor_mm_kwargs: Mapping[str, object],
464 out_mm_kwargs: MultiModalKwargs,
465 - ) -> list[PromptReplacement]:
465 + ) -> Sequence[PromptUpdate]:
466 hf_config = self.info.get_hf_config()
467 image_token_id = hf_config.image_token_index
468
vllm/model_executor/models/blip2.py
+9 -15
@@ -1,8 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from collections.abc import Iterable, Mapping, Sequence
4 from functools import cached_property
4 -from typing import (Iterable, Literal, Mapping, Optional, Set, Tuple,
5 - TypedDict, Union)
5 +from typing import Literal, Optional, Set, Tuple, TypedDict, Union
6
7 import torch
8 import torch.nn as nn
@@ -19,8 +19,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
19 NestedTensors)
20 from vllm.multimodal.parse import MultiModalDataItems
21 from vllm.multimodal.processing import (BaseMultiModalProcessor,
22 - BaseProcessingInfo, PromptReplacement,
23 - PromptReplacementDetails)
22 + BaseProcessingInfo, PromptInsertion,
23 + PromptUpdate)
24 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
25 from vllm.sequence import IntermediateTensors
26
@@ -474,30 +474,24 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
474 image_embeds=MultiModalFieldConfig.batched("image"),
475 )
476
477 - def _get_prompt_replacements(
477 + def _get_prompt_updates(
478 self,
479 mm_items: MultiModalDataItems,
480 hf_processor_mm_kwargs: Mapping[str, object],
481 out_mm_kwargs: MultiModalKwargs,
482 - ) -> list[PromptReplacement]:
482 + ) -> Sequence[PromptUpdate]:
483 tokenizer = self.info.get_tokenizer()
484 vocab = tokenizer.get_vocab()
485
486 - bos_token_id = tokenizer.bos_token_id
487 - assert isinstance(bos_token_id, int)
488 -
486 image_token_id = vocab["<image>"]
487 num_image_tokens = self.info.get_num_image_tokens()
488 image_tokens = [image_token_id] * num_image_tokens
489
490 return [
494 - PromptReplacement(
491 + PromptInsertion(
492 modality="image",
496 - target=[bos_token_id],
497 - replacement=PromptReplacementDetails(
498 - full=image_tokens + [bos_token_id],
499 - features=image_tokens,
500 - ),
493 + target="",
494 + insertion=image_tokens,
495 )
496 ]
497
vllm/model_executor/models/chameleon.py
+7 -7
@@ -1,8 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 +from collections.abc import Iterable, Mapping, Sequence
4 from functools import cached_property
4 -from typing import (Any, Dict, Iterable, Literal, Mapping, Optional, Set,
5 - Tuple, TypedDict, Union)
5 +from typing import Any, Dict, Literal, Optional, Set, Tuple, TypedDict, Union
6
7 import torch
8 import torch.nn as nn
@@ -35,7 +35,7 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
35 from vllm.multimodal.parse import MultiModalDataItems
36 from vllm.multimodal.processing import (BaseMultiModalProcessor,
37 BaseProcessingInfo, PromptReplacement,
38 - PromptReplacementDetails)
38 + PromptUpdate, PromptUpdateDetails)
39 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
40 from vllm.sequence import IntermediateTensors
41
@@ -141,12 +141,12 @@ class ChameleonMultiModalProcessor(
141 ) -> Mapping[str, MultiModalFieldConfig]:
142 return dict(pixel_values=MultiModalFieldConfig.batched("image"))
143
144 - def _get_prompt_replacements(
144 + def _get_prompt_updates(
145 self,
146 mm_items: MultiModalDataItems,
147 hf_processor_mm_kwargs: Mapping[str, object],
148 out_mm_kwargs: MultiModalKwargs,
149 - ) -> list[PromptReplacement]:
149 + ) -> Sequence[PromptUpdate]:
150 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
151 tokenizer = self.info.get_tokenizer()
152 vocab = tokenizer.get_vocab()
@@ -162,7 +162,7 @@ class ChameleonMultiModalProcessor(
162 PromptReplacement(
163 modality="image",
164 target=[image_token_id],
165 - replacement=PromptReplacementDetails(
165 + replacement=PromptUpdateDetails(
166 full=([image_start_id] + image_tokens + [image_end_id]),
167 features=image_tokens,
168 ),
@@ -371,7 +371,7 @@ class ChameleonDecoderLayer(nn.Module):
371 positions: torch.Tensor,
372 hidden_states: torch.Tensor,
373 residual: Optional[torch.Tensor],
374 - ) -> Tuple[torch.Tensor, torch.Tensor]:
374 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
375
376 if residual is None:
377 residual = hidden_states
vllm/model_executor/models/deepseek_vl2.py
+5 -5
@@ -3,9 +3,9 @@
3 # adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
4 """Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
5 import math
6 +from collections.abc import Iterable, Mapping, Sequence
7 from functools import cached_property
7 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
8 - TypedDict, Union)
8 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
9
10 import torch
11 import torch.nn as nn
@@ -26,7 +26,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
26 ImageSize, MultiModalDataItems)
27 from vllm.multimodal.processing import (BaseMultiModalProcessor,
28 BaseProcessingInfo, ProcessingCache,
29 - PromptReplacement)
29 + PromptReplacement, PromptUpdate)
30 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
@@ -281,12 +281,12 @@ class DeepseekVL2MultiModalProcessor(
281 image_embeds=MultiModalFieldConfig.batched("image"),
282 )
283
284 - def _get_prompt_replacements(
284 + def _get_prompt_updates(
285 self,
286 mm_items: MultiModalDataItems,
287 hf_processor_mm_kwargs: Mapping[str, object],
288 out_mm_kwargs: MultiModalKwargs,
289 - ) -> list[PromptReplacement]:
289 + ) -> Sequence[PromptUpdate]:
290 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
291
292 image_token_id = hf_processor.image_token_id
vllm/model_executor/models/florence2.py
+10 -14
@@ -1,9 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 +from collections import OrderedDict
5 +from collections.abc import Iterable, Mapping, Sequence
6 from functools import cached_property
5 -from typing import (Iterable, List, Literal, Mapping, Optional, OrderedDict,
6 - Set, Tuple, TypedDict, Union)
7 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
8
9 import torch
10 import torch.nn as nn
@@ -24,8 +25,7 @@ from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs
25 from vllm.multimodal.parse import MultiModalDataDict, MultiModalDataItems
26 from vllm.multimodal.processing import (BaseProcessingInfo,
27 EncDecMultiModalProcessor,
27 - PromptReplacement,
28 - PromptReplacementDetails)
28 + PromptInsertion, PromptUpdate)
29 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
30 from vllm.sequence import IntermediateTensors
31
@@ -803,7 +803,7 @@ class Florence2DummyInputsBuilder(
803 class Florence2MultiModalProcessor(
804 EncDecMultiModalProcessor[Florence2ProcessingInfo]):
805
806 - def _hf_processor_applies_repl(
806 + def _hf_processor_applies_updates(
807 self,
808 prompt_text: str,
809 mm_items: MultiModalDataItems,
@@ -850,26 +850,22 @@ class Florence2MultiModalProcessor(
850 ) -> Mapping[str, MultiModalFieldConfig]:
851 return dict(pixel_values=MultiModalFieldConfig.batched("image"))
852
853 - def _get_prompt_replacements(
853 + def _get_prompt_updates(
854 self,
855 mm_items: MultiModalDataItems,
856 hf_processor_mm_kwargs: Mapping[str, object],
857 out_mm_kwargs: MultiModalKwargs,
858 - ) -> list[PromptReplacement]:
858 + ) -> Sequence[PromptUpdate]:
859 hf_config = self.info.get_hf_config()
860 pad_token_id = hf_config.pad_token_id
861 - bos_token_id = hf_config.bos_token_id
861 num_image_tokens = self.info.get_max_image_tokens()
862 image_tokens = [pad_token_id] * num_image_tokens
863
864 return [
866 - PromptReplacement(
865 + PromptInsertion(
866 modality="image",
868 - target=[bos_token_id],
869 - replacement=PromptReplacementDetails(
870 - full=image_tokens + [bos_token_id],
871 - features=image_tokens,
872 - ),
867 + target="",
868 + insertion=image_tokens,
869 )
870 ]
871
vllm/model_executor/models/fuyu.py
+6 -6
@@ -17,8 +17,8 @@
17 # limitations under the License.
18 """ PyTorch Fuyu model."""
19 import math
20 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
21 - TypedDict)
20 +from collections.abc import Iterable, Mapping, Sequence
21 +from typing import List, Literal, Optional, Set, Tuple, TypedDict
22
23 import torch
24 import torch.nn as nn
@@ -37,7 +37,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
37 MultiModalDataItems)
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 BaseProcessingInfo, PromptReplacement,
40 - PromptReplacementDetails)
40 + PromptUpdate, PromptUpdateDetails)
41 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42 from vllm.sequence import IntermediateTensors
43
@@ -203,12 +203,12 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
203 ) -> Mapping[str, MultiModalFieldConfig]:
204 return dict(image_patches=MultiModalFieldConfig.batched("image"))
205
206 - def _get_prompt_replacements(
206 + def _get_prompt_updates(
207 self,
208 mm_items: MultiModalDataItems,
209 hf_processor_mm_kwargs: Mapping[str, object],
210 out_mm_kwargs: MultiModalKwargs,
211 - ) -> list[PromptReplacement]:
211 + ) -> Sequence[PromptUpdate]:
212 hf_config = self.info.get_hf_config()
213 bos_token_id = hf_config.bos_token_id
214 assert isinstance(bos_token_id, int)
@@ -228,7 +228,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor[FuyuProcessingInfo]):
228 image_tokens = ([_IMAGE_TOKEN_ID] * ncols +
229 [_NEWLINE_TOKEN_ID]) * nrows
230
231 - return PromptReplacementDetails(
231 + return PromptUpdateDetails(
232 full=image_tokens + [bos_token_id],
233 features=image_tokens,
234 )
vllm/model_executor/models/glm4v.py
+6 -5
@@ -4,7 +4,8 @@
4 # https://github.com/THUDM/CogAgent
5 """Inference-only CogAgent model compatible with THUDM weights."""
6 from argparse import Namespace
7 -from typing import Literal, Mapping, Optional, TypedDict, Union
7 +from collections.abc import Mapping, Sequence
8 +from typing import Literal, Optional, TypedDict, Union
9
10 import torch
11 from torch import nn
@@ -32,7 +33,7 @@ from vllm.multimodal.parse import MultiModalDataItems
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, BatchFeature,
35 MultiModalFieldConfig,
35 - PromptReplacement)
36 + PromptReplacement, PromptUpdate)
37 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 from vllm.sequence import IntermediateTensors
39 from vllm.transformers_utils.configs import ChatGLMConfig
@@ -480,7 +481,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
481
482 class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
483
483 - def _hf_processor_applies_repl(
484 + def _hf_processor_applies_updates(
485 self,
486 prompt_text: str,
487 mm_items: MultiModalDataItems,
@@ -495,12 +496,12 @@ class GLM4VMultiModalProcessor(BaseMultiModalProcessor[GLM4VProcessingInfo]):
496 ) -> Mapping[str, MultiModalFieldConfig]:
497 return dict(pixel_values=MultiModalFieldConfig.batched("image"))
498
498 - def _get_prompt_replacements(
499 + def _get_prompt_updates(
500 self,
501 mm_items: MultiModalDataItems,
502 hf_processor_mm_kwargs: Mapping[str, object],
503 out_mm_kwargs: MultiModalKwargs,
503 - ) -> list[PromptReplacement]:
504 + ) -> Sequence[PromptUpdate]:
505 hf_config = self.info.get_hf_config()
506
507 boi_token_id = hf_config.boi_token_id
vllm/model_executor/models/h2ovl.py
+6 -5
@@ -7,7 +7,8 @@
7 # Copyright (c) 2024 H2O.AI
8 # Licensed under Apache 2.0 License [see LICENSE for details]
9 # --------------------------------------------------------
10 -from typing import Mapping, Optional
10 +from collections.abc import Mapping, Sequence
11 +from typing import Optional
12
13 import torch
14 from PIL import Image
@@ -20,7 +21,7 @@ from vllm.multimodal.inputs import MultiModalKwargs
21 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
22 MultiModalDataItems)
23 from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
23 - PromptReplacementDetails)
24 + PromptUpdate, PromptUpdateDetails)
25 from vllm.multimodal.profiling import BaseDummyInputsBuilder
26 from vllm.transformers_utils.tokenizer import AnyTokenizer
27
@@ -487,12 +488,12 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
488 f"{type(self).__name__} does not support processing cache with "
489 "multi-image support enabled.")
490
490 - def _get_prompt_replacements(
491 + def _get_prompt_updates(
492 self,
493 mm_items: MultiModalDataItems,
494 hf_processor_mm_kwargs: Mapping[str, object],
495 out_mm_kwargs: MultiModalKwargs,
495 - ) -> list[PromptReplacement]:
496 + ) -> Sequence[PromptUpdate]:
497 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
498
499 if "image_num_patches" in out_mm_kwargs:
@@ -527,7 +528,7 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]
528 if num_patches is not None:
529 assert isinstance(num_patches, int)
530
530 - return PromptReplacementDetails(
531 + return PromptUpdateDetails(
532 full=hf_processor.get_image_repl_full(feature_size,
533 num_patches),
534 features=hf_processor.get_image_repl_features(
vllm/model_executor/models/idefics3.py
+5 -5
@@ -16,8 +16,8 @@
16 """Inference-only Idefics3 model compatible with HuggingFace weights."""
17
18 import math
19 -from typing import (Dict, Iterable, List, Literal, Mapping, Optional, Set,
20 - Tuple, TypedDict, Union)
19 +from collections.abc import Iterable, Mapping, Sequence
20 +from typing import Dict, List, Literal, Optional, Set, Tuple, TypedDict, Union
21
22 import torch
23 import torch.utils.checkpoint
@@ -41,7 +41,7 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 BaseProcessingInfo,
42 MultiModalDataItems,
43 MultiModalFieldConfig,
44 - PromptReplacement)
44 + PromptReplacement, PromptUpdate)
45 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
46 from vllm.sequence import IntermediateTensors
47
@@ -274,12 +274,12 @@ class Idefics3MultimodalProcessor(
274 image_embeds=MultiModalFieldConfig.batched("image"),
275 )
276
277 - def _get_prompt_replacements(
277 + def _get_prompt_updates(
278 self,
279 mm_items: MultiModalDataItems,
280 hf_processor_mm_kwargs: Mapping[str, object],
281 out_mm_kwargs: MultiModalKwargs,
282 - ) -> list[PromptReplacement]:
282 + ) -> Sequence[PromptUpdate]:
283 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
284
285 image_token = hf_processor.image_token.content
vllm/model_executor/models/internvl.py
+7 -6
@@ -7,9 +7,10 @@
7 # Licensed under The MIT License [see LICENSE for details]
8 # --------------------------------------------------------
9 from abc import ABC, abstractmethod
10 +from collections.abc import Iterable, Mapping, Sequence
11 from functools import cached_property
11 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
12 - TypedDict, TypeVar, Union)
12 +from typing import (List, Literal, Optional, Set, Tuple, TypedDict, TypeVar,
13 + Union)
14
15 import torch
16 import torch.nn as nn
@@ -31,7 +32,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 ImageSize, MultiModalDataItems)
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, PromptReplacement,
34 - PromptReplacementDetails)
35 + PromptUpdate, PromptUpdateDetails)
36 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
38 from vllm.transformers_utils.tokenizer import AnyTokenizer
@@ -599,12 +600,12 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
600 image_token_id=MultiModalFieldConfig.shared("image", num_images),
601 )
602
602 - def _get_prompt_replacements(
603 + def _get_prompt_updates(
604 self,
605 mm_items: MultiModalDataItems,
606 hf_processor_mm_kwargs: Mapping[str, object],
607 out_mm_kwargs: MultiModalKwargs,
607 - ) -> list[PromptReplacement]:
608 + ) -> Sequence[PromptUpdate]:
609 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
610
611 if "image_num_patches" in out_mm_kwargs:
@@ -636,7 +637,7 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):
637 if num_patches is not None:
638 assert isinstance(num_patches, int)
639
639 - return PromptReplacementDetails(
640 + return PromptUpdateDetails(
641 full=hf_processor.get_image_repl_full(feature_size,
642 num_patches),
643 features=hf_processor.get_image_repl_features(
vllm/model_executor/models/llava.py
+12 -11
@@ -1,9 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 from abc import abstractmethod
4 +from collections.abc import Iterable, Mapping, Sequence
5 from functools import cached_property
5 -from typing import (Final, Iterable, List, Literal, Mapping, Optional,
6 - Protocol, Set, Tuple, TypedDict, TypeVar, Union)
6 +from typing import (Final, List, Literal, Optional, Protocol, Set, Tuple,
7 + TypedDict, TypeVar, Union)
8
9 import torch
10 import torch.nn as nn
@@ -31,7 +32,7 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
32 ImageSize, MultiModalDataItems)
33 from vllm.multimodal.processing import (BaseMultiModalProcessor,
34 BaseProcessingInfo, ProcessingCache,
34 - PromptReplacement)
35 + PromptReplacement, PromptUpdate)
36 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
37 from vllm.sequence import IntermediateTensors
38
@@ -222,12 +223,12 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor[_I]):
223 ) -> Mapping[str, MultiModalFieldConfig]:
224 raise NotImplementedError
225
225 - def _get_prompt_replacements(
226 + def _get_prompt_updates(
227 self,
228 mm_items: MultiModalDataItems,
229 hf_processor_mm_kwargs: Mapping[str, object],
230 out_mm_kwargs: MultiModalKwargs,
230 - ) -> list[PromptReplacement]:
231 + ) -> Sequence[PromptUpdate]:
232 hf_config = self.info.get_hf_config()
233 image_token_id = hf_config.image_token_index
234
@@ -328,12 +329,12 @@ class PixtralHFMultiModalProcessor(
329 image_embeds=MultiModalFieldConfig.batched("image"),
330 )
331
331 - def _get_prompt_replacements(
332 + def _get_prompt_updates(
333 self,
334 mm_items: MultiModalDataItems,
335 hf_processor_mm_kwargs: Mapping[str, object],
336 out_mm_kwargs: MultiModalKwargs,
336 - ) -> list[PromptReplacement]:
337 + ) -> Sequence[PromptUpdate]:
338 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
339 hf_config = self.info.get_hf_config()
340 tokenizer = self.info.get_tokenizer()
@@ -789,7 +790,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
790 "</Image>)", # 3 tokens
791 ])
792
792 - mantis_mm_repls = self._bind_and_group_repls([
793 + mantis_mm_repls = self._bind_and_group_updates([
794 PromptReplacement(
795 modality="image",
796 target=[image_token_id] * num_image_tokens,
@@ -797,18 +798,18 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
798 )
799 ])
800
800 - prompt_ids, prompt, _ = self._apply_prompt_replacements(
801 + prompt_ids, prompt, _ = self._apply_prompt_updates(
802 result["prompt_token_ids"],
803 mantis_mm_repls,
804 mm_item_counts,
805 )
806
806 - unbound_orig_repls = self._get_prompt_replacements(
807 + unbound_orig_repls = self._get_prompt_updates(
808 mm_items,
809 hf_processor_mm_kwargs,
810 mm_kwargs,
811 )
811 - orig_repls = self._bind_and_group_repls(unbound_orig_repls)
812 + orig_repls = self._bind_and_group_updates(unbound_orig_repls)
813
814 mm_placeholders = self._find_mm_placeholders(
815 orig_repls,
vllm/model_executor/models/llava_next_video.py
+6 -5
@@ -1,9 +1,9 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 +from collections.abc import Iterable, Mapping, Sequence
5 from functools import cached_property
5 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
6 - TypedDict, Union)
6 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
7
8 import torch
9 import torch.nn as nn
@@ -21,7 +21,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
21 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
22 VideoEmbeddingItems, VideoProcessorItems)
23 from vllm.multimodal.processing import (BaseMultiModalProcessor,
24 - BaseProcessingInfo, PromptReplacement)
24 + BaseProcessingInfo, PromptReplacement,
25 + PromptUpdate)
26 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
27 from vllm.sequence import IntermediateTensors
28 from vllm.utils import is_list_of
@@ -183,12 +184,12 @@ class LlavaNextVideoMultiModalProcessor(
184 ) -> Mapping[str, MultiModalFieldConfig]:
185 return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
186
186 - def _get_prompt_replacements(
187 + def _get_prompt_updates(
188 self,
189 mm_items: MultiModalDataItems,
190 hf_processor_mm_kwargs: Mapping[str, object],
191 out_mm_kwargs: MultiModalKwargs,
191 - ) -> list[PromptReplacement]:
192 + ) -> Sequence[PromptUpdate]:
193 hf_config = self.info.get_hf_config()
194 video_token_id = hf_config.video_token_index
195
vllm/model_executor/models/llava_onevision.py
+11 -9
@@ -1,9 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 +from collections.abc import Iterable, Mapping, Sequence
5 from functools import cached_property
5 -from typing import (Final, Iterable, List, Literal, Mapping, Optional,
6 - Protocol, Set, Tuple, TypedDict, Union)
6 +from typing import (Final, List, Literal, Optional, Protocol, Set, Tuple,
7 + TypedDict, Union)
8
9 import torch
10 import torch.nn as nn
@@ -22,7 +23,7 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
23 NestedTensors)
24 from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
25 VideoEmbeddingItems, VideoProcessorItems)
25 -from vllm.multimodal.processing import PromptReplacement
26 +from vllm.multimodal.processing import PromptReplacement, PromptUpdate
27 from vllm.multimodal.profiling import ProcessorInputs
28 from vllm.sequence import IntermediateTensors
29 from vllm.utils import is_list_of
@@ -347,13 +348,13 @@ class LlavaOnevisionMultiModalProcessor(
348 )
349 return BatchFeature(combined_outputs)
350
350 - def _hf_processor_applies_repl(
351 + def _hf_processor_applies_updates(
352 self,
353 prompt_text: str,
354 mm_items: MultiModalDataItems,
355 hf_processor_mm_kwargs: Mapping[str, object],
356 ) -> bool:
356 - base_result = super()._hf_processor_applies_repl(
357 + base_result = super()._hf_processor_applies_updates(
358 prompt_text=prompt_text,
359 mm_items=mm_items,
360 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
@@ -361,13 +362,13 @@ class LlavaOnevisionMultiModalProcessor(
362
363 return base_result and mm_items.get_count("video", strict=False) == 0
364
364 - def _get_prompt_replacements(
365 + def _get_prompt_updates(
366 self,
367 mm_items: MultiModalDataItems,
368 hf_processor_mm_kwargs: Mapping[str, object],
369 out_mm_kwargs: MultiModalKwargs,
369 - ) -> list[PromptReplacement]:
370 - image_repls = super()._get_prompt_replacements(
370 + ) -> Sequence[PromptUpdate]:
371 + image_repls = super()._get_prompt_updates(
372 mm_items=mm_items,
373 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
374 out_mm_kwargs=out_mm_kwargs,
@@ -392,7 +393,8 @@ class LlavaOnevisionMultiModalProcessor(
393
394 return [video_token_id] * num_video_tokens
395
395 - return image_repls + [
396 + return [
397 + *image_repls,
398 PromptReplacement(
399 modality="video",
400 target=[video_token_id],
vllm/model_executor/models/minicpmo.py
+5 -4
@@ -22,9 +22,10 @@
22 # See the License for the specific language governing permissions and
23 # limitations under the License.
24 """Inference-only MiniCPM-O model compatible with HuggingFace weights."""
25 +from collections.abc import Iterable, Mapping, Sequence
26 from functools import partial
26 -from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
27 - Optional, Set, Tuple, TypedDict, Union)
27 +from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,
28 + TypedDict, Union)
29
30 import torch
31 from torch import nn
@@ -356,10 +357,10 @@ class MiniCPMOMultiModalProcessor(
357 inputs["audio"]["audio_lens"][index])
358 return super().get_prompt_texts_by_modality(inputs, modality, index)
359
359 - def _get_prompt_replacements(
360 + def _get_prompt_updates(
361 self, mm_items: MultiModalDataItems,
362 hf_processor_mm_kwargs: Mapping[str, Any],
362 - out_mm_kwargs: MultiModalKwargs) -> List[PromptReplacement]:
363 + out_mm_kwargs: MultiModalKwargs) -> Sequence[PromptReplacement]:
364 placeholder = {
365 "image": self.info.image_pattern,
366 "video": self.info.video_pattern,
vllm/model_executor/models/minicpmv.py
+6 -5
@@ -25,9 +25,10 @@
25 import math
26 import re
27 from collections import Counter
28 +from collections.abc import Iterable, Mapping, Sequence
29 from functools import cached_property, partial
29 -from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
30 - Optional, Set, Tuple, TypedDict, Union)
30 +from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,
31 + TypedDict, Union)
32
33 import numpy as np
34 import torch
@@ -732,7 +733,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
733 }
734 }
735
735 - def _hf_processor_applies_repl(
736 + def _hf_processor_applies_updates(
737 self,
738 prompt_text: str,
739 mm_items: MultiModalDataItems,
@@ -740,10 +741,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
741 ) -> bool:
742 return False
743
743 - def _get_prompt_replacements(
744 + def _get_prompt_updates(
745 self, mm_items: MultiModalDataItems,
746 hf_processor_mm_kwargs: Mapping[str, Any],
746 - out_mm_kwargs: MultiModalKwargs) -> List[PromptReplacement]:
747 + out_mm_kwargs: MultiModalKwargs) -> Sequence[PromptReplacement]:
748 placeholder = {
749 "image": self.info.image_pattern,
750 "video": self.info.video_pattern,
vllm/model_executor/models/mllama.py
+5 -5
@@ -15,8 +15,8 @@
15 # limitations under the License.
16 """PyTorch Mllama model."""
17 import math
18 -from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
19 - TypedDict, Union)
18 +from collections.abc import Iterable, Mapping, Sequence
19 +from typing import List, Literal, Optional, Set, Tuple, TypedDict, Union
20
21 import numpy as np
22 import torch
@@ -59,7 +59,7 @@ from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
59 MultiModalDataDict, MultiModalDataItems)
60 from vllm.multimodal.processing import (BaseProcessingInfo,
61 EncDecMultiModalProcessor,
62 - PromptReplacement)
62 + PromptReplacement, PromptUpdate)
63 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
64
65 from .clip import CLIPMLP
@@ -243,12 +243,12 @@ class MllamaMultiModalProcessor(EncDecMultiModalProcessor[MllamaProcessingInfo]
243 image_token_id = self.info.get_hf_config().image_token_index
244 return [image_token_id] * num_images
245
246 - def _get_prompt_replacements(
246 + def _get_prompt_updates(
247 self,
248 mm_items: MultiModalDataItems,
249 hf_processor_mm_kwargs: Mapping[str, object],
250 out_mm_kwargs: MultiModalKwargs,
251 - ) -> list[PromptReplacement]:
251 + ) -> Sequence[PromptUpdate]:
252 token_per_chunk = self.info.get_token_per_chunk_from_config()
253 image_token_id = self.info.get_hf_config().image_token_index
254
vllm/model_executor/models/molmo.py
+13 -22
@@ -1,10 +1,10 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 +from collections.abc import Iterable, Mapping, Sequence
5 from dataclasses import dataclass
6 from functools import cached_property, partial
6 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
7 - Union, cast)
7 +from typing import List, Optional, Set, Tuple, TypedDict, Union, cast
8
9 import numpy as np
10 import torch
@@ -46,8 +46,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
46 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47 MultiModalDataItems)
48 from vllm.multimodal.processing import (BaseMultiModalProcessor,
49 - BaseProcessingInfo, PromptReplacement,
50 - PromptReplacementDetails)
49 + BaseProcessingInfo, PromptInsertion,
50 + PromptUpdate)
51 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52 from vllm.sequence import IntermediateTensors
53 from vllm.utils import JSONTree, json_map_leaves
@@ -1190,6 +1190,8 @@ class MolmoProcessingInfo(BaseProcessingInfo):
1190 return MolmoProcessorWrapper(processor)
1191
1192 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
1193 + # TODO: Investigate different `embed_is_patch` between cache/no-cache
1194 + # in multi-image case
1195 return {"image": 1}
1196
1197 def get_mm_max_tokens_per_item(
@@ -1328,25 +1330,18 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1330 img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1331 )
1332
1331 - def _get_prompt_replacements(
1333 + def _get_prompt_updates(
1334 self,
1335 mm_items: MultiModalDataItems,
1336 hf_processor_mm_kwargs: Mapping[str, object],
1337 out_mm_kwargs: MultiModalKwargs,
1336 - ) -> list[PromptReplacement]:
1338 + ) -> Sequence[PromptUpdate]:
1339 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1338 - tokenizer = self.info.get_tokenizer()
1340
1341 image_token_length_w = processor.image_token_length_w
1342 image_token_length_h = processor.image_token_length_h
1343 pooling_size = processor.pooling_size
1344
1344 - user_str = "User:"
1345 - if processor.always_start_with_space:
1346 - user_str = " " + user_str
1347 -
1348 - user_tokens = tokenizer.encode(user_str, add_special_tokens=False)
1349 -
1345 img_patch_id = processor.image_patch_id
1346 img_col_id = processor.im_col_id
1347 img_start_id = processor.im_start_id
@@ -1356,7 +1351,7 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1351 extra_joint = ([img_start_id] + extra_row * image_token_length_h +
1352 [img_end_id])
1353
1359 - def get_replacement_molmo(item_idx: int):
1354 + def get_insertion_molmo(item_idx: int):
1355 images = mm_items.get_items("image", ImageProcessorItems)
1356 image_size = images.get_image_size(item_idx)
1357
@@ -1371,17 +1366,13 @@ class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1366 ((nrows + 1) // pooling_size) + [img_end_id])
1367
1368 image_tokens = extra_joint + joint
1374 -
1375 - return PromptReplacementDetails(
1376 - full=image_tokens + user_tokens,
1377 - features=image_tokens,
1378 - )
1369 + return image_tokens
1370
1371 return [
1381 - PromptReplacement(
1372 + PromptInsertion(
1373 modality="image",
1383 - target=user_str,
1384 - replacement=get_replacement_molmo,
1374 + target="<|endoftext|>",
1375 + insertion=get_insertion_molmo,
1376 )
1377 ]
1378
vllm/model_executor/models/nvlm_d.py
+7 -6
@@ -6,7 +6,8 @@
6 # Copyright (c) 2024 NVIDIA
7 # Licensed under Apache 2.0 License [see LICENSE for details]
8 # --------------------------------------------------------
9 -from typing import Mapping, Optional
9 +from collections.abc import Mapping, Sequence
10 +from typing import Optional
11
12 import torch
13 import torch.nn as nn
@@ -17,8 +18,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import MultiModalKwargs
19 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
20 MultiModalDataItems)
20 -from vllm.multimodal.processing import (PromptReplacement,
21 - PromptReplacementDetails)
21 +from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,
22 + PromptUpdateDetails)
23 from vllm.multimodal.profiling import ProcessorInputs
24
25 from .intern_vit import InternVisionModel
@@ -142,12 +143,12 @@ class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):
143
144 class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
145
145 - def _get_prompt_replacements(
146 + def _get_prompt_updates(
147 self,
148 mm_items: MultiModalDataItems,
149 hf_processor_mm_kwargs: Mapping[str, object],
150 out_mm_kwargs: MultiModalKwargs,
150 - ) -> list[PromptReplacement]:
151 + ) -> Sequence[PromptUpdate]:
152 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
153
154 if "image_num_patches" in out_mm_kwargs:
@@ -179,7 +180,7 @@ class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):
180 if num_patches is not None:
181 assert isinstance(num_patches, int)
182
182 - return PromptReplacementDetails(
183 + return PromptUpdateDetails(
184 full=hf_processor.get_image_repl_full(feature_size,
185 num_patches) + "\n",
186 features=hf_processor.get_image_repl_features(
vllm/model_executor/models/phi3v.py
+10 -11
@@ -38,11 +38,10 @@ from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
38 # yapf conflicts with isort for this block
39 # yapf: disable
40 from vllm.multimodal.processing import (BaseMultiModalProcessor,
41 - BaseProcessingInfo,
42 - BoundPromptReplacement,
41 + BaseProcessingInfo, BoundPromptUpdate,
42 PlaceholderFeaturesInfo,
44 - PromptReplacement,
45 - PromptReplacementDetails)
43 + PromptReplacement, PromptUpdate,
44 + PromptUpdateDetails)
45 # yapf: enable
46 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
47 from vllm.sequence import IntermediateTensors
@@ -420,12 +419,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
419 image_embeds=MultiModalFieldConfig.batched("image"),
420 )
421
423 - def _get_prompt_replacements(
422 + def _get_prompt_updates(
423 self,
424 mm_items: MultiModalDataItems,
425 hf_processor_mm_kwargs: Mapping[str, Any],
426 out_mm_kwargs: MultiModalKwargs,
428 - ) -> list[PromptReplacement]:
427 + ) -> Sequence[PromptUpdate]:
428 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
429 image_tokens: list[str] = hf_processor.img_tokens # type: ignore
430
@@ -449,7 +448,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
448
449 image_tokens = [_IMAGE_TOKEN_ID] * num_image_tokens
450
452 - return PromptReplacementDetails(
451 + return PromptUpdateDetails(
452 full=image_tokens + [bos_token_id],
453 features=image_tokens,
454 )
@@ -464,15 +463,15 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor[Phi3VProcessingInfo]):
463 ) for image_token in image_tokens[:num_images]
464 ]
465
467 - def _apply_prompt_replacements(
466 + def _apply_prompt_updates(
467 self,
468 token_ids: list[int],
470 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
469 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
470 mm_item_counts: Mapping[str, int],
471 ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
473 - token_ids, text, placeholders = super()._apply_prompt_replacements(
472 + token_ids, text, placeholders = super()._apply_prompt_updates(
473 token_ids=token_ids,
475 - mm_prompt_repls=mm_prompt_repls,
474 + mm_prompt_updates=mm_prompt_updates,
475 mm_item_counts=mm_item_counts,
476 )
477
vllm/model_executor/models/prithvi_geospatial_mae.py
+9 -15
@@ -15,7 +15,8 @@
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
17 """Inference-only IBM/NASA Prithvi Geospatial model."""
18 -from typing import Iterable, Mapping, Optional, Set, Tuple, Union
18 +from collections.abc import Iterable, Mapping, Sequence
19 +from typing import Optional, Set, Tuple, Union
20
21 import torch
22 import torch.nn as nn
@@ -32,7 +33,7 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
33 MultiModalInputs, MultiModalKwargs)
34 from vllm.multimodal.parse import MultiModalDataItems
35 from vllm.multimodal.processing import (BaseMultiModalProcessor,
35 - BaseProcessingInfo, PromptReplacement)
36 + BaseProcessingInfo, PromptUpdate)
37 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
38 from vllm.sequence import (IntermediateTensors, PoolerOutput,
39 PoolingSequenceGroupOutput)
@@ -44,7 +45,7 @@ class PrithviGeoSpatialMAEProcessingInfo(BaseProcessingInfo):
45 return {"image": None}
46
47 def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
47 - pass
48 + return {"image": 0}
49
50
51 class PrithviGeoSpatialMAEInputBuilder(
@@ -78,20 +79,13 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
79 location_coords=MultiModalFieldConfig.batched("image"),
80 )
81
81 - def _get_prompt_replacements(
82 + def _get_prompt_updates(
83 self,
84 mm_items: MultiModalDataItems,
85 hf_processor_mm_kwargs: Mapping[str, object],
86 out_mm_kwargs: MultiModalKwargs,
86 - ) -> list[PromptReplacement]:
87 - pass
88 -
89 - def _get_mm_fields_config(
90 - self,
91 - hf_inputs: BatchFeature,
92 - hf_processor_mm_kwargs: Mapping[str, object],
93 - ) -> Mapping[str, MultiModalFieldConfig]:
94 - pass
87 + ) -> Sequence[PromptUpdate]:
88 + return []
89
90 def apply(
91 self,
@@ -120,7 +114,7 @@ class PrithviGeoSpatialMAEMultiModalProcessor(BaseMultiModalProcessor):
114 class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
115 """ Prithvi Masked Autoencoder"""
116
123 - def _instantiate_model(self, config: dict) -> nn.Module | None:
117 + def _instantiate_model(self, config: dict) -> Optional[nn.Module]:
118
119 # We might be able/need to support different tasks with this same model
120 if config["task_args"]["task"] == "SemanticSegmentationTask":
@@ -158,7 +152,7 @@ class PrithviGeoSpatialMAE(nn.Module, IsAttentionFree, SupportsMultiModal):
152 "by PrithviGeospatialMAE.")
153
154 def _parse_and_validate_multimodal_data(
161 - self, **kwargs) -> Tuple[torch.Tensor, torch.Tensor | None]:
155 + self, **kwargs) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
156
157 pixel_values = kwargs.pop("pixel_values", None)
158 if not isinstance(pixel_values, torch.Tensor):
vllm/model_executor/models/qwen2_audio.py
+6 -6
@@ -21,9 +21,9 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Inference-only Qwen2-Audio model compatible with HuggingFace weights."""
24 +from collections.abc import Iterable, Mapping, Sequence
25 from functools import cached_property
25 -from typing import (Any, Iterable, Mapping, Optional, Set, Tuple, TypedDict,
26 - Union)
26 +from typing import Any, Optional, Set, Tuple, TypedDict, Union
27
28 import torch
29 import torch.nn as nn
@@ -43,7 +43,7 @@ from vllm.multimodal.parse import (AudioProcessorItems, MultiModalDataItems,
43 MultiModalDataParser)
44 from vllm.multimodal.processing import (BaseMultiModalProcessor,
45 BaseProcessingInfo, PromptReplacement,
46 - PromptReplacementDetails)
46 + PromptUpdate, PromptUpdateDetails)
47 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
48 from vllm.sequence import IntermediateTensors
49
@@ -188,12 +188,12 @@ class Qwen2AudioMultiModalProcessor(
188 feature_attention_mask=MultiModalFieldConfig.batched("audio"),
189 )
190
191 - def _get_prompt_replacements(
191 + def _get_prompt_updates(
192 self,
193 mm_items: MultiModalDataItems,
194 hf_processor_mm_kwargs: Mapping[str, object],
195 out_mm_kwargs: MultiModalKwargs,
196 - ) -> list[PromptReplacement]:
196 + ) -> Sequence[PromptUpdate]:
197 processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
198 tokenizer = self.info.get_tokenizer()
199 vocab = tokenizer.get_vocab()
@@ -230,7 +230,7 @@ class Qwen2AudioMultiModalProcessor(
230
231 audio_tokens = [audio_token_id] * num_features
232
233 - return PromptReplacementDetails(
233 + return PromptUpdateDetails(
234 full=[audio_bos_id] + audio_tokens + [audio_eos_id],
235 features=audio_tokens,
236 )
vllm/model_executor/models/qwen2_vl.py
+9 -7
@@ -23,9 +23,10 @@
23 # See the License for the specific language governing permissions and
24 # limitations under the License.
25 """Inference-only Qwen2-VL model compatible with HuggingFace weights."""
26 +from collections.abc import Iterable, Mapping, Sequence
27 from functools import cached_property, partial
27 -from typing import (Any, Callable, Iterable, Literal, Mapping, Optional, Set,
28 - Tuple, Type, TypedDict, Union)
28 +from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,
29 + Union)
30
31 import torch
32 import torch.nn as nn
@@ -61,7 +62,8 @@ from vllm.multimodal.parse import (DictEmbeddingItems, ImageSize,
62 ModalityDataItems, MultiModalDataItems,
63 MultiModalDataParser)
64 from vllm.multimodal.processing import (BaseMultiModalProcessor,
64 - BaseProcessingInfo, PromptReplacement)
65 + BaseProcessingInfo, PromptReplacement,
66 + PromptUpdate)
67 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
68 from vllm.platforms import _Backend
69 from vllm.sequence import IntermediateTensors
@@ -169,7 +171,7 @@ class Qwen2VisionMLP(nn.Module):
171 self,
172 in_features: int,
173 hidden_features: int,
172 - act_layer: Type[nn.Module] = QuickGELU,
174 + act_layer: type[nn.Module] = QuickGELU,
175 quant_config: Optional[QuantizationConfig] = None,
176 prefix: str = "",
177 ):
@@ -383,7 +385,7 @@ class Qwen2VisionBlock(nn.Module):
385 dim: int,
386 num_heads: int,
387 mlp_ratio: float,
386 - act_layer: Type[nn.Module] = QuickGELU,
388 + act_layer: type[nn.Module] = QuickGELU,
389 norm_layer: Optional[Callable[[int], nn.Module]] = None,
390 quant_config: Optional[QuantizationConfig] = None,
391 prefix: str = "",
@@ -987,12 +989,12 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
989 self.info._get_image_processor_kwargs(**mm_kwargs),
990 )
991
990 - def _get_prompt_replacements(
992 + def _get_prompt_updates(
993 self,
994 mm_items: MultiModalDataItems,
995 hf_processor_mm_kwargs: Mapping[str, Any],
996 out_mm_kwargs: MultiModalKwargs,
995 - ) -> list[PromptReplacement]:
997 + ) -> Sequence[PromptUpdate]:
998 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
999 image_processor = self.info.get_image_processor(
1000 **hf_processor_mm_kwargs)
vllm/model_executor/models/qwen_vl.py
+8 -7
@@ -9,9 +9,10 @@ import copy
9 import math
10 import re
11 import unicodedata
12 +from collections.abc import Collection, Mapping, Sequence
13 +from collections.abc import Set as AbstractSet
14 from functools import lru_cache, partial
13 -from typing import (AbstractSet, Callable, Collection, List, Literal, Mapping,
14 - Optional, TypedDict, Union)
15 +from typing import Callable, List, Literal, Optional, TypedDict, Union
16
17 import torch
18 from torch import nn
@@ -36,7 +37,7 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
37 from vllm.multimodal.parse import MultiModalDataItems
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 BaseProcessingInfo, PromptReplacement,
39 - PromptReplacementDetails)
40 + PromptUpdate, PromptUpdateDetails)
41 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
42 from vllm.sequence import IntermediateTensors
43
@@ -606,7 +607,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
607 mm_kwargs=mm_kwargs,
608 )
609
609 - def _hf_processor_applies_repl(
610 + def _hf_processor_applies_updates(
611 self,
612 prompt_text: str,
613 mm_items: MultiModalDataItems,
@@ -624,12 +625,12 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
625 image_embeds=MultiModalFieldConfig.batched("image"),
626 )
627
627 - def _get_prompt_replacements(
628 + def _get_prompt_updates(
629 self,
630 mm_items: MultiModalDataItems,
631 hf_processor_mm_kwargs: Mapping[str, object],
632 out_mm_kwargs: MultiModalKwargs,
632 - ) -> list[PromptReplacement]:
633 + ) -> Sequence[PromptUpdate]:
634 tokenizer = self.info.get_tokenizer()
635 special_tokens: dict[str,
636 int] = tokenizer.special_tokens # type: ignore
@@ -646,7 +647,7 @@ class QwenVLMultiModalProcessor(BaseMultiModalProcessor[QwenVLProcessingInfo]):
647 PromptReplacement(
648 modality="image",
649 target=[img_start_id, img_end_id],
649 - replacement=PromptReplacementDetails(
650 + replacement=PromptUpdateDetails(
651 full=[img_start_id] + image_tokens + [img_end_id],
652 features=image_tokens,
653 ),
vllm/model_executor/models/ultravox.py
+6 -5
@@ -3,9 +3,9 @@
3 # Adapted from https://github.com/fixie-ai/ultravox/blob/ecd58c4041030bae2ad15aa6bcf04ab43199ea02/ultravox/model/ultravox_model.py
4 """PyTorch Ultravox model."""
5 import math
6 +from collections.abc import Iterable, Mapping, Sequence
7 from functools import cached_property
7 -from typing import (Any, Iterable, Literal, Mapping, Optional, Set, Tuple,
8 - TypedDict, Union)
8 +from typing import Any, Literal, Optional, Set, Tuple, TypedDict, Union
9
10 import torch
11 import torch.utils.checkpoint
@@ -29,7 +29,8 @@ from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
29 NestedTensors)
30 from vllm.multimodal.parse import MultiModalDataItems, MultiModalDataParser
31 from vllm.multimodal.processing import (BaseMultiModalProcessor,
32 - BaseProcessingInfo, PromptReplacement)
32 + BaseProcessingInfo, PromptReplacement,
33 + PromptUpdate)
34 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
35 from vllm.sequence import IntermediateTensors
36 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
@@ -197,12 +198,12 @@ class UltravoxMultiModalProcessor(
198 audio_embeds=MultiModalFieldConfig.batched("audio"),
199 )
200
200 - def _get_prompt_replacements(
201 + def _get_prompt_updates(
202 self,
203 mm_items: MultiModalDataItems,
204 hf_processor_mm_kwargs: Mapping[str, Any],
205 out_mm_kwargs: MultiModalKwargs,
205 - ) -> list[PromptReplacement]:
206 + ) -> Sequence[PromptUpdate]:
207 hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
208 tokenizer = self.info.get_tokenizer()
209 vocab = tokenizer.get_vocab()
vllm/model_executor/models/whisper.py
+5 -5
@@ -1,8 +1,8 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 -from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
5 - Union)
4 +from collections.abc import Iterable, Mapping, Sequence
5 +from typing import List, Optional, Set, Tuple, TypedDict, Union
6
7 import torch
8 from torch import nn
@@ -31,7 +31,7 @@ from vllm.multimodal.parse import (MultiModalDataDict, MultiModalDataItems,
31 MultiModalDataParser)
32 from vllm.multimodal.processing import (BaseProcessingInfo,
33 EncDecMultiModalProcessor,
34 - PromptReplacement)
34 + PromptReplacement, PromptUpdate)
35 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
36
37 from .interfaces import SupportsMultiModal, SupportsTranscription
@@ -623,12 +623,12 @@ class WhisperMultiModalProcessor(
623 ) -> Mapping[str, MultiModalFieldConfig]:
624 return dict(input_features=MultiModalFieldConfig.batched("audio"))
625
626 - def _get_prompt_replacements(
626 + def _get_prompt_updates(
627 self,
628 mm_items: MultiModalDataItems,
629 hf_processor_mm_kwargs: Mapping[str, object],
630 out_mm_kwargs: MultiModalKwargs,
631 - ) -> list[PromptReplacement]:
631 + ) -> Sequence[PromptUpdate]:
632 num_tokens = self.info.get_max_audio_tokens()
633 return [
634 PromptReplacement(
vllm/multimodal/processing.py
+303 -183
@@ -6,11 +6,14 @@ from collections import defaultdict
6 from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
7 Sequence)
8 from dataclasses import dataclass, field
9 +from enum import Enum
10 from functools import lru_cache
11 +from itertools import groupby
12 from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
11 - TypeVar, Union)
13 + TypeVar, Union, cast)
14
15 from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
16 +from typing_extensions import assert_never
17
18 import vllm.envs as envs
19 from vllm.inputs import InputProcessingContext
@@ -38,35 +41,129 @@ PromptSeq = Union[str, list[int]]
41
42
43 @dataclass
41 -class PromptReplacementDetails:
42 - """Details about the replacement token sequence or text."""
44 +class PromptUpdateDetails:
45 + """Details about the token sequence or text that are part of the update."""
46
47 full: PromptSeq
45 - """The full replacement."""
48 + """The full content."""
49
50 features: PromptSeq
51 """
49 - The part of the replacement that corresponds to feature placeholders;
52 + The part of the content that corresponds to feature placeholders;
53 this will be replaced by the output of the vision encoder during model
54 inference.
55 """
56
57 @staticmethod
55 - def from_seq(seq: PromptSeq) -> "PromptReplacementDetails":
56 - return PromptReplacementDetails(full=seq, features=seq)
58 + def from_seq(seq: PromptSeq) -> "PromptUpdateDetails":
59 + return PromptUpdateDetails(full=seq, features=seq)
60
61
59 -PromptRepl = Union[PromptSeq, PromptReplacementDetails]
62 +PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
63 """
61 -The replacement token sequence or text.
64 +The token sequence or text that are part of the update.
65
63 -If only part of the replacement corresponds to feature placeholders, you can
64 -use :class:`PromptReplacementDetails` to specify which part.
66 +If only part of the content corresponds to feature placeholders, you can
67 +use :class:`PromptUpdateDetails` to specify which part.
68 """
69
70 +PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
71 + PromptUpdateInfo]
72 +"""
73 +Given the index of the processed item within :attr:`modality`,
74 +output the corresponding token sequence (or text).
75 +
76 +For convenience, you can directly pass in the token sequence (or text)
77 +instead of a function if it does not depend on the input.
78 +"""
79 +
80 +
81 +class UpdateMode(str, Enum):
82 + INSERT = "insert"
83 + REPLACE = "replace"
84 +
85 +
86 +@dataclass
87 +class PromptUpdate:
88 + """
89 + Defines how to update a prompt with placeholder tokens.
90 + """
91 +
92 + modality: str
93 + """The modality for which the update is made."""
94 +
95 + target: PromptSeq
96 + """The token sequence (or text) to update."""
97 +
98 + @property
99 + @abstractmethod
100 + def content(self) -> PromptUpdateContent:
101 + """The placeholder tokens that are part of the update."""
102 + raise NotImplementedError
103 +
104 + @property
105 + @abstractmethod
106 + def mode(self) -> UpdateMode:
107 + """Defines how to update the prompt."""
108 + raise NotImplementedError
109 +
110 + def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptUpdate":
111 + return BoundPromptUpdate(
112 + _origin=self,
113 + tokenizer=tokenizer,
114 + )
115 +
116
117 @dataclass
69 -class PromptReplacement:
118 +class PromptInsertion(PromptUpdate):
119 + """
120 + Defines how to insert placeholder tokens into a prompt.
121 +
122 + Example:
123 +
124 + For each image, insert a number of ``<image>`` feature placeholders
125 + equal to the feature size of the vision encoder at the start of the
126 + prompt:
127 +
128 + .. code-block:: python
129 +
130 + PromptInsertion(
131 + modality="image",
132 + target="",
133 + insertion="<image>" * image_feature_size,
134 + )
135 +
136 + As above, but insert after the ``<s>`` token:
137 +
138 + .. code-block:: python
139 +
140 + PromptInsertion(
141 + modality="image",
142 + target="<s>",
143 + insertion="<image>" * image_feature_size,
144 + )
145 + """
146 +
147 + insertion: PromptUpdateContent = field(repr=False)
148 + """
149 + Given the index of the processed item within :attr:`modality`,
150 + output the token sequence (or text) to insert right after :attr:`target`.
151 +
152 + For convenience, you can directly pass in the token sequence (or text)
153 + instead of a function if it does not depend on the input.
154 + """
155 +
156 + @property
157 + def content(self) -> PromptUpdateContent:
158 + return self.insertion
159 +
160 + @property
161 + def mode(self) -> UpdateMode:
162 + return UpdateMode.INSERT
163 +
164 +
165 +@dataclass
166 +class PromptReplacement(PromptUpdate):
167 """
168 Defines how to replace portions of an input prompt with placeholder tokens.
169
@@ -93,7 +190,7 @@ class PromptReplacement:
190 PromptReplacement(
191 modality="image",
192 target="<image>",
96 - replacement=PromptReplacementDetails(
193 + replacement=PromptUpdateDetails(
194 full="".join([
195 "<image_bos>",
196 "<image>" * image_feature_size,
@@ -111,7 +208,7 @@ class PromptReplacement:
208 PromptReplacement(
209 modality="image",
210 target=[image_token_id],
114 - replacement=PromptReplacementDetails(
211 + replacement=PromptUpdateDetails(
212 full=([image_bos_id] + [image_token_id] * image_feature_size
213 + [image_eos_id]),
214 features=[image_token_id] * image_feature_size,
@@ -119,29 +216,22 @@ class PromptReplacement:
216 )
217 """
218
122 - modality: str
123 - """The modality for which the replacement is made."""
124 -
125 - target: PromptSeq
126 - """The token sequence (or text) to find and replace."""
127 -
128 - replacement: Union[Callable[[int], PromptRepl],
129 - PromptRepl] = field(repr=False)
219 + replacement: PromptUpdateContent = field(repr=False)
220 """
221 Given the index of the processed item within :attr:`modality`,
132 - output the replacement token sequence (or text).
222 + output the token sequence (or text) to replace :attr:`target`.
223
134 - For convenience, you can directly pass in the replacement token sequence
135 - (or text) instead of a function if it does not depend on the input.
224 + For convenience, you can directly pass in the token sequence (or text)
225 + instead of a function if it does not depend on the input.
226 """
227
138 - def bind(self, tokenizer: AnyTokenizer) -> "BoundPromptReplacement":
139 - return BoundPromptReplacement(
140 - tokenizer=tokenizer,
141 - modality=self.modality,
142 - _target=self.target,
143 - _replacement=self.replacement,
144 - )
228 + @property
229 + def content(self) -> PromptUpdateContent:
230 + return self.replacement
231 +
232 + @property
233 + def mode(self) -> UpdateMode:
234 + return UpdateMode.REPLACE
235
236
237 @lru_cache(maxsize=2048)
@@ -232,64 +322,73 @@ class _BoundPromptSequence:
322
323
324 @dataclass
235 -class _BoundPromptReplacementGroup:
325 +class _BoundPromptContent:
326 full: _BoundPromptSequence
327 features: _BoundPromptSequence
328
329
330 @dataclass
241 -class BoundPromptReplacement:
331 +class BoundPromptUpdate:
332 """
243 - A :class:`PromptReplacement` bound to a tokenizer to automatically
244 - convert :attr:`target` and the result of :meth:`get_replacement` between
333 + A :class:`PromptUpdate` bound to a tokenizer to automatically convert
334 + :attr:`target` and the result of :meth:`get_content` between
335 token sequence and text representations.
336 """
337 + _origin: PromptUpdate
338 tokenizer: AnyTokenizer = field(repr=False)
248 - modality: str
249 -
250 - _target: PromptSeq
251 - _replacement: Union[Callable[[int], PromptRepl],
252 - PromptRepl] = field(repr=False)
339
340 def __post_init__(self) -> None:
255 - self._replacement_cache = dict[int, _BoundPromptReplacementGroup]()
341 + self._content_cache = dict[int, _BoundPromptContent]()
342 +
343 + @property
344 + def modality(self) -> str:
345 + return self._origin.modality
346
347 @property
348 def target(self) -> _BoundPromptSequence:
259 - """The token sequence (or text) to find and replace."""
260 - return _BoundPromptSequence.from_seq(self.tokenizer, self._target)
349 + """The token sequence (or text) to update."""
350 + return _BoundPromptSequence.from_seq(self.tokenizer,
351 + self._origin.target)
352
262 - def get_replacement(self, item_idx: int) -> _BoundPromptReplacementGroup:
353 + @property
354 + def content(self) -> PromptUpdateContent:
355 + """The placeholder tokens that are part of the update."""
356 + return self._origin.content
357 +
358 + @property
359 + def mode(self) -> UpdateMode:
360 + """Defines how to update the prompt."""
361 + return self._origin.mode
362 +
363 + def get_content(self, item_idx: int) -> _BoundPromptContent:
364 """
365 Given the index of the processed item within :attr:`modality`,
265 - output the replacement token sequence (or text).
366 + output the token sequence (or text) to update.
367 """
267 - replacement = self._replacement
268 - if callable(replacement):
368 + content = self.content
369 + if callable(content):
370 cache_key = item_idx
270 - if cache_key in self._replacement_cache:
271 - return self._replacement_cache[cache_key]
371 + if cache_key in self._content_cache:
372 + return self._content_cache[cache_key]
373
273 - replacement = replacement(item_idx)
374 + content = content(item_idx)
375 else:
376 cache_key = None
377
277 - if not isinstance(replacement, PromptReplacementDetails):
278 - replacement = PromptReplacementDetails.from_seq(replacement)
378 + if not isinstance(content, PromptUpdateDetails):
379 + content = PromptUpdateDetails.from_seq(content)
380
381 bound_full = _BoundPromptSequence.from_seq(self.tokenizer,
281 - replacement.full)
382 + content.full)
383 bound_features = _BoundPromptSequence.from_seq(self.tokenizer,
283 - replacement.features)
284 - bound_replacement = _BoundPromptReplacementGroup(
285 - full=bound_full,
286 - features=bound_features,
287 - )
384 + content.features)
385 + bound_content = _BoundPromptContent(full=bound_full,
386 + features=bound_features)
387
388 if cache_key is not None:
290 - self._replacement_cache[cache_key] = bound_replacement
389 + self._content_cache[cache_key] = bound_content
390
292 - return bound_replacement
391 + return bound_content
392
393
394 class _TokenMatch(NamedTuple):
@@ -326,12 +425,12 @@ def iter_token_matches(
425
426
427 @dataclass(repr=False)
329 -class _PromptReplacementMatch(ABC):
330 - prompt_repl: BoundPromptReplacement
428 +class _PromptTargetMatch(ABC):
429 + _origin: BoundPromptUpdate
430
431 @property
432 def modality(self) -> str:
334 - return self.prompt_repl.modality
433 + return self._origin.modality
434
435 @property
436 @abstractmethod
@@ -349,7 +448,7 @@ class _PromptReplacementMatch(ABC):
448
449
450 @dataclass(repr=False)
352 -class _PromptReplacementTokenMatch(_PromptReplacementMatch):
451 +class _PromptTargetTokenMatch(_PromptTargetMatch):
452 match: _TokenMatch
453
454 @property
@@ -362,7 +461,7 @@ class _PromptReplacementTokenMatch(_PromptReplacementMatch):
461
462
463 @dataclass(repr=False)
365 -class _PromptReplacementTextMatch(_PromptReplacementMatch):
464 +class _PromptTargetTextMatch(_PromptTargetMatch):
465 match: re.Match[str]
466
467 @property
@@ -394,40 +493,37 @@ class PlaceholderFeaturesInfo:
493
494 def find_token_matches(
495 prompt: list[int],
397 - prompt_repls: Sequence[BoundPromptReplacement],
398 -) -> list[_PromptReplacementTokenMatch]:
399 - """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
496 + prompt_updates: Sequence[BoundPromptUpdate],
497 +) -> Sequence[_PromptTargetMatch]:
498 + """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
499 return [
401 - _PromptReplacementTokenMatch(prompt_repl, match)
402 - for prompt_repl in prompt_repls
403 - for match in iter_token_matches(prompt, prompt_repl.target.token_ids)
500 + _PromptTargetTokenMatch(update, match) for update in prompt_updates
501 + for match in iter_token_matches(prompt, update.target.token_ids)
502 ]
503
504
505 def find_text_matches(
506 prompt: str,
409 - prompt_repls: Sequence[BoundPromptReplacement],
410 -) -> list[_PromptReplacementTextMatch]:
411 - """Return each target of :code:`prompt_repls` found in :code:`prompt`."""
507 + prompt_updates: Sequence[BoundPromptUpdate],
508 +) -> Sequence[_PromptTargetMatch]:
509 + """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
510 return [
413 - _PromptReplacementTextMatch(prompt_repl, match)
414 - for prompt_repl in prompt_repls
415 - for match in re.finditer(re.escape(prompt_repl.target.text), prompt)
511 + _PromptTargetTextMatch(update, match) for update in prompt_updates
512 + for match in re.finditer(re.escape(update.target.text), prompt)
513 ]
514
515
516 def _resolve_matches(
517 prompt: PromptSeq,
421 - mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
422 -) -> list[_PromptReplacementMatch]:
518 + mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
519 +) -> list[_PromptTargetMatch]:
520 """
521 Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
522 and sort them such that earlier matches take priority over later ones.
523 """
524 matches = [m for matches in mm_matches.values() for m in matches]
525
429 - seen_matches: list[Optional[_PromptReplacementMatch]] = [None
430 - ] * len(prompt)
526 + seen_matches: list[Optional[_PromptTargetMatch]] = [None] * len(prompt)
527
528 for match in matches:
529 for idx in range(match.start_idx, match.end_idx):
@@ -441,74 +537,91 @@ def _resolve_matches(
537 return sorted(matches, key=lambda x: x.start_idx)
538
539
444 -def _replace_matches(
540 +def _apply_matches(
541 prompt: _S,
446 - mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
542 + mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
543 mm_item_counts: Mapping[str, int],
544 ) -> list[_S]:
449 - """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
450 - out_seqs = list[_S]()
545 + """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
546 + out_seqs = list[Union[str, list[int]]]()
547 prev_end_idx = 0
548 next_idx_by_modality = defaultdict[str, int](lambda: 0)
549
454 - for match in _resolve_matches(prompt, mm_matches):
455 - modality = match.modality
550 + for (start_idx, end_idx), group in groupby(
551 + _resolve_matches(prompt, mm_matches),
552 + key=lambda x: (x.start_idx, x.end_idx),
553 + ):
554 + matches = tuple(group)
555 + assert len(matches) == 1
556
457 - item_idx = next_idx_by_modality[modality]
458 - if item_idx >= mm_item_counts.get(modality, 0):
459 - continue
557 + for match in matches:
558 + modality = match.modality
559
461 - start_idx = match.start_idx
462 - end_idx = match.end_idx
560 + item_idx = next_idx_by_modality[modality]
561 + if item_idx >= mm_item_counts.get(modality, 0):
562 + continue
563
464 - repl_info = match.prompt_repl
465 - replacement = repl_info.get_replacement(item_idx)
564 + origin = match._origin
565 + content = origin.get_content(item_idx)
566 + mode = origin.mode
567
467 - if isinstance(prompt, str):
468 - repl_seq = replacement.full.text
469 - out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
470 - else:
471 - repl_seq = replacement.full.token_ids
472 - out_seqs.append(prompt[prev_end_idx:start_idx] + repl_seq)
568 + if mode == UpdateMode.INSERT:
569 + out_seqs.append(prompt[prev_end_idx:end_idx])
570 + num_inserts = mm_item_counts.get(modality, 0)
571 + elif mode == UpdateMode.REPLACE:
572 + out_seqs.append(prompt[prev_end_idx:start_idx])
573 + num_inserts = 1
574 + else:
575 + assert_never(mode)
576
474 - prev_end_idx = end_idx
475 - next_idx_by_modality[modality] += 1
577 + for _ in range(num_inserts):
578 + if item_idx >= mm_item_counts.get(modality, 0):
579 + continue
580 +
581 + if isinstance(prompt, str):
582 + out_seqs.append(content.full.text)
583 + else:
584 + out_seqs.append(content.full.token_ids)
585 +
586 + next_idx_by_modality[modality] += 1
587 +
588 + prev_end_idx = end_idx
589
590 out_seqs.append(prompt[prev_end_idx:])
591
479 - return out_seqs
592 + return cast(list[_S], out_seqs)
593
594
482 -def replace_token_matches(
595 +def apply_token_matches(
596 prompt: list[int],
484 - mm_matches: Mapping[str, Sequence[_PromptReplacementTokenMatch]],
597 + mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
598 mm_item_counts: Mapping[str, int],
599 ) -> list[int]:
487 - """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
600 + """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
601 if not mm_matches:
602 return prompt
603
491 - token_id_seqs = _replace_matches(prompt, mm_matches, mm_item_counts)
604 + token_id_seqs = _apply_matches(prompt, mm_matches, mm_item_counts)
605
606 return flatten_2d_lists(token_id_seqs)
607
608
496 -def replace_text_matches(
609 +def apply_text_matches(
610 prompt: str,
498 - mm_matches: Mapping[str, Sequence[_PromptReplacementTextMatch]],
611 + mm_matches: Mapping[str, Sequence[_PromptTargetMatch]],
612 mm_item_counts: Mapping[str, int],
613 ) -> str:
501 - """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
614 + """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
615 if not mm_matches:
616 return prompt
617
505 - texts = _replace_matches(prompt, mm_matches, mm_item_counts)
618 + texts = _apply_matches(prompt, mm_matches, mm_item_counts)
619
620 return "".join(texts)
621
622
623 def _iter_placeholders(
511 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
624 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
625 prompt: list[int],
626 mm_item_counts: Mapping[str, int],
627 ) -> Iterable[PlaceholderFeaturesInfo]:
@@ -517,7 +630,7 @@ def _iter_placeholders(
630
631 Matches are exclusive even when multiple modalities share
632 the same placeholder tokens. In that case, the modality that
520 - appears earlier in `mm_prompt_repls` takes priority.
633 + appears earlier in `mm_prompt_updates` takes priority.
634
635 Note that empty matches are ignored.
636 """
@@ -528,37 +641,37 @@ def _iter_placeholders(
641 while start_idx < prompt_len:
642 found = False
643
531 - for modality, modality_repls in mm_prompt_repls.items():
644 + for modality, modality_updates in mm_prompt_updates.items():
645 item_idx = item_idx_by_modality[modality]
646 if item_idx >= mm_item_counts.get(modality, 0):
647 continue
648
536 - for repl_info in modality_repls:
537 - replacement = repl_info.get_replacement(item_idx)
538 - repl_tokens_full = replacement.full.token_ids
539 - repl_len_full = len(repl_tokens_full)
540 - end_idx_full = start_idx + repl_len_full
649 + for update_info in modality_updates:
650 + content = update_info.get_content(item_idx)
651 + content_tokens_full = content.full.token_ids
652 + content_len_full = len(content_tokens_full)
653 + end_idx_full = start_idx + content_len_full
654
542 - if repl_len_full == 0 or end_idx_full > prompt_len:
655 + if content_len_full == 0 or end_idx_full > prompt_len:
656 continue
657
545 - if prompt[start_idx:end_idx_full] == repl_tokens_full:
546 - repl_tokens_feat = replacement.features.token_ids
658 + if prompt[start_idx:end_idx_full] == content_tokens_full:
659 + content_tokens_feat = content.features.token_ids
660
661 try:
662 match = next(
550 - iter_token_matches(repl_tokens_full,
551 - repl_tokens_feat))
663 + iter_token_matches(content_tokens_full,
664 + content_tokens_feat))
665 yield PlaceholderFeaturesInfo(
666 modality=modality,
667 item_idx=item_idx,
668 start_idx=start_idx + match.start_idx,
556 - tokens=repl_tokens_feat,
669 + tokens=content_tokens_feat,
670 )
671 except StopIteration:
672 raise AssertionError(
560 - f"{repl_tokens_feat=} should be a "
561 - f"subsequence of {repl_tokens_full=}") from None
673 + f"{content_tokens_feat=} should be a "
674 + f"subsequence of {content_tokens_full=}") from None
675
676 # Exclude overlapping matches
677 start_idx = end_idx_full
@@ -574,11 +687,11 @@ def _iter_placeholders(
687
688
689 def find_mm_placeholders(
577 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
690 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
691 prompt: list[int],
692 mm_item_counts: Mapping[str, int],
693 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
581 - it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
694 + it = _iter_placeholders(mm_prompt_updates, prompt, mm_item_counts)
695 return dict(full_groupby_modality(it))
696
697
@@ -712,6 +825,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
825 *,
826 cache: Optional[ProcessingCache] = None,
827 enable_sanity_checks: bool = True) -> None:
828 + if get_repls := getattr(self, "_get_prompt_replacements", None):
829 + logger.warning_once("`_get_prompt_replacements` has been renamed "
830 + "to `_get_prompt_updates`. The old name will "
831 + "be removed in an upcoming release.")
832 + self._get_prompt_updates = get_repls # type: ignore[method-assign]
833 +
834 super().__init__()
835
836 self.info = info
@@ -770,34 +889,34 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
889 raise NotImplementedError
890
891 @abstractmethod
773 - def _get_prompt_replacements(
892 + def _get_prompt_updates(
893 self,
894 mm_items: MultiModalDataItems,
895 hf_processor_mm_kwargs: Mapping[str, object],
896 out_mm_kwargs: MultiModalKwargs,
778 - ) -> list[PromptReplacement]:
897 + ) -> list[PromptUpdate]:
898 """
899 Given the original multi-modal items for this modality
781 - and HF-processed data, output the replacements to perform.
900 + and HF-processed data, output the updates to perform.
901
902 Notes:
903 - You should not assume that HF processor always performs prompt
785 - replacement: in :meth:`_apply_hf_processor_missing`, this method
904 + updates: in :meth:`_apply_hf_processor_missing`, this method
905 is called on text-only and multimodal-only inputs separately,
906 instead of passing them in the same call.
788 - - The replacement information returned by this method is also used
789 - to determine the placeholder token positions for each multi-modal
907 + - The update information returned by this method is also used to
908 + determine the placeholder token positions for each multi-modal
909 item.
910 """
911 raise NotImplementedError
912
913 def _find_mm_placeholders(
914 self,
796 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
915 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
916 new_token_ids: list[int],
917 mm_item_counts: Mapping[str, int],
918 ) -> Mapping[str, list[PlaceholderFeaturesInfo]]:
800 - return find_mm_placeholders(mm_prompt_repls, new_token_ids,
919 + return find_mm_placeholders(mm_prompt_updates, new_token_ids,
920 mm_item_counts)
921
922 def _get_hf_mm_data(
@@ -831,14 +950,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
950 mm_kwargs,
951 )
952
834 - def _hf_processor_applies_repl(
953 + def _hf_processor_applies_updates(
954 self,
955 prompt_text: str,
956 mm_items: MultiModalDataItems,
957 hf_processor_mm_kwargs: Mapping[str, object],
958 ) -> bool:
959 """
841 - Return whether the HF processor applies prompt replacements.
960 + Return whether the HF processor applies prompt updates.
961
962 For most HF processors, this should be :code:`True` when multi-modal
963 data items are passed, but :code:`False` when multi-modal embeddings
@@ -858,7 +977,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
977 Apply the HF processor on the prompt text and multi-modal data
978 together.
979
861 - In addition, return whether prompt replacements have been applied.
980 + In addition, return whether prompt updates have been applied.
981 """
982 processor_data, passthrough_data = self._get_hf_mm_data(mm_items)
983
@@ -876,13 +995,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
995 self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
996 )
997
879 - is_repl_applied = self._hf_processor_applies_repl(
998 + is_update_applied = self._hf_processor_applies_updates(
999 prompt_text=prompt_text,
1000 mm_items=mm_items,
1001 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1002 )
1003
885 - return prompt_ids, mm_kwargs, is_repl_applied
1004 + return prompt_ids, mm_kwargs, is_update_applied
1005
1006 def _apply_hf_processor_text_only(self, prompt_text: str) -> list[int]:
1007 """
@@ -948,21 +1067,21 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1067 mm_items: MultiModalDataItems,
1068 hf_processor_mm_kwargs: Mapping[str, object],
1069 *,
951 - enable_hf_prompt_replacement: bool,
1070 + enable_hf_prompt_update: bool,
1071 ) -> tuple[list[int], MultiModalKwargs, bool]:
1072 """
1073 Apply the HF processor on the prompt text and multi-modal data.
1074
956 - In addition, return whether prompt replacements have been applied
1075 + In addition, return whether prompt updates have been applied
1076 (for most HF processors, this should be :code:`True`).
1077
1078 Note:
960 - If :code:`enable_hf_prompt_replacement=False`, we use HF processor
961 - to perform prompt replacement if available; HF processor requires
1079 + If :code:`enable_hf_prompt_update=False`, we use HF processor
1080 + to perform prompt updates if available; HF processor requires
1081 that the prompt corresponds to multi-modal items.
1082 """
1083 if isinstance(prompt, str):
965 - if enable_hf_prompt_replacement:
1084 + if enable_hf_prompt_update:
1085 return self._apply_hf_processor_text_mm(
1086 prompt_text=prompt,
1087 mm_items=mm_items,
@@ -999,7 +1118,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1118 prompt=prompt,
1119 mm_items=mm_data_items,
1120 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1002 - enable_hf_prompt_replacement=True,
1121 + enable_hf_prompt_update=True,
1122 )
1123
1124 mm_maybe_cached_kw_items = {
@@ -1022,17 +1141,17 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1141 mm_missing_data_items = self._to_mm_items(mm_missing_data)
1142
1143 # NOTE: `prompt` does not correspond to `mm_missing_data_items`,
1025 - # so we can't apply prompt replacements until the new multimodal
1144 + # so we can't apply prompt updates until the new multimodal
1145 # items are combined with the cached multimodal items
1146 (
1147 prompt_ids,
1148 mm_missing_kwargs,
1030 - is_repl_applied,
1149 + is_update_applied,
1150 ) = self._apply_hf_processor_main(
1151 prompt=prompt,
1152 mm_items=mm_missing_data_items,
1153 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
1035 - enable_hf_prompt_replacement=False,
1154 + enable_hf_prompt_update=False,
1155 )
1156
1157 mm_missing_next_idx = {
@@ -1071,28 +1190,28 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1190
1191 mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
1192
1074 - return prompt_ids, mm_kwargs, is_repl_applied
1193 + return prompt_ids, mm_kwargs, is_update_applied
1194
1076 - def _bind_and_group_repls(
1195 + def _bind_and_group_updates(
1196 self,
1078 - prompt_repls: list[PromptReplacement],
1079 - ) -> dict[str, list[BoundPromptReplacement]]:
1197 + prompt_updates: list[PromptUpdate],
1198 + ) -> dict[str, list[BoundPromptUpdate]]:
1199 tokenizer = self.info.get_tokenizer()
1200
1082 - it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
1201 + it = (update.bind(tokenizer) for update in prompt_updates)
1202 return dict(full_groupby_modality(it))
1203
1085 - def _apply_prompt_replacements(
1204 + def _apply_prompt_updates(
1205 self,
1206 token_ids: list[int],
1088 - mm_prompt_repls: Mapping[str, Sequence[BoundPromptReplacement]],
1207 + mm_prompt_updates: Mapping[str, Sequence[BoundPromptUpdate]],
1208 mm_item_counts: Mapping[str, int],
1209 ) -> tuple[list[int], str, Mapping[str, list[PlaceholderFeaturesInfo]]]:
1210 tokenizer = self.info.get_tokenizer()
1211
1212 mm_token_matches = {
1094 - modality: find_token_matches(token_ids, prompt_repls)
1095 - for modality, prompt_repls in mm_prompt_repls.items()
1213 + modality: find_token_matches(token_ids, updates)
1214 + for modality, updates in mm_prompt_updates.items()
1215 }
1216 mm_match_counts = {
1217 modality: len(matches)
@@ -1107,31 +1226,31 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1226 # up a token, then the token ID of "foo" will not appear at all
1227 # ----
1228 # Since it is inefficient to search for all possible tokenizations
1110 - # of the search text in the prompt, we instead perform string
1111 - # replacement on the decoded token IDs, then encode them back.
1229 + # of the search text in the prompt, we instead perform string-based
1230 + # updates on the decoded token IDs, then encode them back.
1231 if all(
1232 mm_match_counts.get(modality, 0) >= item_count
1233 for modality, item_count in mm_item_counts.items()
1234 ): # yapf: disable
1116 - token_ids = replace_token_matches(
1235 + token_ids = apply_token_matches(
1236 token_ids,
1237 mm_token_matches,
1238 mm_item_counts,
1239 )
1240
1241 text = decode_tokens(tokenizer, token_ids)
1123 - matched_repls = {
1124 - modality: [match.prompt_repl for match in token_matches]
1242 + matched_updates = {
1243 + modality: [match._origin for match in token_matches]
1244 for modality, token_matches in mm_token_matches.items()
1245 }
1246 else:
1247 text = decode_tokens(tokenizer, token_ids)
1248
1249 mm_text_matches = {
1131 - modality: find_text_matches(text, prompt_repls)
1132 - for modality, prompt_repls in mm_prompt_repls.items()
1250 + modality: find_text_matches(text, updates)
1251 + for modality, updates in mm_prompt_updates.items()
1252 }
1134 - text = replace_text_matches(
1253 + text = apply_text_matches(
1254 text,
1255 mm_text_matches,
1256 mm_item_counts,
@@ -1140,13 +1259,13 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1259 token_ids = encode_tokens(tokenizer,
1260 text,
1261 add_special_tokens=False)
1143 - matched_repls = {
1144 - modality: [match.prompt_repl for match in token_matches]
1262 + matched_updates = {
1263 + modality: [match._origin for match in token_matches]
1264 for modality, token_matches in mm_text_matches.items()
1265 }
1266
1267 placeholders = self._find_mm_placeholders(
1149 - matched_repls,
1268 + matched_updates,
1269 token_ids,
1270 mm_item_counts,
1271 )
@@ -1184,14 +1303,14 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1303
1304 if len(placeholders) != item_count:
1305 raise RuntimeError(
1187 - f"Expected there to be {item_count} prompt replacements "
1306 + f"Expected there to be {item_count} prompt updates "
1307 f"corresponding to {item_count} {modality} items, but "
1189 - f"instead found {len(placeholders)} prompt replacements! "
1308 + f"instead found {len(placeholders)} prompt updates! "
1309 "Either the prompt text has missing/incorrect tokens for "
1310 "multi-modal inputs, or there is a problem with your "
1311 "implementation of merged multi-modal processor for this "
1312 "model (usually arising from an inconsistency between "
1194 - "`_call_hf_processor` and `_get_prompt_replacements`).")
1313 + "`_call_hf_processor` and `_get_prompt_updates`).")
1314
1315 def apply(
1316 self,
@@ -1206,7 +1325,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1325
1326 1. Apply HF Processor on prompt text and multi-modal data together,
1327 outputting token IDs and processed tensors.
1209 - 2. Find and replace sequences in the token IDs with placeholder tokens.
1328 + 2. Find and update sequences in the token IDs with placeholder tokens.
1329 The number of placeholder tokens equals the feature size of the
1330 multi-modal data outputted by the multi-modal encoder.
1331 3. Extract information about the placeholder tokens from the
@@ -1235,26 +1354,27 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1354 (
1355 prompt_ids,
1356 mm_kwargs,
1238 - is_repl_applied,
1357 + is_update_applied,
1358 ) = self._cached_apply_hf_processor(
1359 prompt,
1360 mm_items,
1361 hf_processor_mm_kwargs,
1362 )
1363
1245 - unbound_prompt_repls = self._get_prompt_replacements(
1364 + unbound_prompt_updates = self._get_prompt_updates(
1365 mm_items,
1366 hf_processor_mm_kwargs,
1367 mm_kwargs,
1368 )
1250 - mm_prompt_repls = self._bind_and_group_repls(unbound_prompt_repls)
1369 + mm_prompt_updates = self._bind_and_group_updates(
1370 + unbound_prompt_updates)
1371
1372 mm_item_counts = mm_items.get_all_counts()
1373 self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1374
1255 - if is_repl_applied:
1375 + if is_update_applied:
1376 mm_placeholders = self._find_mm_placeholders(
1257 - mm_prompt_repls,
1377 + mm_prompt_updates,
1378 prompt_ids,
1379 mm_item_counts,
1380 )
@@ -1267,9 +1387,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1387 prompt_ids,
1388 prompt,
1389 mm_placeholders,
1270 - ) = self._apply_prompt_replacements(
1390 + ) = self._apply_prompt_updates(
1391 prompt_ids,
1272 - mm_prompt_repls,
1392 + mm_prompt_updates,
1393 mm_item_counts,
1394 )
1395 self._validate_mm_placeholders(mm_placeholders, mm_item_counts)