A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Merged multi-modal processors for LLaVA-NeXT-Video and LLaVA-OneVision (#11717)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 4, 2025 at 19:40 UTC
eed11ebee93e9d137ac74d8e6e97427354bd3797
31 files changed
+1114
-983
tests/models/decoder_only/vision_language/processing/__init__.py
renamed
tests/models/decoder_only/vision_language/processing/test_idefics3.py
renamed
tests/models/decoder_only/vision_language/processing/test_internvl.py
renamed
tests/models/decoder_only/vision_language/processing/test_llava_next.py
new
+58
@@ -0,0 +1,58 @@
1
+import pytest
2
+from PIL import Image
3
+from transformers import AutoTokenizer
4
+
5
+from vllm.inputs import InputProcessingContext
6
+
7
+from ....utils import build_model_context
8
+
9
+
10
+# Fixtures lazy import to avoid initializing CUDA during test collection
11
+@pytest.fixture()
12
+def processor_for_llava_next():
13
+ from vllm.model_executor.models.llava_next import (
14
+ LlavaNextMultiModalProcessor)
15
+ return LlavaNextMultiModalProcessor
16
+
17
+
18
+# FIXME: image_size [(198, 176), (176, 198)]
19
+@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
20
+@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21
+ (488, 183)])
22
+@pytest.mark.parametrize("num_imgs", [1, 2])
23
+def test_processor_prompt_replacements(
24
+ processor_for_llava_next,
25
+ model_id: str,
26
+ image_size: tuple[int, int],
27
+ num_imgs: int,
28
+):
29
+ """
30
+ Ensure LlavaNextMultiModalProcessor handles prompt replacement properly.
31
+ """
32
+ ctx = build_model_context(
33
+ model_name=model_id,
34
+ tokenizer_name=model_id,
35
+ mm_processor_kwargs=None,
36
+ limit_mm_per_prompt={"image": num_imgs},
37
+ )
38
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
39
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
40
+
41
+ # Build the image str / prompt based on the number of images we pass
42
+ prompt = "<image>" * num_imgs
43
+ mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
44
+
45
+ # The processor will throw an error if there is a mismatch
46
+ # in the prompt replacements
47
+ processor = processor_for_llava_next(ctx)
48
+ processed_inputs = processor.apply(prompt, mm_data, {})
49
+
50
+ image_placeholders = processed_inputs["mm_placeholders"]["image"]
51
+ assert len(image_placeholders) == num_imgs
52
+
53
+ first_placeholder = image_placeholders[0]
54
+
55
+ # NOTE: There is a BOS token
56
+ assert first_placeholder["offset"] == 1
57
+ assert first_placeholder["length"] == (
58
+ len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py
new
+59
@@ -0,0 +1,59 @@
1
+import pytest
2
+from PIL import Image
3
+from transformers import AutoTokenizer
4
+
5
+from vllm.inputs import InputProcessingContext
6
+
7
+from ....utils import build_model_context
8
+
9
+
10
+# Fixtures lazy import to avoid initializing CUDA during test collection
11
+@pytest.fixture()
12
+def processor_for_llava_onevision():
13
+ from vllm.model_executor.models.llava_onevision import (
14
+ LlavaOnevisionMultiModalProcessor)
15
+ return LlavaOnevisionMultiModalProcessor
16
+
17
+
18
+@pytest.mark.parametrize("model_id",
19
+ ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
20
+@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21
+ (488, 183), (198, 176), (176, 198)])
22
+@pytest.mark.parametrize("num_imgs", [1, 2])
23
+def test_processor_prompt_replacements(
24
+ processor_for_llava_onevision,
25
+ model_id: str,
26
+ image_size: tuple[int, int],
27
+ num_imgs: int,
28
+):
29
+ """
30
+ Ensure LlavaOnevisionMultiModalProcessor handles prompt replacement
31
+ properly.
32
+ """
33
+ ctx = build_model_context(
34
+ model_name=model_id,
35
+ tokenizer_name=model_id,
36
+ mm_processor_kwargs=None,
37
+ limit_mm_per_prompt={"image": num_imgs},
38
+ )
39
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
40
+ ctx = InputProcessingContext(ctx.model_config, tokenizer)
41
+
42
+ # Build the image str / prompt based on the number of images we pass
43
+ prompt = "<image>" * num_imgs
44
+ mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
45
+
46
+ # The processor will throw an error if there is a mismatch
47
+ # in the prompt replacements
48
+ processor = processor_for_llava_onevision(ctx)
49
+ processed_inputs = processor.apply(prompt, mm_data, {})
50
+
51
+ image_placeholders = processed_inputs["mm_placeholders"]["image"]
52
+ assert len(image_placeholders) == num_imgs
53
+
54
+ first_placeholder = image_placeholders[0]
55
+
56
+ # NOTE: There is a BOS token
57
+ assert first_placeholder["offset"] == 0
58
+ assert first_placeholder["length"] == len(
59
+ processed_inputs["prompt_token_ids"]) // num_imgs
tests/models/decoder_only/vision_language/processing/test_phi3v.py
renamed
+20
-24
@@ -1,6 +1,4 @@
1
"""Tests for phi3v's multimodal preprocessing kwargs."""
2
-from typing import Optional
3
-
2
import pytest
3
from transformers import AutoTokenizer
4
@@ -10,8 +8,6 @@ from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
8
from .....conftest import _ImageAssets
9
from ....utils import build_model_context
10
13
-models = ["microsoft/Phi-3.5-vision-instruct"]
14
-
11
12
# Wrap lazy imports to avoid initializing CUDA during test collection
13
@pytest.fixture()
@@ -20,40 +16,40 @@ def processor_for_phi3v():
16
return Phi3VMultiModalProcessor
17
18
23
-@pytest.mark.parametrize("model", models)
19
+@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
20
+# yapf: disable
21
@pytest.mark.parametrize(
25
- "num_crops,expected_toks_per_img",
22
+ ("mm_processor_kwargs", "expected_toks_per_img"),
23
[
27
- (4, 757),
28
- (16, 1921),
24
+ ({"num_crops": 4}, 757),
25
+ ({"num_crops": 16}, 1921),
26
# the default num_crops of phi-3.5-vision is 4
30
- (None, 757),
27
+ ({}, 757),
28
])
29
+# yapf: enable
30
@pytest.mark.parametrize("num_imgs", [1, 2])
33
-def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
34
- model: str, num_crops: Optional[int],
35
- expected_toks_per_img: int, num_imgs: int):
31
+def test_processor_override(
32
+ processor_for_phi3v,
33
+ image_assets: _ImageAssets,
34
+ model_id: str,
35
+ mm_processor_kwargs: dict[str, int],
36
+ expected_toks_per_img: int,
37
+ num_imgs: int,
38
+):
39
"""Ensure input_processor_for_phi3v handles num_crops properly."""
37
- # Same as the previous test - don't initialize mm_processor_kwargs
38
- # in this test and assume that the kwargs will be correctly expanded by
39
- # the partial when calling the custom input processor.
40
ctx = build_model_context(
41
- model_name=model,
42
- tokenizer_name=model,
41
+ model_name=model_id,
42
+ tokenizer_name=model_id,
43
trust_remote_code=True,
44
limit_mm_per_prompt={"image": num_imgs},
45
)
46
- tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
46
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
47
ctx = InputProcessingContext(ctx.model_config, tokenizer)
48
+
49
# Build the image str / prompt based on the number of images we pass
50
img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
51
prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
51
- images = [image_assets[0].pil_image] * num_imgs
52
-
53
- mm_data = {"image": images}
54
- mm_processor_kwargs = {}
55
- if num_crops is not None:
56
- mm_processor_kwargs = {"num_crops": num_crops}
52
+ mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
53
54
processor = processor_for_phi3v(ctx)
55
processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
tests/models/decoder_only/vision_language/processing/test_qwen.py
renamed
tests/models/decoder_only/vision_language/processing/test_qwen2_vl.py
renamed
+13
-26
@@ -1,5 +1,3 @@
1
-from typing import Any, Dict, Tuple
2
-
1
import pytest
2
from transformers import AutoTokenizer
3
@@ -8,56 +6,45 @@ from vllm.inputs import InputProcessingContext
6
from .....conftest import _ImageAssets
7
from ....utils import build_model_context
8
11
-MODEL = "Qwen/Qwen2-VL-2B-Instruct"
12
-MIN_PIXELS = "min_pixels"
13
-MAX_PIXELS = "max_pixels"
14
-
9
10
# Fixtures lazy import to avoid initializing CUDA during test collection
17
-# NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
18
-# input mappers.
11
@pytest.fixture()
12
def processor_for_qwen2_vl():
13
from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
14
return Qwen2VLMultiModalProcessor
15
16
17
+@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
18
+# yapf: disable
19
@pytest.mark.parametrize(
26
- "mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
20
+ ("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"), [
21
({}, 1426, (5704, 1176)),
28
- ({
29
- MIN_PIXELS: 64**2,
30
- MAX_PIXELS: 512**2
31
- }, 330, (1320, 1176)),
22
+ ({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
23
])
33
-@pytest.mark.parametrize("model", [MODEL])
24
+# yapf: enable
25
@pytest.mark.parametrize("num_imgs", [1, 2])
26
def test_processor_override(
27
processor_for_qwen2_vl,
28
image_assets: _ImageAssets,
38
- model: str,
39
- mm_processor_kwargs: Dict[str, Any],
29
+ model_id: str,
30
+ mm_processor_kwargs: dict[str, object],
31
expected_toks_per_img: int,
41
- expected_pixels_shape: Tuple[int, int],
32
+ expected_pixels_shape: tuple[int, int],
33
num_imgs: int,
34
):
35
"""Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
45
- # Same as the previous test - don't initialize mm_processor_kwargs
46
- # in this test and assume that the kwargs will be correctly expanded by
47
- # the partial when calling the custom input processor.
36
ctx = build_model_context(
49
- model_name=model,
50
- tokenizer_name=model,
37
+ model_name=model_id,
38
+ tokenizer_name=model_id,
39
mm_processor_kwargs=None,
40
limit_mm_per_prompt={"image": num_imgs},
41
)
54
- tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
42
+ tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
43
ctx = InputProcessingContext(ctx.model_config, tokenizer)
44
+
45
# Build the image str / prompt based on the number of images we pass
46
prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
58
- images = [image_assets[0].pil_image] * num_imgs
59
-
60
- mm_data = {"image": images}
47
+ mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
48
49
processor = processor_for_qwen2_vl(ctx)
50
processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
tests/models/decoder_only/vision_language/test_models.py
+2
-7
@@ -274,10 +274,8 @@ VLM_TEST_SETTINGS = {
274
),
275
limit_mm_per_prompt={"image": 4},
276
)],
277
- # Llava-next tests fixed sizes & the default size factors
278
- image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
277
),
280
- "llava_one_vision": VLMTestInfo(
278
+ "llava_onevision": VLMTestInfo(
279
models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
280
test_type=VLMTestType.CUSTOM_INPUTS,
281
prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
@@ -288,8 +286,6 @@ VLM_TEST_SETTINGS = {
286
),
287
auto_cls=AutoModelForVision2Seq,
288
vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
291
- # Llava-one-vision tests fixed sizes & the default size factors
292
- image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
289
custom_test_opts=[CustomTestOptions(
290
inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
291
formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
@@ -306,7 +302,6 @@ VLM_TEST_SETTINGS = {
302
max_model_len=4096,
303
auto_cls=AutoModelForVision2Seq,
304
vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
309
- image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
305
),
306
"mantis": VLMTestInfo(
307
models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
@@ -431,7 +426,7 @@ VLM_TEST_SETTINGS = {
426
) for inp in custom_inputs.different_patch_input_cases_internvl()
427
],
428
),
434
- "llava_one_vision-multiple-images": VLMTestInfo(
429
+ "llava_onevision-multiple-images": VLMTestInfo(
430
models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
431
test_type=VLMTestType.CUSTOM_INPUTS,
432
max_model_len=16384,
tests/models/decoder_only/vision_language/test_qwen2_vl.py
-127
@@ -427,130 +427,3 @@ def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,
427
mm_limit=1,
428
tensor_parallel_size=1,
429
)
430
-
431
-
432
-def run_chunked_prefill_test(
433
- vllm_runner: Type[VllmRunner],
434
- inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
435
- model: str,
436
- *,
437
- dtype: str,
438
- max_tokens: int,
439
- num_logprobs: int,
440
- mm_limit: int,
441
- tensor_parallel_size: int,
442
- distributed_executor_backend: Optional[str] = None,
443
-):
444
- """Compare inference result between
445
- chunked prefill disabled and chunked prefill enabled
446
- """
447
-
448
- # NOTE:
449
- # max_model_len should be greater than image_feature_size
450
- with vllm_runner(model,
451
- task="generate",
452
- max_model_len=4000,
453
- max_num_seqs=4,
454
- dtype=dtype,
455
- limit_mm_per_prompt={
456
- "image": mm_limit,
457
- "video": mm_limit
458
- },
459
- tensor_parallel_size=tensor_parallel_size,
460
- distributed_executor_backend=distributed_executor_backend
461
- ) as vllm_model:
462
-
463
- outputs_per_case = [
464
- vllm_model.generate_greedy_logprobs(prompts,
465
- max_tokens,
466
- num_logprobs=num_logprobs,
467
- images=images or None,
468
- videos=videos or None)
469
- for prompts, images, videos in inputs
470
- ]
471
-
472
- with vllm_runner(
473
- model,
474
- task="generate",
475
- max_model_len=4000,
476
- max_num_seqs=4,
477
- dtype=dtype,
478
- limit_mm_per_prompt={
479
- "image": mm_limit,
480
- "video": mm_limit
481
- },
482
- tensor_parallel_size=tensor_parallel_size,
483
- distributed_executor_backend=distributed_executor_backend,
484
- enable_chunked_prefill=True,
485
- # should be small enough to ensure prefilling is chunked
486
- max_num_batched_tokens=32,
487
- mm_processor_kwargs={
488
- "max_pixels": 16 * 28 * 28,
489
- }) as vllm_model_chunked:
490
- outputs_per_case_chunked = [
491
- vllm_model_chunked.generate_greedy_logprobs(
492
- prompts,
493
- max_tokens,
494
- num_logprobs=num_logprobs,
495
- images=images or None,
496
- videos=videos or None) for prompts, images, videos in inputs
497
- ]
498
-
499
- for outputs, \
500
- outputs_chunked \
501
- in zip(outputs_per_case,
502
- outputs_per_case_chunked):
503
- check_logprobs_close(
504
- outputs_0_lst=outputs,
505
- outputs_1_lst=outputs_chunked,
506
- name_0="non_chunked",
507
- name_1="chunked",
508
- )
509
-
510
-
511
-@pytest.mark.core_model
512
-@pytest.mark.parametrize("model", models)
513
-@pytest.mark.parametrize("dtype", [target_dtype])
514
-@pytest.mark.parametrize("max_tokens", [1])
515
-@pytest.mark.parametrize("num_logprobs", [10])
516
-def test_qwen2_vl_mrope_chunked_prefill(vllm_runner, example_prompts,
517
- model: str, dtype: str,
518
- max_tokens: int,
519
- num_logprobs: int) -> None:
520
- """
521
- Test Qwen2-VL's chunked prefill with M-RoPE
522
- """
523
- prompts = [
524
- qwen2_vl_chat_template(IMAGE_PLACEHOLDER, prompt)
525
- for prompt in example_prompts[:1]
526
- ]
527
-
528
- # 1. Qwen2-VL's M-RoPE works only when there are some multi-modal inputs,
529
- # so an image is included in the inputs
530
- # 2. however, Qwen2-VL currently won't work properly
531
- # when chunked prefill is enabled and there are some multi-modal inputs,
532
- # here use a hacky way: provide a **zero-length** image to make it happy
533
- #
534
- # and finally we achieved:
535
- # (1) chunked_prefill enabled; (2) M-RoPE works; to continue our tests
536
- zero_len_image = {
537
- "image_embeds": torch.empty((0, MODEL_HIDDEN_SIZE)),
538
- "image_grid_thw": torch.tensor([[0, 0, 0]])
539
- }
540
- images = [zero_len_image] * len(prompts)
541
-
542
- inputs_per_case: List[Tuple[List[str], PromptImageInput,
543
- PromptVideoInput]] = [
544
- (prompts, images, []),
545
- ]
546
-
547
- run_chunked_prefill_test(
548
- vllm_runner,
549
- inputs_per_case,
550
- model,
551
- dtype=dtype,
552
- max_tokens=max_tokens,
553
- num_logprobs=num_logprobs,
554
- mm_limit=1,
555
- tensor_parallel_size=1,
556
- )
tests/multimodal/test_processing.py
+102
-68
@@ -11,8 +11,8 @@ from vllm.config import ModelConfig
11
from vllm.inputs import InputProcessingContext
12
from vllm.multimodal import MULTIMODAL_REGISTRY
13
from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
14
- _PlaceholderInfo, find_text_matches,
15
- find_token_matches, iter_placeholders,
14
+ _PlaceholderInfo, find_mm_placeholders,
15
+ find_text_matches, find_token_matches,
16
iter_token_matches,
17
replace_text_matches,
18
replace_token_matches)
@@ -314,21 +314,27 @@ def test_find_replace_text(
314
# Should not be used since there is nothing to convert to text
315
mock_tokenizer = cast(AnyTokenizer, object())
316
317
- prompt_repls = [
318
- PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
317
+ mm_prompt_repls = {
318
+ key: [
319
+ PromptReplacement(key, target,
320
+ repl_by_key[key]).bind(mock_tokenizer)
321
+ ]
322
for key, target in target_by_key.items()
320
- ]
321
- matches = find_text_matches(prompt, prompt_repls)
323
+ }
324
+ mm_matches = {
325
+ key: find_text_matches(prompt, prompt_repls)
326
+ for key, prompt_repls in mm_prompt_repls.items()
327
+ }
328
329
result = replace_text_matches(
330
prompt,
325
- matches,
331
+ mm_matches,
332
{key: mm_count
333
for key in repl_by_key},
334
)
335
336
# Only displayed on error
331
- print("matches:", matches)
337
+ print("mm_matches:", mm_matches)
338
print("result:", result)
339
340
# Manually constructed results
@@ -380,21 +386,27 @@ def test_find_replace_tokens(
386
# Should not be used since there is nothing to convert to tokens
387
mock_tokenizer = cast(AnyTokenizer, object())
388
383
- prompt_repls = [
384
- PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
389
+ mm_prompt_repls = {
390
+ key: [
391
+ PromptReplacement(key, target,
392
+ repl_by_key[key]).bind(mock_tokenizer)
393
+ ]
394
for key, target in target_by_key.items()
386
- ]
387
- matches = find_token_matches(prompt, prompt_repls)
395
+ }
396
+ mm_matches = {
397
+ key: find_token_matches(prompt, prompt_repls)
398
+ for key, prompt_repls in mm_prompt_repls.items()
399
+ }
400
401
result = replace_token_matches(
402
prompt,
391
- matches,
403
+ mm_matches,
404
{key: mm_count
405
for key in repl_by_key},
406
)
407
408
# Only displayed on error
397
- print("matches:", matches)
409
+ print("mm_matches:", mm_matches)
410
print("result:", result)
411
412
# Manually constructed results
@@ -417,58 +429,76 @@ def test_find_replace_tokens(
429
[
430
(
431
[1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
420
- [
421
- _PlaceholderInfo(
422
- modality="pattern_1",
423
- start_idx=6,
424
- replacement=[32000, 32000],
425
- ),
426
- ],
432
+ {
433
+ "pattern_1": [
434
+ _PlaceholderInfo(
435
+ modality="pattern_1",
436
+ item_idx=0,
437
+ start_idx=6,
438
+ replacement=[32000, 32000],
439
+ ),
440
+ ],
441
+ }
442
+
443
),
444
(
445
[1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
430
- [
431
- _PlaceholderInfo(
432
- modality="pattern_1",
433
- start_idx=1,
434
- replacement=[32000, 32000],
435
- ),
436
- _PlaceholderInfo(
437
- modality="pattern_1",
438
- start_idx=5,
439
- replacement=[32000, 32000],
440
- ),
441
- _PlaceholderInfo(
442
- modality="pattern_3",
443
- start_idx=7,
444
- replacement=[1550, 918, 1550],
445
- ),
446
- ],
446
+ {
447
+ "pattern_1": [
448
+ _PlaceholderInfo(
449
+ modality="pattern_1",
450
+ item_idx=0,
451
+ start_idx=1,
452
+ replacement=[32000, 32000],
453
+ ),
454
+ _PlaceholderInfo(
455
+ modality="pattern_1",
456
+ item_idx=1,
457
+ start_idx=5,
458
+ replacement=[32000, 32000],
459
+ ),
460
+ ],
461
+ "pattern_3": [
462
+ _PlaceholderInfo(
463
+ modality="pattern_3",
464
+ item_idx=0,
465
+ start_idx=7,
466
+ replacement=[1550, 918, 1550],
467
+ ),
468
+ ],
469
+ }
470
),
471
(
472
[1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
450
- [
451
- _PlaceholderInfo(
452
- modality="pattern_1",
453
- start_idx=1,
454
- replacement=[32000, 32000],
455
- ),
456
- _PlaceholderInfo(
457
- modality="pattern_1",
458
- start_idx=3,
459
- replacement=[32000, 32000],
460
- ),
461
- _PlaceholderInfo(
462
- modality="pattern_3",
463
- start_idx=6,
464
- replacement=[1550, 918, 1550],
465
- ),
466
- ],
473
+ {
474
+ "pattern_1": [
475
+ _PlaceholderInfo(
476
+ modality="pattern_1",
477
+ item_idx=0,
478
+ start_idx=1,
479
+ replacement=[32000, 32000],
480
+ ),
481
+ _PlaceholderInfo(
482
+ modality="pattern_1",
483
+ item_idx=1,
484
+ start_idx=3,
485
+ replacement=[32000, 32000],
486
+ ),
487
+ ],
488
+ "pattern_3": [
489
+ _PlaceholderInfo(
490
+ modality="pattern_3",
491
+ item_idx=0,
492
+ start_idx=6,
493
+ replacement=[1550, 918, 1550],
494
+ ),
495
+ ],
496
+ }
497
),
498
]
499
)
500
# yapf: enable
471
-def test_iter_placeholders(
501
+def test_find_mm_placeholders(
502
repl_by_key,
503
prompt,
504
expected,
@@ -476,19 +506,18 @@ def test_iter_placeholders(
506
# Should not be used since there is nothing to convert to tokens
507
mock_tokenizer = cast(AnyTokenizer, object())
508
479
- prompt_repls = [
480
- PromptReplacement(key, [], repl).bind(mock_tokenizer)
509
+ mm_prompt_repls = {
510
+ key: [PromptReplacement(key, [], repl).bind(mock_tokenizer)]
511
for key, repl in repl_by_key.items()
482
- ]
512
+ }
513
484
- result = list(
485
- iter_placeholders(
486
- prompt_repls,
487
- prompt,
488
- # Effectively match all occurrences in the prompt
489
- {key: 3
490
- for key in repl_by_key},
491
- ))
514
+ result = find_mm_placeholders(
515
+ mm_prompt_repls,
516
+ prompt,
517
+ # Effectively match all occurrences in the prompt
518
+ {key: 3
519
+ for key in repl_by_key},
520
+ )
521
522
# Only displayed on error
523
print("result:", result)
@@ -694,7 +723,10 @@ def _test_processing_cache_correctness(
723
}
724
725
mm_counts = {k: len(vs) for k, vs in mm_data.items()}
697
- prompt = baseline_processor._get_dummy_mm_inputs(mm_counts).prompt_text
726
+ prompt = baseline_processor._get_dummy_processor_inputs(
727
+ model_config.max_model_len,
728
+ mm_counts,
729
+ ).prompt_text
730
731
# Drop unnecessary keys and test single -> multi conversion
732
if rng.rand() < simplify_rate:
@@ -728,6 +760,8 @@ def _test_processing_cache_correctness(
760
("adept/fuyu-8b", {"image": False}),
761
("llava-hf/llava-1.5-7b-hf", {"image": True}),
762
("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
763
+ ("llava-hf/LLaVA-NeXT-Video-7B-hf", {"video": False}),
764
+ ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", {"image": True, "video": True}), # noqa: E501
765
("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
766
("mistral-community/pixtral-12b", {"image": True}),
767
("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
vllm/model_executor/models/aria.py
+3
-2
@@ -456,7 +456,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
456
hf_config = self.ctx.get_hf_config()
457
return max(hf_config.projector_patch_to_query_dict.values())
458
459
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
459
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
460
return {"image": self._get_num_image_tokens()}
461
462
def _get_mm_fields_config(
@@ -488,8 +488,9 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
488
)
489
]
490
491
- def _get_dummy_mm_inputs(
491
+ def _get_dummy_processor_inputs(
492
self,
493
+ seq_len: int,
494
mm_counts: Mapping[str, int],
495
) -> ProcessorInputs:
496
hf_config = self.ctx.get_hf_config()
vllm/model_executor/models/blip2.py
+3
-2
@@ -405,7 +405,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
405
hf_config = self.ctx.get_hf_config(Blip2Config)
406
return hf_config.num_query_tokens
407
408
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
408
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
409
return {"image": self._get_num_image_tokens()}
410
411
def _get_hf_processor(self) -> Blip2Processor:
@@ -457,8 +457,9 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
457
458
return result
459
460
- def _get_dummy_mm_inputs(
460
+ def _get_dummy_processor_inputs(
461
self,
462
+ seq_len: int,
463
mm_counts: Mapping[str, int],
464
) -> ProcessorInputs:
465
hf_config = self.ctx.get_hf_config(Blip2Config)
vllm/model_executor/models/chameleon.py
+3
-2
@@ -57,7 +57,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
57
processor = self._get_hf_processor()
58
return processor.image_seq_length
59
60
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
60
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
61
return {"image": self._get_num_image_tokens()}
62
63
def _get_hf_processor(self) -> ChameleonProcessor:
@@ -90,8 +90,9 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
90
)
91
]
92
93
- def _get_dummy_mm_inputs(
93
+ def _get_dummy_processor_inputs(
94
self,
95
+ seq_len: int,
96
mm_counts: Mapping[str, int],
97
) -> ProcessorInputs:
98
config = self.ctx.get_hf_config(ChameleonConfig)
vllm/model_executor/models/clip.py
+7
-4
@@ -164,15 +164,18 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
164
def get_max_image_tokens(self) -> int:
165
return get_max_clip_image_tokens(self.vision_config)
166
167
- def get_num_patches(self) -> int:
167
+ def get_image_size(self) -> int:
168
+ return self.vision_config.image_size
169
+
170
+ def get_patch_size(self) -> int:
171
+ return self.vision_config.patch_size
172
+
173
+ def get_patch_grid_length(self) -> int:
174
return get_clip_patch_grid_length(
175
image_size=self.vision_config.image_size,
176
patch_size=self.vision_config.patch_size,
177
)
178
173
- def get_image_size(self) -> int:
174
- return self.vision_config.image_size
175
-
179
180
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
181
class CLIPVisionEmbeddings(nn.Module):
vllm/model_executor/models/fuyu.py
+3
-2
@@ -96,7 +96,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
96
nrows = math.ceil(image_height / 30)
97
return ncols, nrows
98
99
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
99
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
100
target_width, target_height = self._get_image_target_size()
101
102
max_ncols, max_nrows = self._get_image_feature_grid_size(
@@ -208,8 +208,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
208
209
return result
210
211
- def _get_dummy_mm_inputs(
211
+ def _get_dummy_processor_inputs(
212
self,
213
+ seq_len: int,
214
mm_counts: Mapping[str, int],
215
) -> ProcessorInputs:
216
target_width, target_height = self._get_image_target_size()
vllm/model_executor/models/llava.py
+33
-42
@@ -25,11 +25,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25
NestedTensors)
26
from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
27
ImageSize)
28
-from vllm.multimodal.processing import (BaseMultiModalProcessor,
29
- InputProcessingContext,
28
+from vllm.multimodal.processing import (InputProcessingContext,
29
MultiModalDataItems, ProcessingCache,
31
- ProcessorInputs, PromptReplacement,
32
- full_groupby_modality)
30
+ ProcessorInputs, PromptReplacement)
31
from vllm.sequence import IntermediateTensors
32
33
from .clip import CLIPVisionModel
@@ -39,7 +37,7 @@ from .pixtral import (PixtralHFVisionModel,
37
from .siglip import SiglipVisionModel
38
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39
maybe_prefix, merge_multimodal_embeddings)
42
-from .vision import vision_encoder_info
40
+from .vision import BaseVisionLanguageMultiModalProcessor
41
42
43
class LlavaImagePixelInputs(TypedDict):
@@ -100,19 +98,7 @@ class LlavaLikeConfig(Protocol):
98
vision_feature_layer: Final[Union[int, List[int]]]
99
100
103
-class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
104
-
105
- def __init__(self,
106
- ctx: InputProcessingContext,
107
- *,
108
- cache: Optional[ProcessingCache] = None,
109
- enable_sanity_checks: bool = True) -> None:
110
- super().__init__(ctx,
111
- cache=cache,
112
- enable_sanity_checks=enable_sanity_checks)
113
-
114
- vision_config = self._get_hf_config().vision_config
115
- self._vision_encoder_info = vision_encoder_info(vision_config)
101
+class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
102
103
@abstractmethod
104
def _get_hf_config(self) -> LlavaLikeConfig:
@@ -121,6 +107,19 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
107
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108
return {"image": None}
109
110
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
111
+ return {"image": self._get_max_image_tokens()}
112
+
113
+ def _get_mm_fields_config(
114
+ self,
115
+ hf_inputs: BatchFeature,
116
+ hf_processor_mm_kwargs: Mapping[str, object],
117
+ ) -> Mapping[str, MultiModalFieldConfig]:
118
+ return dict(
119
+ pixel_values=MultiModalFieldConfig.batched("image"),
120
+ image_embeds=MultiModalFieldConfig.batched("image"),
121
+ )
122
+
123
def _apply_feature_select_strategy(
124
self,
125
strategy: str,
@@ -142,19 +141,6 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
141
self._vision_encoder_info.get_max_image_tokens(),
142
)
143
145
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
146
- return {"image": self._get_max_image_tokens()}
147
-
148
- def _get_mm_fields_config(
149
- self,
150
- hf_inputs: BatchFeature,
151
- hf_processor_mm_kwargs: Mapping[str, object],
152
- ) -> Mapping[str, MultiModalFieldConfig]:
153
- return dict(
154
- pixel_values=MultiModalFieldConfig.batched("image"),
155
- image_embeds=MultiModalFieldConfig.batched("image"),
156
- )
157
-
144
def _get_dummy_image_size(self) -> ImageSize:
145
image_size = self._vision_encoder_info.get_image_size()
146
return ImageSize(image_size, image_size)
@@ -163,8 +149,9 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
149
def _get_image_token(self) -> str:
150
raise NotImplementedError
151
166
- def _get_dummy_mm_inputs(
152
+ def _get_dummy_processor_inputs(
153
self,
154
+ seq_len: int,
155
mm_counts: Mapping[str, int],
156
) -> ProcessorInputs:
157
num_images = mm_counts.get("image", 0)
@@ -709,7 +696,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
696
"</Image>)", # 3 tokens
697
])
698
712
- mantis_repls = self._bind_prompt_replacements([
699
+ mantis_mm_repls = self._bind_and_group_repls([
700
PromptReplacement(
701
modality="image",
702
target=[image_token_id] * num_image_tokens,
@@ -719,7 +706,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
706
707
prompt_ids, prompt_text, _ = self._apply_prompt_replacements(
708
result["prompt_token_ids"],
722
- mantis_repls,
709
+ mantis_mm_repls,
710
mm_item_counts,
711
)
712
@@ -728,15 +715,19 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
715
hf_processor_mm_kwargs,
716
mm_kwargs,
717
)
731
- orig_repls = self._bind_prompt_replacements(unbound_orig_repls)
718
+ orig_repls = self._bind_and_group_repls(unbound_orig_repls)
719
+
720
+ mm_placeholders = self._find_mm_placeholders(
721
+ orig_repls,
722
+ prompt_ids,
723
+ mm_item_counts,
724
+ )
725
733
- all_placeholders = self._find_placeholders(orig_repls, prompt_ids,
734
- mm_item_counts)
735
- assert len(all_placeholders) == mm_item_counts.get("image", 0)
726
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
727
737
- mm_placeholders = {
738
- modality: [item.to_range() for item in items]
739
- for modality, items in full_groupby_modality(all_placeholders)
728
+ mm_placeholder_ranges = {
729
+ modality: [item.to_range() for item in placeholders]
730
+ for modality, placeholders in mm_placeholders.items()
731
}
732
733
return MultiModalInputsV2(
@@ -744,7 +735,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
735
prompt=prompt_text,
736
prompt_token_ids=prompt_ids,
737
mm_kwargs=mm_kwargs,
747
- mm_placeholders=mm_placeholders,
738
+ mm_placeholders=mm_placeholder_ranges,
739
)
740
741
vllm/model_executor/models/llava_next.py
+8
-7
@@ -67,9 +67,6 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
67
def _get_hf_processor(self) -> LlavaNextProcessor:
68
return self.ctx.get_hf_processor(LlavaNextProcessor)
69
70
- def _get_image_token(self) -> str:
71
- return self._get_hf_processor().image_token
72
-
70
def _get_mm_fields_config(
71
self,
72
hf_inputs: BatchFeature,
@@ -81,6 +78,9 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
78
image_embeds=MultiModalFieldConfig.batched("image"),
79
)
80
81
+ def _get_image_token(self) -> str:
82
+ return self._get_hf_processor().image_token
83
+
84
def _get_max_image_tokens(self) -> int:
85
largest_feature_size, _ = self._get_pinpoint_with_most_features()
86
return largest_feature_size
@@ -97,20 +97,20 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
97
image_height: int,
98
) -> int:
99
hf_config = self._get_hf_config()
100
+ vision_encoder_info = self._vision_encoder_info
101
102
base_feature_size = self._apply_feature_select_strategy(
103
hf_config.vision_feature_select_strategy,
103
- self._vision_encoder_info.get_num_image_tokens(
104
+ vision_encoder_info.get_num_image_tokens(
105
image_width=image_width,
106
image_height=image_height,
107
),
108
)
108
- num_patches = self._vision_encoder_info.get_num_patches()
109
110
num_patch_height, num_patch_width = get_anyres_image_grid_shape(
111
image_size=(image_height, image_width),
112
grid_pinpoints=hf_config.image_grid_pinpoints,
113
- patch_size=self._vision_encoder_info.get_image_size(),
113
+ patch_size=vision_encoder_info.get_image_size(),
114
)
115
116
(
@@ -119,7 +119,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
119
) = self._get_num_unpadded_features(
120
original_height=image_height,
121
original_width=image_width,
122
- npatches=num_patches,
122
+ npatches=vision_encoder_info.get_patch_grid_length(),
123
num_patch_height=num_patch_height,
124
num_patch_width=num_patch_width,
125
)
@@ -155,6 +155,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
155
156
unpadded_features = current_height * current_width
157
newline_features = current_height
158
+
159
return (unpadded_features, newline_features)
160
161
def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
vllm/model_executor/models/llava_next_video.py
+134
-139
@@ -3,38 +3,32 @@ from functools import cached_property
3
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
TypedDict, Union)
5
6
-import numpy as np
6
import torch
7
import torch.nn as nn
9
-from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10
- SiglipVisionConfig)
8
+from transformers import (BatchFeature, LlavaNextVideoConfig,
9
+ LlavaNextVideoProcessor)
10
11
from vllm.attention import AttentionMetadata
12
from vllm.config import VllmConfig
14
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15
- InputContext, token_inputs)
13
from vllm.model_executor.layers.activation import get_act_fn
14
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15
from vllm.model_executor.models.clip import CLIPVisionModel
16
from vllm.model_executor.sampling_metadata import SamplingMetadata
17
from vllm.multimodal import MULTIMODAL_REGISTRY
21
-from vllm.multimodal.inputs import NestedTensors
22
-from vllm.multimodal.utils import (cached_get_tokenizer,
23
- repeat_and_pad_placeholder_tokens)
18
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
19
+from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
20
+ VideoEmbeddingItems, VideoProcessorItems)
21
+from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
22
+ PromptReplacement)
23
from vllm.sequence import IntermediateTensors
24
from vllm.utils import is_list_of
25
27
-from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
26
from .interfaces import SupportsMultiModal, SupportsPP
27
from .llava import init_vision_tower_for_llava
30
-from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31
- dummy_seq_data_for_siglip)
28
+from .siglip import SiglipVisionModel
29
from .utils import (AutoWeightsLoader, init_vllm_registered_model,
30
maybe_prefix, merge_multimodal_embeddings)
34
-
35
-# For profile run
36
-_MAX_FRAMES_PER_VIDEO = 32
37
-_MAX_NUM_VIDEOS = 1
31
+from .vision import BaseVisionLanguageMultiModalProcessor
32
33
34
class LlavaNextVideoPixelInputs(TypedDict):
@@ -50,143 +44,148 @@ class LlavaNextVideoPixelInputs(TypedDict):
44
"""
45
46
53
-def get_llava_next_video_frame_feature_size(
54
- hf_config: LlavaNextVideoConfig) -> int:
55
- # Support both CLIPVisionConfig and SiglipVisionConfig
56
- image_size = hf_config.vision_config.image_size
57
- patch_size = hf_config.vision_config.patch_size
58
- spatial_pool_stride = hf_config.spatial_pool_stride
47
+class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
48
60
- return int((image_size / patch_size / spatial_pool_stride)**2)
49
+ def _get_hf_config(self) -> LlavaNextVideoConfig:
50
+ return self.ctx.get_hf_config(LlavaNextVideoConfig)
51
52
+ def _get_hf_processor(self) -> LlavaNextVideoProcessor:
53
+ return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
54
63
-def _get_max_llm_tokens(ctx: InputContext) -> int:
64
- """
65
- Calculated from the maximum video frames under the context length
66
- constraints of the language model.
67
- """
68
- hf_text_config = ctx.model_config.hf_text_config
69
- model_config = ctx.model_config
70
- max_tokens = model_config.max_model_len
71
- rope_scaling = model_config.rope_scaling
72
-
73
- if rope_scaling:
74
- rope_scaling_factor = hf_text_config.rope_scaling["factor"]
75
- else:
76
- rope_scaling_factor = 1
77
-
78
- max_tokens *= rope_scaling_factor
79
-
80
- return max_tokens
81
-
82
-
83
-def get_max_llava_next_video_tokens(ctx: InputContext) -> int:
84
- # Currently set to 32 frames
85
- # TODO: max_tokens = _get_max_llm_tokens(ctx)
86
- hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
87
- tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
88
- return _MAX_FRAMES_PER_VIDEO * tokens_per_frame
89
-
90
-
91
-def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
92
- mm_counts: Mapping[str, int]):
93
- hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
94
- vision_config = hf_config.vision_config
95
-
96
- # TODO: support multiple videos
97
- num_videos = mm_counts["video"]
98
- if num_videos != _MAX_NUM_VIDEOS:
99
- raise NotImplementedError(
100
- f"Only {_MAX_NUM_VIDEOS} videos are supported")
101
-
102
- # TODO: support configuring the number of frames
103
- frames_per_video = _MAX_FRAMES_PER_VIDEO
104
- # num_images = num_videos * frames_per_video
105
-
106
- # fills the sequence with as longer video data as possible
107
- tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
108
- video_feature_size = frames_per_video * tokens_per_frame
109
-
110
- if isinstance(vision_config, CLIPVisionConfig):
111
- seq_data, ranges = dummy_seq_data_for_clip(
112
- vision_config,
113
- seq_len,
114
- num_videos,
115
- image_token_id=hf_config.video_token_index,
116
- image_feature_size_override=video_feature_size,
117
- mm_key="video",
118
- )
55
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
56
+ return {"video": 1}
57
+
58
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
59
+ num_frames = self._get_dummy_num_frames(seq_len)
60
+ max_video_tokens = self._get_max_video_tokens(num_frames)
61
+
62
+ return {"video": max_video_tokens}
63
+
64
+ def _get_mm_fields_config(
65
+ self,
66
+ hf_inputs: BatchFeature,
67
+ hf_processor_mm_kwargs: Mapping[str, object],
68
+ ) -> Mapping[str, MultiModalFieldConfig]:
69
+ return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
70
+
71
+ def _get_num_frame_tokens(
72
+ self,
73
+ *,
74
+ image_width: int,
75
+ image_height: int,
76
+ ) -> int:
77
+ hf_config = self._get_hf_config()
78
+ spatial_pool_stride = hf_config.spatial_pool_stride
79
120
- pil_frame = dummy_image_for_clip(vision_config, num_images=1)
121
- np_frame = np.array(pil_frame["image"])
122
- mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
123
- mm_data = {"video": mm_data_per_video}
124
- return DummyData(seq_data, mm_data, ranges)
125
- elif isinstance(vision_config, SiglipVisionConfig):
126
- seq_data, ranges = dummy_seq_data_for_siglip(
127
- vision_config,
128
- seq_len,
129
- num_videos,
130
- image_token_id=hf_config.video_token_index,
131
- image_feature_size_override=video_feature_size,
132
- mm_key="video",
80
+ patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
81
+ pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
82
+
83
+ return pooled_grid_length * pooled_grid_length
84
+
85
+ def _get_num_video_tokens(
86
+ self,
87
+ *,
88
+ image_width: int,
89
+ image_height: int,
90
+ num_frames: int,
91
+ ) -> int:
92
+ num_frame_tokens = self._get_num_frame_tokens(
93
+ image_width=image_width,
94
+ image_height=image_height,
95
)
96
135
- pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
136
- np_frame = np.array(pil_frame["image"])
137
- mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
138
- mm_data = {"video": mm_data_per_video}
139
- return DummyData(seq_data, mm_data, ranges)
97
+ return num_frame_tokens * num_frames
98
141
- msg = f"Unsupported vision config: {type(vision_config)}"
142
- raise NotImplementedError(msg)
99
+ def _get_max_video_tokens(self, num_frames: int) -> int:
100
+ return self._get_num_video_tokens(image_width=999999,
101
+ image_height=999999,
102
+ num_frames=num_frames)
103
104
+ def _get_max_video_frames(self, max_tokens: int) -> int:
105
+ num_frames = 0
106
145
-def input_processor_for_llava_next_video(ctx: InputContext,
146
- inputs: DecoderOnlyInputs):
147
- multi_modal_data = inputs.get("multi_modal_data")
148
- if multi_modal_data is None or "video" not in multi_modal_data:
149
- return inputs
107
+ while True:
108
+ next_num_frames = num_frames + 1
109
151
- if "multi_modal_placeholders" in inputs and "video" in inputs[
152
- "multi_modal_placeholders"]:
153
- # The inputs already have placeholders.
154
- return inputs
110
+ if self._get_max_video_tokens(next_num_frames) > max_tokens:
111
+ break
112
156
- video_data = multi_modal_data["video"]
113
+ num_frames = next_num_frames
114
158
- model_config = ctx.model_config
159
- hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
160
- vision_config = hf_config.vision_config
115
+ return num_frames
116
162
- if isinstance(video_data, np.ndarray):
163
- # Supports both CLIP and Siglip
164
- num_frames = video_data.shape[0]
165
- frame_feature_size = \
166
- get_llava_next_video_frame_feature_size(hf_config)
167
- video_feature_size = num_frames * frame_feature_size
117
+ def _get_dummy_num_frames(self, seq_len: int) -> int:
118
+ mm_config = self.ctx.get_mm_config()
119
+ max_videos = mm_config.limit_per_prompt.get("video", 1)
120
169
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
121
+ max_total_frames = self._get_max_video_frames(seq_len)
122
171
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
172
- tokenizer,
173
- inputs.get("prompt"),
174
- inputs["prompt_token_ids"],
175
- placeholder_token_id=hf_config.video_token_index,
176
- repeat_count=video_feature_size,
177
- )
123
+ return max(max_total_frames // max(max_videos, 1), 1)
124
179
- return token_inputs(prompt_token_ids=new_token_ids,
180
- prompt=new_prompt,
181
- multi_modal_data=multi_modal_data,
182
- multi_modal_placeholders={"video": ranges})
125
+ def _get_dummy_image_size(self) -> ImageSize:
126
+ image_size = self._vision_encoder_info.get_image_size()
127
+ return ImageSize(image_size, image_size)
128
184
- elif is_list_of(video_data, np.ndarray):
185
- raise NotImplementedError(
186
- "Processing multiple videos is not supported")
129
+ def _get_video_token(self) -> str:
130
+ return self._get_hf_processor().video_token
131
188
- msg = f"Unsupported vision config: {type(vision_config)}"
189
- raise NotImplementedError(msg)
132
+ def _get_prompt_replacements(
133
+ self,
134
+ mm_items: MultiModalDataItems,
135
+ hf_processor_mm_kwargs: Mapping[str, object],
136
+ out_mm_kwargs: MultiModalKwargs,
137
+ ) -> list[PromptReplacement]:
138
+ hf_config = self._get_hf_config()
139
+ video_token_id = hf_config.video_token_index
140
+
141
+ def get_replacement(item_idx: int):
142
+ videos = mm_items.get_items(
143
+ "video", (VideoEmbeddingItems, VideoProcessorItems))
144
+
145
+ if isinstance(videos, VideoEmbeddingItems):
146
+ num_video_tokens = videos.get_feature_size(item_idx)
147
+ else:
148
+ image_size = videos.get_frame_size(item_idx)
149
+ num_video_tokens = self._get_num_video_tokens(
150
+ image_width=image_size.width,
151
+ image_height=image_size.height,
152
+ num_frames=videos.get_num_frames(item_idx),
153
+ )
154
+
155
+ return [video_token_id] * num_video_tokens
156
+
157
+ return [
158
+ PromptReplacement(
159
+ modality="video",
160
+ target=[video_token_id],
161
+ replacement=get_replacement,
162
+ ),
163
+ ]
164
+
165
+ def _get_dummy_processor_inputs(
166
+ self,
167
+ seq_len: int,
168
+ mm_counts: Mapping[str, int],
169
+ ) -> ProcessorInputs:
170
+ num_videos = mm_counts.get("video", 0)
171
+
172
+ video_token = self._get_video_token()
173
+ target_width, target_height = self._get_dummy_image_size()
174
+
175
+ mm_data = {
176
+ "video":
177
+ self._get_dummy_videos(
178
+ width=target_width,
179
+ height=target_height,
180
+ num_frames=self._get_dummy_num_frames(seq_len),
181
+ num_videos=num_videos,
182
+ )
183
+ }
184
+
185
+ return ProcessorInputs(
186
+ prompt_text=video_token * num_videos,
187
+ mm_data=mm_data,
188
+ )
189
190
191
# adopted from transformers modeling_llava_next_video.py
@@ -246,11 +245,7 @@ class LlavaNextMultiModalProjector(nn.Module):
245
return hidden_states
246
247
249
-@MULTIMODAL_REGISTRY.register_input_mapper("video")
250
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
251
- "video", get_max_llava_next_video_tokens)
252
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next_video)
253
-@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next_video)
248
+@MULTIMODAL_REGISTRY.register_processor(LlavaNextVideoMultiModalProcessor)
249
class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
250
SupportsPP):
251
vllm/model_executor/models/llava_onevision.py
+239
-292
@@ -3,47 +3,36 @@ from functools import cached_property
3
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4
TypedDict, Union)
5
6
-import numpy as np
6
import torch
7
import torch.nn as nn
9
-from PIL import Image
10
-from transformers import (CLIPVisionConfig, LlavaOnevisionConfig,
11
- SiglipVisionConfig)
8
+from transformers import (BatchFeature, LlavaOnevisionConfig,
9
+ LlavaOnevisionProcessor)
10
from transformers.models.llava_onevision.modeling_llava_onevision import (
11
get_anyres_image_grid_shape, unpad_image)
12
from typing_extensions import NotRequired
13
14
from vllm.attention import AttentionMetadata
15
from vllm.config import VllmConfig
18
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19
- InputContext, token_inputs)
16
from vllm.model_executor.layers.activation import get_act_fn
17
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18
from vllm.model_executor.sampling_metadata import SamplingMetadata
19
from vllm.multimodal import MULTIMODAL_REGISTRY
24
-from vllm.multimodal.inputs import NestedTensors
25
-from vllm.multimodal.utils import (cached_get_tokenizer,
26
- repeat_and_pad_placeholder_tokens)
20
+from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
21
+from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
22
+ VideoProcessorItems)
23
+from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
24
+ PromptReplacement)
25
from vllm.sequence import IntermediateTensors
26
from vllm.utils import is_list_of
27
30
-from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
31
- dummy_video_for_clip, get_clip_image_feature_size,
32
- get_clip_patch_grid_length, input_processor_for_clip)
28
+from .clip import CLIPVisionModel
29
from .interfaces import SupportsMultiModal, SupportsPP
30
from .llava import init_vision_tower_for_llava
35
-from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36
- dummy_video_for_siglip, get_siglip_image_feature_size,
37
- get_siglip_patch_grid_length, input_processor_for_siglip)
31
+from .llava_next import LlavaNextMultiModalProcessor
32
+from .siglip import SiglipVisionModel
33
from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
34
maybe_prefix, merge_multimodal_embeddings)
35
41
-# Result in the max possible feature size (2x2 grid of 336x336px tiles)
42
-MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
43
-
44
-# For profile run
45
-_MAX_FRAMES_PER_VIDEO = 16
46
-
36
37
class LlavaOnevisionVideoPixelInputs(TypedDict):
38
type: Literal["pixel_values_videos"]
@@ -92,286 +81,251 @@ LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
81
LlavaOnevisionVideoPixelInputs]
82
83
95
-def _get_llava_onevision_image_unppaded_feature_size(height, width, patches,
96
- scale_height,
97
- scale_width):
98
- current_height = patches * scale_height
99
- current_width = patches * scale_width
100
-
101
- original_aspect_ratio = width / height
102
- current_aspect_ratio = current_width / current_height
103
- if original_aspect_ratio > current_aspect_ratio:
104
- new_height = int(height * (current_width / width))
105
- padding = (current_height - new_height) // 2
106
- current_height -= padding * 2
107
- else:
108
- new_width = int(width * (current_height / height))
109
- padding = (current_width - new_width) // 2
110
- current_width -= padding * 2
111
-
112
- unpadded_features = current_height * current_width
113
- newline_features = current_height
114
-
115
- ratio = math.sqrt(current_height * current_width / (9 * patches**2))
116
- if ratio > 1.1:
117
- unpadded_features = int(current_height // ratio) * int(
118
- current_width // ratio)
119
- newline_features = int(current_height // ratio)
120
-
121
- return (unpadded_features, newline_features)
122
-
123
-
124
-def get_llava_onevision_image_feature_size(
125
- hf_config: LlavaOnevisionConfig,
126
- *,
127
- input_height: int,
128
- input_width: int,
129
-) -> int:
130
- vision_config = hf_config.vision_config
131
-
132
- if isinstance(vision_config, CLIPVisionConfig):
133
- num_patches = get_clip_patch_grid_length(
134
- image_size=vision_config.image_size,
135
- patch_size=vision_config.patch_size,
136
- )
137
- base_feature_size = get_clip_image_feature_size(vision_config)
138
- elif isinstance(vision_config, SiglipVisionConfig):
139
- num_patches = get_siglip_patch_grid_length(
140
- image_size=vision_config.image_size,
141
- patch_size=vision_config.patch_size,
142
- )
143
- base_feature_size = get_siglip_image_feature_size(vision_config)
144
- else:
145
- msg = f"Unsupported vision config: {type(vision_config)}"
146
- raise NotImplementedError(msg)
147
-
148
- strategy = hf_config.vision_feature_select_strategy
149
- if strategy == "default":
150
- base_feature_size -= 1
151
- elif strategy == "full":
152
- pass
153
- else:
154
- raise ValueError(f"Unexpected select feature strategy: {strategy}")
84
+class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
85
156
- num_patch_height, num_patch_width = get_anyres_image_grid_shape(
157
- image_size=(input_height, input_width),
158
- grid_pinpoints=hf_config.image_grid_pinpoints,
159
- patch_size=vision_config.image_size,
160
- )
86
+ def _get_hf_config(self) -> LlavaOnevisionConfig:
87
+ return self.ctx.get_hf_config(LlavaOnevisionConfig)
88
162
- (
163
- unpadded_feature_size,
164
- newline_feature_size,
165
- ) = _get_llava_onevision_image_unppaded_feature_size(
166
- input_height, input_width, num_patches, num_patch_height,
167
- num_patch_width)
168
-
169
- return unpadded_feature_size + newline_feature_size + base_feature_size
170
-
171
-
172
-def get_max_llava_onevision_image_tokens(ctx: InputContext):
173
- return get_llava_onevision_image_feature_size(
174
- ctx.get_hf_config(LlavaOnevisionConfig),
175
- input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
176
- input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
177
- )
178
-
179
-
180
-def get_llava_onevision_video_frame_feature_size(
181
- hf_config: LlavaOnevisionConfig) -> int:
182
- # Support both CLIPVisionConfig and SiglipVisionConfig
183
- image_size = hf_config.vision_config.image_size
184
- patch_size = hf_config.vision_config.patch_size
185
- spatial_pool_stride = hf_config.spatial_pool_stride if hasattr(
186
- hf_config, "spatial_pool_stride") else 2
187
-
188
- height = width = image_size // patch_size
189
- return math.ceil(height / spatial_pool_stride) * math.ceil(
190
- width / spatial_pool_stride)
191
-
192
-
193
-def get_llava_onevision_video_tokens(ctx: InputContext,
194
- num_frames: int) -> int:
195
- hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
196
-
197
- # TODO: support configuring (not supported by HF right now)
198
- num_token_image_newline = 1
199
- tokens_per_frame = get_llava_onevision_video_frame_feature_size(hf_config)
200
- video_feature_size = num_frames * tokens_per_frame + num_token_image_newline
201
-
202
- return video_feature_size
203
-
204
-
205
-def get_max_llava_onevision_video_tokens(ctx: InputContext) -> int:
206
- return get_llava_onevision_video_tokens(ctx, _MAX_FRAMES_PER_VIDEO)
207
-
208
-
209
-def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
210
- mm_counts: Mapping[str, int]):
211
- hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
212
- vision_config = hf_config.vision_config
213
-
214
- num_videos = mm_counts["video"]
215
-
216
- # TODO: support configuring the number of frames
217
- num_frames = _MAX_FRAMES_PER_VIDEO
218
- video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
219
-
220
- if isinstance(vision_config, CLIPVisionConfig):
221
- seq_data, ranges = dummy_seq_data_for_clip(
222
- vision_config,
223
- seq_len,
224
- num_videos,
225
- image_token_id=hf_config.video_token_index,
226
- image_feature_size_override=video_feature_size,
227
- mm_key="video")
228
-
229
- mm_data = dummy_video_for_clip(vision_config,
230
- num_frames=num_frames,
231
- num_videos=num_videos)
232
- return DummyData(seq_data, mm_data, ranges)
233
- elif isinstance(vision_config, SiglipVisionConfig):
234
- seq_data, ranges = dummy_seq_data_for_siglip(
235
- vision_config,
236
- seq_len,
237
- num_videos,
238
- image_token_id=hf_config.video_token_index,
239
- image_feature_size_override=video_feature_size,
240
- mm_key="video")
241
-
242
- mm_data = dummy_video_for_siglip(vision_config,
243
- num_frames=num_frames,
244
- num_videos=num_videos)
245
- return DummyData(seq_data, mm_data, ranges)
246
-
247
- msg = f"Unsupported vision config: {type(vision_config)}"
248
- raise NotImplementedError(msg)
249
-
250
-
251
-def input_processor_when_multimodal_input_image(ctx: InputContext,
252
- inputs: DecoderOnlyInputs):
253
- multi_modal_data = inputs.get("multi_modal_data")
254
- if multi_modal_data is None or "image" not in multi_modal_data:
255
- return inputs
256
-
257
- model_config = ctx.model_config
258
- hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
259
- vision_config = hf_config.vision_config
260
-
261
- image_data = multi_modal_data["image"]
262
- if isinstance(image_data, Image.Image):
263
- width, height = image_data.size
264
-
265
- image_feature_size = get_llava_onevision_image_feature_size(
266
- hf_config,
267
- input_height=height,
268
- input_width=width,
269
- )
270
- elif is_list_of(image_data, Image.Image):
271
- image_feature_size = [
272
- get_llava_onevision_image_feature_size(hf_config,
273
- input_height=img.height,
274
- input_width=img.width)
275
- for img in image_data
276
- ]
277
- elif isinstance(image_data, torch.Tensor):
278
- num_images, image_feature_size, hidden_size = image_data.shape
279
- elif is_list_of(image_data, torch.Tensor):
280
- image_feature_size = [item.shape[1] for item in image_data]
281
- else:
282
- raise TypeError(f"Invalid image type: {type(image_data)}")
283
-
284
- vision_config = hf_config.vision_config
285
-
286
- if isinstance(vision_config, CLIPVisionConfig):
287
- return input_processor_for_clip(
288
- model_config,
289
- vision_config,
290
- inputs,
291
- image_token_id=hf_config.image_token_index,
292
- image_feature_size_override=image_feature_size,
89
+ def _get_hf_processor(self) -> LlavaOnevisionProcessor:
90
+ return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
91
+
92
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
93
+ return {"image": None, "video": None}
94
+
95
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
96
+ max_image_tokens = self._get_max_image_tokens()
97
+
98
+ num_frames = self._get_dummy_num_frames(seq_len)
99
+ max_video_tokens = self._get_max_video_tokens(num_frames)
100
+
101
+ return {
102
+ "image": max_image_tokens,
103
+ "video": max_video_tokens,
104
+ }
105
+
106
+ def _get_mm_fields_config(
107
+ self,
108
+ hf_inputs: BatchFeature,
109
+ hf_processor_mm_kwargs: Mapping[str, object],
110
+ ) -> Mapping[str, MultiModalFieldConfig]:
111
+ return dict(
112
+ pixel_values=MultiModalFieldConfig.batched("image"),
113
+ image_sizes=MultiModalFieldConfig.batched("image"),
114
+ image_embeds=MultiModalFieldConfig.batched("image"),
115
+ pixel_values_videos=MultiModalFieldConfig.batched("video"),
116
)
294
- elif isinstance(vision_config, SiglipVisionConfig):
295
- return input_processor_for_siglip(
296
- model_config,
297
- vision_config,
298
- inputs,
299
- image_token_id=hf_config.image_token_index,
300
- image_feature_size_override=image_feature_size,
117
+
118
+ def _get_num_unpadded_features(
119
+ self,
120
+ *,
121
+ original_height: int,
122
+ original_width: int,
123
+ npatches: int,
124
+ num_patch_height: int,
125
+ num_patch_width: int,
126
+ ) -> tuple[int, int]:
127
+ current_height = npatches * num_patch_height
128
+ current_width = npatches * num_patch_width
129
+
130
+ original_aspect_ratio = original_width / original_height
131
+ current_aspect_ratio = current_width / current_height
132
+ if original_aspect_ratio > current_aspect_ratio:
133
+ new_height = int(original_height *
134
+ (current_width / original_width))
135
+ padding = (current_height - new_height) // 2
136
+ current_height -= padding * 2
137
+ else:
138
+ new_width = int(original_width *
139
+ (current_height / original_height))
140
+ padding = (current_width - new_width) // 2
141
+ current_width -= padding * 2
142
+
143
+ unpadded_features = current_height * current_width
144
+ newline_features = current_height
145
+
146
+ ratio = math.sqrt(current_height * current_width / (9 * npatches**2))
147
+ if ratio > 1.1:
148
+ unpadded_features = int(current_height // ratio) * int(
149
+ current_width // ratio)
150
+ newline_features = int(current_height // ratio)
151
+
152
+ return (unpadded_features, newline_features)
153
+
154
+ def _get_num_frame_tokens(
155
+ self,
156
+ *,
157
+ image_width: int,
158
+ image_height: int,
159
+ ) -> int:
160
+ hf_config = self._get_hf_config()
161
+ spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
162
+
163
+ patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
164
+ pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
165
+
166
+ return pooled_grid_length * pooled_grid_length
167
+
168
+ def _get_num_video_tokens(
169
+ self,
170
+ *,
171
+ image_width: int,
172
+ image_height: int,
173
+ num_frames: int,
174
+ ) -> int:
175
+ num_frame_tokens = self._get_num_frame_tokens(
176
+ image_width=image_width,
177
+ image_height=image_height,
178
)
179
303
- msg = f"Unsupported vision config: {type(vision_config)}"
304
- raise NotImplementedError(msg)
180
+ return num_frame_tokens * num_frames + 1 # Newline token
181
+
182
+ def _get_max_video_tokens(self, num_frames: int) -> int:
183
+ return self._get_num_video_tokens(image_width=999999,
184
+ image_height=999999,
185
+ num_frames=num_frames)
186
187
+ def _get_max_video_frames(self, max_tokens: int) -> int:
188
+ num_frames = 0
189
307
-def input_processor_when_multimodal_input_video(ctx: InputContext,
308
- inputs: DecoderOnlyInputs):
309
- multi_modal_data = inputs.get("multi_modal_data")
310
- if multi_modal_data is None or "video" not in multi_modal_data:
311
- return inputs
312
- video_data = multi_modal_data["video"]
190
+ while True:
191
+ next_num_frames = num_frames + 1
192
314
- model_config = ctx.model_config
315
- hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
193
+ if self._get_max_video_tokens(next_num_frames) > max_tokens:
194
+ break
195
317
- if isinstance(video_data, np.ndarray):
318
- # Supports both CLIP and Siglip
319
- num_frames = video_data.shape[0]
320
- video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
321
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
196
+ num_frames = next_num_frames
197
323
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
324
- tokenizer,
325
- inputs.get("prompt"),
326
- inputs["prompt_token_ids"],
327
- placeholder_token_id=hf_config.video_token_index,
328
- repeat_count=video_feature_size,
198
+ return num_frames
199
+
200
+ def _get_dummy_num_frames(self, seq_len: int) -> int:
201
+ mm_config = self.ctx.get_mm_config()
202
+ max_images = mm_config.limit_per_prompt.get("image", 1)
203
+ max_videos = mm_config.limit_per_prompt.get("video", 1)
204
+
205
+ max_image_tokens = self._get_max_image_tokens() * max_images
206
+ max_total_frames = self._get_max_video_frames(seq_len -
207
+ max_image_tokens)
208
+
209
+ return max(max_total_frames // max(max_videos, 1), 1)
210
+
211
+ def _get_video_token(self) -> str:
212
+ return self._get_hf_processor().video_token
213
+
214
+ def _call_hf_processor(
215
+ self,
216
+ prompt: str,
217
+ mm_data: Mapping[str, object],
218
+ mm_kwargs: Mapping[str, object],
219
+ ) -> BatchFeature:
220
+ mm_data = dict(mm_data)
221
+ videos = mm_data.pop("videos", [])
222
+ assert isinstance(videos, list)
223
+
224
+ if not videos:
225
+ return super()._call_hf_processor(
226
+ prompt=prompt,
227
+ mm_data=mm_data,
228
+ mm_kwargs=mm_kwargs,
229
+ )
230
+
231
+ video_token = self._get_video_token()
232
+
233
+ # LLaVA-OneVision processor doesn't support multiple videos
234
+ # with different sizes when converting back to tensors
235
+ text_image_outputs = super()._call_hf_processor(
236
+ prompt=prompt,
237
+ mm_data=mm_data,
238
+ mm_kwargs=mm_kwargs,
239
+ )
240
+
241
+ pixel_values_videos = []
242
+ for video in videos:
243
+ item_processor_data = dict(prompt=video_token, videos=video)
244
+
245
+ item_outputs = super()._call_hf_processor(
246
+ prompt=prompt,
247
+ mm_data=item_processor_data,
248
+ mm_kwargs=mm_kwargs,
249
+ )
250
+
251
+ pixel_values_videos.append(
252
+ item_outputs.pop("pixel_values_videos")[0])
253
+
254
+ combined_outputs = dict(
255
+ **text_image_outputs,
256
+ pixel_values_videos=pixel_values_videos,
257
)
258
+ return BatchFeature(combined_outputs)
259
331
- return token_inputs(prompt_token_ids=new_token_ids,
332
- prompt=new_prompt,
333
- multi_modal_data=multi_modal_data,
334
- multi_modal_placeholders={"video": ranges})
335
-
336
- elif is_list_of(video_data, np.ndarray):
337
- video_feature_size = []
338
- for video in video_data:
339
- num_frames = video.shape[0]
340
- video_feature_size.append(
341
- get_llava_onevision_video_tokens(ctx, num_frames))
342
-
343
- tokenizer = cached_get_tokenizer(model_config.tokenizer)
344
- new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
345
- tokenizer,
346
- inputs.get("prompt"),
347
- inputs["prompt_token_ids"],
348
- placeholder_token_id=hf_config.video_token_index,
349
- repeat_count=video_feature_size,
260
+ def _get_prompt_replacements(
261
+ self,
262
+ mm_items: MultiModalDataItems,
263
+ hf_processor_mm_kwargs: Mapping[str, object],
264
+ out_mm_kwargs: MultiModalKwargs,
265
+ ) -> list[PromptReplacement]:
266
+ image_repls = super()._get_prompt_replacements(
267
+ mm_items=mm_items,
268
+ hf_processor_mm_kwargs=hf_processor_mm_kwargs,
269
+ out_mm_kwargs=out_mm_kwargs,
270
)
351
- return token_inputs(prompt_token_ids=new_token_ids,
352
- prompt=new_prompt,
353
- multi_modal_data=multi_modal_data,
354
- multi_modal_placeholders={"video": ranges})
355
- else:
356
- raise TypeError(f"Invalid video type: {type(video_data)}")
271
358
- msg = f"Unsupported video type: {type(video_data)}"
359
- raise NotImplementedError(msg)
272
+ hf_config = self._get_hf_config()
273
+ video_token_id = hf_config.video_token_index
274
275
+ def get_video_replacement(item_idx: int):
276
+ videos = mm_items.get_items(
277
+ "video", (VideoEmbeddingItems, VideoProcessorItems))
278
362
-def input_processor_for_llava_onevision(ctx: InputContext,
363
- inputs: DecoderOnlyInputs):
364
- multi_modal_data = inputs.get("multi_modal_data")
365
- if multi_modal_data is None or ("video" not in multi_modal_data
366
- and "image" not in multi_modal_data):
367
- return inputs
368
- if "image" in multi_modal_data:
369
- return input_processor_when_multimodal_input_image(ctx, inputs)
370
- if "video" in multi_modal_data:
371
- return input_processor_when_multimodal_input_video(ctx, inputs)
279
+ if isinstance(videos, VideoEmbeddingItems):
280
+ num_video_tokens = videos.get_feature_size(item_idx)
281
+ else:
282
+ image_size = videos.get_frame_size(item_idx)
283
+ num_video_tokens = self._get_num_video_tokens(
284
+ image_width=image_size.width,
285
+ image_height=image_size.height,
286
+ num_frames=videos.get_num_frames(item_idx),
287
+ )
288
+
289
+ return [video_token_id] * num_video_tokens
290
373
- msg = "Unsupported multi data type"
374
- raise NotImplementedError(msg)
291
+ return image_repls + [
292
+ PromptReplacement(
293
+ modality="video",
294
+ target=[video_token_id],
295
+ replacement=get_video_replacement,
296
+ ),
297
+ ]
298
+
299
+ def _get_dummy_processor_inputs(
300
+ self,
301
+ seq_len: int,
302
+ mm_counts: Mapping[str, int],
303
+ ) -> ProcessorInputs:
304
+ num_images = mm_counts.get("image", 0)
305
+ num_videos = mm_counts.get("video", 0)
306
+
307
+ image_token = self._get_image_token()
308
+ video_token = self._get_video_token()
309
+ target_width, target_height = self._get_dummy_image_size()
310
+
311
+ mm_data = {
312
+ "image":
313
+ self._get_dummy_images(width=target_width,
314
+ height=target_height,
315
+ num_images=num_images),
316
+ "video":
317
+ self._get_dummy_videos(
318
+ width=target_width,
319
+ height=target_height,
320
+ num_frames=self._get_dummy_num_frames(seq_len),
321
+ num_videos=num_videos,
322
+ )
323
+ }
324
+
325
+ return ProcessorInputs(
326
+ prompt_text=image_token * num_images + video_token * num_videos,
327
+ mm_data=mm_data,
328
+ )
329
330
331
class LlavaOnevisionMultiModalProjector(nn.Module):
@@ -394,14 +348,7 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
348
return hidden_states
349
350
397
-@MULTIMODAL_REGISTRY.register_image_input_mapper()
398
-@MULTIMODAL_REGISTRY.register_input_mapper("video")
399
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
400
- "image", get_max_llava_onevision_image_tokens)
401
-@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
402
- "video", get_max_llava_onevision_video_tokens)
403
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_onevision)
404
-@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_onevision)
351
+@MULTIMODAL_REGISTRY.register_processor(LlavaOnevisionMultiModalProcessor)
352
class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
353
SupportsPP):
354
vllm/model_executor/models/phi3v.py
+17
-9
@@ -323,7 +323,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
323
height=image_height,
324
)
325
326
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
326
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
327
max_image_tokens = self._get_num_image_tokens(
328
image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329
image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
@@ -415,12 +415,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
415
def _apply_prompt_replacements(
416
self,
417
token_ids: list[int],
418
- prompt_repls: Sequence[_BoundPromptReplacement],
418
+ mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
419
mm_item_counts: Mapping[str, int],
420
- ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
420
+ ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
421
token_ids, text, placeholders = super()._apply_prompt_replacements(
422
token_ids=token_ids,
423
- prompt_repls=prompt_repls,
423
+ mm_prompt_repls=mm_prompt_repls,
424
mm_item_counts=mm_item_counts,
425
)
426
@@ -428,15 +428,23 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
428
if text.startswith("<s> <|image|>"):
429
text = text.replace("<s> <|image|>", "<s><|image|>", 1)
430
token_ids = [token_ids[0], *token_ids[2:]]
431
- placeholders = [
432
- _PlaceholderInfo(p.modality, p.start_idx - 1, p.replacement)
433
- for p in placeholders
434
- ]
431
+ placeholders = {
432
+ modality: [
433
+ _PlaceholderInfo(
434
+ modality=p.modality,
435
+ item_idx=p.item_idx,
436
+ start_idx=p.start_idx - 1,
437
+ replacement=p.replacement,
438
+ ) for p in ps
439
+ ]
440
+ for modality, ps in placeholders.items()
441
+ }
442
443
return token_ids, text, placeholders
444
438
- def _get_dummy_mm_inputs(
445
+ def _get_dummy_processor_inputs(
446
self,
447
+ seq_len: int,
448
mm_counts: Mapping[str, int],
449
) -> ProcessorInputs:
450
num_images = mm_counts.get("image", 0)
vllm/model_executor/models/pixtral.py
+7
-4
@@ -780,15 +780,18 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
780
def get_max_image_tokens(self) -> int:
781
return get_max_pixtral_hf_image_tokens(self.vision_config)
782
783
- def get_num_patches(self) -> int:
783
+ def get_image_size(self) -> int:
784
+ return self.vision_config.image_size
785
+
786
+ def get_patch_size(self) -> int:
787
+ return self.vision_config.patch_size
788
+
789
+ def get_patch_grid_length(self) -> int:
790
return get_pixtral_hf_patch_grid_length(
791
image_size=self.vision_config.image_size,
792
patch_size=self.vision_config.patch_size,
793
)
794
789
- def get_image_size(self) -> int:
790
- return self.vision_config.image_size
791
-
795
796
class PixtralHFMLP(nn.Module):
797
vllm/model_executor/models/qwen2_audio.py
+8
-7
@@ -84,7 +84,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
84
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85
return {"audio": None}
86
87
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
87
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
88
hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
89
max_source_positions = hf_config.audio_config.max_source_positions
90
max_output_lengths = (max_source_positions - 2) // 2 + 1
@@ -184,15 +184,16 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
184
]
185
186
def _always_apply_prompt_replacements(self) -> bool:
187
- # HF never applies prompt replacements, so we have to do it ourselves
188
- # _find_placeholders may incorrectly think that HF has already performed
189
- # processing for multi-audio input when the input audios are short
190
- # (the corresponding placeholders may take up fewer tokens than
191
- # the number of audio items)
187
+ # HF never applies prompt replacements, so we have to do it ourselves.
188
+ # NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
189
+ # has already performed processing for multi-audio input when the input
190
+ # audios are short (the corresponding placeholders may take up fewer
191
+ # tokens than the number of audio items)
192
return True
193
194
- def _get_dummy_mm_inputs(
194
+ def _get_dummy_processor_inputs(
195
self,
196
+ seq_len: int,
197
mm_counts: Mapping[str, int],
198
) -> ProcessorInputs:
199
feature_extractor = self._get_feature_extractor()
vllm/model_executor/models/qwen2_vl.py
+115
-84
@@ -56,7 +56,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
56
from vllm.multimodal.inputs import (ImageItem, ModalityData,
57
MultiModalFieldConfig, MultiModalKwargs,
58
NestedTensors, VideoItem)
59
-from vllm.multimodal.parse import ModalityDataItems, MultiModalDataParser
59
+from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60
+ MultiModalDataParser)
61
from vllm.multimodal.processing import (BaseMultiModalProcessor,
62
MultiModalDataItems, ProcessorInputs,
63
PromptReplacement)
@@ -641,58 +642,6 @@ class Qwen2VisionTransformer(nn.Module):
642
return loaded_params
643
644
644
-# === Vision input helpers === #
645
-
646
-
647
-def _get_vision_info(
648
- vision_config: Qwen2VLVisionConfig,
649
- height: int,
650
- width: int,
651
- min_pixels: int,
652
- max_pixels: int,
653
- *,
654
- do_resize: bool = True,
655
- modality: str = "image",
656
- mm_count: int = 1,
657
-):
658
- """Get information (resized height / width and number of vision tokens)
659
- of input image / video frame."""
660
- patch_size = vision_config.patch_size
661
- merge_size = vision_config.spatial_merge_size
662
- temporal_patch_size = vision_config.temporal_patch_size
663
-
664
- if do_resize:
665
- resized_height, resized_width = smart_resize(
666
- height=height,
667
- width=width,
668
- factor=patch_size * merge_size,
669
- min_pixels=min_pixels,
670
- max_pixels=max_pixels,
671
- )
672
- else:
673
- resized_height, resized_width = height, width
674
-
675
- if modality == "image":
676
- grid_t = mm_count
677
- elif modality == "video":
678
- grid_t = max(mm_count // temporal_patch_size, 1)
679
- else:
680
- raise ValueError(f"Modality {modality} is not supported")
681
-
682
- grid_h = resized_height // patch_size
683
- grid_w = resized_width // patch_size
684
- vision_tokens = grid_t * grid_h * grid_w
685
- llm_num_vision_tokens = vision_tokens // (merge_size**2)
686
-
687
- return resized_height, resized_width, llm_num_vision_tokens
688
-
689
-
690
-def _get_image_processor(hf_processor: Qwen2VLProcessor):
691
- image_processor = hf_processor.image_processor # type: ignore
692
- assert isinstance(image_processor, Qwen2VLImageProcessor)
693
- return image_processor
694
-
695
-
645
class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
646
dict[str, torch.Tensor]]):
647
@@ -764,32 +713,111 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
713
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
714
return {"image": None, "video": None}
715
767
- def _get_max_mm_tokens(self, modality: str) -> int:
716
+ def _get_vision_info(
717
+ self,
718
+ *,
719
+ image_width: int,
720
+ image_height: int,
721
+ num_frames: int = 1,
722
+ do_resize: bool = True,
723
+ ) -> tuple[ImageSize, int]:
724
hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
725
vision_config = hf_config.vision_config
726
+ patch_size = vision_config.patch_size
727
+ merge_size = vision_config.spatial_merge_size
728
+ temporal_patch_size = vision_config.temporal_patch_size
729
730
hf_processor = self._get_hf_processor()
772
- image_processor = _get_image_processor(hf_processor)
773
-
774
- _, _, max_llm_image_tokens = _get_vision_info(
775
- vision_config,
776
- height=9999999,
777
- width=9999999,
778
- min_pixels=image_processor.min_pixels,
779
- max_pixels=image_processor.max_pixels,
780
- modality=modality,
731
+ image_processor = self._get_image_processor(hf_processor)
732
+
733
+ if do_resize:
734
+ resized_height, resized_width = smart_resize(
735
+ height=image_height,
736
+ width=image_width,
737
+ factor=patch_size * merge_size,
738
+ min_pixels=image_processor.min_pixels,
739
+ max_pixels=image_processor.max_pixels,
740
+ )
741
+ preprocessed_size = ImageSize(width=resized_width,
742
+ height=resized_height)
743
+ else:
744
+ preprocessed_size = ImageSize(width=image_width,
745
+ height=image_height)
746
+
747
+ grid_t = max(num_frames // temporal_patch_size, 1)
748
+ grid_h = preprocessed_size.height // patch_size
749
+ grid_w = preprocessed_size.width // patch_size
750
+
751
+ num_patches = grid_t * grid_h * grid_w
752
+ num_vision_tokens = num_patches // (merge_size**2)
753
+
754
+ return preprocessed_size, num_vision_tokens
755
+
756
+ def _get_dummy_image_size(self) -> ImageSize:
757
+ max_image_size, _ = self._get_vision_info(
758
+ image_width=9999999,
759
+ image_height=9999999,
760
+ )
761
+ return max_image_size
762
+
763
+ def _get_max_image_tokens(self) -> int:
764
+ _, max_image_tokens = self._get_vision_info(
765
+ image_width=9999999,
766
+ image_height=9999999,
767
+ )
768
+ return max_image_tokens
769
+
770
+ def _get_max_video_tokens(self, num_frames: int) -> int:
771
+ _, max_video_tokens = self._get_vision_info(
772
+ image_width=9999999,
773
+ image_height=9999999,
774
+ num_frames=num_frames,
775
)
782
- return max_llm_image_tokens
776
+ return max_video_tokens
777
+
778
+ def _get_max_video_frames(self, max_tokens: int) -> int:
779
+ num_frames = 0
780
+
781
+ while True:
782
+ next_num_frames = num_frames + 1
783
+
784
+ if self._get_max_video_tokens(next_num_frames) > max_tokens:
785
+ break
786
+
787
+ num_frames = next_num_frames
788
+
789
+ return num_frames
790
+
791
+ def _get_dummy_num_frames(self, seq_len: int) -> int:
792
+ mm_config = self.ctx.get_mm_config()
793
+ max_images = mm_config.limit_per_prompt.get("image", 1)
794
+ max_videos = mm_config.limit_per_prompt.get("video", 1)
795
+
796
+ max_image_tokens = self._get_max_image_tokens() * max_images
797
+ max_total_frames = self._get_max_video_frames(seq_len -
798
+ max_image_tokens)
799
+
800
+ return max(max_total_frames // max(max_videos, 1), 1)
801
+
802
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
803
+ max_image_tokens = self._get_max_image_tokens()
804
+
805
+ num_frames = self._get_dummy_num_frames(seq_len)
806
+ max_video_tokens = self._get_max_video_tokens(num_frames)
807
784
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
808
return {
786
- "image": self._get_max_mm_tokens("image"),
787
- "video": self._get_max_mm_tokens("video"),
809
+ "image": max_image_tokens,
810
+ "video": max_video_tokens,
811
}
812
813
def _get_data_parser(self) -> MultiModalDataParser:
814
return Qwen2MultiModalDataParser()
815
816
+ def _get_image_processor(self, hf_processor: Qwen2VLProcessor):
817
+ image_processor = hf_processor.image_processor # type: ignore
818
+ assert isinstance(image_processor, Qwen2VLImageProcessor)
819
+ return image_processor
820
+
821
def _get_hf_processor(
822
self,
823
*,
@@ -797,7 +825,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
825
max_pixels: Optional[int] = None,
826
) -> Qwen2VLProcessor:
827
hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
800
- image_processor = _get_image_processor(hf_processor)
828
+ image_processor = self._get_image_processor(hf_processor)
829
830
if min_pixels:
831
image_processor.min_pixels = min_pixels
@@ -818,7 +846,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
846
out_mm_kwargs: MultiModalKwargs,
847
) -> list[PromptReplacement]:
848
hf_processor = self._get_hf_processor()
821
- image_processor = _get_image_processor(hf_processor)
849
+ image_processor = self._get_image_processor(hf_processor)
850
851
# NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
852
# image_token and video_token registered
@@ -873,32 +901,35 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
901
video_grid_thw=MultiModalFieldConfig.batched("video"),
902
)
903
876
- def _get_dummy_mm_inputs(
904
+ def _get_dummy_processor_inputs(
905
self,
906
+ seq_len: int,
907
mm_counts: Mapping[str, int],
908
) -> ProcessorInputs:
880
- hf_processor = self._get_hf_processor()
881
- image_processor = _get_image_processor(hf_processor)
909
+ num_images = mm_counts.get("image", 0)
910
+ num_videos = mm_counts.get("video", 0)
911
912
+ hf_processor = self._get_hf_processor()
913
image_token: str = hf_processor.image_token
884
- resized_height, resized_width = smart_resize(
885
- height=9999999,
886
- width=9999999,
887
- factor=image_processor.patch_size * image_processor.merge_size,
888
- min_pixels=image_processor.min_pixels,
889
- max_pixels=image_processor.max_pixels,
890
- )
891
- num_images = mm_counts.get("image", 0)
914
+ video_token: str = hf_processor.video_token
915
+ target_width, target_height = self._get_dummy_image_size()
916
917
mm_data = {
918
"image":
895
- self._get_dummy_images(width=resized_width,
896
- height=resized_height,
897
- num_images=num_images)
919
+ self._get_dummy_images(width=target_width,
920
+ height=target_height,
921
+ num_images=num_images),
922
+ "video":
923
+ self._get_dummy_videos(
924
+ width=target_width,
925
+ height=target_height,
926
+ num_frames=self._get_dummy_num_frames(seq_len),
927
+ num_videos=num_videos,
928
+ )
929
}
930
931
return ProcessorInputs(
901
- prompt_text=image_token * num_images,
932
+ prompt_text=image_token * num_images + video_token * num_videos,
933
mm_data=mm_data,
934
)
935
vllm/model_executor/models/siglip.py
+7
-4
@@ -171,15 +171,18 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
171
def get_max_image_tokens(self) -> int:
172
return get_max_siglip_image_tokens(self.vision_config)
173
174
- def get_num_patches(self) -> int:
174
+ def get_image_size(self) -> int:
175
+ return self.vision_config.image_size
176
+
177
+ def get_patch_size(self) -> int:
178
+ return self.vision_config.patch_size
179
+
180
+ def get_patch_grid_length(self) -> int:
181
return get_siglip_patch_grid_length(
182
image_size=self.vision_config.image_size,
183
patch_size=self.vision_config.patch_size,
184
)
185
180
- def get_image_size(self) -> int:
181
- return self.vision_config.image_size
182
-
186
187
# Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
188
class SiglipVisionEmbeddings(nn.Module):
vllm/model_executor/models/ultravox.py
+4
-7
@@ -6,7 +6,6 @@ from functools import cached_property
6
from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7
TypedDict, Union)
8
9
-import numpy as np
9
import torch
10
import torch.utils.checkpoint
11
from torch import nn
@@ -31,7 +30,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
30
PromptReplacement)
31
from vllm.sequence import IntermediateTensors
32
from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34
-from vllm.utils import is_list_of
33
34
from .interfaces import SupportsMultiModal, SupportsPP
35
from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -62,7 +60,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
60
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61
return {"audio": None}
62
65
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
63
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
64
feature_extractor = self._get_feature_extractor()
65
max_audio_tokens = math.ceil(feature_extractor.chunk_length *
66
_AUDIO_TOKENS_PER_SECOND)
@@ -103,6 +101,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
101
102
mm_data = dict(mm_data)
103
audios = mm_data.pop("audios", [])
104
+ assert isinstance(audios, list)
105
106
if not audios:
107
return super()._call_hf_processor(
@@ -117,9 +116,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
116
sampling_rate=feature_extractor.sampling_rate,
117
)
118
120
- # Already resampled by _get_hf_mm_data
121
- assert is_list_of(audios, np.ndarray)
122
-
119
# Ultravox processor doesn't support multiple inputs,
120
# therefore we need to input text and audio one by one
121
audio_features, audio_token_len = [], []
@@ -177,8 +173,9 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
173
)
174
]
175
180
- def _get_dummy_mm_inputs(
176
+ def _get_dummy_processor_inputs(
177
self,
178
+ seq_len: int,
179
mm_counts: Mapping[str, int],
180
) -> ProcessorInputs:
181
feature_extractor = self._get_feature_extractor()
vllm/model_executor/models/vision.py
+34
-3
@@ -1,8 +1,12 @@
1
from abc import ABC, abstractmethod
2
-from typing import Generic, TypeVar
2
+from typing import Final, Generic, Optional, Protocol, TypeVar
3
4
from transformers import PretrainedConfig
5
6
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
7
+ InputProcessingContext,
8
+ ProcessingCache)
9
+
10
_C = TypeVar("_C", bound=PretrainedConfig)
11
12
@@ -27,11 +31,15 @@ class VisionEncoderInfo(ABC, Generic[_C]):
31
raise NotImplementedError
32
33
@abstractmethod
30
- def get_num_patches(self) -> int:
34
+ def get_image_size(self) -> int:
35
raise NotImplementedError
36
37
@abstractmethod
34
- def get_image_size(self) -> int:
38
+ def get_patch_size(self) -> int:
39
+ raise NotImplementedError
40
+
41
+ @abstractmethod
42
+ def get_patch_grid_length(self) -> int:
43
raise NotImplementedError
44
45
@@ -50,3 +58,26 @@ def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
58
59
msg = f"Unsupported vision config: {type(vision_config)}"
60
raise NotImplementedError(msg)
61
+
62
+
63
+class VisionLanguageConfig(Protocol):
64
+ vision_config: Final[PretrainedConfig]
65
+
66
+
67
+class BaseVisionLanguageMultiModalProcessor(BaseMultiModalProcessor):
68
+
69
+ def __init__(self,
70
+ ctx: InputProcessingContext,
71
+ *,
72
+ cache: Optional[ProcessingCache] = None,
73
+ enable_sanity_checks: bool = True) -> None:
74
+ super().__init__(ctx,
75
+ cache=cache,
76
+ enable_sanity_checks=enable_sanity_checks)
77
+
78
+ vision_config = self._get_hf_config().vision_config
79
+ self._vision_encoder_info = vision_encoder_info(vision_config)
80
+
81
+ @abstractmethod
82
+ def _get_hf_config(self) -> VisionLanguageConfig:
83
+ raise NotImplementedError
vllm/multimodal/parse.py
+14
@@ -146,6 +146,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
146
def __init__(self, data: Sequence[HfVideoItem]) -> None:
147
super().__init__(data, "video")
148
149
+ def get_num_frames(self, item_idx: int) -> int:
150
+ return len(self.get(item_idx))
151
+
152
+ def get_frame_size(self, item_idx: int) -> ImageSize:
153
+ image = self.get(item_idx)[0] # Assume that the video isn't empty
154
+
155
+ if isinstance(image, Image):
156
+ return ImageSize(*image.size)
157
+ if isinstance(image, (np.ndarray, torch.Tensor)):
158
+ _, h, w = image.shape
159
+ return ImageSize(w, h)
160
+
161
+ assert_never(image)
162
+
163
164
class VideoEmbeddingItems(EmbeddingItems):
165
vllm/multimodal/processing.py
+206
-120
@@ -16,7 +16,8 @@ from transformers import BatchFeature, ProcessorMixin
16
17
from vllm.inputs import DummyData, InputProcessingContext
18
from vllm.logger import init_logger
19
-from vllm.transformers_utils.tokenizer import AnyTokenizer, encode_tokens
19
+from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
20
+ encode_tokens)
21
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
22
23
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -69,19 +70,6 @@ def _cached_encode(
70
add_special_tokens=add_special_tokens)
71
72
72
-def _decode(
73
- tokenizer: AnyTokenizer,
74
- token_ids: list[int],
75
- *,
76
- skip_special_tokens: bool = False,
77
-) -> str:
78
- """
79
- Backend-agnostic equivalent of HF's
80
- :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
81
- """
82
- return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
83
-
84
-
73
@lru_cache(maxsize=2048)
74
def _cached_decode(
75
tokenizer: AnyTokenizer,
@@ -89,9 +77,9 @@ def _cached_decode(
77
*,
78
skip_special_tokens: bool = False,
79
) -> str:
92
- return _decode(tokenizer,
93
- list(token_ids),
94
- skip_special_tokens=skip_special_tokens)
80
+ return decode_tokens(tokenizer,
81
+ list(token_ids),
82
+ skip_special_tokens=skip_special_tokens)
83
84
85
class _HasModalityAttr(Protocol):
@@ -269,8 +257,10 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
257
return self.match.end()
258
259
272
-class _PlaceholderInfo(NamedTuple):
260
+@dataclass
261
+class _PlaceholderInfo:
262
modality: str
263
+ item_idx: int
264
start_idx: int
265
replacement: list[int]
266
@@ -311,12 +301,14 @@ def find_text_matches(
301
302
def _resolve_matches(
303
prompt: _PromptSeq,
314
- matches: Sequence[_PromptReplacementMatch],
304
+ mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
305
) -> list[_PromptReplacementMatch]:
306
"""
317
- Resolve :code:`matches` to ensure that there are no overlapping matches,
307
+ Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
308
and sort them such that earlier matches take priority over later ones.
309
"""
310
+ matches = [m for matches in mm_matches.values() for m in matches]
311
+
312
seen_matches: list[Optional[_PromptReplacementMatch]] = [None
313
] * len(prompt)
314
@@ -334,14 +326,15 @@ def _resolve_matches(
326
327
def _replace_matches(
328
prompt: _S,
337
- matches: Sequence[_PromptReplacementMatch],
329
+ mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
330
mm_item_counts: Mapping[str, int],
331
) -> list[_S]:
332
+ """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
333
out_seqs = list[_S]()
334
prev_end_idx = 0
335
next_idx_by_modality = defaultdict[str, int](lambda: 0)
336
344
- for match in _resolve_matches(prompt, matches):
337
+ for match in _resolve_matches(prompt, mm_matches):
338
modality = match.modality
339
340
item_idx = next_idx_by_modality[modality]
@@ -371,28 +364,28 @@ def _replace_matches(
364
365
def replace_token_matches(
366
prompt: list[int],
374
- matches: Sequence[_PromptReplacementTokenMatch],
367
+ mm_matches: Mapping[str, Sequence[_PromptReplacementTokenMatch]],
368
mm_item_counts: Mapping[str, int],
369
) -> list[int]:
377
- """Apply :code:`prompt_repls` to :code:`prompt`."""
378
- if not matches:
370
+ """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
371
+ if not mm_matches:
372
return prompt
373
381
- token_id_seqs = _replace_matches(prompt, matches, mm_item_counts)
374
+ token_id_seqs = _replace_matches(prompt, mm_matches, mm_item_counts)
375
376
return flatten_2d_lists(token_id_seqs)
377
378
379
def replace_text_matches(
380
prompt: str,
388
- matches: Sequence[_PromptReplacementTextMatch],
381
+ mm_matches: Mapping[str, Sequence[_PromptReplacementTextMatch]],
382
mm_item_counts: Mapping[str, int],
383
) -> str:
391
- """Apply :code:`prompt_repls` to :code:`prompt`."""
392
- if not matches:
384
+ """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
385
+ if not mm_matches:
386
return prompt
387
395
- texts = _replace_matches(prompt, matches, mm_item_counts)
388
+ texts = _replace_matches(prompt, mm_matches, mm_item_counts)
389
390
return "".join(texts)
391
@@ -407,14 +400,14 @@ def _iter_modality_placeholders(
400
return
401
402
prompt_len = len(prompt)
410
- item_index = 0
403
+ item_idx = 0
404
405
start_idx = 0
406
while start_idx < prompt_len:
407
found = False
408
409
for repl_info in modality_repls:
417
- replacement = repl_info.get_replacement(item_index)
410
+ replacement = repl_info.get_replacement(item_idx)
411
repl_tokens = replacement.token_ids
412
repl_len = len(repl_tokens)
413
end_idx = start_idx + repl_len
@@ -425,12 +418,13 @@ def _iter_modality_placeholders(
418
if prompt[start_idx:end_idx] == repl_tokens:
419
yield _PlaceholderInfo(
420
modality=modality,
421
+ item_idx=item_idx,
422
start_idx=start_idx,
423
replacement=repl_tokens,
424
)
425
432
- item_index += 1
433
- if item_index >= modal_item_count:
426
+ item_idx += 1
427
+ if item_idx >= modal_item_count:
428
return
429
430
# Exclude overlapping matches
@@ -442,28 +436,36 @@ def _iter_modality_placeholders(
436
start_idx += 1
437
438
445
-def iter_placeholders(
446
- prompt_repls: Sequence[_BoundPromptReplacement],
439
+def _iter_placeholders(
440
+ mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
441
prompt: list[int],
442
mm_item_counts: Mapping[str, int],
443
) -> Iterable[_PlaceholderInfo]:
444
"""
451
- Yield each set of placeholder tokens found in :code:`prompt`.
445
+ For each modality, yield each set of placeholder tokens found in
446
+ :code:`prompt`.
447
448
Note that empty matches are ignored.
449
"""
455
- repls_by_modality = dict(full_groupby_modality(prompt_repls))
456
-
450
for modality, modal_item_count in mm_item_counts.items():
458
- if modality in repls_by_modality:
451
+ if modality in mm_prompt_repls:
452
yield from _iter_modality_placeholders(
453
prompt,
454
modality,
462
- repls_by_modality[modality],
455
+ mm_prompt_repls[modality],
456
modal_item_count,
457
)
458
459
460
+def find_mm_placeholders(
461
+ mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
462
+ prompt: list[int],
463
+ mm_item_counts: Mapping[str, int],
464
+) -> Mapping[str, list[_PlaceholderInfo]]:
465
+ it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
466
+ return dict(full_groupby_modality(it))
467
+
468
+
469
@dataclass
470
class ProcessorInputs:
471
"""Keyword arguments to :meth:`BaseMultiModalProcessor`."""
@@ -620,7 +622,7 @@ class BaseMultiModalProcessor(ABC):
622
raise NotImplementedError
623
624
@abstractmethod
623
- def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
625
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
626
"""
627
Get the maximum possible number of tokens per data item
628
for each modality.
@@ -703,14 +705,14 @@ class BaseMultiModalProcessor(ABC):
705
"""
706
raise NotImplementedError
707
706
- def _find_placeholders(
708
+ def _find_mm_placeholders(
709
self,
708
- all_prompt_repls: Sequence[_BoundPromptReplacement],
710
+ mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
711
new_token_ids: list[int],
712
mm_item_counts: Mapping[str, int],
711
- ) -> list[_PlaceholderInfo]:
712
- return list(
713
- iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
713
+ ) -> Mapping[str, list[_PlaceholderInfo]]:
714
+ return find_mm_placeholders(mm_prompt_repls, new_token_ids,
715
+ mm_item_counts)
716
717
def _get_hf_mm_data(
718
self,
@@ -797,7 +799,10 @@ class BaseMultiModalProcessor(ABC):
799
800
# Some HF processors (e.g. Qwen2-VL) expect corresponding
801
# multi-modal tokens to be in the prompt text
800
- dummy_inputs = self._get_dummy_mm_inputs(mm_missing_counts)
802
+ dummy_inputs = self._get_dummy_processor_inputs(
803
+ self.ctx.model_config.max_model_len,
804
+ mm_missing_counts,
805
+ )
806
807
_, mm_missing_kwargs = self._apply_hf_processor(
808
prompt_text=dummy_inputs.prompt_text,
@@ -889,50 +894,44 @@ class BaseMultiModalProcessor(ABC):
894
895
mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
896
892
- if self.enable_sanity_checks:
893
- mm_item_counts = mm_data_items.get_all_counts()
894
-
895
- for modality, item_count in mm_item_counts.items():
896
- for item_idx in range(item_count):
897
- try:
898
- mm_kwargs.get_item(modality, item_idx)
899
- except Exception as e:
900
- # Make it easy to set a breakpoint in the debugger
901
- raise e
902
-
897
return prompt_ids, mm_kwargs
898
905
- def _bind_prompt_replacements(
899
+ def _bind_and_group_repls(
900
self,
901
prompt_repls: list[PromptReplacement],
908
- ) -> list[_BoundPromptReplacement]:
902
+ ) -> dict[str, list[_BoundPromptReplacement]]:
903
tokenizer = self._get_tokenizer()
904
911
- return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
905
+ it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
906
+ return dict(full_groupby_modality(it))
907
908
def _always_apply_prompt_replacements(self) -> bool:
909
"""
910
A flag which can be overridden so that
911
:meth:`_apply_prompt_replacements` is always called even if we
917
- detect that HF has performed processing via :meth:`_find_placeholders`.
912
+ detect that HF has performed processing via
913
+ :meth:`_find_placeholders_by_modality`.
914
919
- This is useful in cases where :meth:`_find_placeholders` cannot be
920
- reliably used to detect whether HF has performed processing or not.
915
+ This is useful in cases where :meth:`_find_placeholders_by_modality`
916
+ cannot be reliably used to detect whether HF has performed processing.
917
"""
918
return False
919
920
def _apply_prompt_replacements(
921
self,
922
token_ids: list[int],
927
- prompt_repls: Sequence[_BoundPromptReplacement],
923
+ mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
924
mm_item_counts: Mapping[str, int],
929
- ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
925
+ ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
926
tokenizer = self._get_tokenizer()
927
932
- token_matches = find_token_matches(token_ids, prompt_repls)
928
+ mm_token_matches = {
929
+ modality: find_token_matches(token_ids, prompt_repls)
930
+ for modality, prompt_repls in mm_prompt_repls.items()
931
+ }
932
mm_match_counts = {
933
modality: len(matches)
935
- for modality, matches in full_groupby_modality(token_matches)
934
+ for modality, matches in mm_token_matches.items()
935
}
936
937
# If the search text does not represent a special token,
@@ -951,32 +950,92 @@ class BaseMultiModalProcessor(ABC):
950
): # yapf: disable
951
token_ids = replace_token_matches(
952
token_ids,
954
- token_matches,
953
+ mm_token_matches,
954
mm_item_counts,
955
)
956
958
- text = _decode(tokenizer, token_ids)
959
- matched_repls = [match.prompt_repl for match in token_matches]
957
+ text = decode_tokens(tokenizer, token_ids)
958
+ matched_repls = {
959
+ modality: [match.prompt_repl for match in token_matches]
960
+ for modality, token_matches in mm_token_matches.items()
961
+ }
962
else:
961
- text = _decode(tokenizer, token_ids)
963
+ text = decode_tokens(tokenizer, token_ids)
964
963
- text_matches = find_text_matches(text, prompt_repls)
965
+ mm_text_matches = {
966
+ modality: find_text_matches(text, prompt_repls)
967
+ for modality, prompt_repls in mm_prompt_repls.items()
968
+ }
969
text = replace_text_matches(
970
text,
966
- text_matches,
971
+ mm_text_matches,
972
mm_item_counts,
973
)
974
975
token_ids = encode_tokens(tokenizer,
976
text,
977
add_special_tokens=False)
973
- matched_repls = [match.prompt_repl for match in text_matches]
974
-
975
- placeholders = self._find_placeholders(matched_repls, token_ids,
976
- mm_item_counts)
978
+ matched_repls = {
979
+ modality: [match.prompt_repl for match in token_matches]
980
+ for modality, token_matches in mm_text_matches.items()
981
+ }
982
+
983
+ placeholders = self._find_mm_placeholders(
984
+ matched_repls,
985
+ token_ids,
986
+ mm_item_counts,
987
+ )
988
989
return token_ids, text, placeholders
990
991
+ def _validate_mm_kwargs(
992
+ self,
993
+ mm_kwargs: MultiModalKwargs,
994
+ mm_item_counts: Mapping[str, int],
995
+ ) -> None:
996
+ for modality, item_count in mm_item_counts.items():
997
+ if modality in mm_kwargs.modalities:
998
+ items = mm_kwargs.get_items(modality)
999
+ else:
1000
+ items = []
1001
+
1002
+ if len(items) != item_count:
1003
+ raise RuntimeError(
1004
+ f"Expected there to be {item_count} {modality} items in "
1005
+ f"keyword arguments corresponding to {item_count} "
1006
+ f"{modality} data items, but only found {len(items)}! "
1007
+ "There is likely a problem with your "
1008
+ "implementation of merged multi-modal processor for this "
1009
+ "model (usually arising from an inconsistency between "
1010
+ "`_call_hf_processor` and `_get_mm_fields_config`).")
1011
+
1012
+ def _validate_mm_placeholders(
1013
+ self,
1014
+ mm_placeholders: Mapping[str, list[_PlaceholderInfo]],
1015
+ mm_item_counts: Mapping[str, int],
1016
+ *,
1017
+ allow_missing: bool = False,
1018
+ ) -> Mapping[str, int]:
1019
+ missing_repl_counts = dict[str, int]()
1020
+
1021
+ for modality, item_count in mm_item_counts.items():
1022
+ placeholders = mm_placeholders.get(modality, [])
1023
+
1024
+ if len(placeholders) != item_count and not allow_missing:
1025
+ raise RuntimeError(
1026
+ f"Expected there to be {item_count} prompt replacements "
1027
+ f"corresponding to {item_count} {modality} items, but only "
1028
+ f"found {len(placeholders)} prompt replacements! Either "
1029
+ "the prompt text has missing/incorrect tokens for "
1030
+ "multi-modal inputs, or there is a problem with your "
1031
+ "implementation of merged multi-modal processor for this "
1032
+ "model (usually arising from an inconsistency between "
1033
+ "`_call_hf_processor` and `_get_prompt_replacements`).")
1034
+
1035
+ missing_repl_counts[modality] = item_count - len(placeholders)
1036
+
1037
+ return missing_repl_counts
1038
+
1039
def apply(
1040
self,
1041
prompt_text: str,
@@ -1009,56 +1068,69 @@ class BaseMultiModalProcessor(ABC):
1068
hf_processor_mm_kwargs,
1069
mm_kwargs,
1070
)
1012
- prompt_repls = self._bind_prompt_replacements(unbound_prompt_repls)
1071
+ mm_prompt_repls = self._bind_and_group_repls(unbound_prompt_repls)
1072
1014
- # If HF processor already inserts placeholder tokens,
1015
- # there is no need for us to insert them
1073
mm_item_counts = mm_items.get_all_counts()
1017
- all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
1018
- mm_item_counts)
1074
+ self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1075
+
1076
+ hf_mm_placeholders = self._find_mm_placeholders(
1077
+ mm_prompt_repls,
1078
+ prompt_ids,
1079
+ mm_item_counts,
1080
+ )
1081
+
1082
+ if self._always_apply_prompt_replacements():
1083
+ mm_missing_repl_counts = mm_item_counts
1084
+ mm_missing_repls = dict(mm_prompt_repls)
1085
+ else:
1086
+ mm_missing_repl_counts = self._validate_mm_placeholders(
1087
+ hf_mm_placeholders,
1088
+ mm_item_counts,
1089
+ allow_missing=True,
1090
+ )
1091
+
1092
+ mm_missing_repls = dict[str, list[_BoundPromptReplacement]]()
1093
+ for modality, missing_repl_count in mm_missing_repl_counts.items():
1094
+ if missing_repl_count == 0:
1095
+ mm_missing_repls[modality] = []
1096
+ elif missing_repl_count == mm_item_counts.get(modality, 0):
1097
+ mm_missing_repls[modality] = mm_prompt_repls[modality]
1098
+ else:
1099
+ raise ValueError("Partial prompt replacement within "
1100
+ f"{modality=} is not supported")
1101
1020
- if all_placeholders and not self._always_apply_prompt_replacements():
1102
+ # If HF processor already inserts placeholder tokens,
1103
+ # there is no need for us to insert them
1104
+ if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1105
tokenizer = self._get_tokenizer()
1022
- prompt_text = _decode(tokenizer, prompt_ids)
1106
+ prompt_text = decode_tokens(tokenizer, prompt_ids)
1107
+ mm_placeholders = hf_mm_placeholders
1108
else:
1109
(
1110
prompt_ids,
1111
prompt_text,
1027
- all_placeholders,
1112
+ missing_mm_placeholders,
1113
) = self._apply_prompt_replacements(
1114
prompt_ids,
1030
- prompt_repls,
1031
- mm_item_counts,
1115
+ mm_missing_repls,
1116
+ mm_missing_repl_counts,
1117
)
1118
1034
- mm_placeholders = dict[str, list[PlaceholderRange]]()
1035
- err_suffix = ("This suggests a problem with your implementation of "
1036
- "the merged multi-modal processor for this model, "
1037
- "particularly in the `_get_prompt_replacements` method.")
1038
-
1039
- for modality, placeholders in full_groupby_modality(all_placeholders):
1040
- if modality not in mm_items:
1041
- raise AssertionError(
1042
- f"Expected no placeholders for {modality=}, "
1043
- f"but found {placeholders=}. Input items: {mm_items}"
1044
- f"\n{err_suffix}")
1045
-
1046
- if len(placeholders) != len(mm_items[modality]):
1047
- raise AssertionError(
1048
- f"Expected length of {placeholders=} for {modality=} "
1049
- f"to equal that of input items: {mm_items[modality]}"
1050
- f"\n{err_suffix}")
1051
-
1052
- mm_placeholders[modality] = [
1053
- item.to_range() for item in placeholders
1054
- ]
1119
+ mm_placeholders = {**hf_mm_placeholders, **missing_mm_placeholders}
1120
+
1121
+ self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1122
+
1123
+ mm_placeholder_ranges = {
1124
+ modality: [item.to_range() for item in placeholders]
1125
+ for modality, placeholders in mm_placeholders.items()
1126
+ }
1127
1128
return MultiModalInputsV2(
1129
type="multimodal",
1130
prompt=prompt_text,
1131
prompt_token_ids=prompt_ids,
1132
mm_kwargs=mm_kwargs,
1061
- mm_placeholders=mm_placeholders,
1133
+ mm_placeholders=mm_placeholder_ranges,
1134
)
1135
1136
def _get_dummy_audios(
@@ -1092,8 +1164,9 @@ class BaseMultiModalProcessor(ABC):
1164
return [video] * num_videos
1165
1166
@abstractmethod
1095
- def _get_dummy_mm_inputs(
1167
+ def _get_dummy_processor_inputs(
1168
self,
1169
+ seq_len: int,
1170
mm_counts: Mapping[str, int],
1171
) -> ProcessorInputs:
1172
"""
@@ -1121,12 +1194,25 @@ class BaseMultiModalProcessor(ABC):
1194
1195
return mm_limits
1196
1197
+ def _get_dummy_mm_inputs(
1198
+ self,
1199
+ seq_len: int,
1200
+ mm_counts: Mapping[str, int],
1201
+ ) -> MultiModalInputsV2:
1202
+ processor_inputs = self._get_dummy_processor_inputs(seq_len, mm_counts)
1203
+
1204
+ return self.apply(
1205
+ prompt_text=processor_inputs.prompt_text,
1206
+ mm_data=processor_inputs.mm_data,
1207
+ hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1208
+ )
1209
+
1210
def get_dummy_data(self, seq_len: int) -> DummyData:
1211
# Avoid circular import
1212
from vllm.sequence import SequenceData
1213
1214
mm_counts = self._get_and_validate_dummy_mm_counts()
1129
- mm_max_tokens_per_item = self.get_mm_max_tokens_per_item()
1215
+ mm_max_tokens_per_item = self.get_mm_max_tokens_per_item(seq_len)
1216
if mm_counts.keys() != mm_max_tokens_per_item.keys():
1217
raise AssertionError(
1218
"The keys returned by `get_supported_mm_limits`"
@@ -1134,13 +1220,7 @@ class BaseMultiModalProcessor(ABC):
1220
"returned by `get_mm_max_tokens_per_item` "
1221
f"({set(mm_max_tokens_per_item.keys())})")
1222
1137
- processor_inputs = self._get_dummy_mm_inputs(mm_counts)
1138
- mm_inputs = self.apply(
1139
- prompt_text=processor_inputs.prompt_text,
1140
- mm_data=processor_inputs.mm_data,
1141
- hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1142
- )
1143
-
1223
+ mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
1224
prompt_token_ids = mm_inputs["prompt_token_ids"]
1225
placeholders_by_modality = mm_inputs["mm_placeholders"]
1226
@@ -1171,6 +1251,12 @@ class BaseMultiModalProcessor(ABC):
1251
"reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
1252
total_len, total_placeholders_by_modality)
1253
1254
+ return DummyData(
1255
+ seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
1256
+ multi_modal_data=None,
1257
+ multi_modal_placeholders=None,
1258
+ )
1259
+
1260
prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
1261
1262
return DummyData(
vllm/multimodal/registry.py
+2
-1
@@ -223,7 +223,8 @@ class MultiModalRegistry:
223
if self.has_processor(model_config):
224
tokenizer = cached_get_tokenizer(model_config.tokenizer)
225
processor = self.create_processor(model_config, tokenizer)
226
- return processor.get_mm_max_tokens_per_item()
226
+ seq_len = model_config.max_model_len
227
+ return processor.get_mm_max_tokens_per_item(seq_len)
228
229
return {
230
key: plugin.get_max_multimodal_tokens(model_config)
vllm/transformers_utils/tokenizer.py
+13
@@ -21,6 +21,19 @@ AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
21
MistralTokenizer]
22
23
24
+def decode_tokens(
25
+ tokenizer: AnyTokenizer,
26
+ token_ids: list[int],
27
+ *,
28
+ skip_special_tokens: bool = False,
29
+) -> str:
30
+ """
31
+ Backend-agnostic equivalent of HF's
32
+ :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
33
+ """
34
+ return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
35
+
36
+
37
def encode_tokens(
38
tokenizer: AnyTokenizer,
39
text: str,