@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / eed11ebee9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Merged multi-modal processors for LLaVA-NeXT-Video and LLaVA-OneVision (#11717)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 4, 2025 at 19:40 UTC eed11ebee93e9d137ac74d8e6e97427354bd3797
31 files changed +1114 -983
tests/models/decoder_only/vision_language/processing/__init__.py renamed
tests/models/decoder_only/vision_language/processing/test_idefics3.py renamed
tests/models/decoder_only/vision_language/processing/test_internvl.py renamed
tests/models/decoder_only/vision_language/processing/test_llava_next.py new
+58
@@ -0,0 +1,58 @@
1 +import pytest
2 +from PIL import Image
3 +from transformers import AutoTokenizer
4 +
5 +from vllm.inputs import InputProcessingContext
6 +
7 +from ....utils import build_model_context
8 +
9 +
10 +# Fixtures lazy import to avoid initializing CUDA during test collection
11 +@pytest.fixture()
12 +def processor_for_llava_next():
13 + from vllm.model_executor.models.llava_next import (
14 + LlavaNextMultiModalProcessor)
15 + return LlavaNextMultiModalProcessor
16 +
17 +
18 +# FIXME: image_size [(198, 176), (176, 198)]
19 +@pytest.mark.parametrize("model_id", ["llava-hf/llava-v1.6-mistral-7b-hf"])
20 +@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21 + (488, 183)])
22 +@pytest.mark.parametrize("num_imgs", [1, 2])
23 +def test_processor_prompt_replacements(
24 + processor_for_llava_next,
25 + model_id: str,
26 + image_size: tuple[int, int],
27 + num_imgs: int,
28 +):
29 + """
30 + Ensure LlavaNextMultiModalProcessor handles prompt replacement properly.
31 + """
32 + ctx = build_model_context(
33 + model_name=model_id,
34 + tokenizer_name=model_id,
35 + mm_processor_kwargs=None,
36 + limit_mm_per_prompt={"image": num_imgs},
37 + )
38 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
39 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
40 +
41 + # Build the image str / prompt based on the number of images we pass
42 + prompt = "<image>" * num_imgs
43 + mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
44 +
45 + # The processor will throw an error if there is a mismatch
46 + # in the prompt replacements
47 + processor = processor_for_llava_next(ctx)
48 + processed_inputs = processor.apply(prompt, mm_data, {})
49 +
50 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
51 + assert len(image_placeholders) == num_imgs
52 +
53 + first_placeholder = image_placeholders[0]
54 +
55 + # NOTE: There is a BOS token
56 + assert first_placeholder["offset"] == 1
57 + assert first_placeholder["length"] == (
58 + len(processed_inputs["prompt_token_ids"]) - 1) // num_imgs
tests/models/decoder_only/vision_language/processing/test_llava_onevision.py new
+59
@@ -0,0 +1,59 @@
1 +import pytest
2 +from PIL import Image
3 +from transformers import AutoTokenizer
4 +
5 +from vllm.inputs import InputProcessingContext
6 +
7 +from ....utils import build_model_context
8 +
9 +
10 +# Fixtures lazy import to avoid initializing CUDA during test collection
11 +@pytest.fixture()
12 +def processor_for_llava_onevision():
13 + from vllm.model_executor.models.llava_onevision import (
14 + LlavaOnevisionMultiModalProcessor)
15 + return LlavaOnevisionMultiModalProcessor
16 +
17 +
18 +@pytest.mark.parametrize("model_id",
19 + ["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"])
20 +@pytest.mark.parametrize("image_size", [(1669, 2560), (2560, 1669), (183, 488),
21 + (488, 183), (198, 176), (176, 198)])
22 +@pytest.mark.parametrize("num_imgs", [1, 2])
23 +def test_processor_prompt_replacements(
24 + processor_for_llava_onevision,
25 + model_id: str,
26 + image_size: tuple[int, int],
27 + num_imgs: int,
28 +):
29 + """
30 + Ensure LlavaOnevisionMultiModalProcessor handles prompt replacement
31 + properly.
32 + """
33 + ctx = build_model_context(
34 + model_name=model_id,
35 + tokenizer_name=model_id,
36 + mm_processor_kwargs=None,
37 + limit_mm_per_prompt={"image": num_imgs},
38 + )
39 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
40 + ctx = InputProcessingContext(ctx.model_config, tokenizer)
41 +
42 + # Build the image str / prompt based on the number of images we pass
43 + prompt = "<image>" * num_imgs
44 + mm_data = {"image": [Image.new("RGB", size=image_size)] * num_imgs}
45 +
46 + # The processor will throw an error if there is a mismatch
47 + # in the prompt replacements
48 + processor = processor_for_llava_onevision(ctx)
49 + processed_inputs = processor.apply(prompt, mm_data, {})
50 +
51 + image_placeholders = processed_inputs["mm_placeholders"]["image"]
52 + assert len(image_placeholders) == num_imgs
53 +
54 + first_placeholder = image_placeholders[0]
55 +
56 + # NOTE: There is a BOS token
57 + assert first_placeholder["offset"] == 0
58 + assert first_placeholder["length"] == len(
59 + processed_inputs["prompt_token_ids"]) // num_imgs
tests/models/decoder_only/vision_language/processing/test_phi3v.py renamed
+20 -24
@@ -1,6 +1,4 @@
1 """Tests for phi3v's multimodal preprocessing kwargs."""
2 -from typing import Optional
3 -
2 import pytest
3 from transformers import AutoTokenizer
4
@@ -10,8 +8,6 @@ from vllm.model_executor.models.phi3v import _IMAGE_TOKEN_ID
8 from .....conftest import _ImageAssets
9 from ....utils import build_model_context
10
13 -models = ["microsoft/Phi-3.5-vision-instruct"]
14 -
11
12 # Wrap lazy imports to avoid initializing CUDA during test collection
13 @pytest.fixture()
@@ -20,40 +16,40 @@ def processor_for_phi3v():
16 return Phi3VMultiModalProcessor
17
18
23 -@pytest.mark.parametrize("model", models)
19 +@pytest.mark.parametrize("model_id", ["microsoft/Phi-3.5-vision-instruct"])
20 +# yapf: disable
21 @pytest.mark.parametrize(
25 - "num_crops,expected_toks_per_img",
22 + ("mm_processor_kwargs", "expected_toks_per_img"),
23 [
27 - (4, 757),
28 - (16, 1921),
24 + ({"num_crops": 4}, 757),
25 + ({"num_crops": 16}, 1921),
26 # the default num_crops of phi-3.5-vision is 4
30 - (None, 757),
27 + ({}, 757),
28 ])
29 +# yapf: enable
30 @pytest.mark.parametrize("num_imgs", [1, 2])
33 -def test_processor_override(processor_for_phi3v, image_assets: _ImageAssets,
34 - model: str, num_crops: Optional[int],
35 - expected_toks_per_img: int, num_imgs: int):
31 +def test_processor_override(
32 + processor_for_phi3v,
33 + image_assets: _ImageAssets,
34 + model_id: str,
35 + mm_processor_kwargs: dict[str, int],
36 + expected_toks_per_img: int,
37 + num_imgs: int,
38 +):
39 """Ensure input_processor_for_phi3v handles num_crops properly."""
37 - # Same as the previous test - don't initialize mm_processor_kwargs
38 - # in this test and assume that the kwargs will be correctly expanded by
39 - # the partial when calling the custom input processor.
40 ctx = build_model_context(
41 - model_name=model,
42 - tokenizer_name=model,
41 + model_name=model_id,
42 + tokenizer_name=model_id,
43 trust_remote_code=True,
44 limit_mm_per_prompt={"image": num_imgs},
45 )
46 - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
46 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
47 ctx = InputProcessingContext(ctx.model_config, tokenizer)
48 +
49 # Build the image str / prompt based on the number of images we pass
50 img_str = "".join([f"<|image_{idx}|>\n" for idx in range(1, num_imgs + 1)])
51 prompt = f"<|user|>\n{img_str}<|end|>\n<|assistant|>\n"
51 - images = [image_assets[0].pil_image] * num_imgs
52 -
53 - mm_data = {"image": images}
54 - mm_processor_kwargs = {}
55 - if num_crops is not None:
56 - mm_processor_kwargs = {"num_crops": num_crops}
52 + mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
53
54 processor = processor_for_phi3v(ctx)
55 processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
tests/models/decoder_only/vision_language/processing/test_qwen.py renamed
tests/models/decoder_only/vision_language/processing/test_qwen2_vl.py renamed
+13 -26
@@ -1,5 +1,3 @@
1 -from typing import Any, Dict, Tuple
2 -
1 import pytest
2 from transformers import AutoTokenizer
3
@@ -8,56 +6,45 @@ from vllm.inputs import InputProcessingContext
6 from .....conftest import _ImageAssets
7 from ....utils import build_model_context
8
11 -MODEL = "Qwen/Qwen2-VL-2B-Instruct"
12 -MIN_PIXELS = "min_pixels"
13 -MAX_PIXELS = "max_pixels"
14 -
9
10 # Fixtures lazy import to avoid initializing CUDA during test collection
17 -# NOTE: Qwen2VL supports multiple input modalities, so it registers multiple
18 -# input mappers.
11 @pytest.fixture()
12 def processor_for_qwen2_vl():
13 from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalProcessor
14 return Qwen2VLMultiModalProcessor
15
16
17 +@pytest.mark.parametrize("model_id", ["Qwen/Qwen2-VL-2B-Instruct"])
18 +# yapf: disable
19 @pytest.mark.parametrize(
26 - "mm_processor_kwargs, expected_toks_per_img, expected_pixels_shape", [
20 + ("mm_processor_kwargs", "expected_toks_per_img", "expected_pixels_shape"), [
21 ({}, 1426, (5704, 1176)),
28 - ({
29 - MIN_PIXELS: 64**2,
30 - MAX_PIXELS: 512**2
31 - }, 330, (1320, 1176)),
22 + ({"min_pixels": 64**2, "max_pixels": 512**2}, 330, (1320, 1176)),
23 ])
33 -@pytest.mark.parametrize("model", [MODEL])
24 +# yapf: enable
25 @pytest.mark.parametrize("num_imgs", [1, 2])
26 def test_processor_override(
27 processor_for_qwen2_vl,
28 image_assets: _ImageAssets,
38 - model: str,
39 - mm_processor_kwargs: Dict[str, Any],
29 + model_id: str,
30 + mm_processor_kwargs: dict[str, object],
31 expected_toks_per_img: int,
41 - expected_pixels_shape: Tuple[int, int],
32 + expected_pixels_shape: tuple[int, int],
33 num_imgs: int,
34 ):
35 """Ensure Qwen2VLMultiModalProcessor handles min/max pixels properly."""
45 - # Same as the previous test - don't initialize mm_processor_kwargs
46 - # in this test and assume that the kwargs will be correctly expanded by
47 - # the partial when calling the custom input processor.
36 ctx = build_model_context(
49 - model_name=model,
50 - tokenizer_name=model,
37 + model_name=model_id,
38 + tokenizer_name=model_id,
39 mm_processor_kwargs=None,
40 limit_mm_per_prompt={"image": num_imgs},
41 )
54 - tokenizer = AutoTokenizer.from_pretrained(model, trust_remote_code=True)
42 + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
43 ctx = InputProcessingContext(ctx.model_config, tokenizer)
44 +
45 # Build the image str / prompt based on the number of images we pass
46 prompt = "<|vision_start|><|image_pad|><|vision_end|>" * num_imgs
58 - images = [image_assets[0].pil_image] * num_imgs
59 -
60 - mm_data = {"image": images}
47 + mm_data = {"image": [image_assets[0].pil_image] * num_imgs}
48
49 processor = processor_for_qwen2_vl(ctx)
50 processed_inputs = processor.apply(prompt, mm_data, mm_processor_kwargs)
tests/models/decoder_only/vision_language/test_models.py
+2 -7
@@ -274,10 +274,8 @@ VLM_TEST_SETTINGS = {
274 ),
275 limit_mm_per_prompt={"image": 4},
276 )],
277 - # Llava-next tests fixed sizes & the default size factors
278 - image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
277 ),
280 - "llava_one_vision": VLMTestInfo(
278 + "llava_onevision": VLMTestInfo(
279 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
280 test_type=VLMTestType.CUSTOM_INPUTS,
281 prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
@@ -288,8 +286,6 @@ VLM_TEST_SETTINGS = {
286 ),
287 auto_cls=AutoModelForVision2Seq,
288 vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
291 - # Llava-one-vision tests fixed sizes & the default size factors
292 - image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
289 custom_test_opts=[CustomTestOptions(
290 inputs=custom_inputs.multi_video_multi_aspect_ratio_inputs(
291 formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
@@ -306,7 +302,6 @@ VLM_TEST_SETTINGS = {
302 max_model_len=4096,
303 auto_cls=AutoModelForVision2Seq,
304 vllm_output_post_proc=model_utils.llava_video_vllm_to_hf_output,
309 - image_sizes=[((1669, 2560), (2560, 1669), (183, 488), (488, 183))],
305 ),
306 "mantis": VLMTestInfo(
307 models=["TIGER-Lab/Mantis-8B-siglip-llama3"],
@@ -431,7 +426,7 @@ VLM_TEST_SETTINGS = {
426 ) for inp in custom_inputs.different_patch_input_cases_internvl()
427 ],
428 ),
434 - "llava_one_vision-multiple-images": VLMTestInfo(
429 + "llava_onevision-multiple-images": VLMTestInfo(
430 models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
431 test_type=VLMTestType.CUSTOM_INPUTS,
432 max_model_len=16384,
tests/models/decoder_only/vision_language/test_qwen2_vl.py
-127
@@ -427,130 +427,3 @@ def test_qwen2_vl_video_embeddings_input(vllm_runner, video_assets, model,
427 mm_limit=1,
428 tensor_parallel_size=1,
429 )
430 -
431 -
432 -def run_chunked_prefill_test(
433 - vllm_runner: Type[VllmRunner],
434 - inputs: List[Tuple[List[str], PromptImageInput, PromptVideoInput]],
435 - model: str,
436 - *,
437 - dtype: str,
438 - max_tokens: int,
439 - num_logprobs: int,
440 - mm_limit: int,
441 - tensor_parallel_size: int,
442 - distributed_executor_backend: Optional[str] = None,
443 -):
444 - """Compare inference result between
445 - chunked prefill disabled and chunked prefill enabled
446 - """
447 -
448 - # NOTE:
449 - # max_model_len should be greater than image_feature_size
450 - with vllm_runner(model,
451 - task="generate",
452 - max_model_len=4000,
453 - max_num_seqs=4,
454 - dtype=dtype,
455 - limit_mm_per_prompt={
456 - "image": mm_limit,
457 - "video": mm_limit
458 - },
459 - tensor_parallel_size=tensor_parallel_size,
460 - distributed_executor_backend=distributed_executor_backend
461 - ) as vllm_model:
462 -
463 - outputs_per_case = [
464 - vllm_model.generate_greedy_logprobs(prompts,
465 - max_tokens,
466 - num_logprobs=num_logprobs,
467 - images=images or None,
468 - videos=videos or None)
469 - for prompts, images, videos in inputs
470 - ]
471 -
472 - with vllm_runner(
473 - model,
474 - task="generate",
475 - max_model_len=4000,
476 - max_num_seqs=4,
477 - dtype=dtype,
478 - limit_mm_per_prompt={
479 - "image": mm_limit,
480 - "video": mm_limit
481 - },
482 - tensor_parallel_size=tensor_parallel_size,
483 - distributed_executor_backend=distributed_executor_backend,
484 - enable_chunked_prefill=True,
485 - # should be small enough to ensure prefilling is chunked
486 - max_num_batched_tokens=32,
487 - mm_processor_kwargs={
488 - "max_pixels": 16 * 28 * 28,
489 - }) as vllm_model_chunked:
490 - outputs_per_case_chunked = [
491 - vllm_model_chunked.generate_greedy_logprobs(
492 - prompts,
493 - max_tokens,
494 - num_logprobs=num_logprobs,
495 - images=images or None,
496 - videos=videos or None) for prompts, images, videos in inputs
497 - ]
498 -
499 - for outputs, \
500 - outputs_chunked \
501 - in zip(outputs_per_case,
502 - outputs_per_case_chunked):
503 - check_logprobs_close(
504 - outputs_0_lst=outputs,
505 - outputs_1_lst=outputs_chunked,
506 - name_0="non_chunked",
507 - name_1="chunked",
508 - )
509 -
510 -
511 -@pytest.mark.core_model
512 -@pytest.mark.parametrize("model", models)
513 -@pytest.mark.parametrize("dtype", [target_dtype])
514 -@pytest.mark.parametrize("max_tokens", [1])
515 -@pytest.mark.parametrize("num_logprobs", [10])
516 -def test_qwen2_vl_mrope_chunked_prefill(vllm_runner, example_prompts,
517 - model: str, dtype: str,
518 - max_tokens: int,
519 - num_logprobs: int) -> None:
520 - """
521 - Test Qwen2-VL's chunked prefill with M-RoPE
522 - """
523 - prompts = [
524 - qwen2_vl_chat_template(IMAGE_PLACEHOLDER, prompt)
525 - for prompt in example_prompts[:1]
526 - ]
527 -
528 - # 1. Qwen2-VL's M-RoPE works only when there are some multi-modal inputs,
529 - # so an image is included in the inputs
530 - # 2. however, Qwen2-VL currently won't work properly
531 - # when chunked prefill is enabled and there are some multi-modal inputs,
532 - # here use a hacky way: provide a **zero-length** image to make it happy
533 - #
534 - # and finally we achieved:
535 - # (1) chunked_prefill enabled; (2) M-RoPE works; to continue our tests
536 - zero_len_image = {
537 - "image_embeds": torch.empty((0, MODEL_HIDDEN_SIZE)),
538 - "image_grid_thw": torch.tensor([[0, 0, 0]])
539 - }
540 - images = [zero_len_image] * len(prompts)
541 -
542 - inputs_per_case: List[Tuple[List[str], PromptImageInput,
543 - PromptVideoInput]] = [
544 - (prompts, images, []),
545 - ]
546 -
547 - run_chunked_prefill_test(
548 - vllm_runner,
549 - inputs_per_case,
550 - model,
551 - dtype=dtype,
552 - max_tokens=max_tokens,
553 - num_logprobs=num_logprobs,
554 - mm_limit=1,
555 - tensor_parallel_size=1,
556 - )
tests/multimodal/test_processing.py
+102 -68
@@ -11,8 +11,8 @@ from vllm.config import ModelConfig
11 from vllm.inputs import InputProcessingContext
12 from vllm.multimodal import MULTIMODAL_REGISTRY
13 from vllm.multimodal.processing import (ProcessingCache, PromptReplacement,
14 - _PlaceholderInfo, find_text_matches,
15 - find_token_matches, iter_placeholders,
14 + _PlaceholderInfo, find_mm_placeholders,
15 + find_text_matches, find_token_matches,
16 iter_token_matches,
17 replace_text_matches,
18 replace_token_matches)
@@ -314,21 +314,27 @@ def test_find_replace_text(
314 # Should not be used since there is nothing to convert to text
315 mock_tokenizer = cast(AnyTokenizer, object())
316
317 - prompt_repls = [
318 - PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
317 + mm_prompt_repls = {
318 + key: [
319 + PromptReplacement(key, target,
320 + repl_by_key[key]).bind(mock_tokenizer)
321 + ]
322 for key, target in target_by_key.items()
320 - ]
321 - matches = find_text_matches(prompt, prompt_repls)
323 + }
324 + mm_matches = {
325 + key: find_text_matches(prompt, prompt_repls)
326 + for key, prompt_repls in mm_prompt_repls.items()
327 + }
328
329 result = replace_text_matches(
330 prompt,
325 - matches,
331 + mm_matches,
332 {key: mm_count
333 for key in repl_by_key},
334 )
335
336 # Only displayed on error
331 - print("matches:", matches)
337 + print("mm_matches:", mm_matches)
338 print("result:", result)
339
340 # Manually constructed results
@@ -380,21 +386,27 @@ def test_find_replace_tokens(
386 # Should not be used since there is nothing to convert to tokens
387 mock_tokenizer = cast(AnyTokenizer, object())
388
383 - prompt_repls = [
384 - PromptReplacement(key, target, repl_by_key[key]).bind(mock_tokenizer)
389 + mm_prompt_repls = {
390 + key: [
391 + PromptReplacement(key, target,
392 + repl_by_key[key]).bind(mock_tokenizer)
393 + ]
394 for key, target in target_by_key.items()
386 - ]
387 - matches = find_token_matches(prompt, prompt_repls)
395 + }
396 + mm_matches = {
397 + key: find_token_matches(prompt, prompt_repls)
398 + for key, prompt_repls in mm_prompt_repls.items()
399 + }
400
401 result = replace_token_matches(
402 prompt,
391 - matches,
403 + mm_matches,
404 {key: mm_count
405 for key in repl_by_key},
406 )
407
408 # Only displayed on error
397 - print("matches:", matches)
409 + print("mm_matches:", mm_matches)
410 print("result:", result)
411
412 # Manually constructed results
@@ -417,58 +429,76 @@ def test_find_replace_tokens(
429 [
430 (
431 [1, 9833, 28747, 32000, 9833, 28747, 32000, 32000, 918],
420 - [
421 - _PlaceholderInfo(
422 - modality="pattern_1",
423 - start_idx=6,
424 - replacement=[32000, 32000],
425 - ),
426 - ],
432 + {
433 + "pattern_1": [
434 + _PlaceholderInfo(
435 + modality="pattern_1",
436 + item_idx=0,
437 + start_idx=6,
438 + replacement=[32000, 32000],
439 + ),
440 + ],
441 + }
442 +
443 ),
444 (
445 [1, 32000, 32000, 9833, 28747, 32000, 32000, 1550, 918, 1550],
430 - [
431 - _PlaceholderInfo(
432 - modality="pattern_1",
433 - start_idx=1,
434 - replacement=[32000, 32000],
435 - ),
436 - _PlaceholderInfo(
437 - modality="pattern_1",
438 - start_idx=5,
439 - replacement=[32000, 32000],
440 - ),
441 - _PlaceholderInfo(
442 - modality="pattern_3",
443 - start_idx=7,
444 - replacement=[1550, 918, 1550],
445 - ),
446 - ],
446 + {
447 + "pattern_1": [
448 + _PlaceholderInfo(
449 + modality="pattern_1",
450 + item_idx=0,
451 + start_idx=1,
452 + replacement=[32000, 32000],
453 + ),
454 + _PlaceholderInfo(
455 + modality="pattern_1",
456 + item_idx=1,
457 + start_idx=5,
458 + replacement=[32000, 32000],
459 + ),
460 + ],
461 + "pattern_3": [
462 + _PlaceholderInfo(
463 + modality="pattern_3",
464 + item_idx=0,
465 + start_idx=7,
466 + replacement=[1550, 918, 1550],
467 + ),
468 + ],
469 + }
470 ),
471 (
472 [1, 32000, 32000, 32000, 32000, 32000, 1550, 918, 1550],
450 - [
451 - _PlaceholderInfo(
452 - modality="pattern_1",
453 - start_idx=1,
454 - replacement=[32000, 32000],
455 - ),
456 - _PlaceholderInfo(
457 - modality="pattern_1",
458 - start_idx=3,
459 - replacement=[32000, 32000],
460 - ),
461 - _PlaceholderInfo(
462 - modality="pattern_3",
463 - start_idx=6,
464 - replacement=[1550, 918, 1550],
465 - ),
466 - ],
473 + {
474 + "pattern_1": [
475 + _PlaceholderInfo(
476 + modality="pattern_1",
477 + item_idx=0,
478 + start_idx=1,
479 + replacement=[32000, 32000],
480 + ),
481 + _PlaceholderInfo(
482 + modality="pattern_1",
483 + item_idx=1,
484 + start_idx=3,
485 + replacement=[32000, 32000],
486 + ),
487 + ],
488 + "pattern_3": [
489 + _PlaceholderInfo(
490 + modality="pattern_3",
491 + item_idx=0,
492 + start_idx=6,
493 + replacement=[1550, 918, 1550],
494 + ),
495 + ],
496 + }
497 ),
498 ]
499 )
500 # yapf: enable
471 -def test_iter_placeholders(
501 +def test_find_mm_placeholders(
502 repl_by_key,
503 prompt,
504 expected,
@@ -476,19 +506,18 @@ def test_iter_placeholders(
506 # Should not be used since there is nothing to convert to tokens
507 mock_tokenizer = cast(AnyTokenizer, object())
508
479 - prompt_repls = [
480 - PromptReplacement(key, [], repl).bind(mock_tokenizer)
509 + mm_prompt_repls = {
510 + key: [PromptReplacement(key, [], repl).bind(mock_tokenizer)]
511 for key, repl in repl_by_key.items()
482 - ]
512 + }
513
484 - result = list(
485 - iter_placeholders(
486 - prompt_repls,
487 - prompt,
488 - # Effectively match all occurrences in the prompt
489 - {key: 3
490 - for key in repl_by_key},
491 - ))
514 + result = find_mm_placeholders(
515 + mm_prompt_repls,
516 + prompt,
517 + # Effectively match all occurrences in the prompt
518 + {key: 3
519 + for key in repl_by_key},
520 + )
521
522 # Only displayed on error
523 print("result:", result)
@@ -694,7 +723,10 @@ def _test_processing_cache_correctness(
723 }
724
725 mm_counts = {k: len(vs) for k, vs in mm_data.items()}
697 - prompt = baseline_processor._get_dummy_mm_inputs(mm_counts).prompt_text
726 + prompt = baseline_processor._get_dummy_processor_inputs(
727 + model_config.max_model_len,
728 + mm_counts,
729 + ).prompt_text
730
731 # Drop unnecessary keys and test single -> multi conversion
732 if rng.rand() < simplify_rate:
@@ -728,6 +760,8 @@ def _test_processing_cache_correctness(
760 ("adept/fuyu-8b", {"image": False}),
761 ("llava-hf/llava-1.5-7b-hf", {"image": True}),
762 ("llava-hf/llava-v1.6-mistral-7b-hf", {"image": True}),
763 + ("llava-hf/LLaVA-NeXT-Video-7B-hf", {"video": False}),
764 + ("llava-hf/llava-onevision-qwen2-0.5b-ov-hf", {"image": True, "video": True}), # noqa: E501
765 ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
766 ("mistral-community/pixtral-12b", {"image": True}),
767 ("Qwen/Qwen2-VL-2B-Instruct", {"image": True, "video": True}),
vllm/model_executor/models/aria.py
+3 -2
@@ -456,7 +456,7 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
456 hf_config = self.ctx.get_hf_config()
457 return max(hf_config.projector_patch_to_query_dict.values())
458
459 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
459 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
460 return {"image": self._get_num_image_tokens()}
461
462 def _get_mm_fields_config(
@@ -488,8 +488,9 @@ class AriaMultiModalProcessor(BaseMultiModalProcessor):
488 )
489 ]
490
491 - def _get_dummy_mm_inputs(
491 + def _get_dummy_processor_inputs(
492 self,
493 + seq_len: int,
494 mm_counts: Mapping[str, int],
495 ) -> ProcessorInputs:
496 hf_config = self.ctx.get_hf_config()
vllm/model_executor/models/blip2.py
+3 -2
@@ -405,7 +405,7 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
405 hf_config = self.ctx.get_hf_config(Blip2Config)
406 return hf_config.num_query_tokens
407
408 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
408 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
409 return {"image": self._get_num_image_tokens()}
410
411 def _get_hf_processor(self) -> Blip2Processor:
@@ -457,8 +457,9 @@ class Blip2MultiModalProcessor(BaseMultiModalProcessor):
457
458 return result
459
460 - def _get_dummy_mm_inputs(
460 + def _get_dummy_processor_inputs(
461 self,
462 + seq_len: int,
463 mm_counts: Mapping[str, int],
464 ) -> ProcessorInputs:
465 hf_config = self.ctx.get_hf_config(Blip2Config)
vllm/model_executor/models/chameleon.py
+3 -2
@@ -57,7 +57,7 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
57 processor = self._get_hf_processor()
58 return processor.image_seq_length
59
60 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
60 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
61 return {"image": self._get_num_image_tokens()}
62
63 def _get_hf_processor(self) -> ChameleonProcessor:
@@ -90,8 +90,9 @@ class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
90 )
91 ]
92
93 - def _get_dummy_mm_inputs(
93 + def _get_dummy_processor_inputs(
94 self,
95 + seq_len: int,
96 mm_counts: Mapping[str, int],
97 ) -> ProcessorInputs:
98 config = self.ctx.get_hf_config(ChameleonConfig)
vllm/model_executor/models/clip.py
+7 -4
@@ -164,15 +164,18 @@ class CLIPEncoderInfo(VisionEncoderInfo[CLIPVisionConfig]):
164 def get_max_image_tokens(self) -> int:
165 return get_max_clip_image_tokens(self.vision_config)
166
167 - def get_num_patches(self) -> int:
167 + def get_image_size(self) -> int:
168 + return self.vision_config.image_size
169 +
170 + def get_patch_size(self) -> int:
171 + return self.vision_config.patch_size
172 +
173 + def get_patch_grid_length(self) -> int:
174 return get_clip_patch_grid_length(
175 image_size=self.vision_config.image_size,
176 patch_size=self.vision_config.patch_size,
177 )
178
173 - def get_image_size(self) -> int:
174 - return self.vision_config.image_size
175 -
179
180 # Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/clip/modeling_clip.py#L164 # noqa
181 class CLIPVisionEmbeddings(nn.Module):
vllm/model_executor/models/fuyu.py
+3 -2
@@ -96,7 +96,7 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
96 nrows = math.ceil(image_height / 30)
97 return ncols, nrows
98
99 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
99 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
100 target_width, target_height = self._get_image_target_size()
101
102 max_ncols, max_nrows = self._get_image_feature_grid_size(
@@ -208,8 +208,9 @@ class FuyuMultiModalProcessor(BaseMultiModalProcessor):
208
209 return result
210
211 - def _get_dummy_mm_inputs(
211 + def _get_dummy_processor_inputs(
212 self,
213 + seq_len: int,
214 mm_counts: Mapping[str, int],
215 ) -> ProcessorInputs:
216 target_width, target_height = self._get_image_target_size()
vllm/model_executor/models/llava.py
+33 -42
@@ -25,11 +25,9 @@ from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
25 NestedTensors)
26 from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
27 ImageSize)
28 -from vllm.multimodal.processing import (BaseMultiModalProcessor,
29 - InputProcessingContext,
28 +from vllm.multimodal.processing import (InputProcessingContext,
29 MultiModalDataItems, ProcessingCache,
31 - ProcessorInputs, PromptReplacement,
32 - full_groupby_modality)
30 + ProcessorInputs, PromptReplacement)
31 from vllm.sequence import IntermediateTensors
32
33 from .clip import CLIPVisionModel
@@ -39,7 +37,7 @@ from .pixtral import (PixtralHFVisionModel,
37 from .siglip import SiglipVisionModel
38 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
39 maybe_prefix, merge_multimodal_embeddings)
42 -from .vision import vision_encoder_info
40 +from .vision import BaseVisionLanguageMultiModalProcessor
41
42
43 class LlavaImagePixelInputs(TypedDict):
@@ -100,19 +98,7 @@ class LlavaLikeConfig(Protocol):
98 vision_feature_layer: Final[Union[int, List[int]]]
99
100
103 -class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
104 -
105 - def __init__(self,
106 - ctx: InputProcessingContext,
107 - *,
108 - cache: Optional[ProcessingCache] = None,
109 - enable_sanity_checks: bool = True) -> None:
110 - super().__init__(ctx,
111 - cache=cache,
112 - enable_sanity_checks=enable_sanity_checks)
113 -
114 - vision_config = self._get_hf_config().vision_config
115 - self._vision_encoder_info = vision_encoder_info(vision_config)
101 +class BaseLlavaMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
102
103 @abstractmethod
104 def _get_hf_config(self) -> LlavaLikeConfig:
@@ -121,6 +107,19 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
107 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
108 return {"image": None}
109
110 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
111 + return {"image": self._get_max_image_tokens()}
112 +
113 + def _get_mm_fields_config(
114 + self,
115 + hf_inputs: BatchFeature,
116 + hf_processor_mm_kwargs: Mapping[str, object],
117 + ) -> Mapping[str, MultiModalFieldConfig]:
118 + return dict(
119 + pixel_values=MultiModalFieldConfig.batched("image"),
120 + image_embeds=MultiModalFieldConfig.batched("image"),
121 + )
122 +
123 def _apply_feature_select_strategy(
124 self,
125 strategy: str,
@@ -142,19 +141,6 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
141 self._vision_encoder_info.get_max_image_tokens(),
142 )
143
145 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
146 - return {"image": self._get_max_image_tokens()}
147 -
148 - def _get_mm_fields_config(
149 - self,
150 - hf_inputs: BatchFeature,
151 - hf_processor_mm_kwargs: Mapping[str, object],
152 - ) -> Mapping[str, MultiModalFieldConfig]:
153 - return dict(
154 - pixel_values=MultiModalFieldConfig.batched("image"),
155 - image_embeds=MultiModalFieldConfig.batched("image"),
156 - )
157 -
144 def _get_dummy_image_size(self) -> ImageSize:
145 image_size = self._vision_encoder_info.get_image_size()
146 return ImageSize(image_size, image_size)
@@ -163,8 +149,9 @@ class BaseLlavaMultiModalProcessor(BaseMultiModalProcessor):
149 def _get_image_token(self) -> str:
150 raise NotImplementedError
151
166 - def _get_dummy_mm_inputs(
152 + def _get_dummy_processor_inputs(
153 self,
154 + seq_len: int,
155 mm_counts: Mapping[str, int],
156 ) -> ProcessorInputs:
157 num_images = mm_counts.get("image", 0)
@@ -709,7 +696,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
696 "</Image>)", # 3 tokens
697 ])
698
712 - mantis_repls = self._bind_prompt_replacements([
699 + mantis_mm_repls = self._bind_and_group_repls([
700 PromptReplacement(
701 modality="image",
702 target=[image_token_id] * num_image_tokens,
@@ -719,7 +706,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
706
707 prompt_ids, prompt_text, _ = self._apply_prompt_replacements(
708 result["prompt_token_ids"],
722 - mantis_repls,
709 + mantis_mm_repls,
710 mm_item_counts,
711 )
712
@@ -728,15 +715,19 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
715 hf_processor_mm_kwargs,
716 mm_kwargs,
717 )
731 - orig_repls = self._bind_prompt_replacements(unbound_orig_repls)
718 + orig_repls = self._bind_and_group_repls(unbound_orig_repls)
719 +
720 + mm_placeholders = self._find_mm_placeholders(
721 + orig_repls,
722 + prompt_ids,
723 + mm_item_counts,
724 + )
725
733 - all_placeholders = self._find_placeholders(orig_repls, prompt_ids,
734 - mm_item_counts)
735 - assert len(all_placeholders) == mm_item_counts.get("image", 0)
726 + self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
727
737 - mm_placeholders = {
738 - modality: [item.to_range() for item in items]
739 - for modality, items in full_groupby_modality(all_placeholders)
728 + mm_placeholder_ranges = {
729 + modality: [item.to_range() for item in placeholders]
730 + for modality, placeholders in mm_placeholders.items()
731 }
732
733 return MultiModalInputsV2(
@@ -744,7 +735,7 @@ class MantisMultiModalProcessor(LlavaMultiModalProcessor):
735 prompt=prompt_text,
736 prompt_token_ids=prompt_ids,
737 mm_kwargs=mm_kwargs,
747 - mm_placeholders=mm_placeholders,
738 + mm_placeholders=mm_placeholder_ranges,
739 )
740
741
vllm/model_executor/models/llava_next.py
+8 -7
@@ -67,9 +67,6 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
67 def _get_hf_processor(self) -> LlavaNextProcessor:
68 return self.ctx.get_hf_processor(LlavaNextProcessor)
69
70 - def _get_image_token(self) -> str:
71 - return self._get_hf_processor().image_token
72 -
70 def _get_mm_fields_config(
71 self,
72 hf_inputs: BatchFeature,
@@ -81,6 +78,9 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
78 image_embeds=MultiModalFieldConfig.batched("image"),
79 )
80
81 + def _get_image_token(self) -> str:
82 + return self._get_hf_processor().image_token
83 +
84 def _get_max_image_tokens(self) -> int:
85 largest_feature_size, _ = self._get_pinpoint_with_most_features()
86 return largest_feature_size
@@ -97,20 +97,20 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
97 image_height: int,
98 ) -> int:
99 hf_config = self._get_hf_config()
100 + vision_encoder_info = self._vision_encoder_info
101
102 base_feature_size = self._apply_feature_select_strategy(
103 hf_config.vision_feature_select_strategy,
103 - self._vision_encoder_info.get_num_image_tokens(
104 + vision_encoder_info.get_num_image_tokens(
105 image_width=image_width,
106 image_height=image_height,
107 ),
108 )
108 - num_patches = self._vision_encoder_info.get_num_patches()
109
110 num_patch_height, num_patch_width = get_anyres_image_grid_shape(
111 image_size=(image_height, image_width),
112 grid_pinpoints=hf_config.image_grid_pinpoints,
113 - patch_size=self._vision_encoder_info.get_image_size(),
113 + patch_size=vision_encoder_info.get_image_size(),
114 )
115
116 (
@@ -119,7 +119,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
119 ) = self._get_num_unpadded_features(
120 original_height=image_height,
121 original_width=image_width,
122 - npatches=num_patches,
122 + npatches=vision_encoder_info.get_patch_grid_length(),
123 num_patch_height=num_patch_height,
124 num_patch_width=num_patch_width,
125 )
@@ -155,6 +155,7 @@ class LlavaNextMultiModalProcessor(LlavaMultiModalProcessor):
155
156 unpadded_features = current_height * current_width
157 newline_features = current_height
158 +
159 return (unpadded_features, newline_features)
160
161 def _get_pinpoint_with_most_features(self) -> tuple[int, ImageSize]:
vllm/model_executor/models/llava_next_video.py
+134 -139
@@ -3,38 +3,32 @@ from functools import cached_property
3 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 TypedDict, Union)
5
6 -import numpy as np
6 import torch
7 import torch.nn as nn
9 -from transformers import (CLIPVisionConfig, LlavaNextVideoConfig,
10 - SiglipVisionConfig)
8 +from transformers import (BatchFeature, LlavaNextVideoConfig,
9 + LlavaNextVideoProcessor)
10
11 from vllm.attention import AttentionMetadata
12 from vllm.config import VllmConfig
14 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
15 - InputContext, token_inputs)
13 from vllm.model_executor.layers.activation import get_act_fn
14 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
15 from vllm.model_executor.models.clip import CLIPVisionModel
16 from vllm.model_executor.sampling_metadata import SamplingMetadata
17 from vllm.multimodal import MULTIMODAL_REGISTRY
21 -from vllm.multimodal.inputs import NestedTensors
22 -from vllm.multimodal.utils import (cached_get_tokenizer,
23 - repeat_and_pad_placeholder_tokens)
18 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
19 +from vllm.multimodal.parse import (ImageSize, MultiModalDataItems,
20 + VideoEmbeddingItems, VideoProcessorItems)
21 +from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
22 + PromptReplacement)
23 from vllm.sequence import IntermediateTensors
24 from vllm.utils import is_list_of
25
27 -from .clip import dummy_image_for_clip, dummy_seq_data_for_clip
26 from .interfaces import SupportsMultiModal, SupportsPP
27 from .llava import init_vision_tower_for_llava
30 -from .siglip import (SiglipVisionModel, dummy_image_for_siglip,
31 - dummy_seq_data_for_siglip)
28 +from .siglip import SiglipVisionModel
29 from .utils import (AutoWeightsLoader, init_vllm_registered_model,
30 maybe_prefix, merge_multimodal_embeddings)
34 -
35 -# For profile run
36 -_MAX_FRAMES_PER_VIDEO = 32
37 -_MAX_NUM_VIDEOS = 1
31 +from .vision import BaseVisionLanguageMultiModalProcessor
32
33
34 class LlavaNextVideoPixelInputs(TypedDict):
@@ -50,143 +44,148 @@ class LlavaNextVideoPixelInputs(TypedDict):
44 """
45
46
53 -def get_llava_next_video_frame_feature_size(
54 - hf_config: LlavaNextVideoConfig) -> int:
55 - # Support both CLIPVisionConfig and SiglipVisionConfig
56 - image_size = hf_config.vision_config.image_size
57 - patch_size = hf_config.vision_config.patch_size
58 - spatial_pool_stride = hf_config.spatial_pool_stride
47 +class LlavaNextVideoMultiModalProcessor(BaseVisionLanguageMultiModalProcessor):
48
60 - return int((image_size / patch_size / spatial_pool_stride)**2)
49 + def _get_hf_config(self) -> LlavaNextVideoConfig:
50 + return self.ctx.get_hf_config(LlavaNextVideoConfig)
51
52 + def _get_hf_processor(self) -> LlavaNextVideoProcessor:
53 + return self.ctx.get_hf_processor(LlavaNextVideoProcessor)
54
63 -def _get_max_llm_tokens(ctx: InputContext) -> int:
64 - """
65 - Calculated from the maximum video frames under the context length
66 - constraints of the language model.
67 - """
68 - hf_text_config = ctx.model_config.hf_text_config
69 - model_config = ctx.model_config
70 - max_tokens = model_config.max_model_len
71 - rope_scaling = model_config.rope_scaling
72 -
73 - if rope_scaling:
74 - rope_scaling_factor = hf_text_config.rope_scaling["factor"]
75 - else:
76 - rope_scaling_factor = 1
77 -
78 - max_tokens *= rope_scaling_factor
79 -
80 - return max_tokens
81 -
82 -
83 -def get_max_llava_next_video_tokens(ctx: InputContext) -> int:
84 - # Currently set to 32 frames
85 - # TODO: max_tokens = _get_max_llm_tokens(ctx)
86 - hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
87 - tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
88 - return _MAX_FRAMES_PER_VIDEO * tokens_per_frame
89 -
90 -
91 -def dummy_data_for_llava_next_video(ctx: InputContext, seq_len: int,
92 - mm_counts: Mapping[str, int]):
93 - hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
94 - vision_config = hf_config.vision_config
95 -
96 - # TODO: support multiple videos
97 - num_videos = mm_counts["video"]
98 - if num_videos != _MAX_NUM_VIDEOS:
99 - raise NotImplementedError(
100 - f"Only {_MAX_NUM_VIDEOS} videos are supported")
101 -
102 - # TODO: support configuring the number of frames
103 - frames_per_video = _MAX_FRAMES_PER_VIDEO
104 - # num_images = num_videos * frames_per_video
105 -
106 - # fills the sequence with as longer video data as possible
107 - tokens_per_frame = get_llava_next_video_frame_feature_size(hf_config)
108 - video_feature_size = frames_per_video * tokens_per_frame
109 -
110 - if isinstance(vision_config, CLIPVisionConfig):
111 - seq_data, ranges = dummy_seq_data_for_clip(
112 - vision_config,
113 - seq_len,
114 - num_videos,
115 - image_token_id=hf_config.video_token_index,
116 - image_feature_size_override=video_feature_size,
117 - mm_key="video",
118 - )
55 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
56 + return {"video": 1}
57 +
58 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
59 + num_frames = self._get_dummy_num_frames(seq_len)
60 + max_video_tokens = self._get_max_video_tokens(num_frames)
61 +
62 + return {"video": max_video_tokens}
63 +
64 + def _get_mm_fields_config(
65 + self,
66 + hf_inputs: BatchFeature,
67 + hf_processor_mm_kwargs: Mapping[str, object],
68 + ) -> Mapping[str, MultiModalFieldConfig]:
69 + return dict(pixel_values_videos=MultiModalFieldConfig.batched("video"))
70 +
71 + def _get_num_frame_tokens(
72 + self,
73 + *,
74 + image_width: int,
75 + image_height: int,
76 + ) -> int:
77 + hf_config = self._get_hf_config()
78 + spatial_pool_stride = hf_config.spatial_pool_stride
79
120 - pil_frame = dummy_image_for_clip(vision_config, num_images=1)
121 - np_frame = np.array(pil_frame["image"])
122 - mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
123 - mm_data = {"video": mm_data_per_video}
124 - return DummyData(seq_data, mm_data, ranges)
125 - elif isinstance(vision_config, SiglipVisionConfig):
126 - seq_data, ranges = dummy_seq_data_for_siglip(
127 - vision_config,
128 - seq_len,
129 - num_videos,
130 - image_token_id=hf_config.video_token_index,
131 - image_feature_size_override=video_feature_size,
132 - mm_key="video",
80 + patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
81 + pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
82 +
83 + return pooled_grid_length * pooled_grid_length
84 +
85 + def _get_num_video_tokens(
86 + self,
87 + *,
88 + image_width: int,
89 + image_height: int,
90 + num_frames: int,
91 + ) -> int:
92 + num_frame_tokens = self._get_num_frame_tokens(
93 + image_width=image_width,
94 + image_height=image_height,
95 )
96
135 - pil_frame = dummy_image_for_siglip(vision_config, num_images=1)
136 - np_frame = np.array(pil_frame["image"])
137 - mm_data_per_video = np.repeat([np_frame], frames_per_video, axis=0)
138 - mm_data = {"video": mm_data_per_video}
139 - return DummyData(seq_data, mm_data, ranges)
97 + return num_frame_tokens * num_frames
98
141 - msg = f"Unsupported vision config: {type(vision_config)}"
142 - raise NotImplementedError(msg)
99 + def _get_max_video_tokens(self, num_frames: int) -> int:
100 + return self._get_num_video_tokens(image_width=999999,
101 + image_height=999999,
102 + num_frames=num_frames)
103
104 + def _get_max_video_frames(self, max_tokens: int) -> int:
105 + num_frames = 0
106
145 -def input_processor_for_llava_next_video(ctx: InputContext,
146 - inputs: DecoderOnlyInputs):
147 - multi_modal_data = inputs.get("multi_modal_data")
148 - if multi_modal_data is None or "video" not in multi_modal_data:
149 - return inputs
107 + while True:
108 + next_num_frames = num_frames + 1
109
151 - if "multi_modal_placeholders" in inputs and "video" in inputs[
152 - "multi_modal_placeholders"]:
153 - # The inputs already have placeholders.
154 - return inputs
110 + if self._get_max_video_tokens(next_num_frames) > max_tokens:
111 + break
112
156 - video_data = multi_modal_data["video"]
113 + num_frames = next_num_frames
114
158 - model_config = ctx.model_config
159 - hf_config = ctx.get_hf_config(LlavaNextVideoConfig)
160 - vision_config = hf_config.vision_config
115 + return num_frames
116
162 - if isinstance(video_data, np.ndarray):
163 - # Supports both CLIP and Siglip
164 - num_frames = video_data.shape[0]
165 - frame_feature_size = \
166 - get_llava_next_video_frame_feature_size(hf_config)
167 - video_feature_size = num_frames * frame_feature_size
117 + def _get_dummy_num_frames(self, seq_len: int) -> int:
118 + mm_config = self.ctx.get_mm_config()
119 + max_videos = mm_config.limit_per_prompt.get("video", 1)
120
169 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
121 + max_total_frames = self._get_max_video_frames(seq_len)
122
171 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
172 - tokenizer,
173 - inputs.get("prompt"),
174 - inputs["prompt_token_ids"],
175 - placeholder_token_id=hf_config.video_token_index,
176 - repeat_count=video_feature_size,
177 - )
123 + return max(max_total_frames // max(max_videos, 1), 1)
124
179 - return token_inputs(prompt_token_ids=new_token_ids,
180 - prompt=new_prompt,
181 - multi_modal_data=multi_modal_data,
182 - multi_modal_placeholders={"video": ranges})
125 + def _get_dummy_image_size(self) -> ImageSize:
126 + image_size = self._vision_encoder_info.get_image_size()
127 + return ImageSize(image_size, image_size)
128
184 - elif is_list_of(video_data, np.ndarray):
185 - raise NotImplementedError(
186 - "Processing multiple videos is not supported")
129 + def _get_video_token(self) -> str:
130 + return self._get_hf_processor().video_token
131
188 - msg = f"Unsupported vision config: {type(vision_config)}"
189 - raise NotImplementedError(msg)
132 + def _get_prompt_replacements(
133 + self,
134 + mm_items: MultiModalDataItems,
135 + hf_processor_mm_kwargs: Mapping[str, object],
136 + out_mm_kwargs: MultiModalKwargs,
137 + ) -> list[PromptReplacement]:
138 + hf_config = self._get_hf_config()
139 + video_token_id = hf_config.video_token_index
140 +
141 + def get_replacement(item_idx: int):
142 + videos = mm_items.get_items(
143 + "video", (VideoEmbeddingItems, VideoProcessorItems))
144 +
145 + if isinstance(videos, VideoEmbeddingItems):
146 + num_video_tokens = videos.get_feature_size(item_idx)
147 + else:
148 + image_size = videos.get_frame_size(item_idx)
149 + num_video_tokens = self._get_num_video_tokens(
150 + image_width=image_size.width,
151 + image_height=image_size.height,
152 + num_frames=videos.get_num_frames(item_idx),
153 + )
154 +
155 + return [video_token_id] * num_video_tokens
156 +
157 + return [
158 + PromptReplacement(
159 + modality="video",
160 + target=[video_token_id],
161 + replacement=get_replacement,
162 + ),
163 + ]
164 +
165 + def _get_dummy_processor_inputs(
166 + self,
167 + seq_len: int,
168 + mm_counts: Mapping[str, int],
169 + ) -> ProcessorInputs:
170 + num_videos = mm_counts.get("video", 0)
171 +
172 + video_token = self._get_video_token()
173 + target_width, target_height = self._get_dummy_image_size()
174 +
175 + mm_data = {
176 + "video":
177 + self._get_dummy_videos(
178 + width=target_width,
179 + height=target_height,
180 + num_frames=self._get_dummy_num_frames(seq_len),
181 + num_videos=num_videos,
182 + )
183 + }
184 +
185 + return ProcessorInputs(
186 + prompt_text=video_token * num_videos,
187 + mm_data=mm_data,
188 + )
189
190
191 # adopted from transformers modeling_llava_next_video.py
@@ -246,11 +245,7 @@ class LlavaNextMultiModalProjector(nn.Module):
245 return hidden_states
246
247
249 -@MULTIMODAL_REGISTRY.register_input_mapper("video")
250 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
251 - "video", get_max_llava_next_video_tokens)
252 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_next_video)
253 -@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_next_video)
248 +@MULTIMODAL_REGISTRY.register_processor(LlavaNextVideoMultiModalProcessor)
249 class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal,
250 SupportsPP):
251
vllm/model_executor/models/llava_onevision.py
+239 -292
@@ -3,47 +3,36 @@ from functools import cached_property
3 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
4 TypedDict, Union)
5
6 -import numpy as np
6 import torch
7 import torch.nn as nn
9 -from PIL import Image
10 -from transformers import (CLIPVisionConfig, LlavaOnevisionConfig,
11 - SiglipVisionConfig)
8 +from transformers import (BatchFeature, LlavaOnevisionConfig,
9 + LlavaOnevisionProcessor)
10 from transformers.models.llava_onevision.modeling_llava_onevision import (
11 get_anyres_image_grid_shape, unpad_image)
12 from typing_extensions import NotRequired
13
14 from vllm.attention import AttentionMetadata
15 from vllm.config import VllmConfig
18 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
19 - InputContext, token_inputs)
16 from vllm.model_executor.layers.activation import get_act_fn
17 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
18 from vllm.model_executor.sampling_metadata import SamplingMetadata
19 from vllm.multimodal import MULTIMODAL_REGISTRY
24 -from vllm.multimodal.inputs import NestedTensors
25 -from vllm.multimodal.utils import (cached_get_tokenizer,
26 - repeat_and_pad_placeholder_tokens)
20 +from vllm.multimodal.inputs import MultiModalKwargs, NestedTensors
21 +from vllm.multimodal.parse import (MultiModalDataItems, VideoEmbeddingItems,
22 + VideoProcessorItems)
23 +from vllm.multimodal.processing import (MultiModalFieldConfig, ProcessorInputs,
24 + PromptReplacement)
25 from vllm.sequence import IntermediateTensors
26 from vllm.utils import is_list_of
27
30 -from .clip import (CLIPVisionModel, dummy_seq_data_for_clip,
31 - dummy_video_for_clip, get_clip_image_feature_size,
32 - get_clip_patch_grid_length, input_processor_for_clip)
28 +from .clip import CLIPVisionModel
29 from .interfaces import SupportsMultiModal, SupportsPP
30 from .llava import init_vision_tower_for_llava
35 -from .siglip import (SiglipVisionModel, dummy_seq_data_for_siglip,
36 - dummy_video_for_siglip, get_siglip_image_feature_size,
37 - get_siglip_patch_grid_length, input_processor_for_siglip)
31 +from .llava_next import LlavaNextMultiModalProcessor
32 +from .siglip import SiglipVisionModel
33 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,
34 maybe_prefix, merge_multimodal_embeddings)
35
41 -# Result in the max possible feature size (2x2 grid of 336x336px tiles)
42 -MAX_IMAGE_FEATURE_SIZE_HEIGHT = MAX_IMAGE_FEATURE_SIZE_WIDTH = 448
43 -
44 -# For profile run
45 -_MAX_FRAMES_PER_VIDEO = 16
46 -
36
37 class LlavaOnevisionVideoPixelInputs(TypedDict):
38 type: Literal["pixel_values_videos"]
@@ -92,286 +81,251 @@ LlavaOnevisionMultiInputs = Union[LlavaOnevisionImageInputs,
81 LlavaOnevisionVideoPixelInputs]
82
83
95 -def _get_llava_onevision_image_unppaded_feature_size(height, width, patches,
96 - scale_height,
97 - scale_width):
98 - current_height = patches * scale_height
99 - current_width = patches * scale_width
100 -
101 - original_aspect_ratio = width / height
102 - current_aspect_ratio = current_width / current_height
103 - if original_aspect_ratio > current_aspect_ratio:
104 - new_height = int(height * (current_width / width))
105 - padding = (current_height - new_height) // 2
106 - current_height -= padding * 2
107 - else:
108 - new_width = int(width * (current_height / height))
109 - padding = (current_width - new_width) // 2
110 - current_width -= padding * 2
111 -
112 - unpadded_features = current_height * current_width
113 - newline_features = current_height
114 -
115 - ratio = math.sqrt(current_height * current_width / (9 * patches**2))
116 - if ratio > 1.1:
117 - unpadded_features = int(current_height // ratio) * int(
118 - current_width // ratio)
119 - newline_features = int(current_height // ratio)
120 -
121 - return (unpadded_features, newline_features)
122 -
123 -
124 -def get_llava_onevision_image_feature_size(
125 - hf_config: LlavaOnevisionConfig,
126 - *,
127 - input_height: int,
128 - input_width: int,
129 -) -> int:
130 - vision_config = hf_config.vision_config
131 -
132 - if isinstance(vision_config, CLIPVisionConfig):
133 - num_patches = get_clip_patch_grid_length(
134 - image_size=vision_config.image_size,
135 - patch_size=vision_config.patch_size,
136 - )
137 - base_feature_size = get_clip_image_feature_size(vision_config)
138 - elif isinstance(vision_config, SiglipVisionConfig):
139 - num_patches = get_siglip_patch_grid_length(
140 - image_size=vision_config.image_size,
141 - patch_size=vision_config.patch_size,
142 - )
143 - base_feature_size = get_siglip_image_feature_size(vision_config)
144 - else:
145 - msg = f"Unsupported vision config: {type(vision_config)}"
146 - raise NotImplementedError(msg)
147 -
148 - strategy = hf_config.vision_feature_select_strategy
149 - if strategy == "default":
150 - base_feature_size -= 1
151 - elif strategy == "full":
152 - pass
153 - else:
154 - raise ValueError(f"Unexpected select feature strategy: {strategy}")
84 +class LlavaOnevisionMultiModalProcessor(LlavaNextMultiModalProcessor):
85
156 - num_patch_height, num_patch_width = get_anyres_image_grid_shape(
157 - image_size=(input_height, input_width),
158 - grid_pinpoints=hf_config.image_grid_pinpoints,
159 - patch_size=vision_config.image_size,
160 - )
86 + def _get_hf_config(self) -> LlavaOnevisionConfig:
87 + return self.ctx.get_hf_config(LlavaOnevisionConfig)
88
162 - (
163 - unpadded_feature_size,
164 - newline_feature_size,
165 - ) = _get_llava_onevision_image_unppaded_feature_size(
166 - input_height, input_width, num_patches, num_patch_height,
167 - num_patch_width)
168 -
169 - return unpadded_feature_size + newline_feature_size + base_feature_size
170 -
171 -
172 -def get_max_llava_onevision_image_tokens(ctx: InputContext):
173 - return get_llava_onevision_image_feature_size(
174 - ctx.get_hf_config(LlavaOnevisionConfig),
175 - input_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
176 - input_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
177 - )
178 -
179 -
180 -def get_llava_onevision_video_frame_feature_size(
181 - hf_config: LlavaOnevisionConfig) -> int:
182 - # Support both CLIPVisionConfig and SiglipVisionConfig
183 - image_size = hf_config.vision_config.image_size
184 - patch_size = hf_config.vision_config.patch_size
185 - spatial_pool_stride = hf_config.spatial_pool_stride if hasattr(
186 - hf_config, "spatial_pool_stride") else 2
187 -
188 - height = width = image_size // patch_size
189 - return math.ceil(height / spatial_pool_stride) * math.ceil(
190 - width / spatial_pool_stride)
191 -
192 -
193 -def get_llava_onevision_video_tokens(ctx: InputContext,
194 - num_frames: int) -> int:
195 - hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
196 -
197 - # TODO: support configuring (not supported by HF right now)
198 - num_token_image_newline = 1
199 - tokens_per_frame = get_llava_onevision_video_frame_feature_size(hf_config)
200 - video_feature_size = num_frames * tokens_per_frame + num_token_image_newline
201 -
202 - return video_feature_size
203 -
204 -
205 -def get_max_llava_onevision_video_tokens(ctx: InputContext) -> int:
206 - return get_llava_onevision_video_tokens(ctx, _MAX_FRAMES_PER_VIDEO)
207 -
208 -
209 -def dummy_data_for_llava_onevision(ctx: InputContext, seq_len: int,
210 - mm_counts: Mapping[str, int]):
211 - hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
212 - vision_config = hf_config.vision_config
213 -
214 - num_videos = mm_counts["video"]
215 -
216 - # TODO: support configuring the number of frames
217 - num_frames = _MAX_FRAMES_PER_VIDEO
218 - video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
219 -
220 - if isinstance(vision_config, CLIPVisionConfig):
221 - seq_data, ranges = dummy_seq_data_for_clip(
222 - vision_config,
223 - seq_len,
224 - num_videos,
225 - image_token_id=hf_config.video_token_index,
226 - image_feature_size_override=video_feature_size,
227 - mm_key="video")
228 -
229 - mm_data = dummy_video_for_clip(vision_config,
230 - num_frames=num_frames,
231 - num_videos=num_videos)
232 - return DummyData(seq_data, mm_data, ranges)
233 - elif isinstance(vision_config, SiglipVisionConfig):
234 - seq_data, ranges = dummy_seq_data_for_siglip(
235 - vision_config,
236 - seq_len,
237 - num_videos,
238 - image_token_id=hf_config.video_token_index,
239 - image_feature_size_override=video_feature_size,
240 - mm_key="video")
241 -
242 - mm_data = dummy_video_for_siglip(vision_config,
243 - num_frames=num_frames,
244 - num_videos=num_videos)
245 - return DummyData(seq_data, mm_data, ranges)
246 -
247 - msg = f"Unsupported vision config: {type(vision_config)}"
248 - raise NotImplementedError(msg)
249 -
250 -
251 -def input_processor_when_multimodal_input_image(ctx: InputContext,
252 - inputs: DecoderOnlyInputs):
253 - multi_modal_data = inputs.get("multi_modal_data")
254 - if multi_modal_data is None or "image" not in multi_modal_data:
255 - return inputs
256 -
257 - model_config = ctx.model_config
258 - hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
259 - vision_config = hf_config.vision_config
260 -
261 - image_data = multi_modal_data["image"]
262 - if isinstance(image_data, Image.Image):
263 - width, height = image_data.size
264 -
265 - image_feature_size = get_llava_onevision_image_feature_size(
266 - hf_config,
267 - input_height=height,
268 - input_width=width,
269 - )
270 - elif is_list_of(image_data, Image.Image):
271 - image_feature_size = [
272 - get_llava_onevision_image_feature_size(hf_config,
273 - input_height=img.height,
274 - input_width=img.width)
275 - for img in image_data
276 - ]
277 - elif isinstance(image_data, torch.Tensor):
278 - num_images, image_feature_size, hidden_size = image_data.shape
279 - elif is_list_of(image_data, torch.Tensor):
280 - image_feature_size = [item.shape[1] for item in image_data]
281 - else:
282 - raise TypeError(f"Invalid image type: {type(image_data)}")
283 -
284 - vision_config = hf_config.vision_config
285 -
286 - if isinstance(vision_config, CLIPVisionConfig):
287 - return input_processor_for_clip(
288 - model_config,
289 - vision_config,
290 - inputs,
291 - image_token_id=hf_config.image_token_index,
292 - image_feature_size_override=image_feature_size,
89 + def _get_hf_processor(self) -> LlavaOnevisionProcessor:
90 + return self.ctx.get_hf_processor(LlavaOnevisionProcessor)
91 +
92 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
93 + return {"image": None, "video": None}
94 +
95 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
96 + max_image_tokens = self._get_max_image_tokens()
97 +
98 + num_frames = self._get_dummy_num_frames(seq_len)
99 + max_video_tokens = self._get_max_video_tokens(num_frames)
100 +
101 + return {
102 + "image": max_image_tokens,
103 + "video": max_video_tokens,
104 + }
105 +
106 + def _get_mm_fields_config(
107 + self,
108 + hf_inputs: BatchFeature,
109 + hf_processor_mm_kwargs: Mapping[str, object],
110 + ) -> Mapping[str, MultiModalFieldConfig]:
111 + return dict(
112 + pixel_values=MultiModalFieldConfig.batched("image"),
113 + image_sizes=MultiModalFieldConfig.batched("image"),
114 + image_embeds=MultiModalFieldConfig.batched("image"),
115 + pixel_values_videos=MultiModalFieldConfig.batched("video"),
116 )
294 - elif isinstance(vision_config, SiglipVisionConfig):
295 - return input_processor_for_siglip(
296 - model_config,
297 - vision_config,
298 - inputs,
299 - image_token_id=hf_config.image_token_index,
300 - image_feature_size_override=image_feature_size,
117 +
118 + def _get_num_unpadded_features(
119 + self,
120 + *,
121 + original_height: int,
122 + original_width: int,
123 + npatches: int,
124 + num_patch_height: int,
125 + num_patch_width: int,
126 + ) -> tuple[int, int]:
127 + current_height = npatches * num_patch_height
128 + current_width = npatches * num_patch_width
129 +
130 + original_aspect_ratio = original_width / original_height
131 + current_aspect_ratio = current_width / current_height
132 + if original_aspect_ratio > current_aspect_ratio:
133 + new_height = int(original_height *
134 + (current_width / original_width))
135 + padding = (current_height - new_height) // 2
136 + current_height -= padding * 2
137 + else:
138 + new_width = int(original_width *
139 + (current_height / original_height))
140 + padding = (current_width - new_width) // 2
141 + current_width -= padding * 2
142 +
143 + unpadded_features = current_height * current_width
144 + newline_features = current_height
145 +
146 + ratio = math.sqrt(current_height * current_width / (9 * npatches**2))
147 + if ratio > 1.1:
148 + unpadded_features = int(current_height // ratio) * int(
149 + current_width // ratio)
150 + newline_features = int(current_height // ratio)
151 +
152 + return (unpadded_features, newline_features)
153 +
154 + def _get_num_frame_tokens(
155 + self,
156 + *,
157 + image_width: int,
158 + image_height: int,
159 + ) -> int:
160 + hf_config = self._get_hf_config()
161 + spatial_pool_stride = getattr(hf_config, "spatial_pool_stride", 2)
162 +
163 + patch_grid_length = self._vision_encoder_info.get_patch_grid_length()
164 + pooled_grid_length = math.ceil(patch_grid_length / spatial_pool_stride)
165 +
166 + return pooled_grid_length * pooled_grid_length
167 +
168 + def _get_num_video_tokens(
169 + self,
170 + *,
171 + image_width: int,
172 + image_height: int,
173 + num_frames: int,
174 + ) -> int:
175 + num_frame_tokens = self._get_num_frame_tokens(
176 + image_width=image_width,
177 + image_height=image_height,
178 )
179
303 - msg = f"Unsupported vision config: {type(vision_config)}"
304 - raise NotImplementedError(msg)
180 + return num_frame_tokens * num_frames + 1 # Newline token
181 +
182 + def _get_max_video_tokens(self, num_frames: int) -> int:
183 + return self._get_num_video_tokens(image_width=999999,
184 + image_height=999999,
185 + num_frames=num_frames)
186
187 + def _get_max_video_frames(self, max_tokens: int) -> int:
188 + num_frames = 0
189
307 -def input_processor_when_multimodal_input_video(ctx: InputContext,
308 - inputs: DecoderOnlyInputs):
309 - multi_modal_data = inputs.get("multi_modal_data")
310 - if multi_modal_data is None or "video" not in multi_modal_data:
311 - return inputs
312 - video_data = multi_modal_data["video"]
190 + while True:
191 + next_num_frames = num_frames + 1
192
314 - model_config = ctx.model_config
315 - hf_config = ctx.get_hf_config(LlavaOnevisionConfig)
193 + if self._get_max_video_tokens(next_num_frames) > max_tokens:
194 + break
195
317 - if isinstance(video_data, np.ndarray):
318 - # Supports both CLIP and Siglip
319 - num_frames = video_data.shape[0]
320 - video_feature_size = get_llava_onevision_video_tokens(ctx, num_frames)
321 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
196 + num_frames = next_num_frames
197
323 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
324 - tokenizer,
325 - inputs.get("prompt"),
326 - inputs["prompt_token_ids"],
327 - placeholder_token_id=hf_config.video_token_index,
328 - repeat_count=video_feature_size,
198 + return num_frames
199 +
200 + def _get_dummy_num_frames(self, seq_len: int) -> int:
201 + mm_config = self.ctx.get_mm_config()
202 + max_images = mm_config.limit_per_prompt.get("image", 1)
203 + max_videos = mm_config.limit_per_prompt.get("video", 1)
204 +
205 + max_image_tokens = self._get_max_image_tokens() * max_images
206 + max_total_frames = self._get_max_video_frames(seq_len -
207 + max_image_tokens)
208 +
209 + return max(max_total_frames // max(max_videos, 1), 1)
210 +
211 + def _get_video_token(self) -> str:
212 + return self._get_hf_processor().video_token
213 +
214 + def _call_hf_processor(
215 + self,
216 + prompt: str,
217 + mm_data: Mapping[str, object],
218 + mm_kwargs: Mapping[str, object],
219 + ) -> BatchFeature:
220 + mm_data = dict(mm_data)
221 + videos = mm_data.pop("videos", [])
222 + assert isinstance(videos, list)
223 +
224 + if not videos:
225 + return super()._call_hf_processor(
226 + prompt=prompt,
227 + mm_data=mm_data,
228 + mm_kwargs=mm_kwargs,
229 + )
230 +
231 + video_token = self._get_video_token()
232 +
233 + # LLaVA-OneVision processor doesn't support multiple videos
234 + # with different sizes when converting back to tensors
235 + text_image_outputs = super()._call_hf_processor(
236 + prompt=prompt,
237 + mm_data=mm_data,
238 + mm_kwargs=mm_kwargs,
239 + )
240 +
241 + pixel_values_videos = []
242 + for video in videos:
243 + item_processor_data = dict(prompt=video_token, videos=video)
244 +
245 + item_outputs = super()._call_hf_processor(
246 + prompt=prompt,
247 + mm_data=item_processor_data,
248 + mm_kwargs=mm_kwargs,
249 + )
250 +
251 + pixel_values_videos.append(
252 + item_outputs.pop("pixel_values_videos")[0])
253 +
254 + combined_outputs = dict(
255 + **text_image_outputs,
256 + pixel_values_videos=pixel_values_videos,
257 )
258 + return BatchFeature(combined_outputs)
259
331 - return token_inputs(prompt_token_ids=new_token_ids,
332 - prompt=new_prompt,
333 - multi_modal_data=multi_modal_data,
334 - multi_modal_placeholders={"video": ranges})
335 -
336 - elif is_list_of(video_data, np.ndarray):
337 - video_feature_size = []
338 - for video in video_data:
339 - num_frames = video.shape[0]
340 - video_feature_size.append(
341 - get_llava_onevision_video_tokens(ctx, num_frames))
342 -
343 - tokenizer = cached_get_tokenizer(model_config.tokenizer)
344 - new_prompt, new_token_ids, ranges = repeat_and_pad_placeholder_tokens(
345 - tokenizer,
346 - inputs.get("prompt"),
347 - inputs["prompt_token_ids"],
348 - placeholder_token_id=hf_config.video_token_index,
349 - repeat_count=video_feature_size,
260 + def _get_prompt_replacements(
261 + self,
262 + mm_items: MultiModalDataItems,
263 + hf_processor_mm_kwargs: Mapping[str, object],
264 + out_mm_kwargs: MultiModalKwargs,
265 + ) -> list[PromptReplacement]:
266 + image_repls = super()._get_prompt_replacements(
267 + mm_items=mm_items,
268 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,
269 + out_mm_kwargs=out_mm_kwargs,
270 )
351 - return token_inputs(prompt_token_ids=new_token_ids,
352 - prompt=new_prompt,
353 - multi_modal_data=multi_modal_data,
354 - multi_modal_placeholders={"video": ranges})
355 - else:
356 - raise TypeError(f"Invalid video type: {type(video_data)}")
271
358 - msg = f"Unsupported video type: {type(video_data)}"
359 - raise NotImplementedError(msg)
272 + hf_config = self._get_hf_config()
273 + video_token_id = hf_config.video_token_index
274
275 + def get_video_replacement(item_idx: int):
276 + videos = mm_items.get_items(
277 + "video", (VideoEmbeddingItems, VideoProcessorItems))
278
362 -def input_processor_for_llava_onevision(ctx: InputContext,
363 - inputs: DecoderOnlyInputs):
364 - multi_modal_data = inputs.get("multi_modal_data")
365 - if multi_modal_data is None or ("video" not in multi_modal_data
366 - and "image" not in multi_modal_data):
367 - return inputs
368 - if "image" in multi_modal_data:
369 - return input_processor_when_multimodal_input_image(ctx, inputs)
370 - if "video" in multi_modal_data:
371 - return input_processor_when_multimodal_input_video(ctx, inputs)
279 + if isinstance(videos, VideoEmbeddingItems):
280 + num_video_tokens = videos.get_feature_size(item_idx)
281 + else:
282 + image_size = videos.get_frame_size(item_idx)
283 + num_video_tokens = self._get_num_video_tokens(
284 + image_width=image_size.width,
285 + image_height=image_size.height,
286 + num_frames=videos.get_num_frames(item_idx),
287 + )
288 +
289 + return [video_token_id] * num_video_tokens
290
373 - msg = "Unsupported multi data type"
374 - raise NotImplementedError(msg)
291 + return image_repls + [
292 + PromptReplacement(
293 + modality="video",
294 + target=[video_token_id],
295 + replacement=get_video_replacement,
296 + ),
297 + ]
298 +
299 + def _get_dummy_processor_inputs(
300 + self,
301 + seq_len: int,
302 + mm_counts: Mapping[str, int],
303 + ) -> ProcessorInputs:
304 + num_images = mm_counts.get("image", 0)
305 + num_videos = mm_counts.get("video", 0)
306 +
307 + image_token = self._get_image_token()
308 + video_token = self._get_video_token()
309 + target_width, target_height = self._get_dummy_image_size()
310 +
311 + mm_data = {
312 + "image":
313 + self._get_dummy_images(width=target_width,
314 + height=target_height,
315 + num_images=num_images),
316 + "video":
317 + self._get_dummy_videos(
318 + width=target_width,
319 + height=target_height,
320 + num_frames=self._get_dummy_num_frames(seq_len),
321 + num_videos=num_videos,
322 + )
323 + }
324 +
325 + return ProcessorInputs(
326 + prompt_text=image_token * num_images + video_token * num_videos,
327 + mm_data=mm_data,
328 + )
329
330
331 class LlavaOnevisionMultiModalProjector(nn.Module):
@@ -394,14 +348,7 @@ class LlavaOnevisionMultiModalProjector(nn.Module):
348 return hidden_states
349
350
397 -@MULTIMODAL_REGISTRY.register_image_input_mapper()
398 -@MULTIMODAL_REGISTRY.register_input_mapper("video")
399 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
400 - "image", get_max_llava_onevision_image_tokens)
401 -@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
402 - "video", get_max_llava_onevision_video_tokens)
403 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_llava_onevision)
404 -@INPUT_REGISTRY.register_input_processor(input_processor_for_llava_onevision)
351 +@MULTIMODAL_REGISTRY.register_processor(LlavaOnevisionMultiModalProcessor)
352 class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
353 SupportsPP):
354
vllm/model_executor/models/phi3v.py
+17 -9
@@ -323,7 +323,7 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
323 height=image_height,
324 )
325
326 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
326 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
327 max_image_tokens = self._get_num_image_tokens(
328 image_width=MAX_IMAGE_FEATURE_SIZE_WIDTH,
329 image_height=MAX_IMAGE_FEATURE_SIZE_HEIGHT,
@@ -415,12 +415,12 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
415 def _apply_prompt_replacements(
416 self,
417 token_ids: list[int],
418 - prompt_repls: Sequence[_BoundPromptReplacement],
418 + mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
419 mm_item_counts: Mapping[str, int],
420 - ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
420 + ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
421 token_ids, text, placeholders = super()._apply_prompt_replacements(
422 token_ids=token_ids,
423 - prompt_repls=prompt_repls,
423 + mm_prompt_repls=mm_prompt_repls,
424 mm_item_counts=mm_item_counts,
425 )
426
@@ -428,15 +428,23 @@ class Phi3VMultiModalProcessor(BaseMultiModalProcessor):
428 if text.startswith("<s> <|image|>"):
429 text = text.replace("<s> <|image|>", "<s><|image|>", 1)
430 token_ids = [token_ids[0], *token_ids[2:]]
431 - placeholders = [
432 - _PlaceholderInfo(p.modality, p.start_idx - 1, p.replacement)
433 - for p in placeholders
434 - ]
431 + placeholders = {
432 + modality: [
433 + _PlaceholderInfo(
434 + modality=p.modality,
435 + item_idx=p.item_idx,
436 + start_idx=p.start_idx - 1,
437 + replacement=p.replacement,
438 + ) for p in ps
439 + ]
440 + for modality, ps in placeholders.items()
441 + }
442
443 return token_ids, text, placeholders
444
438 - def _get_dummy_mm_inputs(
445 + def _get_dummy_processor_inputs(
446 self,
447 + seq_len: int,
448 mm_counts: Mapping[str, int],
449 ) -> ProcessorInputs:
450 num_images = mm_counts.get("image", 0)
vllm/model_executor/models/pixtral.py
+7 -4
@@ -780,15 +780,18 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):
780 def get_max_image_tokens(self) -> int:
781 return get_max_pixtral_hf_image_tokens(self.vision_config)
782
783 - def get_num_patches(self) -> int:
783 + def get_image_size(self) -> int:
784 + return self.vision_config.image_size
785 +
786 + def get_patch_size(self) -> int:
787 + return self.vision_config.patch_size
788 +
789 + def get_patch_grid_length(self) -> int:
790 return get_pixtral_hf_patch_grid_length(
791 image_size=self.vision_config.image_size,
792 patch_size=self.vision_config.patch_size,
793 )
794
789 - def get_image_size(self) -> int:
790 - return self.vision_config.image_size
791 -
795
796 class PixtralHFMLP(nn.Module):
797
vllm/model_executor/models/qwen2_audio.py
+8 -7
@@ -84,7 +84,7 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
84 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
85 return {"audio": None}
86
87 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
87 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
88 hf_config = self.ctx.get_hf_config(Qwen2AudioConfig)
89 max_source_positions = hf_config.audio_config.max_source_positions
90 max_output_lengths = (max_source_positions - 2) // 2 + 1
@@ -184,15 +184,16 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor):
184 ]
185
186 def _always_apply_prompt_replacements(self) -> bool:
187 - # HF never applies prompt replacements, so we have to do it ourselves
188 - # _find_placeholders may incorrectly think that HF has already performed
189 - # processing for multi-audio input when the input audios are short
190 - # (the corresponding placeholders may take up fewer tokens than
191 - # the number of audio items)
187 + # HF never applies prompt replacements, so we have to do it ourselves.
188 + # NOTE: `_find_placeholders_by_modality` may incorrectly think that HF
189 + # has already performed processing for multi-audio input when the input
190 + # audios are short (the corresponding placeholders may take up fewer
191 + # tokens than the number of audio items)
192 return True
193
194 - def _get_dummy_mm_inputs(
194 + def _get_dummy_processor_inputs(
195 self,
196 + seq_len: int,
197 mm_counts: Mapping[str, int],
198 ) -> ProcessorInputs:
199 feature_extractor = self._get_feature_extractor()
vllm/model_executor/models/qwen2_vl.py
+115 -84
@@ -56,7 +56,8 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
56 from vllm.multimodal.inputs import (ImageItem, ModalityData,
57 MultiModalFieldConfig, MultiModalKwargs,
58 NestedTensors, VideoItem)
59 -from vllm.multimodal.parse import ModalityDataItems, MultiModalDataParser
59 +from vllm.multimodal.parse import (ImageSize, ModalityDataItems,
60 + MultiModalDataParser)
61 from vllm.multimodal.processing import (BaseMultiModalProcessor,
62 MultiModalDataItems, ProcessorInputs,
63 PromptReplacement)
@@ -641,58 +642,6 @@ class Qwen2VisionTransformer(nn.Module):
642 return loaded_params
643
644
644 -# === Vision input helpers === #
645 -
646 -
647 -def _get_vision_info(
648 - vision_config: Qwen2VLVisionConfig,
649 - height: int,
650 - width: int,
651 - min_pixels: int,
652 - max_pixels: int,
653 - *,
654 - do_resize: bool = True,
655 - modality: str = "image",
656 - mm_count: int = 1,
657 -):
658 - """Get information (resized height / width and number of vision tokens)
659 - of input image / video frame."""
660 - patch_size = vision_config.patch_size
661 - merge_size = vision_config.spatial_merge_size
662 - temporal_patch_size = vision_config.temporal_patch_size
663 -
664 - if do_resize:
665 - resized_height, resized_width = smart_resize(
666 - height=height,
667 - width=width,
668 - factor=patch_size * merge_size,
669 - min_pixels=min_pixels,
670 - max_pixels=max_pixels,
671 - )
672 - else:
673 - resized_height, resized_width = height, width
674 -
675 - if modality == "image":
676 - grid_t = mm_count
677 - elif modality == "video":
678 - grid_t = max(mm_count // temporal_patch_size, 1)
679 - else:
680 - raise ValueError(f"Modality {modality} is not supported")
681 -
682 - grid_h = resized_height // patch_size
683 - grid_w = resized_width // patch_size
684 - vision_tokens = grid_t * grid_h * grid_w
685 - llm_num_vision_tokens = vision_tokens // (merge_size**2)
686 -
687 - return resized_height, resized_width, llm_num_vision_tokens
688 -
689 -
690 -def _get_image_processor(hf_processor: Qwen2VLProcessor):
691 - image_processor = hf_processor.image_processor # type: ignore
692 - assert isinstance(image_processor, Qwen2VLImageProcessor)
693 - return image_processor
694 -
695 -
645 class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
646 dict[str, torch.Tensor]]):
647
@@ -764,32 +713,111 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
713 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
714 return {"image": None, "video": None}
715
767 - def _get_max_mm_tokens(self, modality: str) -> int:
716 + def _get_vision_info(
717 + self,
718 + *,
719 + image_width: int,
720 + image_height: int,
721 + num_frames: int = 1,
722 + do_resize: bool = True,
723 + ) -> tuple[ImageSize, int]:
724 hf_config = self.ctx.get_hf_config(Qwen2VLConfig)
725 vision_config = hf_config.vision_config
726 + patch_size = vision_config.patch_size
727 + merge_size = vision_config.spatial_merge_size
728 + temporal_patch_size = vision_config.temporal_patch_size
729
730 hf_processor = self._get_hf_processor()
772 - image_processor = _get_image_processor(hf_processor)
773 -
774 - _, _, max_llm_image_tokens = _get_vision_info(
775 - vision_config,
776 - height=9999999,
777 - width=9999999,
778 - min_pixels=image_processor.min_pixels,
779 - max_pixels=image_processor.max_pixels,
780 - modality=modality,
731 + image_processor = self._get_image_processor(hf_processor)
732 +
733 + if do_resize:
734 + resized_height, resized_width = smart_resize(
735 + height=image_height,
736 + width=image_width,
737 + factor=patch_size * merge_size,
738 + min_pixels=image_processor.min_pixels,
739 + max_pixels=image_processor.max_pixels,
740 + )
741 + preprocessed_size = ImageSize(width=resized_width,
742 + height=resized_height)
743 + else:
744 + preprocessed_size = ImageSize(width=image_width,
745 + height=image_height)
746 +
747 + grid_t = max(num_frames // temporal_patch_size, 1)
748 + grid_h = preprocessed_size.height // patch_size
749 + grid_w = preprocessed_size.width // patch_size
750 +
751 + num_patches = grid_t * grid_h * grid_w
752 + num_vision_tokens = num_patches // (merge_size**2)
753 +
754 + return preprocessed_size, num_vision_tokens
755 +
756 + def _get_dummy_image_size(self) -> ImageSize:
757 + max_image_size, _ = self._get_vision_info(
758 + image_width=9999999,
759 + image_height=9999999,
760 + )
761 + return max_image_size
762 +
763 + def _get_max_image_tokens(self) -> int:
764 + _, max_image_tokens = self._get_vision_info(
765 + image_width=9999999,
766 + image_height=9999999,
767 + )
768 + return max_image_tokens
769 +
770 + def _get_max_video_tokens(self, num_frames: int) -> int:
771 + _, max_video_tokens = self._get_vision_info(
772 + image_width=9999999,
773 + image_height=9999999,
774 + num_frames=num_frames,
775 )
782 - return max_llm_image_tokens
776 + return max_video_tokens
777 +
778 + def _get_max_video_frames(self, max_tokens: int) -> int:
779 + num_frames = 0
780 +
781 + while True:
782 + next_num_frames = num_frames + 1
783 +
784 + if self._get_max_video_tokens(next_num_frames) > max_tokens:
785 + break
786 +
787 + num_frames = next_num_frames
788 +
789 + return num_frames
790 +
791 + def _get_dummy_num_frames(self, seq_len: int) -> int:
792 + mm_config = self.ctx.get_mm_config()
793 + max_images = mm_config.limit_per_prompt.get("image", 1)
794 + max_videos = mm_config.limit_per_prompt.get("video", 1)
795 +
796 + max_image_tokens = self._get_max_image_tokens() * max_images
797 + max_total_frames = self._get_max_video_frames(seq_len -
798 + max_image_tokens)
799 +
800 + return max(max_total_frames // max(max_videos, 1), 1)
801 +
802 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
803 + max_image_tokens = self._get_max_image_tokens()
804 +
805 + num_frames = self._get_dummy_num_frames(seq_len)
806 + max_video_tokens = self._get_max_video_tokens(num_frames)
807
784 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
808 return {
786 - "image": self._get_max_mm_tokens("image"),
787 - "video": self._get_max_mm_tokens("video"),
809 + "image": max_image_tokens,
810 + "video": max_video_tokens,
811 }
812
813 def _get_data_parser(self) -> MultiModalDataParser:
814 return Qwen2MultiModalDataParser()
815
816 + def _get_image_processor(self, hf_processor: Qwen2VLProcessor):
817 + image_processor = hf_processor.image_processor # type: ignore
818 + assert isinstance(image_processor, Qwen2VLImageProcessor)
819 + return image_processor
820 +
821 def _get_hf_processor(
822 self,
823 *,
@@ -797,7 +825,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
825 max_pixels: Optional[int] = None,
826 ) -> Qwen2VLProcessor:
827 hf_processor = self.ctx.get_hf_processor(Qwen2VLProcessor)
800 - image_processor = _get_image_processor(hf_processor)
828 + image_processor = self._get_image_processor(hf_processor)
829
830 if min_pixels:
831 image_processor.min_pixels = min_pixels
@@ -818,7 +846,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
846 out_mm_kwargs: MultiModalKwargs,
847 ) -> list[PromptReplacement]:
848 hf_processor = self._get_hf_processor()
821 - image_processor = _get_image_processor(hf_processor)
849 + image_processor = self._get_image_processor(hf_processor)
850
851 # NOTE: Only Qwen2VLProcessor in transformers 4.47.0 has
852 # image_token and video_token registered
@@ -873,32 +901,35 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor):
901 video_grid_thw=MultiModalFieldConfig.batched("video"),
902 )
903
876 - def _get_dummy_mm_inputs(
904 + def _get_dummy_processor_inputs(
905 self,
906 + seq_len: int,
907 mm_counts: Mapping[str, int],
908 ) -> ProcessorInputs:
880 - hf_processor = self._get_hf_processor()
881 - image_processor = _get_image_processor(hf_processor)
909 + num_images = mm_counts.get("image", 0)
910 + num_videos = mm_counts.get("video", 0)
911
912 + hf_processor = self._get_hf_processor()
913 image_token: str = hf_processor.image_token
884 - resized_height, resized_width = smart_resize(
885 - height=9999999,
886 - width=9999999,
887 - factor=image_processor.patch_size * image_processor.merge_size,
888 - min_pixels=image_processor.min_pixels,
889 - max_pixels=image_processor.max_pixels,
890 - )
891 - num_images = mm_counts.get("image", 0)
914 + video_token: str = hf_processor.video_token
915 + target_width, target_height = self._get_dummy_image_size()
916
917 mm_data = {
918 "image":
895 - self._get_dummy_images(width=resized_width,
896 - height=resized_height,
897 - num_images=num_images)
919 + self._get_dummy_images(width=target_width,
920 + height=target_height,
921 + num_images=num_images),
922 + "video":
923 + self._get_dummy_videos(
924 + width=target_width,
925 + height=target_height,
926 + num_frames=self._get_dummy_num_frames(seq_len),
927 + num_videos=num_videos,
928 + )
929 }
930
931 return ProcessorInputs(
901 - prompt_text=image_token * num_images,
932 + prompt_text=image_token * num_images + video_token * num_videos,
933 mm_data=mm_data,
934 )
935
vllm/model_executor/models/siglip.py
+7 -4
@@ -171,15 +171,18 @@ class SiglipEncoderInfo(VisionEncoderInfo[SiglipVisionConfig]):
171 def get_max_image_tokens(self) -> int:
172 return get_max_siglip_image_tokens(self.vision_config)
173
174 - def get_num_patches(self) -> int:
174 + def get_image_size(self) -> int:
175 + return self.vision_config.image_size
176 +
177 + def get_patch_size(self) -> int:
178 + return self.vision_config.patch_size
179 +
180 + def get_patch_grid_length(self) -> int:
181 return get_siglip_patch_grid_length(
182 image_size=self.vision_config.image_size,
183 patch_size=self.vision_config.patch_size,
184 )
185
180 - def get_image_size(self) -> int:
181 - return self.vision_config.image_size
182 -
186
187 # Adapted from https://github.com/huggingface/transformers/blob/v4.43.3/src/transformers/models/siglip/modeling_siglip.py#L249 # noqa
188 class SiglipVisionEmbeddings(nn.Module):
vllm/model_executor/models/ultravox.py
+4 -7
@@ -6,7 +6,6 @@ from functools import cached_property
6 from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
7 TypedDict, Union)
8
9 -import numpy as np
9 import torch
10 import torch.utils.checkpoint
11 from torch import nn
@@ -31,7 +30,6 @@ from vllm.multimodal.processing import (BaseMultiModalProcessor,
30 PromptReplacement)
31 from vllm.sequence import IntermediateTensors
32 from vllm.transformers_utils.configs.ultravox import UltravoxConfig
34 -from vllm.utils import is_list_of
33
34 from .interfaces import SupportsMultiModal, SupportsPP
35 from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
@@ -62,7 +60,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
60 def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
61 return {"audio": None}
62
65 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
63 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
64 feature_extractor = self._get_feature_extractor()
65 max_audio_tokens = math.ceil(feature_extractor.chunk_length *
66 _AUDIO_TOKENS_PER_SECOND)
@@ -103,6 +101,7 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
101
102 mm_data = dict(mm_data)
103 audios = mm_data.pop("audios", [])
104 + assert isinstance(audios, list)
105
106 if not audios:
107 return super()._call_hf_processor(
@@ -117,9 +116,6 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
116 sampling_rate=feature_extractor.sampling_rate,
117 )
118
120 - # Already resampled by _get_hf_mm_data
121 - assert is_list_of(audios, np.ndarray)
122 -
119 # Ultravox processor doesn't support multiple inputs,
120 # therefore we need to input text and audio one by one
121 audio_features, audio_token_len = [], []
@@ -177,8 +173,9 @@ class UltravoxMultiModalProcessor(BaseMultiModalProcessor):
173 )
174 ]
175
180 - def _get_dummy_mm_inputs(
176 + def _get_dummy_processor_inputs(
177 self,
178 + seq_len: int,
179 mm_counts: Mapping[str, int],
180 ) -> ProcessorInputs:
181 feature_extractor = self._get_feature_extractor()
vllm/model_executor/models/vision.py
+34 -3
@@ -1,8 +1,12 @@
1 from abc import ABC, abstractmethod
2 -from typing import Generic, TypeVar
2 +from typing import Final, Generic, Optional, Protocol, TypeVar
3
4 from transformers import PretrainedConfig
5
6 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
7 + InputProcessingContext,
8 + ProcessingCache)
9 +
10 _C = TypeVar("_C", bound=PretrainedConfig)
11
12
@@ -27,11 +31,15 @@ class VisionEncoderInfo(ABC, Generic[_C]):
31 raise NotImplementedError
32
33 @abstractmethod
30 - def get_num_patches(self) -> int:
34 + def get_image_size(self) -> int:
35 raise NotImplementedError
36
37 @abstractmethod
34 - def get_image_size(self) -> int:
38 + def get_patch_size(self) -> int:
39 + raise NotImplementedError
40 +
41 + @abstractmethod
42 + def get_patch_grid_length(self) -> int:
43 raise NotImplementedError
44
45
@@ -50,3 +58,26 @@ def vision_encoder_info(vision_config: PretrainedConfig) -> VisionEncoderInfo:
58
59 msg = f"Unsupported vision config: {type(vision_config)}"
60 raise NotImplementedError(msg)
61 +
62 +
63 +class VisionLanguageConfig(Protocol):
64 + vision_config: Final[PretrainedConfig]
65 +
66 +
67 +class BaseVisionLanguageMultiModalProcessor(BaseMultiModalProcessor):
68 +
69 + def __init__(self,
70 + ctx: InputProcessingContext,
71 + *,
72 + cache: Optional[ProcessingCache] = None,
73 + enable_sanity_checks: bool = True) -> None:
74 + super().__init__(ctx,
75 + cache=cache,
76 + enable_sanity_checks=enable_sanity_checks)
77 +
78 + vision_config = self._get_hf_config().vision_config
79 + self._vision_encoder_info = vision_encoder_info(vision_config)
80 +
81 + @abstractmethod
82 + def _get_hf_config(self) -> VisionLanguageConfig:
83 + raise NotImplementedError
vllm/multimodal/parse.py
+14
@@ -146,6 +146,20 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):
146 def __init__(self, data: Sequence[HfVideoItem]) -> None:
147 super().__init__(data, "video")
148
149 + def get_num_frames(self, item_idx: int) -> int:
150 + return len(self.get(item_idx))
151 +
152 + def get_frame_size(self, item_idx: int) -> ImageSize:
153 + image = self.get(item_idx)[0] # Assume that the video isn't empty
154 +
155 + if isinstance(image, Image):
156 + return ImageSize(*image.size)
157 + if isinstance(image, (np.ndarray, torch.Tensor)):
158 + _, h, w = image.shape
159 + return ImageSize(w, h)
160 +
161 + assert_never(image)
162 +
163
164 class VideoEmbeddingItems(EmbeddingItems):
165
vllm/multimodal/processing.py
+206 -120
@@ -16,7 +16,8 @@ from transformers import BatchFeature, ProcessorMixin
16
17 from vllm.inputs import DummyData, InputProcessingContext
18 from vllm.logger import init_logger
19 -from vllm.transformers_utils.tokenizer import AnyTokenizer, encode_tokens
19 +from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
20 + encode_tokens)
21 from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
22
23 from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
@@ -69,19 +70,6 @@ def _cached_encode(
70 add_special_tokens=add_special_tokens)
71
72
72 -def _decode(
73 - tokenizer: AnyTokenizer,
74 - token_ids: list[int],
75 - *,
76 - skip_special_tokens: bool = False,
77 -) -> str:
78 - """
79 - Backend-agnostic equivalent of HF's
80 - :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
81 - """
82 - return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
83 -
84 -
73 @lru_cache(maxsize=2048)
74 def _cached_decode(
75 tokenizer: AnyTokenizer,
@@ -89,9 +77,9 @@ def _cached_decode(
77 *,
78 skip_special_tokens: bool = False,
79 ) -> str:
92 - return _decode(tokenizer,
93 - list(token_ids),
94 - skip_special_tokens=skip_special_tokens)
80 + return decode_tokens(tokenizer,
81 + list(token_ids),
82 + skip_special_tokens=skip_special_tokens)
83
84
85 class _HasModalityAttr(Protocol):
@@ -269,8 +257,10 @@ class _PromptReplacementTextMatch(_PromptReplacementMatch):
257 return self.match.end()
258
259
272 -class _PlaceholderInfo(NamedTuple):
260 +@dataclass
261 +class _PlaceholderInfo:
262 modality: str
263 + item_idx: int
264 start_idx: int
265 replacement: list[int]
266
@@ -311,12 +301,14 @@ def find_text_matches(
301
302 def _resolve_matches(
303 prompt: _PromptSeq,
314 - matches: Sequence[_PromptReplacementMatch],
304 + mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
305 ) -> list[_PromptReplacementMatch]:
306 """
317 - Resolve :code:`matches` to ensure that there are no overlapping matches,
307 + Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
308 and sort them such that earlier matches take priority over later ones.
309 """
310 + matches = [m for matches in mm_matches.values() for m in matches]
311 +
312 seen_matches: list[Optional[_PromptReplacementMatch]] = [None
313 ] * len(prompt)
314
@@ -334,14 +326,15 @@ def _resolve_matches(
326
327 def _replace_matches(
328 prompt: _S,
337 - matches: Sequence[_PromptReplacementMatch],
329 + mm_matches: Mapping[str, Sequence[_PromptReplacementMatch]],
330 mm_item_counts: Mapping[str, int],
331 ) -> list[_S]:
332 + """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
333 out_seqs = list[_S]()
334 prev_end_idx = 0
335 next_idx_by_modality = defaultdict[str, int](lambda: 0)
336
344 - for match in _resolve_matches(prompt, matches):
337 + for match in _resolve_matches(prompt, mm_matches):
338 modality = match.modality
339
340 item_idx = next_idx_by_modality[modality]
@@ -371,28 +364,28 @@ def _replace_matches(
364
365 def replace_token_matches(
366 prompt: list[int],
374 - matches: Sequence[_PromptReplacementTokenMatch],
367 + mm_matches: Mapping[str, Sequence[_PromptReplacementTokenMatch]],
368 mm_item_counts: Mapping[str, int],
369 ) -> list[int]:
377 - """Apply :code:`prompt_repls` to :code:`prompt`."""
378 - if not matches:
370 + """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
371 + if not mm_matches:
372 return prompt
373
381 - token_id_seqs = _replace_matches(prompt, matches, mm_item_counts)
374 + token_id_seqs = _replace_matches(prompt, mm_matches, mm_item_counts)
375
376 return flatten_2d_lists(token_id_seqs)
377
378
379 def replace_text_matches(
380 prompt: str,
388 - matches: Sequence[_PromptReplacementTextMatch],
381 + mm_matches: Mapping[str, Sequence[_PromptReplacementTextMatch]],
382 mm_item_counts: Mapping[str, int],
383 ) -> str:
391 - """Apply :code:`prompt_repls` to :code:`prompt`."""
392 - if not matches:
384 + """Apply the replacements in :code:`mm_matches` to :code:`prompt`."""
385 + if not mm_matches:
386 return prompt
387
395 - texts = _replace_matches(prompt, matches, mm_item_counts)
388 + texts = _replace_matches(prompt, mm_matches, mm_item_counts)
389
390 return "".join(texts)
391
@@ -407,14 +400,14 @@ def _iter_modality_placeholders(
400 return
401
402 prompt_len = len(prompt)
410 - item_index = 0
403 + item_idx = 0
404
405 start_idx = 0
406 while start_idx < prompt_len:
407 found = False
408
409 for repl_info in modality_repls:
417 - replacement = repl_info.get_replacement(item_index)
410 + replacement = repl_info.get_replacement(item_idx)
411 repl_tokens = replacement.token_ids
412 repl_len = len(repl_tokens)
413 end_idx = start_idx + repl_len
@@ -425,12 +418,13 @@ def _iter_modality_placeholders(
418 if prompt[start_idx:end_idx] == repl_tokens:
419 yield _PlaceholderInfo(
420 modality=modality,
421 + item_idx=item_idx,
422 start_idx=start_idx,
423 replacement=repl_tokens,
424 )
425
432 - item_index += 1
433 - if item_index >= modal_item_count:
426 + item_idx += 1
427 + if item_idx >= modal_item_count:
428 return
429
430 # Exclude overlapping matches
@@ -442,28 +436,36 @@ def _iter_modality_placeholders(
436 start_idx += 1
437
438
445 -def iter_placeholders(
446 - prompt_repls: Sequence[_BoundPromptReplacement],
439 +def _iter_placeholders(
440 + mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
441 prompt: list[int],
442 mm_item_counts: Mapping[str, int],
443 ) -> Iterable[_PlaceholderInfo]:
444 """
451 - Yield each set of placeholder tokens found in :code:`prompt`.
445 + For each modality, yield each set of placeholder tokens found in
446 + :code:`prompt`.
447
448 Note that empty matches are ignored.
449 """
455 - repls_by_modality = dict(full_groupby_modality(prompt_repls))
456 -
450 for modality, modal_item_count in mm_item_counts.items():
458 - if modality in repls_by_modality:
451 + if modality in mm_prompt_repls:
452 yield from _iter_modality_placeholders(
453 prompt,
454 modality,
462 - repls_by_modality[modality],
455 + mm_prompt_repls[modality],
456 modal_item_count,
457 )
458
459
460 +def find_mm_placeholders(
461 + mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
462 + prompt: list[int],
463 + mm_item_counts: Mapping[str, int],
464 +) -> Mapping[str, list[_PlaceholderInfo]]:
465 + it = _iter_placeholders(mm_prompt_repls, prompt, mm_item_counts)
466 + return dict(full_groupby_modality(it))
467 +
468 +
469 @dataclass
470 class ProcessorInputs:
471 """Keyword arguments to :meth:`BaseMultiModalProcessor`."""
@@ -620,7 +622,7 @@ class BaseMultiModalProcessor(ABC):
622 raise NotImplementedError
623
624 @abstractmethod
623 - def get_mm_max_tokens_per_item(self) -> Mapping[str, int]:
625 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
626 """
627 Get the maximum possible number of tokens per data item
628 for each modality.
@@ -703,14 +705,14 @@ class BaseMultiModalProcessor(ABC):
705 """
706 raise NotImplementedError
707
706 - def _find_placeholders(
708 + def _find_mm_placeholders(
709 self,
708 - all_prompt_repls: Sequence[_BoundPromptReplacement],
710 + mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
711 new_token_ids: list[int],
712 mm_item_counts: Mapping[str, int],
711 - ) -> list[_PlaceholderInfo]:
712 - return list(
713 - iter_placeholders(all_prompt_repls, new_token_ids, mm_item_counts))
713 + ) -> Mapping[str, list[_PlaceholderInfo]]:
714 + return find_mm_placeholders(mm_prompt_repls, new_token_ids,
715 + mm_item_counts)
716
717 def _get_hf_mm_data(
718 self,
@@ -797,7 +799,10 @@ class BaseMultiModalProcessor(ABC):
799
800 # Some HF processors (e.g. Qwen2-VL) expect corresponding
801 # multi-modal tokens to be in the prompt text
800 - dummy_inputs = self._get_dummy_mm_inputs(mm_missing_counts)
802 + dummy_inputs = self._get_dummy_processor_inputs(
803 + self.ctx.model_config.max_model_len,
804 + mm_missing_counts,
805 + )
806
807 _, mm_missing_kwargs = self._apply_hf_processor(
808 prompt_text=dummy_inputs.prompt_text,
@@ -889,50 +894,44 @@ class BaseMultiModalProcessor(ABC):
894
895 mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
896
892 - if self.enable_sanity_checks:
893 - mm_item_counts = mm_data_items.get_all_counts()
894 -
895 - for modality, item_count in mm_item_counts.items():
896 - for item_idx in range(item_count):
897 - try:
898 - mm_kwargs.get_item(modality, item_idx)
899 - except Exception as e:
900 - # Make it easy to set a breakpoint in the debugger
901 - raise e
902 -
897 return prompt_ids, mm_kwargs
898
905 - def _bind_prompt_replacements(
899 + def _bind_and_group_repls(
900 self,
901 prompt_repls: list[PromptReplacement],
908 - ) -> list[_BoundPromptReplacement]:
902 + ) -> dict[str, list[_BoundPromptReplacement]]:
903 tokenizer = self._get_tokenizer()
904
911 - return [prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls]
905 + it = (prompt_repl.bind(tokenizer) for prompt_repl in prompt_repls)
906 + return dict(full_groupby_modality(it))
907
908 def _always_apply_prompt_replacements(self) -> bool:
909 """
910 A flag which can be overridden so that
911 :meth:`_apply_prompt_replacements` is always called even if we
917 - detect that HF has performed processing via :meth:`_find_placeholders`.
912 + detect that HF has performed processing via
913 + :meth:`_find_placeholders_by_modality`.
914
919 - This is useful in cases where :meth:`_find_placeholders` cannot be
920 - reliably used to detect whether HF has performed processing or not.
915 + This is useful in cases where :meth:`_find_placeholders_by_modality`
916 + cannot be reliably used to detect whether HF has performed processing.
917 """
918 return False
919
920 def _apply_prompt_replacements(
921 self,
922 token_ids: list[int],
927 - prompt_repls: Sequence[_BoundPromptReplacement],
923 + mm_prompt_repls: Mapping[str, Sequence[_BoundPromptReplacement]],
924 mm_item_counts: Mapping[str, int],
929 - ) -> tuple[list[int], str, list[_PlaceholderInfo]]:
925 + ) -> tuple[list[int], str, Mapping[str, list[_PlaceholderInfo]]]:
926 tokenizer = self._get_tokenizer()
927
932 - token_matches = find_token_matches(token_ids, prompt_repls)
928 + mm_token_matches = {
929 + modality: find_token_matches(token_ids, prompt_repls)
930 + for modality, prompt_repls in mm_prompt_repls.items()
931 + }
932 mm_match_counts = {
933 modality: len(matches)
935 - for modality, matches in full_groupby_modality(token_matches)
934 + for modality, matches in mm_token_matches.items()
935 }
936
937 # If the search text does not represent a special token,
@@ -951,32 +950,92 @@ class BaseMultiModalProcessor(ABC):
950 ): # yapf: disable
951 token_ids = replace_token_matches(
952 token_ids,
954 - token_matches,
953 + mm_token_matches,
954 mm_item_counts,
955 )
956
958 - text = _decode(tokenizer, token_ids)
959 - matched_repls = [match.prompt_repl for match in token_matches]
957 + text = decode_tokens(tokenizer, token_ids)
958 + matched_repls = {
959 + modality: [match.prompt_repl for match in token_matches]
960 + for modality, token_matches in mm_token_matches.items()
961 + }
962 else:
961 - text = _decode(tokenizer, token_ids)
963 + text = decode_tokens(tokenizer, token_ids)
964
963 - text_matches = find_text_matches(text, prompt_repls)
965 + mm_text_matches = {
966 + modality: find_text_matches(text, prompt_repls)
967 + for modality, prompt_repls in mm_prompt_repls.items()
968 + }
969 text = replace_text_matches(
970 text,
966 - text_matches,
971 + mm_text_matches,
972 mm_item_counts,
973 )
974
975 token_ids = encode_tokens(tokenizer,
976 text,
977 add_special_tokens=False)
973 - matched_repls = [match.prompt_repl for match in text_matches]
974 -
975 - placeholders = self._find_placeholders(matched_repls, token_ids,
976 - mm_item_counts)
978 + matched_repls = {
979 + modality: [match.prompt_repl for match in token_matches]
980 + for modality, token_matches in mm_text_matches.items()
981 + }
982 +
983 + placeholders = self._find_mm_placeholders(
984 + matched_repls,
985 + token_ids,
986 + mm_item_counts,
987 + )
988
989 return token_ids, text, placeholders
990
991 + def _validate_mm_kwargs(
992 + self,
993 + mm_kwargs: MultiModalKwargs,
994 + mm_item_counts: Mapping[str, int],
995 + ) -> None:
996 + for modality, item_count in mm_item_counts.items():
997 + if modality in mm_kwargs.modalities:
998 + items = mm_kwargs.get_items(modality)
999 + else:
1000 + items = []
1001 +
1002 + if len(items) != item_count:
1003 + raise RuntimeError(
1004 + f"Expected there to be {item_count} {modality} items in "
1005 + f"keyword arguments corresponding to {item_count} "
1006 + f"{modality} data items, but only found {len(items)}! "
1007 + "There is likely a problem with your "
1008 + "implementation of merged multi-modal processor for this "
1009 + "model (usually arising from an inconsistency between "
1010 + "`_call_hf_processor` and `_get_mm_fields_config`).")
1011 +
1012 + def _validate_mm_placeholders(
1013 + self,
1014 + mm_placeholders: Mapping[str, list[_PlaceholderInfo]],
1015 + mm_item_counts: Mapping[str, int],
1016 + *,
1017 + allow_missing: bool = False,
1018 + ) -> Mapping[str, int]:
1019 + missing_repl_counts = dict[str, int]()
1020 +
1021 + for modality, item_count in mm_item_counts.items():
1022 + placeholders = mm_placeholders.get(modality, [])
1023 +
1024 + if len(placeholders) != item_count and not allow_missing:
1025 + raise RuntimeError(
1026 + f"Expected there to be {item_count} prompt replacements "
1027 + f"corresponding to {item_count} {modality} items, but only "
1028 + f"found {len(placeholders)} prompt replacements! Either "
1029 + "the prompt text has missing/incorrect tokens for "
1030 + "multi-modal inputs, or there is a problem with your "
1031 + "implementation of merged multi-modal processor for this "
1032 + "model (usually arising from an inconsistency between "
1033 + "`_call_hf_processor` and `_get_prompt_replacements`).")
1034 +
1035 + missing_repl_counts[modality] = item_count - len(placeholders)
1036 +
1037 + return missing_repl_counts
1038 +
1039 def apply(
1040 self,
1041 prompt_text: str,
@@ -1009,56 +1068,69 @@ class BaseMultiModalProcessor(ABC):
1068 hf_processor_mm_kwargs,
1069 mm_kwargs,
1070 )
1012 - prompt_repls = self._bind_prompt_replacements(unbound_prompt_repls)
1071 + mm_prompt_repls = self._bind_and_group_repls(unbound_prompt_repls)
1072
1014 - # If HF processor already inserts placeholder tokens,
1015 - # there is no need for us to insert them
1073 mm_item_counts = mm_items.get_all_counts()
1017 - all_placeholders = self._find_placeholders(prompt_repls, prompt_ids,
1018 - mm_item_counts)
1074 + self._validate_mm_kwargs(mm_kwargs, mm_item_counts)
1075 +
1076 + hf_mm_placeholders = self._find_mm_placeholders(
1077 + mm_prompt_repls,
1078 + prompt_ids,
1079 + mm_item_counts,
1080 + )
1081 +
1082 + if self._always_apply_prompt_replacements():
1083 + mm_missing_repl_counts = mm_item_counts
1084 + mm_missing_repls = dict(mm_prompt_repls)
1085 + else:
1086 + mm_missing_repl_counts = self._validate_mm_placeholders(
1087 + hf_mm_placeholders,
1088 + mm_item_counts,
1089 + allow_missing=True,
1090 + )
1091 +
1092 + mm_missing_repls = dict[str, list[_BoundPromptReplacement]]()
1093 + for modality, missing_repl_count in mm_missing_repl_counts.items():
1094 + if missing_repl_count == 0:
1095 + mm_missing_repls[modality] = []
1096 + elif missing_repl_count == mm_item_counts.get(modality, 0):
1097 + mm_missing_repls[modality] = mm_prompt_repls[modality]
1098 + else:
1099 + raise ValueError("Partial prompt replacement within "
1100 + f"{modality=} is not supported")
1101
1020 - if all_placeholders and not self._always_apply_prompt_replacements():
1102 + # If HF processor already inserts placeholder tokens,
1103 + # there is no need for us to insert them
1104 + if all(len(repls) == 0 for repls in mm_missing_repls.items()):
1105 tokenizer = self._get_tokenizer()
1022 - prompt_text = _decode(tokenizer, prompt_ids)
1106 + prompt_text = decode_tokens(tokenizer, prompt_ids)
1107 + mm_placeholders = hf_mm_placeholders
1108 else:
1109 (
1110 prompt_ids,
1111 prompt_text,
1027 - all_placeholders,
1112 + missing_mm_placeholders,
1113 ) = self._apply_prompt_replacements(
1114 prompt_ids,
1030 - prompt_repls,
1031 - mm_item_counts,
1115 + mm_missing_repls,
1116 + mm_missing_repl_counts,
1117 )
1118
1034 - mm_placeholders = dict[str, list[PlaceholderRange]]()
1035 - err_suffix = ("This suggests a problem with your implementation of "
1036 - "the merged multi-modal processor for this model, "
1037 - "particularly in the `_get_prompt_replacements` method.")
1038 -
1039 - for modality, placeholders in full_groupby_modality(all_placeholders):
1040 - if modality not in mm_items:
1041 - raise AssertionError(
1042 - f"Expected no placeholders for {modality=}, "
1043 - f"but found {placeholders=}. Input items: {mm_items}"
1044 - f"\n{err_suffix}")
1045 -
1046 - if len(placeholders) != len(mm_items[modality]):
1047 - raise AssertionError(
1048 - f"Expected length of {placeholders=} for {modality=} "
1049 - f"to equal that of input items: {mm_items[modality]}"
1050 - f"\n{err_suffix}")
1051 -
1052 - mm_placeholders[modality] = [
1053 - item.to_range() for item in placeholders
1054 - ]
1119 + mm_placeholders = {**hf_mm_placeholders, **missing_mm_placeholders}
1120 +
1121 + self._validate_mm_placeholders(mm_placeholders, mm_item_counts)
1122 +
1123 + mm_placeholder_ranges = {
1124 + modality: [item.to_range() for item in placeholders]
1125 + for modality, placeholders in mm_placeholders.items()
1126 + }
1127
1128 return MultiModalInputsV2(
1129 type="multimodal",
1130 prompt=prompt_text,
1131 prompt_token_ids=prompt_ids,
1132 mm_kwargs=mm_kwargs,
1061 - mm_placeholders=mm_placeholders,
1133 + mm_placeholders=mm_placeholder_ranges,
1134 )
1135
1136 def _get_dummy_audios(
@@ -1092,8 +1164,9 @@ class BaseMultiModalProcessor(ABC):
1164 return [video] * num_videos
1165
1166 @abstractmethod
1095 - def _get_dummy_mm_inputs(
1167 + def _get_dummy_processor_inputs(
1168 self,
1169 + seq_len: int,
1170 mm_counts: Mapping[str, int],
1171 ) -> ProcessorInputs:
1172 """
@@ -1121,12 +1194,25 @@ class BaseMultiModalProcessor(ABC):
1194
1195 return mm_limits
1196
1197 + def _get_dummy_mm_inputs(
1198 + self,
1199 + seq_len: int,
1200 + mm_counts: Mapping[str, int],
1201 + ) -> MultiModalInputsV2:
1202 + processor_inputs = self._get_dummy_processor_inputs(seq_len, mm_counts)
1203 +
1204 + return self.apply(
1205 + prompt_text=processor_inputs.prompt_text,
1206 + mm_data=processor_inputs.mm_data,
1207 + hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1208 + )
1209 +
1210 def get_dummy_data(self, seq_len: int) -> DummyData:
1211 # Avoid circular import
1212 from vllm.sequence import SequenceData
1213
1214 mm_counts = self._get_and_validate_dummy_mm_counts()
1129 - mm_max_tokens_per_item = self.get_mm_max_tokens_per_item()
1215 + mm_max_tokens_per_item = self.get_mm_max_tokens_per_item(seq_len)
1216 if mm_counts.keys() != mm_max_tokens_per_item.keys():
1217 raise AssertionError(
1218 "The keys returned by `get_supported_mm_limits`"
@@ -1134,13 +1220,7 @@ class BaseMultiModalProcessor(ABC):
1220 "returned by `get_mm_max_tokens_per_item` "
1221 f"({set(mm_max_tokens_per_item.keys())})")
1222
1137 - processor_inputs = self._get_dummy_mm_inputs(mm_counts)
1138 - mm_inputs = self.apply(
1139 - prompt_text=processor_inputs.prompt_text,
1140 - mm_data=processor_inputs.mm_data,
1141 - hf_processor_mm_kwargs=processor_inputs.hf_processor_mm_kwargs,
1142 - )
1143 -
1223 + mm_inputs = self._get_dummy_mm_inputs(seq_len, mm_counts)
1224 prompt_token_ids = mm_inputs["prompt_token_ids"]
1225 placeholders_by_modality = mm_inputs["mm_placeholders"]
1226
@@ -1171,6 +1251,12 @@ class BaseMultiModalProcessor(ABC):
1251 "reduce `max_num_seqs`, and/or reduce `mm_counts`.", seq_len,
1252 total_len, total_placeholders_by_modality)
1253
1254 + return DummyData(
1255 + seq_data=SequenceData.from_prompt_token_counts((0, seq_len)),
1256 + multi_modal_data=None,
1257 + multi_modal_placeholders=None,
1258 + )
1259 +
1260 prompt_token_ids.extend([0] * (seq_len - len(prompt_token_ids)))
1261
1262 return DummyData(
vllm/multimodal/registry.py
+2 -1
@@ -223,7 +223,8 @@ class MultiModalRegistry:
223 if self.has_processor(model_config):
224 tokenizer = cached_get_tokenizer(model_config.tokenizer)
225 processor = self.create_processor(model_config, tokenizer)
226 - return processor.get_mm_max_tokens_per_item()
226 + seq_len = model_config.max_model_len
227 + return processor.get_mm_max_tokens_per_item(seq_len)
228
229 return {
230 key: plugin.get_max_multimodal_tokens(model_config)
vllm/transformers_utils/tokenizer.py
+13
@@ -21,6 +21,19 @@ AnyTokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast,
21 MistralTokenizer]
22
23
24 +def decode_tokens(
25 + tokenizer: AnyTokenizer,
26 + token_ids: list[int],
27 + *,
28 + skip_special_tokens: bool = False,
29 +) -> str:
30 + """
31 + Backend-agnostic equivalent of HF's
32 + :code:`tokenizer.decode(token_ids, skip_special_tokens=...)`.
33 + """
34 + return tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
35 +
36 +
37 def encode_tokens(
38 tokenizer: AnyTokenizer,
39 text: str,