@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75f81750f3

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 08:30 UTC

[VLM] Initialize video input support for InternVL models (#18499)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed May 25, 2025 at 12:51 UTC 75f81750f3a9071b23f7f2d5c9f9f1c2cd0091b1

10 files changed +596 -62

docs/models/supported_models.md

+4 -1

@@ -527,7 +527,7 @@ Specified using `--task generate`.

527 | `GraniteSpeechForConditionalGeneration` | Granite Speech | T + A | `ibm-granite/granite-speech-3.3-8b` | ✅︎ | ✅︎ | ✅︎ |

528 | `H2OVLChatModel` | H2OVL | T + IE+ | `h2oai/h2ovl-mississippi-800m`, `h2oai/h2ovl-mississippi-2b`, etc. | ✅︎ | ✅︎\* | |

529 | `Idefics3ForConditionalGeneration` | Idefics3 | T + I | `HuggingFaceM4/Idefics3-8B-Llama3` etc. | ✅︎ | ✅︎ | |

530 -| `InternVLChatModel` | InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + IE+ | `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ | |

530 +| `InternVLChatModel` | InternVL 3.0, InternVideo 2.5, InternVL 2.5, Mono-InternVL, InternVL 2.0 | T + IE+ + (VE+) | `OpenGVLab/InternVL3-9B`, `OpenGVLab/InternVideo2_5_Chat_8B`, `OpenGVLab/InternVL2_5-4B`, `OpenGVLab/Mono-InternVL-2B`, `OpenGVLab/InternVL2-4B`, etc. | ✅︎ | ✅︎ | |

531 | `KimiVLForConditionalGeneration` | Kimi-VL-A3B-Instruct, Kimi-VL-A3B-Thinking | T + I+ | `moonshotai/Kimi-VL-A3B-Instruct`, `moonshotai/Kimi-VL-A3B-Thinking` | ✅︎ | | |

532 | `Llama4ForConditionalGeneration` | Llama 4 | T + I+ | `meta-llama/Llama-4-Scout-17B-16E-Instruct`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8`, `meta-llama/Llama-4-Maverick-17B-128E-Instruct`, etc. | ✅︎ | ✅︎ | |

533 | `LlavaForConditionalGeneration` | LLaVA-1.5 | T + IE+ | `llava-hf/llava-1.5-7b-hf`, `TIGER-Lab/Mantis-8B-siglip-llama3` (see note), etc. | ✅︎ | ✅︎ | |

@@ -577,6 +577,9 @@ Specified using `--task generate`.

577

578 This limitation exists because the model's mixed attention pattern (bidirectional for images, causal otherwise) is not yet supported by vLLM's attention backends.

579

580 +!!! note

581 + Only `InternVLChatModel` with Qwen2.5 text backbone (`OpenGVLab/InternVL3-2B`, `OpenGVLab/InternVL2.5-1B` etc) has video inputs support currently.

582 +

583 !!! note

584 `h2oai/h2ovl-mississippi-2b` will be available in V1 once we support head size 80.

585

examples/offline_inference/vision_language.py

+11 -4

@@ -330,22 +330,26 @@ def run_smolvlm(questions: list[str], modality: str) -> ModelRequestData:

330

331 # InternVL

332 def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

333 - assert modality == "image"

333

335 - model_name = "OpenGVLab/InternVL2-2B"

334 + model_name = "OpenGVLab/InternVL3-2B"

335

336 engine_args = EngineArgs(

337 model=model_name,

338 trust_remote_code=True,

340 - max_model_len=4096,

339 + max_model_len=8192,

340 limit_mm_per_prompt={modality: 1},

341 )

342

343 + if modality == "image":

344 + placeholder = "<image>"

345 + elif modality == "video":

346 + placeholder = "<video>"

347 +

348 tokenizer = AutoTokenizer.from_pretrained(model_name,

349 trust_remote_code=True)

350 messages = [[{

351 'role': 'user',

348 - 'content': f"<image>\n{question}"

352 + 'content': f"{placeholder}\n{question}"

353 }] for question in questions]

354 prompts = tokenizer.apply_chat_template(messages,

355 tokenize=False,

@@ -357,6 +361,9 @@ def run_internvl(questions: list[str], modality: str) -> ModelRequestData:

361 # https://huggingface.co/OpenGVLab/InternVL2-2B/blob/main/conversation.py

362 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]

363 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]

364 + stop_token_ids = [

365 + token_id for token_id in stop_token_ids if token_id is not None

366 + ]

367

368 return ModelRequestData(

369 engine_args=engine_args,

tests/models/multimodal/generation/test_common.py

+11

@@ -349,6 +349,17 @@ VLM_TEST_SETTINGS = {

349 use_tokenizer_eos=True,

350 patch_hf_runner=model_utils.internvl_patch_hf_runner,

351 ),

352 + "intern_vl-video": VLMTestInfo(

353 + models=[

354 + "OpenGVLab/InternVL3-1B",

355 + ],

356 + test_type=VLMTestType.VIDEO,

357 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501

358 + video_idx_to_prompt=lambda idx: "<video>",

359 + max_model_len=8192,

360 + use_tokenizer_eos=True,

361 + patch_hf_runner=model_utils.internvl_patch_hf_runner,

362 + ),

363 "kimi_vl": VLMTestInfo(

364 models=["moonshotai/Kimi-VL-A3B-Instruct"],

365 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),

tests/models/multimodal/generation/vlm_utils/model_utils.py

+66 -20

@@ -7,6 +7,8 @@ import types

7 from pathlib import PosixPath

8 from typing import Optional, Union

10 +import numpy as np

11 +import numpy.typing as npt

12 import regex as re

13 import torch

14 from PIL.Image import Image

@@ -495,30 +497,74 @@ def internvl_patch_hf_runner(hf_model: HfRunner) -> HfRunner:

497 self.max_num = self.config.max_dynamic_patch

498 self.image_size = self.vision_config.image_size

499

498 - def __call__(self, text: str, images: Union[Image, list[Image]],

499 - **kwargs):

500 + def __call__(

501 + self,

502 + text: str,

503 + images: Union[Image, list[Image]] = None,

504 + videos: Union[npt.NDArray, list[npt.NDArray]] = None,

505 + **kwargs,

506 + ):

507 from vllm.model_executor.models.internvl import (

508 IMG_CONTEXT, IMG_END, IMG_START,

502 - image_to_pixel_values_internvl)

509 + image_to_pixel_values_internvl, video_to_pixel_values_internvl)

510 images = [images] if isinstance(images, Image) else images

504 - pixel_values = [

505 - image_to_pixel_values_internvl(

506 - image,

507 - input_size=self.image_size,

508 - min_num=self.min_num,

509 - max_num=self.max_num,

510 - use_thumbnail=self.use_thumbnail,

511 - ) for image in images

512 - ]

513 - num_patches_list = [

514 - pixel_value.shape[0] for pixel_value in pixel_values

515 - ]

511 + videos = [videos] if isinstance(videos, np.ndarray) else videos

512 + if images is not None:

513 + pixel_values_images = [

514 + image_to_pixel_values_internvl(

515 + image,

516 + input_size=self.image_size,

517 + min_num=self.min_num,

518 + max_num=self.max_num,

519 + use_thumbnail=self.use_thumbnail,

520 + ) for image in images

521 + ]

522 + num_patches_images = [

523 + pixel_value.shape[0] for pixel_value in pixel_values_images

524 + ]

525 + else:

526 + pixel_values_images, num_patches_images = [], []

527 +

528 + if videos is not None:

529 + pixel_values_videos = [

530 + video_to_pixel_values_internvl(

531 + video,

532 + input_size=self.image_size,

533 + min_num=1,

534 + max_num=1,

535 + use_thumbnail=False,

536 + ) for video in videos

537 + ]

538 + num_patches_videos = [

539 + pixel_value.shape[0] for pixel_value in pixel_values_videos

540 + ]

541 + else:

542 + pixel_values_videos, num_patches_videos = [], []

543 +

544 + pixel_values = []

545 + while ("<image>" in text) or ("<video>" in text):

546 + image_index = text.find("<image>")

547 + video_index = text.find("<video>")

548 + if image_index == -1 or (video_index > -1

549 + and video_index < image_index):

550 + num_patches = num_patches_videos.pop(0)

551 + pixel_values.append(pixel_values_videos.pop(0))

552 + context_tokens = IMG_START + \

553 + IMG_CONTEXT * self.num_image_token + IMG_END

554 + video_tokens = ''.join([

555 + f'Frame{i+1}: {context_tokens}'

556 + for i in range(num_patches)

557 + ])

558 + text = text.replace('<video>', video_tokens, 1)

559 + else:

560 + num_patches = num_patches_images.pop(0)

561 + pixel_values.append(pixel_values_images.pop(0))

562 + context_tokens = IMG_CONTEXT * self.num_image_token \

563 + * num_patches

564 + image_tokens = IMG_START + context_tokens + IMG_END

565 + text = text.replace('<image>', image_tokens, 1)

566 pixel_values = torch.cat(pixel_values, dim=0)

517 - for num_patches in num_patches_list:

518 - context_tokens = IMG_CONTEXT * self.num_image_token \

519 - * num_patches

520 - image_tokens = IMG_START + context_tokens + IMG_END

521 - text = text.replace('<image>', image_tokens, 1)

567 +

568 prompt = self.tokenizer(text, return_tensors="pt")

569 prompt.update({"pixel_values": pixel_values})

570 return prompt

tests/models/multimodal/processing/test_common.py

@@ -258,6 +258,7 @@ def _test_processing_correctness_mistral(

258 "ibm-granite/granite-speech-3.3-8b",

259 "h2oai/h2ovl-mississippi-800m",

260 "OpenGVLab/InternVL2-1B",

261 + "OpenGVLab/InternVL3-1B",

262 "HuggingFaceM4/Idefics3-8B-Llama3",

263 "HuggingFaceTB/SmolVLM2-2.2B-Instruct",

264 "moonshotai/Kimi-VL-A3B-Instruct",

tests/models/registry.py

+2 -1

@@ -334,7 +334,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {

334 max_transformers_version="4.48", # noqa: E501

335 transformers_version_reason="HF model is not compatible."), # noqa: E501

336 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",

337 - extras={"2B": "OpenGVLab/InternVL2-2B"}, # noqa: E501

337 + extras={"2B": "OpenGVLab/InternVL2-2B",

338 + "3.0": "OpenGVLab/InternVL3-1B"}, # noqa: E501

339 trust_remote_code=True),

340 "Idefics3ForConditionalGeneration": _HfExamplesInfo("HuggingFaceM4/Idefics3-8B-Llama3", # noqa: E501

341 {"tiny": "HuggingFaceTB/SmolVLM-256M-Instruct"}), # noqa: E501

vllm/entrypoints/chat_utils.py

@@ -556,6 +556,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):

556 return "(<audio>./</audio>)"

557 raise TypeError(f"Unknown model type: {model_type}")

558 elif modality == "video":

559 + if model_type == "internvl_chat":

560 + return "<video>"

561 if model_type in ("qwen2_vl", "qwen2_5_vl"):

562 return "<|vision_start|><|video_pad|><|vision_end|>"

563 if model_type == "qwen2_5_omni":

vllm/model_executor/models/h2ovl.py

+6 -5

@@ -25,9 +25,10 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer

26 from .intern_vit import InternVisionModel

27 from .internvl import (IMG_CONTEXT, IMG_END, IMG_START,

28 + BaseInternVLDummyInputsBuilder,

29 + BaseInternVLMultiModalProcessor,

30 BaseInternVLProcessingInfo, BaseInternVLProcessor,

29 - InternVLChatModel, InternVLDummyInputsBuilder,

30 - InternVLMultiModalProcessor, build_transform,

31 + InternVLChatModel, build_transform,

32 find_closest_aspect_ratio, get_internvl_target_ratios)

@@ -430,8 +431,8 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):

431 )

432

433

433 -class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]

434 - ):

434 +class H2OVLMultiModalProcessor(

435 + BaseInternVLMultiModalProcessor[H2OVLProcessingInfo]):

436

437 def _get_prompt_updates(

438 self,

@@ -514,7 +515,7 @@ class H2OVLMultiModalProcessor(InternVLMultiModalProcessor[H2OVLProcessingInfo]

515 @MULTIMODAL_REGISTRY.register_processor(

516 H2OVLMultiModalProcessor,

517 info=H2OVLProcessingInfo,

517 - dummy_inputs=InternVLDummyInputsBuilder)

518 + dummy_inputs=BaseInternVLDummyInputsBuilder)

519 class H2OVLChatModel(InternVLChatModel):

520

521 def _init_vision_model(

vllm/model_executor/models/internvl.py

+485 -26

@@ -8,8 +8,9 @@

8 # --------------------------------------------------------

9 from abc import ABC, abstractmethod

10 from collections.abc import Iterable, Mapping, Sequence

11 -from typing import Literal, Optional, TypedDict, TypeVar, Union

11 +from typing import Any, Literal, Optional, TypedDict, TypeVar, Union

13 +import numpy.typing as npt

14 import torch

15 import torch.nn as nn

16 import torchvision.transforms as T

@@ -74,6 +75,33 @@ InternVLImageInputs = Union[InternVLImagePixelInputs,

75 InternVLImageEmbeddingInputs]

78 +class InternVLVideoPixelInputs(TypedDict):

79 + type: Literal["pixel_values_videos"]

80 + pixel_values_flat: torch.Tensor

81 + """

82 + Shape:

83 + `(batch_size * num_video * num_frames, num_channels, height, width)`

84 + """

85 +

86 + num_patches: torch.Tensor

87 + """Shape: `(batch_size * num_images)`"""

88 +

89 +

90 +class InternVLVideoEmbeddingInputs(TypedDict):

91 + type: Literal["video_embeds"]

92 + data: Union[torch.Tensor, list[torch.Tensor]]

93 + """

94 + A tensor of shape `(num_videos, total_video_feature_size, hidden_size)`

95 + or a list of tensors of shape `(total_video_feature_size, hidden_size)`

96 +

97 + `hidden_size` must match the hidden size of language model backbone.

98 + """

99 +

100 +

101 +InternVLVideoInputs = Union[InternVLVideoPixelInputs,

102 + InternVLVideoEmbeddingInputs]

103 +

104 +

105 # adapted from https://huggingface.co/OpenGVLab/InternVL2-1B

106 def build_transform(input_size: int):

107 MEAN, STD = IMAGENET_MEAN, IMAGENET_STD

@@ -231,6 +259,33 @@ def image_to_pixel_values_internvl(

259 return pixel_values

260

261

262 +# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B

263 +def video_to_pixel_values_internvl(

264 + video: npt.NDArray,

265 + *,

266 + input_size: int,

267 + min_num: int,

268 + max_num: int,

269 + use_thumbnail: bool,

270 +) -> torch.Tensor:

271 + target_ratios = get_internvl_target_ratios(min_num, max_num)

272 +

273 + transform = build_transform(input_size=input_size)

274 + frames_list = list[Image.Image]()

275 + for frame in video:

276 + pil_frame = dynamic_preprocess_internvl(

277 + Image.fromarray(frame, mode="RGB"),

278 + target_ratios=target_ratios,

279 + image_size=input_size,

280 + use_thumbnail=use_thumbnail,

281 + )

282 + assert len(pil_frame) == 1

283 + frames_list.extend(pil_frame)

284 +

285 + pixel_values = torch.stack([transform(image) for image in frames_list])

286 + return pixel_values

287 +

288 +

289 class BaseInternVLProcessor(ABC):

290 """

291 This model doesn't define its own HF processor,

@@ -375,24 +430,14 @@ class BaseInternVLProcessor(ABC):

430 ) for image in images

431 ]

432

378 - def __call__(

433 + def _preprocess_image(

434 self,

380 - text: Optional[Union[str, list[str]]] = None,

381 - images: Optional[Union[Image.Image, list[Image.Image]]] = None,

435 + text: list[str],

436 + images: list[Image.Image],

437 min_dynamic_patch: Optional[int] = None,

438 max_dynamic_patch: Optional[int] = None,

439 dynamic_image_size: Optional[bool] = None,

385 - return_tensors: Optional[Union[str, TensorType]] = None,

386 - ) -> Mapping[str, NestedTensors]:

387 - if text is None:

388 - text = []

389 - if not isinstance(text, list):

390 - text = [text]

391 - if images is None:

392 - images = []

393 - if not isinstance(images, list):

394 - images = [images]

395 -

440 + ) -> tuple[list[str], dict[str, torch.Tensor]]:

441 if len(images) == 0:

442 image_inputs = {}

443 else:

@@ -415,6 +460,34 @@ class BaseInternVLProcessor(ABC):

460

461 image_repl = self.get_image_repl(feature_size, num_patches)

462 text = [t.replace('<image>', image_repl.full, 1) for t in text]

463 + return text, image_inputs

464 +

465 + def _make_batch_input(self,

466 + input_item: Optional[Union[Any, list[Any]]] = None):

467 + if input_item is None:

468 + input_item = []

469 + if not isinstance(input_item, list):

470 + input_item = [input_item]

471 + return input_item

472 +

473 + def __call__(

474 + self,

475 + text: Optional[Union[str, list[str]]] = None,

476 + images: Optional[Union[Image.Image, list[Image.Image]]] = None,

477 + min_dynamic_patch: Optional[int] = None,

478 + max_dynamic_patch: Optional[int] = None,

479 + dynamic_image_size: Optional[bool] = None,

480 + return_tensors: Optional[Union[str, TensorType]] = None,

481 + ) -> Mapping[str, NestedTensors]:

482 + text, images = [self._make_batch_input(x) for x in (text, images)]

483 +

484 + text, image_inputs = self._preprocess_image(

485 + text=text,

486 + images=images,

487 + min_dynamic_patch=min_dynamic_patch,

488 + max_dynamic_patch=max_dynamic_patch,

489 + dynamic_image_size=dynamic_image_size,

490 + )

491

492 text_inputs = self.tokenizer(text)

493

@@ -425,11 +498,133 @@ class BaseInternVLProcessor(ABC):

498

499

500 class InternVLProcessor(BaseInternVLProcessor):

501 + """

502 + HF Processor for InternVLChatModel with extended video processing logic.

503 +

504 + Code for video processing is adapted from video example:

505 + https://huggingface.co/OpenGVLab/InternVL3-1B#inference-with-transformers

506 + """

507 +

508 + def __init__(

509 + self,

510 + config: PretrainedConfig,

511 + tokenizer: AnyTokenizer,

512 + *,

513 + min_dynamic_patch: Optional[int] = None,

514 + max_dynamic_patch: Optional[int] = None,

515 + dynamic_image_size: Optional[bool] = None,

516 + video_token: Optional[str] = None,

517 + ) -> None:

518 + super().__init__(

519 + config=config,

520 + tokenizer=tokenizer,

521 + min_dynamic_patch=min_dynamic_patch,

522 + max_dynamic_patch=max_dynamic_patch,

523 + dynamic_image_size=dynamic_image_size,

524 + )

525 + # add extra video token for video processing

526 + self.video_token = video_token

527

528 @property

529 def image_token_id(self) -> int:

530 return self.tokenizer.get_vocab()[IMG_CONTEXT]

531

532 + @property

533 + def video_token_id(self) -> Optional[int]:

534 + if self.video_token is None:

535 + return None

536 + return self.tokenizer.get_vocab().get(self.video_token, None)

537 +

538 + @property

539 + def supports_video(self) -> bool:

540 + return self.video_token_id is not None

541 +

542 + def _videos_to_pixel_values_lst(

543 + self,

544 + videos: list[npt.NDArray],

545 + dynamic_image_size: Optional[bool] = None,

546 + ) -> list[torch.Tensor]:

547 + min_num, max_num = self.resolve_min_max_num(

548 + min_dynamic_patch=1,

549 + max_dynamic_patch=1,

550 + dynamic_image_size=dynamic_image_size,

551 + use_thumbnail=False, # Applied in image_to_pixel_values

552 + )

553 +

554 + return [

555 + video_to_pixel_values_internvl(

556 + video,

557 + input_size=self.image_size,

558 + min_num=min_num,

559 + max_num=max_num,

560 + use_thumbnail=False,

561 + ) for video in videos

562 + ]

563 +

564 + def _preprocess_video(

565 + self,

566 + text: list[str],

567 + videos: list[npt.NDArray],

568 + dynamic_image_size: Optional[bool] = None,

569 + ):

570 + if len(videos) == 0 or not self.supports_video:

571 + video_inputs = {}

572 + else:

573 + pixel_values_lst_video = self._videos_to_pixel_values_lst(

574 + videos,

575 + dynamic_image_size=dynamic_image_size,

576 + )

577 + video_inputs: dict[str, NestedTensors] = {

578 + "pixel_values_flat_video":

579 + torch.cat(pixel_values_lst_video),

580 + "video_num_patches":

581 + torch.tensor([len(item) for item in pixel_values_lst_video]),

582 + }

583 +

584 + for pixel_values in pixel_values_lst_video:

585 + num_patches = pixel_values.shape[0]

586 +

587 + video_repl = self.get_video_repl(self.num_image_token,

588 + num_patches, self.video_token)

589 + text = [t.replace('<video>', video_repl.full, 1) for t in text]

590 + return text, video_inputs

591 +

592 + def __call__(

593 + self,

594 + text: Optional[Union[str, list[str]]] = None,

595 + images: Optional[Union[Image.Image, list[Image.Image]]] = None,

596 + videos: Optional[Union[npt.NDArray, list[npt.NDArray]]] = None,

597 + min_dynamic_patch: Optional[int] = None,

598 + max_dynamic_patch: Optional[int] = None,

599 + dynamic_image_size: Optional[bool] = None,

600 + return_tensors: Optional[Union[str, TensorType]] = None,

601 + ) -> Mapping[str, NestedTensors]:

602 + text, images, videos = [

603 + self._make_batch_input(x) for x in (text, images, videos)

604 + ]

605 +

606 + text, image_inputs = self._preprocess_image(

607 + text=text,

608 + images=images,

609 + min_dynamic_patch=min_dynamic_patch,

610 + max_dynamic_patch=max_dynamic_patch,

611 + dynamic_image_size=dynamic_image_size,

612 + )

613 +

614 + text, video_inputs = self._preprocess_video(

615 + text=text,

616 + videos=videos,

617 + dynamic_image_size=dynamic_image_size,

618 + )

619 +

620 + text_inputs = self.tokenizer(text)

621 +

622 + return {

623 + **BatchEncoding(text_inputs, tensor_type=return_tensors),

624 + **image_inputs,

625 + **video_inputs,

626 + }

627 +

628 def get_image_repl(

629 self,

630 feature_size: int,

@@ -440,8 +635,24 @@ class InternVLProcessor(BaseInternVLProcessor):

635

636 return PromptUpdateDetails.select_text(repl_full, IMG_CONTEXT)

637

638 + def get_video_repl(

639 + self,

640 + feature_size: int,

641 + num_patches: Optional[int] = None,

642 + video_context_token: str = IMG_CONTEXT,

643 + ) -> PromptUpdateDetails[str]:

644 + repl_features = video_context_token * self.num_image_token

645 + repl_features_with_sep = IMG_START + repl_features + IMG_END

646 + # num_patches is equal to num_frames

647 + repl_full = ''.join([

648 + f'Frame{i+1}: {repl_features_with_sep}' for i in range(num_patches)

649 + ])

650 +

651 + return PromptUpdateDetails.select_text(repl_full, video_context_token)

652 +

653

654 class BaseInternVLProcessingInfo(BaseProcessingInfo):

655 + """Basic image-only ProcessingInfo for InternVL-style models."""

656

657 @abstractmethod

658 def get_hf_processor(

@@ -497,11 +708,22 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):

708

709 return largest_feature_pinpoint

710

711 + def get_max_image_tokens(self) -> int:

712 + processor = self.get_hf_processor()

713 + target_width, target_height = self.get_image_size_with_most_features()

714 +

715 + return self.get_num_image_tokens(

716 + image_width=target_width,

717 + image_height=target_height,

718 + processor=processor,

719 + )

720 +

721

722 _I = TypeVar("_I", bound=BaseInternVLProcessingInfo)

723

724

504 -class InternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

725 +class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

726 + """Basic image-only DummyInputsBuilder for InternVL-style models."""

727

728 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

729 num_images = mm_counts.get("image", 0)

@@ -525,7 +747,8 @@ class InternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):

747 }

748

749

528 -class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

750 +class BaseInternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

751 + """ Basic image-only MultiModalProcessor for InternVL-style models."""

752

753 def _call_hf_processor(

754 self,

@@ -614,6 +837,38 @@ class InternVLMultiModalProcessor(BaseMultiModalProcessor[_I]):

837

838

839 class InternVLProcessingInfo(BaseInternVLProcessingInfo):

840 + """InternVL ProcessingInfo extended for video processing"""

841 +

842 + @property

843 + def supports_video(self):

844 + return self.get_hf_processor().supports_video

845 +

846 + def get_supported_mm_limits(self):

847 + video_limit = {"video": None} if self.supports_video else {}

848 + return {**super().get_supported_mm_limits(), **video_limit}

849 +

850 + def get_video_token(self) -> Optional[str]:

851 + text_model_type = self.get_hf_config().get_text_config().model_type

852 + if text_model_type == "qwen2":

853 + return "<|video_pad|>"

854 + return None

855 +

856 + def get_num_frames_with_most_features(

857 + self,

858 + seq_len: int,

859 + mm_counts: Mapping[str, int],

860 + ) -> int:

861 + max_images = mm_counts.get("image", 0)

862 + max_videos = mm_counts.get("video", 0)

863 +

864 + processor = self.get_hf_processor()

865 +

866 + max_image_tokens = self.get_max_image_tokens() * max_images

867 + max_total_frames = (seq_len -

868 + max_image_tokens) // processor.num_image_token

869 + max_frames_per_video = max_total_frames // max(max_videos, 1)

870 +

871 + return max(max_frames_per_video, 1)

872

873 def get_hf_processor(

874 self,

@@ -630,6 +885,8 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):

885 if dynamic_image_size is not None:

886 kwargs["dynamic_image_size"] = dynamic_image_size

887

888 + kwargs["video_token"] = self.get_video_token()

889 +

890 return self.ctx.init_processor(

891 InternVLProcessor,

892 config=self.get_hf_config(),

@@ -638,6 +895,121 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):

895 )

896

897

898 +class InternVLDummyInputsBuilder(

899 + BaseInternVLDummyInputsBuilder[InternVLProcessingInfo]):

900 + """InternVL DummyInputsBuilder extended for video support"""

901 +

902 + def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

903 + num_videos = mm_counts.get("video", 0)

904 +

905 + return super().get_dummy_text(mm_counts) + "<video>" * num_videos

906 +

907 + def get_dummy_mm_data(

908 + self,

909 + seq_len: int,

910 + mm_counts: Mapping[str, int],

911 + ) -> MultiModalDataDict:

912 + dummy_image = super().get_dummy_mm_data(seq_len=seq_len,

913 + mm_counts=mm_counts)

914 + if self.info.supports_video:

915 + config = self.info.get_hf_config()

916 + image_size: int = config.vision_config.image_size

917 + target_num_frames = \

918 + self.info.get_num_frames_with_most_features(seq_len, mm_counts)

919 + num_videos = mm_counts.get("video", 0)

920 + dummy_video = {

921 + "video":

922 + self._get_dummy_videos(width=image_size,

923 + height=image_size,

924 + num_frames=target_num_frames,

925 + num_videos=num_videos)

926 + }

927 + else:

928 + dummy_video = {}

929 + return {**dummy_image, **dummy_video}

930 +

931 +

932 +class InternVLMultiModalProcessor(

933 + BaseInternVLMultiModalProcessor[InternVLProcessingInfo]):

934 + """InternVL MultiModalProcessor extended for video support"""

935 +

936 + def _call_hf_processor(

937 + self,

938 + prompt: str,

939 + mm_data: Mapping[str, object],

940 + mm_kwargs: Mapping[str, object],

941 + ) -> Mapping[str, NestedTensors]:

942 + processed_outputs = super()._call_hf_processor(prompt, mm_data,

943 + mm_kwargs)

944 +

945 + hf_processor = self.info.get_hf_processor(**mm_kwargs)

946 + if self.info.supports_video and (

947 + video_token_id := hf_processor.video_token_id) is not None:

948 + processed_outputs["video_token_id"] = torch.tensor(video_token_id)

949 + return processed_outputs

950 +

951 + def _get_mm_fields_config(

952 + self,

953 + hf_inputs: Mapping[str, NestedTensors],

954 + hf_processor_mm_kwargs: Mapping[str, object],

955 + ) -> Mapping[str, MultiModalFieldConfig]:

956 + image_fields = super()._get_mm_fields_config(hf_inputs,

957 + hf_processor_mm_kwargs)

958 + if self.info.supports_video:

959 + video_num_patches = hf_inputs.get("video_num_patches",

960 + torch.empty(0))

961 + num_videos = len(video_num_patches)

962 + video_fields = dict(

963 + pixel_values_flat_video=MultiModalFieldConfig.flat_from_sizes(

964 + "video", video_num_patches),

965 + video_num_patches=MultiModalFieldConfig.batched("video"),

966 + video_token_id=MultiModalFieldConfig.shared(

967 + "video", num_videos),

968 + )

969 + else:

970 + video_fields = {}

971 +

972 + return image_fields | video_fields

973 +

974 + def _get_prompt_updates(

975 + self,

976 + mm_items: MultiModalDataItems,

977 + hf_processor_mm_kwargs: Mapping[str, object],

978 + out_mm_kwargs: MultiModalKwargs,

979 + ) -> Sequence[PromptUpdate]:

980 + prompt_repl: list[PromptUpdate] = super()._get_prompt_updates(

981 + mm_items, hf_processor_mm_kwargs, out_mm_kwargs)

982 +

983 + hf_processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)

984 +

985 + if "video_num_patches" in out_mm_kwargs:

986 + video_num_patches = out_mm_kwargs["video_num_patches"]

987 + assert isinstance(video_num_patches, torch.Tensor)

988 + video_num_patches = video_num_patches.tolist()

989 + else:

990 + video_num_patches = []

991 +

992 + def get_video_replacement_internvl(item_idx: int):

993 + feature_size = hf_processor.num_image_token

994 + num_patches = video_num_patches[item_idx]

995 + if num_patches is not None:

996 + assert isinstance(num_patches, int)

997 +

998 + return hf_processor.get_video_repl(

999 + feature_size,

1000 + num_patches,

1001 + video_context_token=hf_processor.video_token)

1002 +

1003 + if self.info.supports_video:

1004 + prompt_repl.append(

1005 + PromptReplacement(

1006 + modality="video",

1007 + target="<video>",

1008 + replacement=get_video_replacement_internvl,

1009 + ))

1010 + return prompt_repl

1011 +

1012 +

1013 @MULTIMODAL_REGISTRY.register_processor(

1014 InternVLMultiModalProcessor,

1015 info=InternVLProcessingInfo,

@@ -681,6 +1053,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1053 self.mlp1 = self._init_mlp1(config)

1054

1055 self.img_context_token_id = None

1056 + self.video_context_token_id = None

1057 +

1058 self.visual_token_mask = None

1059 self.make_empty_intermediate_tensors = (

1060 self.language_model.make_empty_intermediate_tensors)

@@ -825,10 +1199,55 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1199

1200 raise AssertionError("This line should be unreachable.")

1201

1202 + def _parse_and_validate_video_input(

1203 + self, **kwargs: object) -> Optional[InternVLVideoPixelInputs]:

1204 + pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)

1205 + video_num_patches = kwargs.pop("video_num_patches", None)

1206 + video_embeds = kwargs.pop("image_embeds", None)

1207 +

1208 + if pixel_values_flat_video is None and video_embeds is None:

1209 + return None

1210 +

1211 + if video_embeds is not None:

1212 + if not isinstance(video_embeds, (torch.Tensor, list)):

1213 + raise ValueError("Incorrect type of video embeddings. "

1214 + f"Got type: {type(video_embeds)}")

1215 +

1216 + return InternVLImageEmbeddingInputs(

1217 + type="video_embeds",

1218 + data=flatten_bn(video_embeds),

1219 + )

1220 +

1221 + video_token_id = kwargs["video_token_id"]

1222 + assert isinstance(video_token_id, torch.Tensor)

1223 + self.video_context_token_id = video_token_id.flatten().unique().item()

1224 +

1225 + if pixel_values_flat_video is not None:

1226 + if not isinstance(pixel_values_flat_video, (torch.Tensor, list)):

1227 + raise ValueError("Incorrect type of pixel values. "

1228 + f"Got type: {type(pixel_values_flat_video)}")

1229 +

1230 + if not isinstance(video_num_patches, (torch.Tensor, list)):

1231 + raise ValueError("Incorrect type of image_num_patches. "

1232 + f"Got type: {type(video_num_patches)}")

1233 +

1234 + pixel_values_flat_video = flatten_bn(pixel_values_flat_video,

1235 + concat=True)

1236 + video_num_patches = flatten_bn(video_num_patches, concat=True)

1237 +

1238 + return InternVLVideoPixelInputs(

1239 + type="pixel_values_videos",

1240 + pixel_values_flat=self._validate_pixel_values(

1241 + pixel_values_flat_video),

1242 + num_patches=video_num_patches,

1243 + )

1244 +

1245 + raise AssertionError("This line should be unreachable.")

1246 +

1247 def _process_image_input(

1248 self,

830 - image_input: InternVLImageInputs,

831 - ) -> Union[torch.Tensor, list[torch.Tensor], tuple[torch.Tensor, ...]]:

1249 + image_input: Union[InternVLImageInputs, InternVLVideoPixelInputs],

1250 + ) -> tuple[torch.Tensor, ...]:

1251 if image_input["type"] == "image_embeds":

1252 return image_input["data"]

1253

@@ -840,8 +1259,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1259

1260 # Only one image in the current batch

1261 if len(num_patches) == 1:

843 - return image_embeds.view(

844 - -1, self.config.text_config.hidden_size).unsqueeze(0)

1262 + return (image_embeds.view(-1,

1263 + self.config.text_config.hidden_size), )

1264

1265 # NOTE: Image embeddings are split into separate tensors for each image

1266 # by the size of each embedding.

@@ -853,8 +1272,26 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1272 ]

1273 return image_embeds.split(image_feature_sizes)

1274

1275 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

1276 + modalities = {}

1277 +

1278 + # Preserve the order of modalities if there are multiple of them

1279 + # from the order of kwargs.

1280 + for input_key in kwargs:

1281 + if input_key in ("pixel_values_flat",

1282 + "image_embeds") and "images" not in modalities:

1283 + modalities["images"] = self._parse_and_validate_image_input(

1284 + **kwargs)

1285 + if input_key in ("pixel_values_flat_video",

1286 + ) and "videos" not in modalities:

1287 + modalities["videos"] = self._parse_and_validate_video_input(

1288 + **kwargs)

1289 +

1290 + return modalities

1291 +

1292 def _set_visual_token_mask(self, input_ids: torch.Tensor) -> None:

1293 if self.is_mono:

1294 + assert self.img_context_token_id is not None

1295 self.visual_token_mask = (

1296 input_ids == self.img_context_token_id).reshape(-1, 1)

1297 else:

@@ -865,11 +1302,28 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1302

1303 def get_multimodal_embeddings(

1304 self, **kwargs: object) -> Optional[MultiModalEmbeddings]:

868 - image_input = self._parse_and_validate_image_input(**kwargs)

869 - if image_input is None:

1305 +

1306 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)

1307 + if not modalities:

1308 return None

1309

872 - return self._process_image_input(image_input)

1310 + # The result multimodal_embeddings is tuple of tensors, with each

1311 + # tensor correspoending to a multimodal data item (image or video).

1312 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()

1313 +

1314 + # NOTE: It is important to iterate over the keys in this dictionary

1315 + # to preserve the order of the modalities.

1316 + for modality in modalities:

1317 + if modality == "images":

1318 + image_input = modalities["images"]

1319 + vision_embeddings = self._process_image_input(image_input)

1320 + multimodal_embeddings += vision_embeddings

1321 + if modality == "videos":

1322 + video_input = modalities["videos"]

1323 + video_embeddings = self._process_image_input(video_input)

1324 + multimodal_embeddings += video_embeddings

1325 +

1326 + return multimodal_embeddings

1327

1328 def get_input_embeddings(

1329 self,

@@ -878,13 +1332,18 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP):

1332 ) -> torch.Tensor:

1333 inputs_embeds = self.language_model.get_input_embeddings(input_ids)

1334 if multimodal_embeddings is not None:

881 - assert self.img_context_token_id is not None

1335 + context_token_ids = [

1336 + token_id for token_id in (self.img_context_token_id,

1337 + self.video_context_token_id)

1338 + if token_id is not None

1339 + ]

1340 + assert len(context_token_ids) >= 1

1341 self._set_visual_token_mask(input_ids)

1342 inputs_embeds = merge_multimodal_embeddings(

1343 input_ids,

1344 inputs_embeds,

1345 multimodal_embeddings,

887 - self.img_context_token_id,

1346 + context_token_ids,

1347 )

1348 return inputs_embeds

1349

vllm/model_executor/models/nvlm_d.py

+8 -5

@@ -22,9 +22,10 @@ from vllm.multimodal.processing import (PromptReplacement, PromptUpdate,

22 PromptUpdateDetails)

24 from .intern_vit import InternVisionModel

25 -from .internvl import (BaseInternVLProcessingInfo, BaseInternVLProcessor,

26 - InternVLChatModel, InternVLDummyInputsBuilder,

27 - InternVLMultiModalProcessor)

25 +from .internvl import (BaseInternVLDummyInputsBuilder,

26 + BaseInternVLMultiModalProcessor,

27 + BaseInternVLProcessingInfo, BaseInternVLProcessor,

28 + InternVLChatModel)

30 IMG_PAD = "<|vision_pad|>"

@@ -84,7 +85,8 @@ class NVLMProcessingInfo(BaseInternVLProcessingInfo):

85 )

87 -class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):

88 +class NVLMDummyInputsBuilder(BaseInternVLDummyInputsBuilder[NVLMProcessingInfo]

89 + ):

91 def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:

92 num_images = mm_counts.get("image", 0)

@@ -110,7 +112,8 @@ class NVLMDummyInputsBuilder(InternVLDummyInputsBuilder[NVLMProcessingInfo]):

112 }

113

114

113 -class NVLMMultiModalProcessor(InternVLMultiModalProcessor[NVLMProcessingInfo]):

115 +class NVLMMultiModalProcessor(

116 + BaseInternVLMultiModalProcessor[NVLMProcessingInfo]):

117

118 def _get_prompt_updates(

119 self,