@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab93f1360f

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 19:45 UTC

[VLM] Various cleanup and fixes (#14806)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Mar 14, 2025 at 20:58 UTC ab93f1360fb7289bf78b1880e5dfbcd3aec6be36

14 files changed +283 -273

vllm/entrypoints/chat_utils.py

+14 -1

@@ -37,6 +37,7 @@ from vllm.config import ModelConfig

37 from vllm.logger import init_logger

38 from vllm.multimodal import MultiModalDataDict

39 from vllm.multimodal.utils import MediaConnector

40 +from vllm.transformers_utils.processor import cached_get_processor

41 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer

43 logger = init_logger(__name__)

@@ -1070,7 +1071,19 @@ def apply_hf_chat_template(

1071 tokenize: bool = False, # Different from HF's default

1072 **kwargs: Any,

1073 ) -> str:

1073 - if chat_template is None and tokenizer.chat_template is None:

1074 + if chat_template is None:

1075 + chat_template = tokenizer.chat_template

1076 +

1077 + # FIXME: Temporary workaround for

1078 + # https://huggingface.co/mistral-community/pixtral-12b/discussions/31

1079 + if chat_template is None:

1080 + try:

1081 + processor = cached_get_processor(tokenizer.name_or_path)

1082 + chat_template = processor.chat_template

1083 + except Exception:

1084 + pass

1085 +

1086 + if chat_template is None:

1087 raise ValueError(

1088 "As of transformers v4.44, default chat template is no longer "

1089 "allowed, so you must provide a chat template if the tokenizer "

vllm/model_executor/models/fuyu.py

+8 -7

@@ -18,7 +18,7 @@

18 """ PyTorch Fuyu model."""

19 import math

20 from collections.abc import Iterable, Mapping, Sequence

21 -from typing import List, Literal, Optional, Set, Tuple, TypedDict

21 +from typing import Literal, Optional, Set, Tuple, TypedDict

23 import torch

24 import torch.nn as nn

@@ -31,8 +31,7 @@ from vllm.model_executor.layers.sampler import SamplerOutput

31 from vllm.model_executor.models.persimmon import PersimmonForCausalLM

32 from vllm.model_executor.sampling_metadata import SamplingMetadata

33 from vllm.multimodal import MULTIMODAL_REGISTRY

34 -from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,

35 - NestedTensors)

34 +from vllm.multimodal.inputs import MultiModalFieldConfig, MultiModalKwargs

35 from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,

36 MultiModalDataItems)

37 from vllm.multimodal.processing import (BaseMultiModalProcessor,

@@ -58,10 +57,12 @@ class FuyuImagePatchInputs(TypedDict):

57 `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`

58 """

61 - patches_per_image: List[int]

60 + patches_per_image: list[int]

61 """

63 - List of number of total patches for each image in the batch.

64 - This is used to restore the first two dimensions of `flat_data`.

62 + The number of total patches for each image in the batch.

63 +

64 + This is used to split the embeddings which has the first two dimensions

65 + flattened just like `flat_data`.

66 """

@@ -317,7 +318,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):

318 return None

319

320 def _process_image_input(

320 - self, image_input: FuyuImagePatchInputs) -> NestedTensors:

321 + self, image_input: FuyuImagePatchInputs) -> MultiModalEmbeddings:

322 image_patches_flat = image_input["flat_data"]

323 patches_per_image = image_input["patches_per_image"]

324

vllm/model_executor/models/interfaces.py

+2 -2

@@ -5,7 +5,7 @@ from typing import (TYPE_CHECKING, ClassVar, Dict, List, Literal, Optional,

6 import torch

7 from torch import Tensor

8 -from typing_extensions import TypeIs

8 +from typing_extensions import Self, TypeIs

10 from vllm.logger import init_logger

11 from vllm.model_executor.layers.quantization.base_config import (

@@ -451,7 +451,7 @@ class SupportsQuant:

451 packed_modules_mapping: ClassVar[Dict[str, List[str]]] = {}

452 quant_config: Optional[QuantizationConfig] = None

453

454 - def __new__(cls, *args, **kwargs) -> "SupportsQuant":

454 + def __new__(cls, *args, **kwargs) -> Self:

455 instance = super().__new__(cls)

456 quant_config = cls._find_quant_config(*args, **kwargs)

457 if quant_config is not None:

vllm/model_executor/models/llava.py

+64 -72

@@ -3,8 +3,8 @@

3 from abc import abstractmethod

4 from collections.abc import Iterable, Mapping, Sequence

5 from functools import cached_property

6 -from typing import (Final, List, Literal, Optional, Protocol, Set, Tuple,

7 - TypedDict, TypeVar, Union, cast)

6 +from typing import (Final, Literal, Optional, Protocol, Set, Tuple, TypedDict,

7 + TypeVar, Union, cast)

9 import torch

10 import torch.nn as nn

@@ -39,8 +39,7 @@ from vllm.utils import JSONTree, flatten_2d_lists, json_map_leaves

40 from .clip import CLIPVisionModel

41 from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP

42 -from .pixtral import (PixtralHFVisionModel,

43 - get_pixtral_hf_image_feature_grid_size)

42 +from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel

43 from .siglip import SiglipVisionModel

44 from .utils import (AutoWeightsLoader, flatten_bn, init_vllm_registered_model,

45 maybe_prefix, merge_multimodal_embeddings)

@@ -49,7 +48,7 @@ from .vision import get_vision_encoder_info

49 class LlavaImagePixelInputs(TypedDict):

50 type: Literal["pixel_values"]

52 - data: Union[torch.Tensor, List[torch.Tensor]]

51 + pixel_values: torch.Tensor

52 """

53 Shape: `(batch_size * num_images, num_channels, height, width)`

@@ -57,7 +56,18 @@ class LlavaImagePixelInputs(TypedDict):

56 in which case the data is passed as a list instead of a batched tensor.

57 """

60 - feat_is_patch: Union[torch.Tensor, List[torch.Tensor]]

59 +

60 +class PixtralHFImagePixelInputs(TypedDict):

61 + type: Literal["pixel_values_pixtral"]

62 + pixel_values: Union[torch.Tensor, list[torch.Tensor]]

63 + """

64 + Shape: `(batch_size * num_images, num_channels, height, width)`

65 +

66 + Note that `height` or `width` may be different per batch and image,

67 + in which case the data is passed as a list instead of a batched tensor.

68 + """

69 +

70 + feat_is_patch: Union[torch.Tensor, list[torch.Tensor]]

71 """

72 A boolean mask indicating which image features correspond

73 to patch tokens.

@@ -65,7 +75,7 @@ class LlavaImagePixelInputs(TypedDict):

75 Shape: `(batch_size, num_crops, num_patch)`

76 """

68 - embed_is_patch: Union[torch.Tensor, List[torch.Tensor]]

78 + embed_is_patch: Union[torch.Tensor, list[torch.Tensor]]

79 """

80 A boolean mask indicating which image embeddings correspond

81 to patch tokens.

@@ -73,7 +83,7 @@ class LlavaImagePixelInputs(TypedDict):

83 Shape: `(batch_size, num_embeds)`

84 """

76 - num_crops: torch.Tensor

86 + num_crops: Union[torch.Tensor, list[torch.Tensor]]

87 """Shape: `(batch_size, num_images)`"""

@@ -85,27 +95,9 @@ class LlavaImageEmbeddingInputs(TypedDict):

95 `hidden_size` must match the hidden size of language model backbone.

96 """

88 - feat_is_patch: Union[torch.Tensor, List[torch.Tensor]]

89 - """

90 - A boolean mask indicating which image features correspond

91 - to patch tokens.

92 -

93 - Shape: `(batch_size, num_crops, num_patch)`

94 - """

95 -

96 - embed_is_patch: Union[torch.Tensor, List[torch.Tensor]]

97 - """

98 - A boolean mask indicating which image embeddings correspond

99 - to patch tokens.

100 -

101 - Shape: `(batch_size, num_embeds)`

102 - """

103 -

104 - num_crops: torch.Tensor

105 - """Shape: `(batch_size, num_images)`"""

106 -

108 -LlavaImageInputs = Union[LlavaImagePixelInputs, LlavaImageEmbeddingInputs]

99 +LlavaImageInputs = Union[LlavaImagePixelInputs, PixtralHFImagePixelInputs,

100 + LlavaImageEmbeddingInputs]

101

102

103 class LlavaMultiModalProjector(nn.Module):

@@ -357,13 +349,15 @@ class PixtralHFMultiModalProcessor(

349 ]

350

351 hf_config = self.info.get_hf_config()

352 + vision_config = hf_config.vision_config

353 + assert isinstance(vision_config, PixtralVisionConfig)

354 + encoder_info = PixtralHFEncoderInfo(vision_config)

355

356 tile_sizes = [

362 - get_pixtral_hf_image_feature_grid_size(

363 - hf_config.vision_config,

357 + encoder_info.get_patch_grid_size(

358 image_width=pixel_value.shape[-1],

365 - image_height=pixel_value.shape[-2])

366 - for pixel_value in processed_outputs["pixel_values"]

359 + image_height=pixel_value.shape[-2],

360 + ) for pixel_value in processed_outputs["pixel_values"]

361 ]

362 num_crops = torch.tensor([(ncols + 1) * nrows

363 for ncols, nrows in tile_sizes])

@@ -411,13 +405,13 @@ class PixtralHFMultiModalProcessor(

405

406 vision_config = hf_config.vision_config

407 assert isinstance(vision_config, PixtralVisionConfig)

408 + encoder_info = PixtralHFEncoderInfo(vision_config)

409

410 def get_replacement(item_idx: int):

411 images = mm_items.get_items("image", ImageProcessorItems)

412 image_size = images.get_image_size(item_idx)

413

419 - ncols, nrows = get_pixtral_hf_image_feature_grid_size(

420 - vision_config,

414 + ncols, nrows = encoder_info.get_patch_grid_size(

415 image_width=image_size.width,

416 image_height=image_size.height,

417 )

@@ -512,7 +506,7 @@ def init_vision_tower_for_llava(

506 *,

507 require_post_norm: Optional[bool] = None,

508 prefix: str = "",

515 -):

509 +) -> Union[CLIPVisionModel, SiglipVisionModel, PixtralHFVisionModel]:

510 vision_config = hf_config.vision_config

511

512 # Initialize the vision tower only up to the deepest required feature layer

@@ -627,32 +621,30 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

621 if pixel_values is None and image_embeds is None:

622 return None

623

630 - feat_is_patch = kwargs.pop("feat_is_patch", None)

631 - if feat_is_patch is not None and not isinstance(

632 - feat_is_patch, (torch.Tensor, list)):

633 - raise ValueError("Incorrect type of feat_is_patch. "

634 - f"Got type: {type(feat_is_patch)}")

635 -

636 - embed_is_patch = kwargs.pop("embed_is_patch", None)

637 - if embed_is_patch is not None and not isinstance(

638 - embed_is_patch, (torch.Tensor, list)):

639 - raise ValueError("Incorrect type of embed_is_patch. "

640 - f"Got type: {type(embed_is_patch)}")

641 -

642 - num_crops = kwargs.pop("num_crops", None)

643 - if num_crops is not None and not isinstance(num_crops, torch.Tensor):

644 - raise ValueError("Incorrect type of num_crops. "

645 - f"Got type: {type(num_crops)}")

646 -

624 if pixel_values is not None:

625 if not isinstance(pixel_values, (torch.Tensor, list)):

626 raise ValueError("Incorrect type of pixel values. "

627 f"Got type: {type(pixel_values)}")

628

629 if self.config.vision_config.model_type == "pixtral":

653 - return LlavaImagePixelInputs(

654 - type="pixel_values",

655 - data=flatten_bn(pixel_values),

630 + feat_is_patch = kwargs.pop("feat_is_patch")

631 + if not isinstance(feat_is_patch, (torch.Tensor, list)):

632 + raise ValueError("Incorrect type of feat_is_patch. "

633 + f"Got type: {type(feat_is_patch)}")

634 +

635 + embed_is_patch = kwargs.pop("embed_is_patch")

636 + if not isinstance(embed_is_patch, (torch.Tensor, list)):

637 + raise ValueError("Incorrect type of embed_is_patch. "

638 + f"Got type: {type(embed_is_patch)}")

639 +

640 + num_crops = kwargs.pop("num_crops")

641 + if not isinstance(num_crops, (torch.Tensor, list)):

642 + raise ValueError("Incorrect type of num_crops. "

643 + f"Got type: {type(num_crops)}")

644 +

645 + return PixtralHFImagePixelInputs(

646 + type="pixel_values_pixtral",

647 + pixel_values=flatten_bn(pixel_values),

648 feat_is_patch=feat_is_patch,

649 embed_is_patch=embed_is_patch,

650 num_crops=num_crops,

@@ -660,11 +652,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

652

653 return LlavaImagePixelInputs(

654 type="pixel_values",

663 - data=self._validate_pixel_values(

655 + pixel_values=self._validate_pixel_values(

656 flatten_bn(pixel_values, concat=True)),

665 - feat_is_patch=feat_is_patch,

666 - embed_is_patch=embed_is_patch,

667 - num_crops=num_crops,

657 )

658

659 if image_embeds is not None:

@@ -672,12 +661,12 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

661 raise ValueError("Incorrect type of image embeddings. "

662 f"Got type: {type(image_embeds)}")

663

664 + if self.config.vision_config.model_type == "pixtral":

665 + raise ValueError("Pixtral-HF does not support image_embeds.")

666 +

667 return LlavaImageEmbeddingInputs(

668 type="image_embeds",

669 data=flatten_bn(image_embeds, concat=True),

678 - feat_is_patch=feat_is_patch,

679 - embed_is_patch=embed_is_patch,

680 - num_crops=num_crops,

670 )

671

672 raise AssertionError("This line should be unreachable.")

@@ -696,7 +685,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

685 self,

686 vision_tower: Union[CLIPVisionModel, SiglipVisionModel,

687 PixtralHFVisionModel],

699 - pixel_values: torch.Tensor,

688 + pixel_values: Union[torch.Tensor, list[torch.Tensor]],

689 ) -> torch.Tensor:

690

691 # NOTE: we skip the step to select the vision feature layer since

@@ -708,17 +697,20 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

697 strategy=self.config.vision_feature_select_strategy,

698 )

699

711 - def _process_image_pixels(self,

712 - inputs: LlavaImagePixelInputs) -> torch.Tensor:

700 + def _process_image_pixels(

701 + self,

702 + inputs: Union[LlavaImagePixelInputs, PixtralHFImagePixelInputs],

703 + ) -> torch.Tensor:

704 assert self.vision_tower is not None

705

715 - pixel_values = inputs["data"]

706 + pixel_values = inputs["pixel_values"]

707

708 return self._image_pixels_to_features(self.vision_tower, pixel_values)

709

719 - def _process_image_input(self,

720 - image_input: LlavaImageInputs) -> torch.Tensor:

721 -

710 + def _process_image_input(

711 + self,

712 + image_input: LlavaImageInputs,

713 + ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:

714 if image_input["type"] == "image_embeds":

715 return image_input["data"]

716

@@ -783,11 +775,11 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):

775 image_input = self._parse_and_validate_image_input(**kwargs)

776 if image_input is None:

777 return None

778 +

779 vision_embeddings = self._process_image_input(image_input)

780

788 - if kwargs.get("v0_path", False) or \

789 - image_input.get("feat_is_patch") is None or \

790 - image_input.get("embed_is_patch") is None:

781 + if (kwargs.get("v0_path", False)

782 + or image_input["type"] != "pixel_values_pixtral"):

783 # The path is used for pixtral (V0 only) and llava (V0/V1)

784 return vision_embeddings

785

vllm/model_executor/models/llava_next.py

+4 -3

@@ -32,7 +32,7 @@ from .utils import (AutoWeightsLoader, embed_multimodal, flatten_bn,

33 class LlavaNextImagePixelInputs(TypedDict):

34 type: Literal["pixel_values"]

35 - data: Union[torch.Tensor, List[torch.Tensor]]

35 + pixel_values: Union[torch.Tensor, list[torch.Tensor]]

36 """

37 Shape:

38 `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`

@@ -315,7 +315,8 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

315

316 return LlavaNextImagePixelInputs(

317 type="pixel_values",

318 - data=self._validate_pixel_values(flatten_bn(pixel_values)),

318 + pixel_values=self._validate_pixel_values(

319 + flatten_bn(pixel_values)),

320 image_sizes=self._validate_image_sizes(

321 flatten_bn(image_sizes, concat=True)),

322 )

@@ -434,7 +435,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,

435 ) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:

436 assert self.vision_tower is not None

437

437 - pixel_values = inputs["data"]

438 + pixel_values = inputs["pixel_values"]

439

440 if isinstance(pixel_values, torch.Tensor):

441 b, num_patches, c, h, w = pixel_values.shape

vllm/model_executor/models/llava_onevision.py

+14 -15

@@ -42,7 +42,7 @@ _MAX_FRAMES_PER_VIDEO = 16

43 class LlavaOnevisionVideoPixelInputs(TypedDict):

44 type: Literal["pixel_values_videos"]

45 - data: Union[torch.Tensor, List[torch.Tensor]]

45 + pixel_values_videos: Union[torch.Tensor, list[torch.Tensor]]

46 """

47 Shape: `(batch_size, num_videos, num_frames, num_channels, height, width)`

@@ -54,7 +54,7 @@ class LlavaOnevisionVideoPixelInputs(TypedDict):

55 class LlavaOnevisionImagePixelInputs(TypedDict):

56 type: Literal["pixel_values"]

57 - data: Union[torch.Tensor, List[torch.Tensor]]

57 + pixel_values: Union[torch.Tensor, list[torch.Tensor]]

58 """

59 Shape:

60 `(batch_size * num_images, 1 + num_patches, num_channels, height, width)`

@@ -521,7 +521,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

521

522 return LlavaOnevisionImagePixelInputs(

523 type="pixel_values",

524 - data=self._validate_image_pixel_values(

524 + pixel_values=self._validate_image_pixel_values(

525 flatten_bn(pixel_values)),

526 image_sizes=self._validate_image_sizes(

527 flatten_bn(image_sizes, concat=True)),

@@ -570,21 +570,20 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

570 List[b, Tensor(nb_frames, nb_channels, height, width)]

571 }

572 """

573 - pixel_values = kwargs.pop("pixel_values_videos", None)

574 -

575 - if pixel_values is None:

573 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)

574 + if pixel_values_videos is None:

575 return None

576

578 - if not (is_list_of(pixel_values,

579 - (torch.Tensor)) # different shape videos

580 - or isinstance(pixel_values,

577 + if not (is_list_of(pixel_values_videos,

578 + torch.Tensor) # different shape videos

579 + or isinstance(pixel_values_videos,

580 torch.Tensor)): # same shape videos

582 - raise ValueError("Incorrect type of pixel values. "

583 - f"Got type: {type(pixel_values)}")

581 + raise ValueError("Incorrect type of pixel_values_videos. "

582 + f"Got type: {type(pixel_values_videos)}")

583

584 return LlavaOnevisionVideoPixelInputs(

585 type="pixel_values_videos",

587 - data=pixel_values,

586 + pixel_values_videos=pixel_values_videos,

587 )

588

589 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:

@@ -723,7 +722,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

722 ) -> Union[torch.Tensor, List[torch.Tensor]]:

723 assert self.vision_tower is not None

724

726 - pixel_values = inputs["data"]

725 + pixel_values = inputs["pixel_values"]

726

727 if isinstance(pixel_values, torch.Tensor):

728 b, num_patches, c, h, w = pixel_values.shape

@@ -757,7 +756,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

756

757 image_sizes = image_input.get("image_sizes")

758 if image_sizes is None:

760 - batch_size = len(image_input["data"])

759 + batch_size = len(image_input["pixel_values"])

760 vision_config = self.config.vision_config

761 default_height = default_width = vision_config.image_size

762 image_sizes = torch.as_tensor([[default_height, default_width]

@@ -808,7 +807,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,

807 def _process_video_pixels(self, inputs: LlavaOnevisionVideoPixelInputs):

808 assert self.vision_tower is not None

809

811 - video_pixels = inputs["data"]

810 + video_pixels = inputs["pixel_values_videos"]

811

812 if isinstance(video_pixels, torch.Tensor):

813 b, num_videos, frames, c, h, w = video_pixels.shape

vllm/model_executor/models/minicpmo.py

+46 -38

@@ -23,7 +23,6 @@

23 # limitations under the License.

24 """Inference-only MiniCPM-O model compatible with HuggingFace weights."""

25 from collections.abc import Iterable, Mapping, Sequence

26 -from functools import partial

26 from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,

27 TypedDict, Union)

@@ -36,11 +35,12 @@ from transformers.models.whisper.modeling_whisper import (

36 from vllm.config import VllmConfig

37 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

39 -from vllm.multimodal.inputs import MultiModalFieldConfig

40 -from vllm.multimodal.parse import (AudioItem, DictEmbeddingItems, ModalityData,

38 +from vllm.multimodal.inputs import MultiModalFieldConfig, NestedTensors

39 +from vllm.multimodal.parse import (AudioItem, AudioProcessorItems,

40 + DictEmbeddingItems, ModalityData,

41 ModalityDataItems, MultiModalDataItems,

42 MultiModalDataParser)

43 -from vllm.multimodal.processing import PromptReplacement

43 +from vllm.multimodal.processing import PromptReplacement, PromptUpdate

44 from vllm.multimodal.profiling import ProcessorInputs

45 from vllm.sequence import IntermediateTensors

@@ -272,8 +272,13 @@ class MiniCPMOMultiModalProcessor(

272 tokenizer.audio_end_id)

273 return special_tokens

274

275 - def process_audios(self, mm_data: Mapping[str, object],

276 - mm_kwargs: Mapping[str, object]) -> Dict[str, object]:

275 + def process_audios(

276 + self,

277 + mm_data: Mapping[str, object],

278 + mm_kwargs: Mapping[str, object],

279 + ) -> Mapping[str, NestedTensors]:

280 + mm_data = dict(mm_data)

281 +

282 audios = mm_data.pop("audios", [])

283 audio_embeds = mm_data.pop("audio_embeds", [])

284 if isinstance(audios, (list, torch.Tensor)) and len(audios) > 0:

@@ -332,11 +337,15 @@ class MiniCPMOMultiModalProcessor(

337 def get_placeholder_split_pattern(self) -> str:

338 return r"\(<(?:image|video|audio)>./</(?:image|video|audio)>\)"

339

335 - def process_mm_inputs(self, mm_data, mm_kwargs) -> object:

340 + def process_mm_inputs(

341 + self,

342 + mm_data: Mapping[str, object],

343 + mm_kwargs: Mapping[str, object],

344 + ) -> Mapping[str, Mapping[str, NestedTensors]]:

345 return {

346 "image": self.process_images(mm_data, mm_kwargs),

347 "video": self.process_videos(mm_data, mm_kwargs),

339 - "audio": self.process_audios(mm_data, mm_kwargs)

348 + "audio": self.process_audios(mm_data, mm_kwargs),

349 }

350

351 def get_modality_num_counter(self, modality: str) -> str:

@@ -358,39 +367,38 @@ class MiniCPMOMultiModalProcessor(

367 return super().get_prompt_texts_by_modality(inputs, modality, index)

368

369 def _get_prompt_updates(

361 - self, mm_items: MultiModalDataItems,

362 - hf_processor_mm_kwargs: Mapping[str, Any],

363 - out_mm_kwargs: MultiModalKwargs) -> Sequence[PromptReplacement]:

364 - placeholder = {

365 - "image": self.info.image_pattern,

366 - "video": self.info.video_pattern,

367 - "audio": self.info.audio_pattern

368 - }

370 + self,

371 + mm_items: MultiModalDataItems,

372 + hf_processor_mm_kwargs: Mapping[str, object],

373 + out_mm_kwargs: MultiModalKwargs,

374 + ) -> Sequence[PromptUpdate]:

375 + base_updates = super()._get_prompt_updates(

376 + mm_items=mm_items,

377 + hf_processor_mm_kwargs=hf_processor_mm_kwargs,

378 + out_mm_kwargs=out_mm_kwargs,

379 + )

380

370 - def get_replacement_minicpmv(item_idx: int, modality: str):

371 - if modality == "image":

372 - return self.get_image_prompt_texts(

373 - mm_items["image"].get_image_size(item_idx), item_idx)

374 - elif modality == "video":

375 - return self.get_video_prompt_texts(

376 - mm_items["video"].get_frame_size(item_idx),

377 - mm_items["video"].get_num_frames(item_idx))

378 - else: # audio

379 - if isinstance(mm_items["audio"], MiniCPMOAudioEmbeddingItems):

380 - single_audio_embeds = mm_items["audio"].get(item_idx)

381 - audio_len = self.info.get_audio_len_by_num_chunks(

382 - sum(chunk_embeds.shape[0]

383 - for chunk_embeds in single_audio_embeds))

384 - return self.get_audio_prompt_texts(audio_len)

385 - return self.get_audio_prompt_texts(

386 - len(mm_items["audio"].get(item_idx)))

381 + audio_placeholder = self.info.audio_pattern

382 +

383 + def get_audio_replacement(item_idx: int):

384 + audios = mm_items.get_items(

385 + "audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems))

386 +

387 + if isinstance(audios, MiniCPMOAudioEmbeddingItems):

388 + single_audio_embeds = audios.get(item_idx)["audio_embeds"]

389 + audio_len = self.info.get_audio_len_by_num_chunks(

390 + sum(chunk_embeds.shape[0]

391 + for chunk_embeds in single_audio_embeds))

392 + else:

393 + audio_len = audios.get_audio_length(item_idx)

394 +

395 + return self.get_audio_prompt_texts(audio_len)

396

397 return [

389 - PromptReplacement(modality=modality,

390 - target=placeholder[modality],

391 - replacement=partial(get_replacement_minicpmv,

392 - modality=modality))

393 - for modality in ("image", "video", "audio")

398 + *base_updates,

399 + PromptReplacement(modality="audio",

400 + target=audio_placeholder,

401 + replacement=get_audio_replacement),

402 ]

403

404 def _get_mm_fields_config(

vllm/model_executor/models/minicpmv.py

+74 -46

@@ -24,7 +24,6 @@

24 """Inference-only MiniCPM-V model compatible with HuggingFace weights."""

25 import math

26 import re

27 -from collections import Counter

27 from collections.abc import Iterable, Mapping, Sequence

28 from functools import cached_property, partial

29 from typing import (Any, Callable, Dict, List, Literal, Optional, Set, Tuple,

@@ -51,13 +50,16 @@ from vllm.model_executor.models.qwen2 import Qwen2ForCausalLM

50 from vllm.model_executor.sampling_metadata import SamplingMetadata

51 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs

52 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,

54 - MultiModalInputs, PlaceholderRange)

55 -from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem, ImageSize,

53 + MultiModalInputs, NestedTensors,

54 + PlaceholderRange)

55 +from vllm.multimodal.parse import (DictEmbeddingItems, ImageItem,

56 + ImageProcessorItems, ImageSize,

57 ModalityData, ModalityDataItems,

58 MultiModalDataItems, MultiModalDataParser,

58 - VideoItem)

59 + VideoItem, VideoProcessorItems)

60 from vllm.multimodal.processing import (BaseMultiModalProcessor,

60 - BaseProcessingInfo, PromptReplacement)

61 + BaseProcessingInfo, PromptReplacement,

62 + PromptUpdate)

63 from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs

64 from vllm.platforms import current_platform

65 from vllm.sequence import IntermediateTensors

@@ -557,8 +559,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

559 outputs = {key: outputs[key][0] for key in valid_keys}

560 return outputs

561

560 - def process_images(self, mm_data: Mapping[str, object],

561 - mm_kwargs: Mapping[str, object]) -> Dict[str, object]:

562 + def process_images(

563 + self,

564 + mm_data: Mapping[str, object],

565 + mm_kwargs: Mapping[str, object],

566 + ) -> Mapping[str, NestedTensors]:

567 + mm_data = dict(mm_data)

568 +

569 images = mm_data.pop("images", [])

570 image_embeds = mm_data.pop("image_embeds", [])

571 if isinstance(images, Image.Image):

@@ -568,8 +575,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

575 prompt=self.info.image_pattern * len(images),

576 mm_data={"images": images},

577 mm_kwargs=mm_kwargs)

571 - image_outputs = MiniCPMVMultiModalProcessor.\

572 - repack_processor_outputs(image_outputs)

578 + image_outputs = self.repack_processor_outputs(image_outputs)

579 elif len(image_embeds) > 0:

580 image_sizes = mm_data.pop("image_sizes", None)

581 image_outputs = {

@@ -580,8 +586,13 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

586 image_outputs = {}

587 return image_outputs

588

583 - def process_videos(self, mm_data: Mapping[str, object],

584 - mm_kwargs: Mapping[str, object]) -> Dict[str, object]:

589 + def process_videos(

590 + self,

591 + mm_data: Mapping[str, object],

592 + mm_kwargs: Mapping[str, object],

593 + ) -> Mapping[str, NestedTensors]:

594 + mm_data = dict(mm_data)

595 +

596 videos = mm_data.pop("videos", [])

597 video_embeds = mm_data.pop("video_embeds", [])

598 if len(videos) > 0 and isinstance(videos[0], Image.Image):

@@ -635,10 +646,14 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

646 def get_placeholder_split_pattern(self) -> str:

647 return r"\(<(?:image|video)>./</(?:image|video)>\)"

648

638 - def process_mm_inputs(self, mm_data, mm_kwargs) -> object:

649 + def process_mm_inputs(

650 + self,

651 + mm_data: Mapping[str, object],

652 + mm_kwargs: Mapping[str, object],

653 + ) -> Mapping[str, Mapping[str, NestedTensors]]:

654 return {

655 "image": self.process_images(mm_data, mm_kwargs),

641 - "video": self.process_videos(mm_data, mm_kwargs)

656 + "video": self.process_videos(mm_data, mm_kwargs),

657 }

658

659 def get_input_modalities(self, mm_data) -> List[str]:

@@ -655,8 +670,10 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

670 elif modality == "video":

671 return "video_image_sizes"

672

658 - def get_num_slices_by_modality(self, inputs: Dict[str, object],

659 - modality: str, index: int) -> int:

673 + raise NotImplementedError(modality)

674 +

675 + def get_num_slices_by_modality(self, inputs: dict[str, Any], modality: str,

676 + index: int) -> int:

677 if modality == "image":

678 return self.info.get_image_slice_nums(

679 inputs[modality]["image_sizes"][index],

@@ -669,20 +686,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

686 else:

687 raise ValueError(f"Unexpected modality: {modality}")

688

672 - def check_mm_inputs(self, inputs: Dict[str, object],

673 - matches: List[str]) -> None:

674 - counts = Counter(matches)

675 - for modality, count in counts.items():

676 - if modality not in inputs or not inputs[modality]:

677 - raise ValueError(f"None input data of {modality}."

678 - " But prompt requires.")

679 - counter_key = self.get_modality_num_counter(modality)

680 - if len(inputs[modality][counter_key]) != count:

681 - raise ValueError(f"The prompt requires {count} "

682 - f"{modality} inputs while you pass "

683 - f"{len(inputs[modality][counter_key])}")

684 -

685 - def get_prompt_texts_by_modality(self, inputs: Dict[str, object],

689 + def get_prompt_texts_by_modality(self, inputs: dict[str, Any],

690 modality: str, index: int) -> str:

691 if modality == "image":

692 return self.get_image_prompt_texts(

@@ -715,13 +719,23 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

719 tokenizer = self.info.get_tokenizer()

720 inputs = self.process_mm_inputs(mm_data, mm_kwargs)

721 mm_input_modalities = self.get_input_modalities(inputs)

718 - num_mm_slices = {modality: [] for modality in mm_input_modalities}

722 +

723 + num_mm_slices_lst = {

724 + modality: list[int]()

725 + for modality in mm_input_modalities

726 + }

727 for modality in mm_input_modalities:

728 num_counter_key = self.get_modality_num_counter(modality)

729 for index in range(len(inputs[modality][num_counter_key])):

722 - num_mm_slices[modality].append(

730 + num_mm_slices_lst[modality].append(

731 self.get_num_slices_by_modality(inputs, modality, index))

724 - return {

732 +

733 + num_mm_slices = {

734 + modality: torch.tensor(v)

735 + for modality, v in num_mm_slices_lst.items()

736 + }

737 +

738 + return BatchFeature({

739 "input_ids": np.array([tokenizer.encode(prompt)]),

740 **{

741 key: value

@@ -732,7 +746,7 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

746 f"{modality}_num_slices": num_mm_slices[modality]

747 for modality in mm_input_modalities

748 }

735 - }

749 + })

750

751 def _hf_processor_applies_updates(

752 self,

@@ -743,28 +757,42 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):

757 return False

758

759 def _get_prompt_updates(

746 - self, mm_items: MultiModalDataItems,

747 - hf_processor_mm_kwargs: Mapping[str, Any],

748 - out_mm_kwargs: MultiModalKwargs) -> Sequence[PromptReplacement]:

760 + self,

761 + mm_items: MultiModalDataItems,

762 + hf_processor_mm_kwargs: Mapping[str, object],

763 + out_mm_kwargs: MultiModalKwargs,

764 + ) -> Sequence[PromptUpdate]:

765 placeholder = {

766 "image": self.info.image_pattern,

767 "video": self.info.video_pattern,

768 }

769

754 - def get_replacement_minicpmv(item_idx: int, modality: str):

755 - if modality == "image":

756 - return self.get_image_prompt_texts(

757 - mm_items["image"].get_image_size(item_idx), item_idx)

758 - else: # video

759 - return self.get_video_prompt_texts(

760 - mm_items["video"].get_frame_size(item_idx),

761 - mm_items["video"].get_num_frames(item_idx))

770 + def get_image_replacement(item_idx: int):

771 + images = mm_items.get_items(

772 + "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems))

773 +

774 + image_size = images.get_image_size(item_idx)

775 +

776 + return self.get_image_prompt_texts(image_size, item_idx)

777 +

778 + def get_video_replacement(item_idx: int):

779 + videos = mm_items.get_items(

780 + "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems))

781 +

782 + frame_size = videos.get_frame_size(item_idx)

783 + num_frames = videos.get_num_frames(item_idx)

784 +

785 + return self.get_video_prompt_texts(frame_size, num_frames)

786 +

787 + get_replacement = {

788 + "image": get_image_replacement,

789 + "video": get_video_replacement,

790 + }

791

792 return [

793 PromptReplacement(modality=modality,

794 target=placeholder[modality],

766 - replacement=partial(get_replacement_minicpmv,

767 - modality=modality))

795 + replacement=get_replacement[modality])

796 for modality in ("image", "video")

797 ]

798

vllm/model_executor/models/molmo.py

+1 -1

@@ -1478,7 +1478,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,

1478 f"Got type: {type(embed_is_patch)}")

1479

1480 num_crops = kwargs.pop("num_crops", None)

1481 - if not isinstance(num_crops, torch.Tensor):

1481 + if not isinstance(num_crops, (torch.Tensor, list)):

1482 raise ValueError("Incorrect type of num_crops. "

1483 f"Got type: {type(num_crops)}")

1484

vllm/model_executor/models/pixtral.py

+42 -82

@@ -1,9 +1,10 @@

1 # SPDX-License-Identifier: Apache-2.0

3 import math

4 +from collections.abc import Iterable, Mapping

5 from dataclasses import dataclass, fields

6 from functools import cached_property

6 -from typing import Iterable, List, Mapping, Optional, Set, Tuple, Union

7 +from typing import List, Optional, Set, Tuple, Union

9 import torch

10 import torch.nn as nn

@@ -683,79 +684,6 @@ class VisionLanguageAdapter(nn.Module):

684 # and [`MistralForCausalLM`] for its language decoder.

685

686

686 -def get_pixtral_hf_patch_grid_length(*, image_size: int,

687 - patch_size: int) -> int:

688 - # Since interpolation is applied, the image size need not be divisible

689 - # assert image_size % patch_size == 0

690 - return image_size // patch_size

691 -

692 -

693 -def get_pixtral_hf_image_feature_size(

694 - *,

695 - image_size: int,

696 - patch_size: int,

697 -) -> int:

698 - grid_length = get_pixtral_hf_patch_grid_length(

699 - image_size=image_size,

700 - patch_size=patch_size,

701 - )

702 -

703 - # Consider the image_break_token

704 - return (grid_length + 1) * grid_length

705 -

706 -

707 -def get_max_pixtral_hf_image_tokens(hf_config: PixtralVisionConfig) -> int:

708 - grid_length = get_pixtral_hf_patch_grid_length(

709 - image_size=hf_config.image_size,

710 - patch_size=hf_config.patch_size,

711 - )

712 -

713 - # Consider the image_break_token

714 - return (grid_length + 1) * grid_length

715 -

716 -

717 -def dummy_image_for_pixtral_hf(

718 - hf_config: PixtralVisionConfig,

719 - num_images: int,

720 - *,

721 - image_width_override: Optional[int] = None,

722 - image_height_override: Optional[int] = None,

723 -):

724 - width = height = hf_config.image_size

725 - if image_width_override is not None:

726 - width = image_width_override

727 - if image_height_override is not None:

728 - height = image_height_override

729 -

730 - image = Image.new("RGB", (width, height), color=0)

731 - return {"image": image if num_images == 1 else [image] * num_images}

732 -

733 -

734 -# Adapted from transformers.models.pixtral.image_processing_pixtral.get_resize_output_image_size # noqa: E501

735 -# https://github.com/huggingface/transformers/blob/2bd4d5897dc73e8b172832070a6f9e567a0df017/src/transformers/models/pixtral/image_processing_pixtral.py#L180

736 -def get_pixtral_hf_image_feature_grid_size(

737 - hf_config: PixtralVisionConfig,

738 - *,

739 - image_width: int,

740 - image_height: int,

741 -) -> tuple[int, int]:

742 - max_width = max_height = hf_config.image_size

743 - patch_width = patch_height = hf_config.patch_size

744 -

745 - ratio = max(image_width / max_width, image_height / max_height)

746 -

747 - if ratio > 1:

748 - image_width = int(math.ceil(image_width / ratio))

749 - image_height = int(math.ceil(image_height / ratio))

750 -

751 - nrows, ncols = _get_pixtral_hf_num_image_tokens(

752 - (image_height, image_width),

753 - (patch_height, patch_width),

754 - ) # type: ignore

755 -

756 - return ncols, nrows

757 -

758 -

687 class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):

688

689 def get_num_image_tokens(

@@ -764,13 +692,21 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):

692 image_width: int,

693 image_height: int,

694 ) -> int:

767 - return get_pixtral_hf_image_feature_size(

768 - image_size=self.vision_config.image_size,

769 - patch_size=self.vision_config.patch_size,

695 + ncols, nrows = self.get_patch_grid_size(

696 + image_width=image_width,

697 + image_height=image_height,

698 )

699

700 + # Consider the image_break_token

701 + return (ncols + 1) * nrows

702 +

703 def get_max_image_tokens(self) -> int:

773 - return get_max_pixtral_hf_image_tokens(self.vision_config)

704 + image_size = self.get_image_size()

705 +

706 + return self.get_num_image_tokens(

707 + image_width=image_size,

708 + image_height=image_size,

709 + )

710

711 def get_image_size(self) -> int:

712 return self.vision_config.image_size

@@ -779,10 +715,34 @@ class PixtralHFEncoderInfo(VisionEncoderInfo[PixtralVisionConfig]):

715 return self.vision_config.patch_size

716

717 def get_patch_grid_length(self) -> int:

782 - return get_pixtral_hf_patch_grid_length(

783 - image_size=self.vision_config.image_size,

784 - patch_size=self.vision_config.patch_size,

785 - )

718 + image_size, patch_size = self.get_image_size(), self.get_patch_size()

719 +

720 + # Since interpolation is applied, the image size need not be divisible

721 + # assert image_size % patch_size == 0

722 + return image_size // patch_size

723 +

724 + # Adapted from: https://github.com/huggingface/transformers/blob/v4.49.0/src/transformers/models/pixtral/image_processing_pixtral.py#L99

725 + def get_patch_grid_size(

726 + self,

727 + *,

728 + image_width: int,

729 + image_height: int,

730 + ) -> tuple[int, int]:

731 + max_width = max_height = self.get_image_size()

732 + patch_width = patch_height = self.get_patch_size()

733 +

734 + ratio = max(image_width / max_width, image_height / max_height)

735 +

736 + if ratio > 1:

737 + image_width = int(math.ceil(image_width / ratio))

738 + image_height = int(math.ceil(image_height / ratio))

739 +

740 + nrows, ncols = _get_pixtral_hf_num_image_tokens(

741 + (image_height, image_width),

742 + (patch_height, patch_width),

743 + ) # type: ignore

744 +

745 + return ncols, nrows

746

747

748 class PixtralHFMLP(nn.Module):

vllm/model_executor/models/qwen2_audio.py

+4 -4

@@ -222,10 +222,10 @@ class Qwen2AudioMultiModalProcessor(

222 num_features = audio_output_lengths[item_idx]

223 if num_features == 0:

224 audios = mm_items.get_items("audio", AudioProcessorItems)

225 - audio = audios.get(item_idx)

226 - raise ValueError(

227 - f"The audio {audio} (len={len(audio)}) is too short "

228 - "to be represented inside the model")

225 + audio_len = audios.get_audio_length(item_idx)

226 +

227 + raise ValueError(f"The audio (len={audio_len}) is too short "

228 + "to be represented inside the model")

229

230 audio_tokens = [audio_token_id] * num_features

231

vllm/multimodal/inputs.py

@@ -433,6 +433,10 @@ class MultiModalFieldConfig:

433 :func:`MultiModalFieldConfig.flat`

434 """

435

436 + if size_per_item.ndim != 1:

437 + raise ValueError("size_per_item should be a 1-D tensor, "

438 + f"but found shape: {size_per_item.shape}")

439 +

440 slice_idxs = [0, *accumulate(size_per_item)]

441 slices = [

442 slice(slice_idxs[i], slice_idxs[i + 1])

vllm/multimodal/parse.py

@@ -176,6 +176,10 @@ class AudioProcessorItems(ProcessorBatchItems[HfAudioItem]):

176 def __init__(self, data: Sequence[HfAudioItem]) -> None:

177 super().__init__(data, "audio")

178

179 + def get_audio_length(self, item_idx: int) -> int:

180 + audio = self.get(item_idx)

181 + return len(audio)

182 +

183

184 class AudioEmbeddingItems(EmbeddingItems):

185

vllm/multimodal/processing.py

+2 -2

@@ -1311,8 +1311,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1311

1312 def _bind_and_group_updates(

1313 self,

1314 - prompt_updates: list[PromptUpdate],

1315 - ) -> dict[str, list[BoundPromptUpdate]]:

1314 + prompt_updates: Sequence[PromptUpdate],

1315 + ) -> dict[str, Sequence[BoundPromptUpdate]]:

1316 tokenizer = self.info.get_tokenizer()

1317

1318 it = (update.bind(tokenizer) for update in prompt_updates)