@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 749f792553

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 09:45 UTC

[Frontend] decrease import time of vllm.multimodal (#18031)

Co-authored-by: Aaron Pham <Aaronpham0103@gmail.com>

David Xia committed May 14, 2025 at 18:43 UTC 749f792553d48dad68855f32910fafa61a28297e

3 files changed +45 -34

vllm/multimodal/inputs.py

+24 -21

@@ -10,40 +10,43 @@ from typing import (TYPE_CHECKING, Any, Literal, Optional, TypedDict, TypeVar,

10 Union, cast, final)

12 import numpy as np

13 -import torch

14 -import torch.types

15 -from PIL.Image import Image

16 -from transformers import BatchFeature

13 from typing_extensions import NotRequired, TypeAlias

15 from vllm.jsontree import JSONTree, json_map_leaves

20 -from vllm.utils import full_groupby, is_list_of

16 +from vllm.utils import LazyLoader, full_groupby, is_list_of

18 if TYPE_CHECKING:

19 + import torch

20 + import torch.types

21 + from PIL.Image import Image

22 + from transformers.feature_extraction_utils import BatchFeature

23 +

24 from .hasher import MultiModalHashDict

25 +else:

26 + torch = LazyLoader("torch", globals(), "torch")

28 _T = TypeVar("_T")

27 -HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]

30 +HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]

31 """

32 A {class}`transformers.image_utils.ImageInput` representing a single image

33 item, which can be passed to a HuggingFace `ImageProcessor`.

34 """

33 -HfVideoItem: TypeAlias = Union[list[Image], np.ndarray, torch.Tensor,

34 - list[np.ndarray], list[torch.Tensor]]

36 +HfVideoItem: TypeAlias = Union[list["Image"], np.ndarray, "torch.Tensor",

37 + list[np.ndarray], list["torch.Tensor"]]

38 """

39 A {class}`transformers.image_utils.VideoInput` representing a single video

40 item, which can be passed to a HuggingFace `VideoProcessor`.

41 """

40 -HfAudioItem: TypeAlias = Union[list[float], np.ndarray, torch.Tensor]

43 +HfAudioItem: TypeAlias = Union[list[float], np.ndarray, "torch.Tensor"]

44 """

45 Represents a single audio

46 item, which can be passed to a HuggingFace `AudioProcessor`.

47 """

46 -ImageItem: TypeAlias = Union[HfImageItem, torch.Tensor]

49 +ImageItem: TypeAlias = Union[HfImageItem, "torch.Tensor"]

50 """

51 A {class}`transformers.image_utils.ImageInput` representing a single image

52 item, which can be passed to a HuggingFace `ImageProcessor`.

@@ -53,7 +56,7 @@ which are treated as image embeddings;

56 these are directly passed to the model without HF processing.

57 """

56 -VideoItem: TypeAlias = Union[HfVideoItem, torch.Tensor]

59 +VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor"]

60 """

61 A {class}`transformers.image_utils.VideoInput` representing a single video

62 item, which can be passed to a HuggingFace `VideoProcessor`.

@@ -64,7 +67,7 @@ these are directly passed to the model without HF processing.

67 """

69 AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],

67 - torch.Tensor]

70 + "torch.Tensor"]

71 """

72 Represents a single audio

73 item, which can be passed to a HuggingFace `AudioProcessor`.

@@ -132,7 +135,7 @@ class PlaceholderRange:

135 length: int

136 """The length of the placeholder."""

137

135 - is_embed: Optional[torch.Tensor] = None

138 + is_embed: Optional["torch.Tensor"] = None

139 """

140 A boolean mask of shape `(length,)` indicating which positions

141 between `offset` and `offset + length` to assign embeddings to.

@@ -158,8 +161,8 @@ class PlaceholderRange:

161 return nested_tensors_equal(self.is_embed, other.is_embed)

162

163

161 -NestedTensors = Union[list["NestedTensors"], list[torch.Tensor], torch.Tensor,

162 - tuple[torch.Tensor, ...]]

164 +NestedTensors: TypeAlias = Union[list["NestedTensors"], list["torch.Tensor"],

165 + "torch.Tensor", tuple["torch.Tensor", ...]]

166 """

167 Uses a list instead of a tensor if the dimensions of each element do not match.

168 """

@@ -261,7 +264,7 @@ class BaseMultiModalField(ABC):

264 """

265 Construct {class}`MultiModalFieldElem` instances to represent

266 the provided data.

264 -

267 +

268 This is the inverse of {meth}`reduce_data`.

269 """

270 raise NotImplementedError

@@ -422,7 +425,7 @@ class MultiModalFieldConfig:

425 modality: The modality of the multi-modal item that uses this

426 keyword argument.

427 slices: For each multi-modal item, a slice (dim=0) or a tuple of

425 - slices (dim>0) that is used to extract the data corresponding

428 + slices (dim>0) that is used to extract the data corresponding

429 to it.

430 dim: The dimension to extract data, default to 0.

431

@@ -465,7 +468,7 @@ class MultiModalFieldConfig:

468

469 @staticmethod

470 def flat_from_sizes(modality: str,

468 - size_per_item: torch.Tensor,

471 + size_per_item: "torch.Tensor",

472 dim: int = 0):

473 """

474 Defines a field where an element in the batch is obtained by

@@ -602,7 +605,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):

605

606 @staticmethod

607 def from_hf_inputs(

605 - hf_inputs: BatchFeature,

608 + hf_inputs: "BatchFeature",

609 config_by_key: Mapping[str, MultiModalFieldConfig],

610 ):

611 # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`

@@ -792,7 +795,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):

795 return self._items_by_modality[modality]

796

797

795 -MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]

798 +MultiModalPlaceholderDict: TypeAlias = Mapping[str, Sequence[PlaceholderRange]]

799 """

800 A dictionary containing placeholder ranges for each modality.

801 """

@@ -823,7 +826,7 @@ class MultiModalInputs(TypedDict):

826 mm_hashes: Optional["MultiModalHashDict"]

827 """The hashes of the multi-modal data."""

828

826 - mm_placeholders: MultiModalPlaceholderDict

829 + mm_placeholders: "MultiModalPlaceholderDict"

830 """

831 For each modality, information about the placeholder tokens in

832 `prompt_token_ids`.

vllm/multimodal/parse.py

+13 -8

@@ -8,11 +8,9 @@ from typing import (TYPE_CHECKING, Any, Generic, Literal, NamedTuple, Optional,

9 import numpy as np

10 import torch

11 -from PIL.Image import Image

12 -from transformers import BatchFeature

11 from typing_extensions import TypeAlias, TypeGuard, assert_never

15 -from vllm.utils import is_list_of

13 +from vllm.utils import LazyLoader, is_list_of

15 from .audio import AudioResampler

16 from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,

@@ -22,6 +20,11 @@ from .inputs import (AudioItem, HfAudioItem, HfImageItem, HfVideoItem,

20 _T = TypeVar("_T")

21 _I = TypeVar("_I")

23 +if TYPE_CHECKING:

24 + import PIL.Image as PILImage

25 +else:

26 + PILImage = LazyLoader("PILImage", globals(), "PIL.Image")

27 +

29 class ModalityDataItems(ABC, Generic[_T, _I]):

30 """

@@ -131,6 +134,8 @@ class DictEmbeddingItems(ModalityDataItems[Mapping[str, torch.Tensor],

134 Mapping[str, MultiModalFieldConfig],

135 ],

136 ) -> None:

137 + from transformers.feature_extraction_utils import BatchFeature

138 +

139 super().__init__(data, modality)

140

141 missing_required_data_keys = required_fields - data.keys()

@@ -200,7 +205,7 @@ class ImageProcessorItems(ProcessorBatchItems[HfImageItem]):

205 def get_image_size(self, item_idx: int) -> ImageSize:

206 image = self.get(item_idx)

207

203 - if isinstance(image, Image):

208 + if isinstance(image, PILImage.Image):

209 return ImageSize(*image.size)

210 if isinstance(image, (np.ndarray, torch.Tensor)):

211 _, h, w = image.shape

@@ -226,7 +231,7 @@ class VideoProcessorItems(ProcessorBatchItems[HfVideoItem]):

231 def get_frame_size(self, item_idx: int) -> ImageSize:

232 image = self.get(item_idx)[0] # Assume that the video isn't empty

233

229 - if isinstance(image, Image):

234 + if isinstance(image, PILImage.Image):

235 return ImageSize(*image.size)

236 if isinstance(image, (np.ndarray, torch.Tensor)):

237 _, h, w = image.shape

@@ -253,7 +258,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):

258 def get_count(self, modality: str, *, strict: bool = True) -> int:

259 """

260 Get the number of data items belonging to a modality.

256 -

261 +

262 If `strict=False`, return `0` instead of raising {exc}`KeyError`

263 even if the modality is not found.

264 """

@@ -399,7 +404,7 @@ class MultiModalDataParser:

404 if self._is_embeddings(data):

405 return ImageEmbeddingItems(data)

406

402 - if (isinstance(data, Image)

407 + if (isinstance(data, PILImage.Image)

408 or isinstance(data,

409 (np.ndarray, torch.Tensor)) and data.ndim == 3):

410 data_items = [data]

@@ -420,7 +425,7 @@ class MultiModalDataParser:

425 if self._is_embeddings(data):

426 return VideoEmbeddingItems(data)

427

423 - if (is_list_of(data, Image)

428 + if (is_list_of(data, PILImage.Image)

429 or isinstance(data,

430 (np.ndarray, torch.Tensor)) and data.ndim == 4):

431 data_items = [data]

vllm/multimodal/processing.py

+8 -5

@@ -13,7 +13,6 @@ from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,

13 TypeVar, Union, cast)

15 import torch

16 -from transformers import BatchFeature, PretrainedConfig, ProcessorMixin

16 from typing_extensions import assert_never

18 from vllm.inputs import InputProcessingContext

@@ -31,6 +30,10 @@ from .parse import (DictEmbeddingItems, EmbeddingItems, MultiModalDataItems,

30 MultiModalDataParser)

32 if TYPE_CHECKING:

33 + from transformers.configuration_utils import PretrainedConfig

34 + from transformers.feature_extraction_utils import BatchFeature

35 + from transformers.processing_utils import ProcessorMixin

36 +

37 from .profiling import BaseDummyInputsBuilder

39 logger = init_logger(__name__)

@@ -1047,10 +1050,10 @@ class BaseProcessingInfo:

1050 def get_tokenizer(self) -> AnyTokenizer:

1051 return self.ctx.tokenizer

1052

1050 - def get_hf_config(self) -> PretrainedConfig:

1053 + def get_hf_config(self) -> "PretrainedConfig":

1054 return self.ctx.get_hf_config()

1055

1053 - def get_hf_processor(self, **kwargs: object) -> ProcessorMixin:

1056 + def get_hf_processor(self, **kwargs: object) -> "ProcessorMixin":

1057 """

1058 Subclasses can override this method to handle

1059 specific kwargs from model config or user inputs.

@@ -1165,7 +1168,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1168 @abstractmethod

1169 def _get_mm_fields_config(

1170 self,

1168 - hf_inputs: BatchFeature,

1171 + hf_inputs: "BatchFeature",

1172 hf_processor_mm_kwargs: Mapping[str, object],

1173 ) -> Mapping[str, MultiModalFieldConfig]:

1174 """Given the HF-processed data, output the metadata of each field."""

@@ -1222,7 +1225,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):

1225 # This refers to the data to be passed to HF processor.

1226 mm_data: Mapping[str, object],

1227 mm_kwargs: Mapping[str, object],

1225 - ) -> BatchFeature:

1228 + ) -> "BatchFeature":

1229 """

1230 Call the HF processor on the prompt text and

1231 associated multi-modal data.