@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 23c1b10a4c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM][Bugfix] Multi-modal processor compatible with V1 multi-input (#11674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 2, 2025 at 17:00 UTC 23c1b10a4c8cd77c5b13afa9242d67ffd055296b
3 files changed +151 -168
vllm/multimodal/inputs.py
+116 -136
@@ -2,7 +2,8 @@ from abc import ABC, abstractmethod
2 from collections import UserDict, defaultdict
3 from collections.abc import Mapping, Sequence
4 from dataclasses import dataclass
5 -from typing import Any, Literal, TypedDict, TypeVar, Union, cast, final
5 +from typing import (Any, Literal, Optional, TypedDict, TypeVar, Union, cast,
6 + final)
7
8 import numpy as np
9 import torch
@@ -11,7 +12,7 @@ from PIL.Image import Image
12 from transformers import BatchFeature
13 from typing_extensions import NotRequired, TypeAlias
14
14 -from vllm.utils import JSONTree, is_list_of, json_map_leaves
15 +from vllm.utils import JSONTree, full_groupby, is_list_of, json_map_leaves
16
17 _T = TypeVar("_T")
18
@@ -160,11 +161,8 @@ A dictionary containing nested tensors which have been batched via
161
162
163 @dataclass(frozen=True)
163 -class MultiModalFieldItem:
164 - """
165 - Contains metadata and data in :class:`MultiModalKwargs`
166 - corresponding to a data item in :class:`MultiModalDataItems`.
167 - """
164 +class MultiModalFieldElem:
165 + """Contains metadata and data of an item in :class:`MultiModalKwargs`."""
166 field: "BaseMultiModalField"
167 data: NestedTensors
168
@@ -186,34 +184,34 @@ class BaseMultiModalField(ABC):
184 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
185 raise NotImplementedError
186
189 - def _build_item(self, data: NestedTensors) -> MultiModalFieldItem:
190 - return MultiModalFieldItem(self, data)
187 + def _build_elem(self, data: NestedTensors) -> MultiModalFieldElem:
188 + return MultiModalFieldElem(self, data)
189
192 - def reduce(self, batch: list[MultiModalFieldItem]) -> MultiModalFieldItem:
193 - """Merge multiple instances of :class:`MultiModalFieldItem` together."""
190 + def reduce(self, batch: list[MultiModalFieldElem]) -> MultiModalFieldElem:
191 + """Merge multiple instances of :class:`MultiModalFieldElem` together."""
192 fields = [item.field for item in batch]
193 if len(set(fields)) > 1:
194 raise ValueError(f"Cannot merge different {fields=}")
195
196 data = self._reduce_data([item.data for item in batch])
197
200 - return self._build_item(data)
198 + return self._build_elem(data)
199
200
201 @dataclass(frozen=True)
202 class MultiModalBatchedField(BaseMultiModalField):
203 """
206 - A :class:`BaseMultiModalField` implementation where an item is obtained by
207 - directly indexing into the first dimension of the underlying data.
204 + A :class:`BaseMultiModalField` implementation where an element in the batch
205 + is obtained by indexing into the first dimension of the underlying data.
206 """
207
210 - def build_items(self, batch: NestedTensors) -> list[MultiModalFieldItem]:
211 - return [self._build_item(item) for item in batch]
208 + def build_elems(self, batch: NestedTensors) -> list[MultiModalFieldElem]:
209 + return [self._build_elem(item) for item in batch]
210
211 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
212 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
213 first_shape = batch[0].shape
216 - if all(item.shape == first_shape for item in batch):
214 + if all(elem.shape == first_shape for elem in batch):
215 return torch.stack(batch)
216
217 return batch
@@ -222,24 +220,24 @@ class MultiModalBatchedField(BaseMultiModalField):
220 @dataclass(frozen=True)
221 class MultiModalFlatField(BaseMultiModalField):
222 """
225 - A :class:`BaseMultiModalField` implementation where an item is obtained by
226 - slicing along the first dimension of the underlying data.
223 + A :class:`BaseMultiModalField` implementation where an element in the batch
224 + is obtained by slicing along the first dimension of the underlying data.
225 """
226
229 - def build_items(
227 + def build_elems(
228 self,
229 batch: NestedTensors,
230 slices: Sequence[slice],
233 - ) -> list[MultiModalFieldItem]:
234 - return [self._build_item(batch[slice_]) for slice_ in slices]
231 + ) -> list[MultiModalFieldElem]:
232 + return [self._build_elem(batch[slice_]) for slice_ in slices]
233
234 def _reduce_data(self, batch: list[NestedTensors]) -> NestedTensors:
235 if len(batch) > 0 and is_list_of(batch, torch.Tensor, check="all"):
236 first_shape = batch[0].shape
239 - if all(item.shape[1:] == first_shape[1:] for item in batch):
237 + if all(elem.shape[1:] == first_shape[1:] for elem in batch):
238 return torch.concat(batch)
239
242 - return [elem for item in batch for elem in item]
240 + return [e for elem in batch for e in elem]
241
242
243 class MultiModalFieldConfig:
@@ -267,115 +265,111 @@ class MultiModalFieldConfig:
265 ) -> None:
266 super().__init__()
267
270 - self._field_cls = field_cls
271 - self._modality = modality
272 - self._field_config = field_config
268 + self.field_cls = field_cls
269 + self.modality = modality
270 + self.field_config = field_config
271
274 - def build_items(
272 + def build_elems(
273 self,
274 key: str,
275 batch: NestedTensors,
278 - ) -> list[MultiModalFieldItem]:
279 - field = self._field_cls(key=key, modality=self._modality)
280 - return field.build_items(batch, **self._field_config) # type: ignore
276 + ) -> Sequence[MultiModalFieldElem]:
277 + field = self.field_cls(key=key, modality=self.modality)
278 + return field.build_elems(batch, **self.field_config) # type: ignore
279
280
283 -class MultiModalKwargs(UserDict[str, NestedTensors]):
281 +class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
282 + """
283 + A collection of :class:`MultiModalFieldElem`
284 + corresponding to a data item in :class:`MultiModalDataItems`.
285 """
285 - A dictionary that represents the keyword arguments to
286 - :meth:`~torch.nn.Module.forward`.
286
288 - The metadata :code:`items_by_key` defines how to split batched keyword
289 - arguments corresponding to each data item in :class:`MultiModalDataItems`:
287 + @staticmethod
288 + def from_elems(elems: Sequence[MultiModalFieldElem]):
289 + return MultiModalKwargsItem({elem.field.key: elem for elem in elems})
290
291 - - For a keyword argument, we can access the :code:`i` th item in the batch
292 - via :code:`items_by_key[key][i]`.
293 - - We can gather the keyword arguments belonging to a modality by finding
294 - the keys with items that belong to that modality, then accessing
295 - the :code:`i` th item in the batch for each such key.
291 + @property
292 + def modality(self) -> str:
293 + modalities = {elem.field.modality for elem in self.data.values()}
294 + assert len(modalities) == 1, f"Found different modalities={modalities}"
295 + return next(iter(modalities))
296
297 - Example:
297
299 - .. code-block:: python
300 -
301 - # All items belong to the "image" modality
302 - items_by_key={
303 - "pixel_values": [a, b, c, d], # "image" modality
304 - "image_grid_thw": [e, f, g, h], # "image" modality
305 - "pixel_values_video": [h, i, j], # "video" modality
306 - "video_grid_thw": [k, l, m], # "video" modality
307 - }
298 +# NOTE: UserDict is for V0 compatibility.
299 +# V1 should access individual items via `get_item`.
300 +class MultiModalKwargs(UserDict[str, NestedTensors]):
301 + """
302 + A dictionary that represents the keyword arguments to
303 + :meth:`~torch.nn.Module.forward`.
304
309 - - The keyword arguments belonging to the first image are
310 - :code:`{"pixel_values": a, "image_grid_thw": e}`.
311 - - The keyword arguments belonging to the second video are
312 - :code:`{"pixel_values_video": i, "video_grid_thw": l}`.
305 + The metadata :code:`items` enables us to obtain the keyword arguments
306 + corresponding to each data item in :class:`MultiModalDataItems`, via
307 + :meth:`get_item` and :meth:`get_items`.
308 """
309
310 @staticmethod
311 def from_hf_inputs(
312 hf_inputs: BatchFeature,
313 config_by_key: Mapping[str, MultiModalFieldConfig],
319 - *,
320 - enable_sanity_checks: bool = False,
314 ):
315 # NOTE: This skips fields in `hf_inputs` that are not in `config_by_key`
316 # We assume that those fields are not used in vLLM
324 - items_by_key = {
325 - key: config.build_items(key, batch)
326 - for key, config in config_by_key.items()
327 - if (batch := hf_inputs.get(key)) is not None
328 - }
329 -
330 - return MultiModalKwargs.from_items_by_key(
331 - items_by_key,
332 - enable_sanity_checks=enable_sanity_checks,
333 - )
317 + elems_by_key = dict[str, Sequence[MultiModalFieldElem]]()
318 + keys_by_modality = defaultdict[str, set[str]](set)
319 + for key, config in config_by_key.items():
320 + batch = hf_inputs.get(key)
321 + if batch is not None:
322 + elems = config.build_elems(key, batch)
323 + if len(elems) > 0:
324 + elems_by_key[key] = elems
325 + keys_by_modality[config.modality].add(key)
326 +
327 + items = list[MultiModalKwargsItem]()
328 + for modality, keys in keys_by_modality.items():
329 + elems_in_modality = {k: elems_by_key[k] for k in keys}
330 + batch_sizes = {k: len(v) for k, v in elems_in_modality.items()}
331 +
332 + if len(set(batch_sizes.values())) > 1:
333 + raise ValueError(
334 + f"Cannot merge different batch sizes for {modality=}! "
335 + f"Found: {batch_sizes=}")
336 +
337 + batch_size = next(iter(batch_sizes.values()))
338 + for item_idx in range(batch_size):
339 + elems = [v[item_idx] for v in elems_in_modality.values()]
340 + items.append(MultiModalKwargsItem.from_elems(elems))
341 +
342 + return MultiModalKwargs.from_items(items)
343
344 @staticmethod
336 - def from_items_by_key(
337 - items_by_key: Mapping[str, list[MultiModalFieldItem]],
338 - *,
339 - enable_sanity_checks: bool = False,
340 - ) -> "MultiModalKwargs":
345 + def from_items(items: Sequence[MultiModalKwargsItem]):
346 + """Construct a new :class:`MultiModalKwargs` from multiple items."""
347 + elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
348 + for item in items:
349 + for key, elem in item.items():
350 + elems_by_key[key].append(elem)
351 +
352 data = {
342 - key: items[0].field.reduce(items).data
343 - for key, items in items_by_key.items() if len(items) > 0
353 + key: elems[0].field.reduce(elems).data
354 + for key, elems in elems_by_key.items() if len(elems) > 0
355 }
356
346 - return MultiModalKwargs(data,
347 - items_by_key=items_by_key,
348 - enable_sanity_checks=enable_sanity_checks)
357 + return MultiModalKwargs(data, items=items)
358
359 def __init__(
360 self,
361 data: Mapping[str, NestedTensors],
362 *,
354 - items_by_key: Mapping[str, list[MultiModalFieldItem]] = {},
355 - enable_sanity_checks: bool = False,
363 + items: Optional[Sequence[MultiModalKwargsItem]] = None,
364 ) -> None:
365 super().__init__(data)
366
359 - # Shallow copy to avoid footgun in case a defaultdict is passed in
360 - self._items_by_key = dict(items_by_key)
367 + items_by_modality = full_groupby(items or [], key=lambda x: x.modality)
368 + self._items_by_modality = dict(items_by_modality)
369
362 - keys_by_modality = defaultdict[str, set[str]](set)
363 - for key, items in items_by_key.items():
364 - for item in items:
365 - keys_by_modality[item.field.modality].add(key)
366 -
367 - self._keys_by_modality = dict(keys_by_modality)
368 -
369 - if enable_sanity_checks:
370 - for modality, keys in keys_by_modality.items():
371 - items_in_modality = {k: items_by_key[k] for k in keys}
372 - batch_sizes = {k: len(v) for k, v in items_in_modality.items()}
373 - batch_size = next(iter(batch_sizes.values()), 0)
374 - assert all(bs == batch_size
375 - for bs in batch_sizes.values()), dict(
376 - modality=modality,
377 - batch_sizes=batch_sizes,
378 - items_by_key=items_by_key)
370 + @property
371 + def modalities(self):
372 + return self._items_by_modality.keys()
373
374 @staticmethod
375 def _try_stack(nested_tensors: NestedTensors) -> NestedTensors:
@@ -452,58 +446,44 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
446 def __eq__(self, other: object) -> bool:
447 if not isinstance(other, self.__class__):
448 return False
455 - if self._items_by_key != other._items_by_key:
449 + if self._items_by_modality != other._items_by_modality:
450 return False
451
452 ks = self.keys()
453 return (ks == other.keys()
454 and all(nested_tensors_equal(self[k], other[k]) for k in ks))
455
462 - def get_item(self, key: str, item_index: int) -> MultiModalFieldItem:
463 - return self._items_by_key[key][item_index]
456 + def _validate_modality(self, method_name: str, modality: str) -> None:
457 + if not self._items_by_modality:
458 + raise RuntimeError(
459 + f"`{method_name}` is not supported when "
460 + "MultiModalKwargs is not initialized with `items`")
461
465 - def get_items_by_modality(
466 - self,
467 - modality: str,
468 - item_index: int,
469 - ) -> Mapping[str, MultiModalFieldItem]:
470 - """
471 - Get the keyword arguments corresponding to an item identified by
472 - its modality and index.
473 - """
474 - if modality not in self._keys_by_modality:
475 - available_modalities = set(self._keys_by_modality.keys())
462 + if modality not in self._items_by_modality:
463 + available_modalities = set(self._items_by_modality.keys())
464 raise KeyError(f"Modality {modality!r} not found. "
465 f"Available modalities: {available_modalities}")
466
479 - keys_to_gather = self._keys_by_modality[modality]
467 + def get_item_count(self, modality: str) -> int:
468 + """Get the number of items belonging to a modality."""
469 + self._validate_modality("get_item_count", modality)
470 + return len(self._items_by_modality[modality])
471
481 - return {
482 - key: self.get_item(key, item_index)
483 - for key in keys_to_gather if key in self
484 - }
472 + def get_item(self, modality: str, item_index: int) -> MultiModalKwargsItem:
473 + """
474 + Get the keyword arguments corresponding to an item identified by
475 + its modality and index.
476 + """
477 + self._validate_modality("get_item", modality)
478 + return self._items_by_modality[modality][item_index]
479
486 - @staticmethod
487 - def from_items_by_modality(
488 - items_by_modality: Mapping[str, list[Mapping[str,
489 - MultiModalFieldItem]]],
490 - *,
491 - enable_sanity_checks: bool = False,
492 - ) -> "MultiModalKwargs":
480 + def get_items(self, modality: str) -> Sequence[MultiModalKwargsItem]:
481 """
494 - Construct a new :class:`MultiModalKwargs` from multiple items returned
495 - by :meth:`get_fields_by_modality`.
482 + Get the keyword arguments corresponding to each item belonging to
483 + a modality.
484 """
497 - items_by_key = defaultdict[str, list[MultiModalFieldItem]](list)
498 - for fields in items_by_modality.values():
499 - for field in fields:
500 - for k, v in field.items():
501 - items_by_key[k].append(v)
502 -
503 - return MultiModalKwargs.from_items_by_key(
504 - items_by_key,
505 - enable_sanity_checks=enable_sanity_checks,
506 - )
485 + self._validate_modality("get_items", modality)
486 + return self._items_by_modality[modality]
487
488
489 MultiModalPlaceholderDict = Mapping[str, Sequence[PlaceholderRange]]
vllm/multimodal/processing.py
+18 -27
@@ -20,8 +20,8 @@ from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
20 from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
21
22 from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
23 - MultiModalFieldItem, MultiModalInputsV2, MultiModalKwargs,
24 - PlaceholderRange)
23 + MultiModalInputsV2, MultiModalKwargs,
24 + MultiModalKwargsItem, PlaceholderRange)
25 from .parse import MultiModalDataItems, MultiModalDataParser
26
27 logger = init_logger(__name__)
@@ -496,8 +496,7 @@ class ProcessingCache:
496 # DEBUG: Set to None to disable
497 self.debug_cache_hit_ratio_steps: Optional[int] = None
498
499 - self._cache = LRUCache[str, Mapping[str,
500 - MultiModalFieldItem]](capacity)
499 + self._cache = LRUCache[str, MultiModalKwargsItem](capacity)
500
501 def _maybe_log_cache_stats(self) -> None:
502 steps = self.debug_cache_hit_ratio_steps
@@ -565,7 +564,7 @@ class ProcessingCache:
564 modality: str,
565 input_item: object,
566 input_kwargs: Mapping[str, object],
568 - ) -> Optional[Mapping[str, MultiModalFieldItem]]:
567 + ) -> Optional[MultiModalKwargsItem]:
568 """
569 Get a processed multi-modal item from the cache
570 according to its dependencies, including:
@@ -588,7 +587,7 @@ class ProcessingCache:
587 modality: str,
588 input_item: object,
589 input_kwargs: Mapping[str, object],
591 - output_kwargs: Mapping[str, MultiModalFieldItem],
590 + output_kwargs: MultiModalKwargsItem,
591 ) -> None:
592 """
593 Put a processed multi-modal item into the cache
@@ -784,7 +783,6 @@ class BaseMultiModalProcessor(ABC):
783 mm_kwargs = MultiModalKwargs.from_hf_inputs(
784 processed_data,
785 self._get_mm_fields_config(processed_data, hf_processor_mm_kwargs),
787 - enable_sanity_checks=self.enable_sanity_checks,
786 )
787
788 return prompt_ids, mm_kwargs
@@ -846,7 +844,7 @@ class BaseMultiModalProcessor(ABC):
844 hf_processor_mm_kwargs=hf_processor_mm_kwargs,
845 )
846
849 - mm_maybe_cached_field_items = {
847 + mm_maybe_cached_kw_items = {
848 modality: [
849 cache.get(model_id, modality, item, hf_processor_mm_kwargs)
850 for item in items
@@ -855,8 +853,9 @@ class BaseMultiModalProcessor(ABC):
853 }
854
855 mm_missing_idxs = {
858 - modality: [idx for idx, out in enumerate(fields) if out is None]
859 - for modality, fields in mm_maybe_cached_field_items.items()
856 + modality:
857 + [idx for idx, item in enumerate(kw_items) if item is None]
858 + for modality, kw_items in mm_maybe_cached_kw_items.items()
859 }
860 mm_missing_data = {
861 modality: [mm_data_items[modality][idx] for idx in idxs]
@@ -875,14 +874,11 @@ class BaseMultiModalProcessor(ABC):
874 for modality in mm_missing_data_items
875 }
876
878 - mm_merged_field_items = dict[str, list[Mapping[str,
879 - MultiModalFieldItem]]]()
880 - for modality, modal_items_lst in mm_maybe_cached_field_items.items():
881 - merged_modal_items_lst = list[Mapping[str, MultiModalFieldItem]]()
882 -
883 - for idx, modal_items in enumerate(modal_items_lst):
884 - if modal_items is None:
885 - modal_items = mm_missing_kwargs.get_items_by_modality(
877 + merged_kw_items = list[MultiModalKwargsItem]()
878 + for modality, kw_items in mm_maybe_cached_kw_items.items():
879 + for idx, kw_item in enumerate(kw_items):
880 + if kw_item is None:
881 + kw_item = mm_missing_kwargs.get_item(
882 modality,
883 mm_missing_next_idx[modality],
884 )
@@ -892,14 +888,12 @@ class BaseMultiModalProcessor(ABC):
888 modality,
889 mm_data_items[modality][idx],
890 hf_processor_mm_kwargs,
895 - modal_items,
891 + kw_item,
892 )
893
894 mm_missing_next_idx[modality] += 1
895
900 - merged_modal_items_lst.append(modal_items)
901 -
902 - mm_merged_field_items[modality] = merged_modal_items_lst
896 + merged_kw_items.append(kw_item)
897
898 if self.enable_sanity_checks:
899 mm_missing_counts = mm_missing_data_items.get_all_counts()
@@ -909,10 +903,7 @@ class BaseMultiModalProcessor(ABC):
903 mm_missing_next_idx=mm_missing_next_idx,
904 mm_missing_counts=mm_missing_counts)
905
912 - mm_kwargs = MultiModalKwargs.from_items_by_modality(
913 - mm_merged_field_items,
914 - enable_sanity_checks=self.enable_sanity_checks,
915 - )
906 + mm_kwargs = MultiModalKwargs.from_items(merged_kw_items)
907
908 if self.enable_sanity_checks:
909 mm_item_counts = mm_data_items.get_all_counts()
@@ -920,7 +911,7 @@ class BaseMultiModalProcessor(ABC):
911 for modality, item_count in mm_item_counts.items():
912 for item_idx in range(item_count):
913 try:
923 - mm_kwargs.get_items_by_modality(modality, item_idx)
914 + mm_kwargs.get_item(modality, item_idx)
915 except Exception as e:
916 # Make it easy to set a breakpoint in the debugger
917 raise e
vllm/v1/engine/processor.py
+17 -5
@@ -113,15 +113,27 @@ class Processor:
113
114 # For merged preprocessor, mm_data is already mm_inputs
115 precomputed_mm_inputs = None
116 - if isinstance(decoder_inputs.multi_modal_data, MultiModalKwargs):
117 - precomputed_mm_inputs = [decoder_inputs.multi_modal_data]
116 + decoder_mm_data = decoder_inputs.multi_modal_data
117 + if isinstance(decoder_mm_data, MultiModalKwargs):
118 + # The output of merged multi-modal processor (`decoder_mm_data`)
119 + # contains the kwargs for all items from all modalities.
120 + # This code separates them so that there is one set of kwargs
121 + # per item per modality.
122 + precomputed_mm_inputs = [
123 + MultiModalKwargs.from_items([item])
124 + for modality in decoder_mm_data.modalities
125 + for item in decoder_mm_data.get_items(modality)
126 + ]
127
128 # Apply MM mapper
129 mm_inputs = None
121 - if len(decoder_inputs.multi_modal_data) > 0:
130 + if len(decoder_mm_data) > 0:
131 mm_inputs = self.mm_input_mapper_client.process_inputs(
123 - decoder_inputs.multi_modal_data, mm_hashes,
124 - decoder_inputs.mm_processor_kwargs, precomputed_mm_inputs)
132 + decoder_mm_data,
133 + mm_hashes,
134 + decoder_inputs.mm_processor_kwargs,
135 + precomputed_mm_inputs,
136 + )
137
138 return EngineCoreRequest(
139 request_id,