@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b197a5ccfd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[V1][Bugfix] Fix data item ordering in mixed-modality inference (#12259)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Jan 21, 2025 at 05:18 UTC b197a5ccfdaa46b6750feb5efa4c5d8bf4030d44
2 files changed +62 -14
vllm/multimodal/utils.py
+33 -1
@@ -1,4 +1,5 @@
1 from functools import lru_cache
2 +from itertools import groupby
3 from pathlib import Path
4 from typing import TYPE_CHECKING, Optional, TypeVar, Union
5 from urllib.parse import ParseResult, urlparse
@@ -26,7 +27,7 @@ _M = TypeVar("_M")
27
28 if TYPE_CHECKING:
29 from .hasher import MultiModalHashDict
29 - from .inputs import MultiModalPlaceholderDict
30 + from .inputs import MultiModalKwargs, MultiModalPlaceholderDict
31
32
33 class MediaConnector:
@@ -477,3 +478,34 @@ def merge_and_sort_multimodal_metadata(
478 merged_hashes = None
479
480 return sorted_modalities, merged_placeholders, merged_hashes
481 +
482 +
483 +def group_mm_inputs_by_modality(
484 + mm_inputs: list["MultiModalKwargs"]) -> list[list["MultiModalKwargs"]]:
485 + """Group consecutive MultiModalKwargs from mm_inputs with the same modality
486 + together into the same list for batching purpose. For MultiModalKwargs with
487 + multiple modalities, put them into their own list.
488 +
489 + Args:
490 + mm_inputs: List of MultiModalKwargs.
491 +
492 + Returns:
493 + list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each
494 + inner list contains consecutive MultiModalKwargs with same modality, or
495 + one with multimodal modalities.
496 + """
497 + if not mm_inputs:
498 + return []
499 +
500 + def modality_group_func(mm_input: "MultiModalKwargs") -> Union[str, int]:
501 + # If the input has multiple modalities, return a id as the unique key
502 + # for the mm_input input.
503 + if len(mm_input.modalities) > 1:
504 + return id(mm_input)
505 +
506 + # Otherwise return the modality string
507 + return list(mm_input.modalities)[0]
508 +
509 + return [
510 + list(group) for _, group in groupby(mm_inputs, key=modality_group_func)
511 + ]
vllm/v1/worker/gpu_model_runner.py
+29 -13
@@ -17,6 +17,7 @@ from vllm.logger import init_logger
17 from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
18 from vllm.model_executor.model_loader import get_model
19 from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
20 +from vllm.multimodal.utils import group_mm_inputs_by_modality
21 from vllm.sampling_params import SamplingType
22 from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
23 LayerBlockType, cdiv, is_pin_memory_available)
@@ -629,19 +630,34 @@ class GPUModelRunner:
630 for input_id in encoder_input_ids:
631 mm_inputs.append(req_state.mm_inputs[input_id])
632 req_input_ids.append((req_id, input_id))
632 - batched_mm_inputs = MultiModalKwargs.batch(mm_inputs)
633 - batched_mm_inputs = MultiModalKwargs.as_kwargs(batched_mm_inputs,
634 - device=self.device)
635 -
636 - # Run the encoder.
637 - # `encoder_outputs` is either of the following:
638 - # 1. A tensor of shape [num_images, feature_size, hidden_size]
639 - # in case when feature_size is fixed across all images.
640 - # 2. A list (length: num_images) of tensors, each of shape
641 - # [feature_size, hidden_size] in case when the feature size is
642 - # dynamic depending on input images.
643 - encoder_outputs = self.model.get_multimodal_embeddings(
644 - **batched_mm_inputs)
633 +
634 + # Batch mm inputs as much as we can: if a request in the batch has
635 + # multiple modalities or a different modality than the previous one,
636 + # we process it separately to preserve item order.
637 + # FIXME(ywang96): This is a hacky way to deal with multiple modalities
638 + # in the same batch while still being able to benefit from batching
639 + # multimodal inputs. The proper solution should be reordering the
640 + # encoder outputs.
641 + grouped_mm_inputs_list = group_mm_inputs_by_modality(mm_inputs)
642 +
643 + encoder_outputs = []
644 + for grouped_mm_inputs in grouped_mm_inputs_list:
645 + batched_mm_inputs = MultiModalKwargs.batch(grouped_mm_inputs)
646 + batched_mm_inputs = MultiModalKwargs.as_kwargs(batched_mm_inputs,
647 + device=self.device)
648 +
649 + # Run the encoder.
650 + # `curr_group_outputs` is either of the following:
651 + # 1. A tensor of shape (num_items, feature_size, hidden_size)
652 + # in case feature_size is fixed across all multimodal items.
653 + # 2. A list or tuple (length: num_items) of tensors, each of shape
654 + # (feature_size, hidden_size) in case the feature size is dynamic
655 + # depending on the input multimodal items.
656 + curr_group_outputs = self.model.get_multimodal_embeddings(
657 + **batched_mm_inputs)
658 +
659 + for output in curr_group_outputs:
660 + encoder_outputs.append(output)
661
662 # Cache the encoder outputs.
663 for (req_id, input_id), output in zip(req_input_ids, encoder_outputs):