@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 365801fedd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[VLM] Add max-count checking in data parser for single image models (#11661)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Cyrus Leung committed Jan 1, 2025 at 14:15 UTC 365801feddaf5c4448704a1f55269dd992f5a4b1
6 files changed +48 -11
docs/source/models/supported_models.md
+1 -1
@@ -566,7 +566,7 @@ See [this page](#generative-models) for more information on how to use generativ
566 - [V1](gh-issue:8779)
567 * - `AriaForConditionalGeneration`
568 - Aria
569 - - T + I
569 + - T + I<sup>+</sup>
570 - `rhymes-ai/Aria`
571 -
572 - ✅︎
tests/multimodal/test_processing.py
+2 -1
@@ -622,10 +622,11 @@ def _test_processing_cache_correctness(
622
623
624 # yapf: disable
625 +# True if the model supports multiple data items of the modality per request
626 @pytest.mark.parametrize(("model_id", "modalities"), [
627 ("rhymes-ai/Aria", {"image": True}),
628 ("Salesforce/blip2-opt-2.7b", {"image": False}),
628 - ("facebook/chameleon-7b", {"image": True}),
629 + ("facebook/chameleon-7b", {"image": False}),
630 ("adept/fuyu-8b", {"image": False}),
631 ("llava-hf/llava-1.5-7b-hf", {"image": True}),
632 ("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
vllm/model_executor/models/blip2.py
+4
@@ -18,6 +18,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
18 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19 MultiModalInputsV2, MultiModalKwargs,
20 NestedTensors, PlaceholderRange)
21 +from vllm.multimodal.parse import MultiModalDataParser
22 from vllm.multimodal.processing import (BaseMultiModalProcessor,
23 MultiModalDataItems, ProcessorInputs,
24 PromptReplacement)
@@ -404,6 +405,9 @@ def get_max_blip2_image_tokens(ctx: InputContext):
405
406 class Blip2MultiModalProcessor(BaseMultiModalProcessor):
407
408 + def _get_data_parser(self) -> MultiModalDataParser:
409 + return MultiModalDataParser(max_mm_counts={"image": 1})
410 +
411 def _get_hf_processor(self) -> Blip2Processor:
412 return self.ctx.get_hf_processor(Blip2Processor)
413
vllm/model_executor/models/chameleon.py
+4
@@ -31,6 +31,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
31 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32 MultiModalInputsV2, MultiModalKwargs,
33 NestedTensors, PlaceholderRange)
34 +from vllm.multimodal.parse import MultiModalDataParser
35 from vllm.multimodal.processing import (BaseMultiModalProcessor,
36 MultiModalDataItems, ProcessorInputs,
37 PromptReplacement)
@@ -60,6 +61,9 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
61
62 class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
63
64 + def _get_data_parser(self) -> MultiModalDataParser:
65 + return MultiModalDataParser(max_mm_counts={"image": 1})
66 +
67 def _get_hf_processor(self) -> ChameleonProcessor:
68 return self.ctx.get_hf_processor(ChameleonProcessor)
69
vllm/model_executor/models/fuyu.py
+11 -7
@@ -34,7 +34,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
34 from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35 MultiModalInputsV2, MultiModalKwargs,
36 NestedTensors, PlaceholderRange)
37 -from vllm.multimodal.parse import ImageProcessorItems
37 +from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataParser
38 from vllm.multimodal.processing import (BaseMultiModalProcessor,
39 MultiModalDataItems, ProcessorInputs,
40 PromptReplacement)
@@ -54,7 +54,7 @@ MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
54
55 class FuyuImagePatchInputs(TypedDict):
56 type: Literal["image_patches"]
57 - data: torch.Tensor
57 + flat_data: torch.Tensor
58 """
59 Shape:
60 `(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`
@@ -63,7 +63,7 @@ class FuyuImagePatchInputs(TypedDict):
63 patches_per_image: List[int]
64 """
65 List of number of total patches for each image in the batch.
66 - This is used to restore the first two dimensions of `data`.
66 + This is used to restore the first two dimensions of `flat_data`.
67 """
68
69
@@ -102,6 +102,9 @@ def get_max_fuyu_image_tokens(ctx: InputContext):
102
103 class FuyuMultiModalProcessor(BaseMultiModalProcessor):
104
105 + def _get_data_parser(self) -> MultiModalDataParser:
106 + return MultiModalDataParser(max_mm_counts={"image": 1})
107 +
108 def _get_hf_processor(self) -> FuyuProcessor:
109 return self.ctx.get_hf_processor(FuyuProcessor)
110
@@ -304,7 +307,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
307
308 return FuyuImagePatchInputs(
309 type="image_patches",
307 - data=self._validate_pixel_values(
310 + flat_data=self._validate_pixel_values(
311 flatten_bn(image_patches_flat, concat=True)),
312 patches_per_image=[x.size(0) for x in image_patches_flat],
313 )
@@ -313,12 +316,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
316
317 def _process_image_input(
318 self, image_input: FuyuImagePatchInputs) -> NestedTensors:
316 - image_patches = image_input["data"]
319 + image_patches_flat = image_input["flat_data"]
320 patches_per_image = image_input["patches_per_image"]
321
322 assert self.vision_embed_tokens is not None
320 - vision_embeddings, _ = self.vision_embed_tokens(image_patches)
321 - return vision_embeddings.split(patches_per_image, dim=0)
323 + vision_embeddings_flat, _ = self.vision_embed_tokens(
324 + image_patches_flat)
325 + return vision_embeddings_flat.split(patches_per_image, dim=0)
326
327 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
328 image_input = self._parse_and_validate_image_input(**kwargs)
vllm/multimodal/parse.py
+26 -2
@@ -220,11 +220,24 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
220 class MultiModalDataParser:
221 """
222 Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
223 +
224 + Args:
225 + max_mm_counts (Mapping[str, int]): The maximum allowed number of items
226 + belonging to each modality. This effectively sets a hard limit over
227 + `--limit-mm-per-prompt`.
228 + target_sr (float, optional): Enables automatic resampling of audio
229 + items to the model's expected sampling rate.
230 """
231
225 - def __init__(self, *, target_sr: Optional[float] = None) -> None:
232 + def __init__(
233 + self,
234 + *,
235 + max_mm_counts: Mapping[str, int] = {},
236 + target_sr: Optional[float] = None,
237 + ) -> None:
238 super().__init__()
239
240 + self.max_mm_counts = max_mm_counts
241 self.target_sr = target_sr
242
243 def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
@@ -332,6 +345,7 @@ class MultiModalDataParser:
345
346 def parse_mm_data(self,
347 mm_data: MultiModalDataDict) -> MultiModalDataItems:
348 + max_mm_counts = self.max_mm_counts
349 subparsers = self._get_subparsers()
350
351 mm_items = MultiModalDataItems()
@@ -339,6 +353,16 @@ class MultiModalDataParser:
353 if k not in subparsers:
354 raise ValueError(f"Unsupported modality: {k}")
355
342 - mm_items[k] = subparsers[k](v)
356 + modality_items = subparsers[k](v)
357 +
358 + if k in max_mm_counts:
359 + max_count = max_mm_counts[k]
360 + if len(modality_items) > max_count:
361 + raise ValueError(
362 + f"This model supports at most {max_count} {k} items "
363 + f"per prompt, but {len(modality_items)} {k} items "
364 + "were given or set as its limit_mm_per_prompt.")
365 +
366 + mm_items[k] = modality_items
367
368 return mm_items