A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[VLM] Add max-count checking in data parser for single image models (#11661)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Cyrus Leung committed
Jan 1, 2025 at 14:15 UTC
365801feddaf5c4448704a1f55269dd992f5a4b1
6 files changed
+48
-11
docs/source/models/supported_models.md
+1
-1
@@ -566,7 +566,7 @@ See [this page](#generative-models) for more information on how to use generativ
566
- [V1](gh-issue:8779)
567
* - `AriaForConditionalGeneration`
568
- Aria
569
- - T + I
569
+ - T + I<sup>+</sup>
570
- `rhymes-ai/Aria`
571
-
572
- ✅︎
tests/multimodal/test_processing.py
+2
-1
@@ -622,10 +622,11 @@ def _test_processing_cache_correctness(
622
623
624
# yapf: disable
625
+# True if the model supports multiple data items of the modality per request
626
@pytest.mark.parametrize(("model_id", "modalities"), [
627
("rhymes-ai/Aria", {"image": True}),
628
("Salesforce/blip2-opt-2.7b", {"image": False}),
628
- ("facebook/chameleon-7b", {"image": True}),
629
+ ("facebook/chameleon-7b", {"image": False}),
630
("adept/fuyu-8b", {"image": False}),
631
("llava-hf/llava-1.5-7b-hf", {"image": True}),
632
("TIGER-Lab/Mantis-8B-siglip-llama3", {"image": True}),
vllm/model_executor/models/blip2.py
+4
@@ -18,6 +18,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
18
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
19
MultiModalInputsV2, MultiModalKwargs,
20
NestedTensors, PlaceholderRange)
21
+from vllm.multimodal.parse import MultiModalDataParser
22
from vllm.multimodal.processing import (BaseMultiModalProcessor,
23
MultiModalDataItems, ProcessorInputs,
24
PromptReplacement)
@@ -404,6 +405,9 @@ def get_max_blip2_image_tokens(ctx: InputContext):
405
406
class Blip2MultiModalProcessor(BaseMultiModalProcessor):
407
408
+ def _get_data_parser(self) -> MultiModalDataParser:
409
+ return MultiModalDataParser(max_mm_counts={"image": 1})
410
+
411
def _get_hf_processor(self) -> Blip2Processor:
412
return self.ctx.get_hf_processor(Blip2Processor)
413
vllm/model_executor/models/chameleon.py
+4
@@ -31,6 +31,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
31
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
32
MultiModalInputsV2, MultiModalKwargs,
33
NestedTensors, PlaceholderRange)
34
+from vllm.multimodal.parse import MultiModalDataParser
35
from vllm.multimodal.processing import (BaseMultiModalProcessor,
36
MultiModalDataItems, ProcessorInputs,
37
PromptReplacement)
@@ -60,6 +61,9 @@ def get_max_chameleon_image_tokens(ctx: InputContext):
61
62
class ChameleonMultiModalProcessor(BaseMultiModalProcessor):
63
64
+ def _get_data_parser(self) -> MultiModalDataParser:
65
+ return MultiModalDataParser(max_mm_counts={"image": 1})
66
+
67
def _get_hf_processor(self) -> ChameleonProcessor:
68
return self.ctx.get_hf_processor(ChameleonProcessor)
69
vllm/model_executor/models/fuyu.py
+11
-7
@@ -34,7 +34,7 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
34
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
35
MultiModalInputsV2, MultiModalKwargs,
36
NestedTensors, PlaceholderRange)
37
-from vllm.multimodal.parse import ImageProcessorItems
37
+from vllm.multimodal.parse import ImageProcessorItems, MultiModalDataParser
38
from vllm.multimodal.processing import (BaseMultiModalProcessor,
39
MultiModalDataItems, ProcessorInputs,
40
PromptReplacement)
@@ -54,7 +54,7 @@ MAX_IMAGE_FEATURE_SIZE_WIDTH = 1920
54
55
class FuyuImagePatchInputs(TypedDict):
56
type: Literal["image_patches"]
57
- data: torch.Tensor
57
+ flat_data: torch.Tensor
58
"""
59
Shape:
60
`(batch_size * num_patches, patch_size_x * patch_size_y * num_channels)`
@@ -63,7 +63,7 @@ class FuyuImagePatchInputs(TypedDict):
63
patches_per_image: List[int]
64
"""
65
List of number of total patches for each image in the batch.
66
- This is used to restore the first two dimensions of `data`.
66
+ This is used to restore the first two dimensions of `flat_data`.
67
"""
68
69
@@ -102,6 +102,9 @@ def get_max_fuyu_image_tokens(ctx: InputContext):
102
103
class FuyuMultiModalProcessor(BaseMultiModalProcessor):
104
105
+ def _get_data_parser(self) -> MultiModalDataParser:
106
+ return MultiModalDataParser(max_mm_counts={"image": 1})
107
+
108
def _get_hf_processor(self) -> FuyuProcessor:
109
return self.ctx.get_hf_processor(FuyuProcessor)
110
@@ -304,7 +307,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
307
308
return FuyuImagePatchInputs(
309
type="image_patches",
307
- data=self._validate_pixel_values(
310
+ flat_data=self._validate_pixel_values(
311
flatten_bn(image_patches_flat, concat=True)),
312
patches_per_image=[x.size(0) for x in image_patches_flat],
313
)
@@ -313,12 +316,13 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
316
317
def _process_image_input(
318
self, image_input: FuyuImagePatchInputs) -> NestedTensors:
316
- image_patches = image_input["data"]
319
+ image_patches_flat = image_input["flat_data"]
320
patches_per_image = image_input["patches_per_image"]
321
322
assert self.vision_embed_tokens is not None
320
- vision_embeddings, _ = self.vision_embed_tokens(image_patches)
321
- return vision_embeddings.split(patches_per_image, dim=0)
323
+ vision_embeddings_flat, _ = self.vision_embed_tokens(
324
+ image_patches_flat)
325
+ return vision_embeddings_flat.split(patches_per_image, dim=0)
326
327
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
328
image_input = self._parse_and_validate_image_input(**kwargs)
vllm/multimodal/parse.py
+26
-2
@@ -220,11 +220,24 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
220
class MultiModalDataParser:
221
"""
222
Parses :class:`MultiModalDataDict` into :class:`MultiModalDataItems`.
223
+
224
+ Args:
225
+ max_mm_counts (Mapping[str, int]): The maximum allowed number of items
226
+ belonging to each modality. This effectively sets a hard limit over
227
+ `--limit-mm-per-prompt`.
228
+ target_sr (float, optional): Enables automatic resampling of audio
229
+ items to the model's expected sampling rate.
230
"""
231
225
- def __init__(self, *, target_sr: Optional[float] = None) -> None:
232
+ def __init__(
233
+ self,
234
+ *,
235
+ max_mm_counts: Mapping[str, int] = {},
236
+ target_sr: Optional[float] = None,
237
+ ) -> None:
238
super().__init__()
239
240
+ self.max_mm_counts = max_mm_counts
241
self.target_sr = target_sr
242
243
def _is_embeddings(self, data: object) -> TypeGuard[NestedTensors]:
@@ -332,6 +345,7 @@ class MultiModalDataParser:
345
346
def parse_mm_data(self,
347
mm_data: MultiModalDataDict) -> MultiModalDataItems:
348
+ max_mm_counts = self.max_mm_counts
349
subparsers = self._get_subparsers()
350
351
mm_items = MultiModalDataItems()
@@ -339,6 +353,16 @@ class MultiModalDataParser:
353
if k not in subparsers:
354
raise ValueError(f"Unsupported modality: {k}")
355
342
- mm_items[k] = subparsers[k](v)
356
+ modality_items = subparsers[k](v)
357
+
358
+ if k in max_mm_counts:
359
+ max_count = max_mm_counts[k]
360
+ if len(modality_items) > max_count:
361
+ raise ValueError(
362
+ f"This model supports at most {max_count} {k} items "
363
+ f"per prompt, but {len(modality_items)} {k} items "
364
+ "were given or set as its limit_mm_per_prompt.")
365
+
366
+ mm_items[k] = modality_items
367
368
return mm_items