@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d45cbe70f5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Check that number of images matches number of <|image|> tokens with mllama (#11939)

Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>

Travis Johnson committed Jan 10, 2025 at 16:26 UTC d45cbe70f5bf25bb2f490f4152c256e9acb2a62b
1 file changed +9
vllm/model_executor/models/mllama.py
+9
@@ -123,6 +123,13 @@ def input_processor_for_mllama(
123
124 assert is_list_of(image_data, Image.Image)
125
126 + num_image_tokens = dec_inputs['prompt_token_ids'].count(
127 + MLLAMA_IMAGE_TOKEN_ID)
128 + if num_image_tokens != len(image_data):
129 + raise ValueError(
130 + f"The number of image tokens ({num_image_tokens}) must be"
131 + f" the same as the number of images ({len(image_data)})")
132 +
133 # Since only the last group of consecutive images
134 # are attended by the decoded tokens, we only need to
135 # get the number of tiles for those images.
@@ -1493,6 +1500,8 @@ def convert_sparse_cross_attention_mask_to_dense(
1500 dense_mask[seq_start + start:seq_start + end,
1501 tile_start:tile_start + tile] = 1
1502 tile_start += tile
1503 + assert ts != -1
1504 + assert td != 0
1505 tile_range_for_decode.append((ts, ts + td))
1506 seq_start += length
1507