A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Check that number of images matches number of <|image|> tokens with mllama (#11939)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Jan 10, 2025 at 16:26 UTC
d45cbe70f5bf25bb2f490f4152c256e9acb2a62b
1 file changed
+9
vllm/model_executor/models/mllama.py
+9
@@ -123,6 +123,13 @@ def input_processor_for_mllama(
123
124
assert is_list_of(image_data, Image.Image)
125
126
+ num_image_tokens = dec_inputs['prompt_token_ids'].count(
127
+ MLLAMA_IMAGE_TOKEN_ID)
128
+ if num_image_tokens != len(image_data):
129
+ raise ValueError(
130
+ f"The number of image tokens ({num_image_tokens}) must be"
131
+ f" the same as the number of images ({len(image_data)})")
132
+
133
# Since only the last group of consecutive images
134
# are attended by the decoded tokens, we only need to
135
# get the number of tiles for those images.
@@ -1493,6 +1500,8 @@ def convert_sparse_cross_attention_mask_to_dense(
1500
dense_mask[seq_start + start:seq_start + end,
1501
tile_start:tile_start + tile] = 1
1502
tile_start += tile
1503
+ assert ts != -1
1504
+ assert td != 0
1505
tile_range_for_decode.append((ts, ts + td))
1506
seq_start += length
1507