A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Make llama3.2 support multiple and interleaved images (#9095)
Xiang Xu committed
Oct 14, 2024 at 15:24 UTC
f0fe4fe86d45763cb5904ac256ac6241c5eb2fde
3 files changed
+384
-42
examples/offline_inference_vision_language_multi_image.py
+23
@@ -234,12 +234,35 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
234
)
235
236
237
+def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
238
+ model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
239
+
240
+ # The configuration below has been confirmed to launch on a single L40 GPU.
241
+ llm = LLM(
242
+ model=model_name,
243
+ max_model_len=4096,
244
+ max_num_seqs=16,
245
+ enforce_eager=True,
246
+ limit_mm_per_prompt={"image": len(image_urls)},
247
+ )
248
+
249
+ prompt = f"<|image|><|image|><|begin_of_text|>{question}"
250
+ return ModelRequestData(
251
+ llm=llm,
252
+ prompt=prompt,
253
+ stop_token_ids=None,
254
+ image_data=[fetch_image(url) for url in image_urls],
255
+ chat_template=None,
256
+ )
257
+
258
+
259
model_example_map = {
260
"phi3_v": load_phi3v,
261
"internvl_chat": load_internvl,
262
"NVLM_D": load_nvlm_d,
263
"qwen2_vl": load_qwen2_vl,
264
"qwen_vl_chat": load_qwenvl_chat,
265
+ "mllama": load_mllama,
266
}
267
268
tests/models/encoder_decoder/vision_language/test_mllama.py
+82
-3
@@ -12,7 +12,7 @@ from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12
from ....utils import large_gpu_test
13
from ...utils import check_logprobs_close
14
15
-_LIMIT_IMAGE_PER_PROMPT = 1
15
+_LIMIT_IMAGE_PER_PROMPT = 3
16
17
HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18
"stop_sign":
@@ -244,8 +244,9 @@ def _run_test(
244
@pytest.mark.parametrize("dtype", ["bfloat16"])
245
@pytest.mark.parametrize("max_tokens", [128])
246
@pytest.mark.parametrize("num_logprobs", [5])
247
-def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
248
- max_tokens, num_logprobs) -> None:
247
+def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
248
+ model, sizes, dtype, max_tokens,
249
+ num_logprobs) -> None:
250
run_test(
251
hf_runner,
252
vllm_runner,
@@ -257,3 +258,81 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
258
num_logprobs=num_logprobs,
259
tensor_parallel_size=1,
260
)
261
+
262
+
263
+@large_gpu_test(min_gb=48)
264
+@pytest.mark.parametrize("model", models)
265
+@pytest.mark.parametrize("dtype", ["bfloat16"])
266
+@pytest.mark.parametrize("max_tokens", [128])
267
+@pytest.mark.parametrize("num_logprobs", [5])
268
+def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
269
+ model, dtype, max_tokens,
270
+ num_logprobs) -> None:
271
+
272
+ stop_sign = image_assets[0].pil_image
273
+ cherry_blossom = image_assets[1].pil_image
274
+
275
+ inputs = [(
276
+ [
277
+ "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
278
+ "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
279
+ "<|image|><|image|><|image|><|begin_of_text|>Describe 3 images.", # noqa: E501
280
+ ],
281
+ [
282
+ [stop_sign, cherry_blossom],
283
+ # Images with different sizes.
284
+ [
285
+ stop_sign.resize((512, 512)),
286
+ stop_sign,
287
+ ],
288
+ [
289
+ stop_sign,
290
+ stop_sign.resize((512, 1536)),
291
+ cherry_blossom.resize((512, 1024)),
292
+ ],
293
+ ])]
294
+
295
+ _run_test(
296
+ hf_runner,
297
+ vllm_runner,
298
+ inputs,
299
+ model,
300
+ dtype=dtype,
301
+ max_tokens=max_tokens,
302
+ num_logprobs=num_logprobs,
303
+ tensor_parallel_size=1,
304
+ )
305
+
306
+
307
+@large_gpu_test(min_gb=48)
308
+@pytest.mark.parametrize("model", models)
309
+@pytest.mark.parametrize("dtype", ["bfloat16"])
310
+@pytest.mark.parametrize("max_tokens", [128])
311
+@pytest.mark.parametrize("num_logprobs", [5])
312
+def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
313
+ dtype, max_tokens, num_logprobs) -> None:
314
+
315
+ stop_sign = image_assets[0].pil_image
316
+ cherry_blossom = image_assets[1].pil_image
317
+
318
+ inputs = [(
319
+ [
320
+ "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
321
+ "<|begin_of_text|>Between the first image <|image|> and the second image<|image|>, " # noqa: E501
322
+ "which is a stop sign and which is a cherry blossom?", # noqa: E501
323
+ ],
324
+ [
325
+ [stop_sign],
326
+ [stop_sign, cherry_blossom],
327
+ ])]
328
+
329
+ _run_test(
330
+ hf_runner,
331
+ vllm_runner,
332
+ inputs,
333
+ model,
334
+ dtype=dtype,
335
+ max_tokens=max_tokens,
336
+ num_logprobs=num_logprobs,
337
+ tensor_parallel_size=1,
338
+ )
vllm/model_executor/models/mllama.py
+279
-39
@@ -18,6 +18,7 @@ from array import array
18
from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19
TypedDict, Union)
20
21
+import numpy as np
22
import torch
23
import torch.nn.functional as F
24
import torch.utils.checkpoint
@@ -28,9 +29,12 @@ from transformers.modeling_outputs import (BaseModelOutput,
29
CausalLMOutputWithPast)
30
from transformers.models.mllama.image_processing_mllama import (
31
get_optimal_tiled_canvas)
32
+from transformers.models.mllama.processing_mllama import (
33
+ get_cross_attention_token_mask)
34
35
import vllm.distributed.parallel_state as ps
36
from vllm.attention import Attention, AttentionMetadata, AttentionType
37
+from vllm.attention.ops.paged_attn import PagedAttention
38
from vllm.config import CacheConfig, MultiModalConfig
39
from vllm.distributed import get_tensor_model_parallel_world_size
40
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
@@ -72,6 +76,16 @@ class MllamaImagePixelInputs(TypedDict):
76
# TODO: support LlamaImageEmbeddingInputs
77
78
79
+def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
80
+ num_images = 0
81
+ for token_id in prompt_token_ids[::-1]:
82
+ if token_id == MLLAMA_IMAGE_TOKEN_ID:
83
+ num_images += 1
84
+ elif num_images > 0:
85
+ break
86
+ return num_images
87
+
88
+
89
def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
90
# move encoder_prompt to prompt
91
if llm_inputs.get("prompt") is None:
@@ -91,12 +105,16 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
105
llm_inputs["encoder_multi_modal_data"] = {}
106
return llm_inputs
107
94
- # get num_tiles
108
if isinstance(multi_modal_data['image'], Image.Image):
109
multi_modal_data['image'] = [multi_modal_data['image']]
110
+ # Since only the last group of consecutive images
111
+ # are attended by the decoded tokens, we only need to
112
+ # get the number of tiles for those images.
113
+ num_decode_images = _get_num_image_in_last_group(
114
+ llm_inputs["prompt_token_ids"])
115
hf_config = ctx.model_config.hf_config
116
num_tiles = 0
99
- for image in multi_modal_data["image"]:
117
+ for image in multi_modal_data["image"][::-1]:
118
width, height = image.size
119
tile_size = hf_config.vision_config.image_size
120
canvas_height, canvas_width = get_optimal_tiled_canvas(
@@ -108,8 +126,13 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
126
num_tiles_height = canvas_height // tile_size
127
num_tiles_width = canvas_width // tile_size
128
num_tiles += num_tiles_height * num_tiles_width
129
+ num_decode_images -= 1
130
+ if num_decode_images == 0:
131
+ break
132
112
- # set encoder prompt based on num_tiles
133
+ # Set encoder prompt length based on the number of tiles.
134
+ # This tells the block manager to allocate correct number
135
+ # of slots for encoder tokens.
136
assert hf_config.vision_config.image_size % 14 == 0, \
137
"chunk size should be multiple of 14"
138
token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
@@ -675,6 +698,7 @@ class MllamaTextCrossAttention(nn.Module):
698
self,
699
hidden_states: torch.Tensor,
700
attention_mask: Optional[torch.Tensor],
701
+ kv_range_for_decode: Optional[List[Tuple[int, int]]],
702
cross_attention_states: Optional[torch.Tensor],
703
kv_cache: torch.Tensor,
704
attn_metadata: AttentionMetadata,
@@ -697,15 +721,71 @@ class MllamaTextCrossAttention(nn.Module):
721
q = q.view(-1, self.num_local_heads, self.head_dim)
722
q = self.q_norm(q)
723
700
- output = self.attn(q,
701
- k,
702
- v,
703
- kv_cache,
704
- attn_metadata,
705
- attn_type=AttentionType.ENCODER_DECODER)
724
+ if attention_mask is not None:
725
+ output = self.attention_with_mask(q, k, v, kv_cache,
726
+ attention_mask,
727
+ kv_range_for_decode,
728
+ attn_metadata)
729
+ else:
730
+ output = self.attn(q,
731
+ k,
732
+ v,
733
+ kv_cache,
734
+ attn_metadata,
735
+ attn_type=AttentionType.ENCODER_DECODER)
736
out, _ = self.o_proj(output)
737
return out
738
739
+ def attention_with_mask(
740
+ self,
741
+ q: torch.Tensor,
742
+ k: torch.Tensor,
743
+ v: torch.Tensor,
744
+ kv_cache: torch.Tensor,
745
+ attention_mask: torch.Tensor,
746
+ kv_range_for_decode: List[Tuple[int, int]],
747
+ attn_metadata: AttentionMetadata,
748
+ ) -> torch.Tensor:
749
+ # Skip writing kv-cache for the initial profiling run.
750
+ if len(kv_cache.shape) == 3:
751
+ key_cache, value_cache = PagedAttention.split_kv_cache(
752
+ kv_cache, self.num_local_key_value_heads, self.head_dim)
753
+ cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
754
+ cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
755
+ PagedAttention.write_to_paged_cache(
756
+ cached_k, cached_v, key_cache, value_cache,
757
+ attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
758
+ # We have to call torch.sdpa for prefill when using a
759
+ # custom cross-attention mask. Because the mask is not a
760
+ # standard causal mask, neither a block diagonal mask which
761
+ # can be optimized by xformers.BlockDiagonalMask.
762
+ # The mask is specially calculated for supporting multi
763
+ # images and interleaved images.
764
+ q_len = q.shape[0]
765
+ kv_len = k.shape[0]
766
+ q = q.transpose(0, 1).view(self.num_local_key_value_heads,
767
+ self.num_key_value_groups, q_len,
768
+ self.head_dim)
769
+ k = k.transpose(0,
770
+ 1)[:,
771
+ None, :, :].expand(self.num_local_key_value_heads,
772
+ self.num_key_value_groups,
773
+ kv_len, self.head_dim)
774
+ v = v.transpose(0,
775
+ 1)[:,
776
+ None, :, :].expand(self.num_local_key_value_heads,
777
+ self.num_key_value_groups,
778
+ kv_len, self.head_dim)
779
+ attention_mask = attention_mask.view(1, 1, q_len, kv_len)
780
+ output = F.scaled_dot_product_attention(q,
781
+ k,
782
+ v,
783
+ attn_mask=attention_mask,
784
+ is_causal=False)
785
+ output = output.permute(2, 0, 1, 3).reshape(
786
+ q_len, self.num_local_heads * self.head_dim)
787
+ return output
788
+
789
790
class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
791
"""Cross-attention transformer block with tanh-gated attention
@@ -741,6 +821,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
821
hidden_states: torch.Tensor,
822
cross_attention_states: torch.Tensor,
823
cross_attention_mask: torch.Tensor,
824
+ kv_range_for_decode: Optional[List[Tuple[int, int]]],
825
full_text_row_masked_out_mask: torch.Tensor,
826
kv_cache: List[torch.Tensor],
827
attn_metadata: AttentionMetadata,
@@ -751,6 +832,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
832
hidden_states = self.cross_attn(
833
hidden_states=hidden_states,
834
attention_mask=cross_attention_mask,
835
+ kv_range_for_decode=kv_range_for_decode,
836
cross_attention_states=cross_attention_states,
837
kv_cache=kv_cache,
838
attn_metadata=attn_metadata,
@@ -804,6 +886,7 @@ class MllamaTextModel(nn.Module):
886
positions: Optional[torch.LongTensor],
887
cross_attention_states: Optional[torch.LongTensor],
888
cross_attention_mask: Optional[torch.LongTensor],
889
+ kv_range_for_decode: Optional[List[Tuple[int, int]]],
890
full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
891
torch.Tensor]],
892
kv_caches: List[torch.Tensor],
@@ -820,6 +903,7 @@ class MllamaTextModel(nn.Module):
903
hidden_states=hidden_states,
904
cross_attention_states=cross_attention_states,
905
cross_attention_mask=cross_attention_mask,
906
+ kv_range_for_decode=kv_range_for_decode,
907
full_text_row_masked_out_mask=
908
full_text_row_masked_out_mask,
909
kv_cache=kv_caches[idx],
@@ -868,6 +952,7 @@ class MllamaForCausalLM(nn.Module):
952
positions: Optional[torch.LongTensor],
953
cross_attention_states: Optional[torch.LongTensor],
954
cross_attention_mask: Optional[torch.LongTensor],
955
+ kv_range_for_decode: Optional[List[Tuple[int, int]]],
956
full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
957
torch.Tensor]],
958
kv_caches: List[torch.Tensor],
@@ -879,6 +964,7 @@ class MllamaForCausalLM(nn.Module):
964
positions=positions,
965
cross_attention_states=cross_attention_states,
966
cross_attention_mask=cross_attention_mask,
967
+ kv_range_for_decode=kv_range_for_decode,
968
full_text_row_masked_out_mask=full_text_row_masked_out_mask,
969
kv_caches=kv_caches,
970
attn_metadata=attn_metadata,
@@ -1026,36 +1112,102 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1112
raise AssertionError("This line should be unreachable.")
1113
1114
def flat_encoder_result(self, cross_attention_states: torch.Tensor,
1029
- attn_metadata: AttentionMetadata):
1115
+ attn_metadata: AttentionMetadata,
1116
+ actual_encoder_seq_lens: List[int]):
1117
1118
cross_attention_states_flat = torch.zeros(
1032
- sum(attn_metadata.encoder_seq_lens),
1119
+ sum(actual_encoder_seq_lens),
1120
cross_attention_states.shape[-1],
1121
device=cross_attention_states.device,
1122
dtype=cross_attention_states.dtype)
1123
start_pos = 0
1037
- for seq_len, vision_token_in_batch in zip(
1038
- attn_metadata.encoder_seq_lens, cross_attention_states):
1124
+ for seq_len, vision_token_in_batch in zip(actual_encoder_seq_lens,
1125
+ cross_attention_states):
1126
end_pos = start_pos + seq_len
1127
cross_attention_states_flat[
1128
start_pos:end_pos] = vision_token_in_batch[:seq_len]
1129
start_pos = end_pos
1130
cross_attention_states = cross_attention_states_flat
1131
+ return cross_attention_states
1132
+
1133
+ def get_cross_attention_states(
1134
+ self,
1135
+ image_inputs: MllamaImagePixelInputs,
1136
+ attn_metadata: AttentionMetadata,
1137
+ actual_encoder_seq_lens: List[int],
1138
+ ) -> Tuple[torch.Tensor]:
1139
+ # NOTE: llama's reference implementation runs vision model on CPU
1140
+ pixel_values = image_inputs['data']
1141
+ aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1142
+ aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1143
+ cross_attention_states = self.vision_model(pixel_values,
1144
+ aspect_ratio_ids,
1145
+ aspect_ratio_mask)
1146
+ cross_attention_states = self.multi_modal_projector(
1147
+ cross_attention_states)
1148
+
1149
+ bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1150
+ cross_attention_states = cross_attention_states.view(
1151
+ bsz, -1, image_token_dim)
1152
+
1153
+ cross_attention_states = self.flat_encoder_result(
1154
+ cross_attention_states, attn_metadata, actual_encoder_seq_lens)
1155
+
1156
+ return cross_attention_states
1157
+
1158
+ def get_cross_attention_mask(
1159
+ self,
1160
+ input_ids: torch.Tensor,
1161
+ attn_metadata: AttentionMetadata,
1162
+ num_tiles: List[List[int]],
1163
+ num_tokens_per_tile: int,
1164
+ dtype: torch.dtype,
1165
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
1166
+ token_ids = input_ids.tolist()
1167
+ start = 0
1168
+ batch_token_ids = []
1169
+ for seq_len in attn_metadata.seq_lens:
1170
+ batch_token_ids.append(token_ids[start:start + seq_len])
1171
+ start += seq_len
1172
+ sparse_mask = [
1173
+ get_cross_attention_token_mask(t, MLLAMA_IMAGE_TOKEN_ID)
1174
+ for t in batch_token_ids
1175
+ ]
1176
1177
+ # Skip generating cross-attention mask if all samples
1178
+ # are text-only or have only 1 leading image.
1179
+ if skip_attention_mask(sparse_mask):
1180
+ return None, None
1181
+
1182
+ dense_mask, tile_range_for_decode = \
1183
+ convert_sparse_cross_attention_mask_to_dense(
1184
+ sparse_mask, num_tiles, attn_metadata.seq_lens)
1185
+ cross_attention_mask = \
1186
+ convert_dense_cross_attention_mask_to_tensor(
1187
+ dense_mask, num_tokens_per_tile, input_ids.device, dtype)
1188
+ kv_range_for_decode = [[
1189
+ t[0] * num_tokens_per_tile, t[1] * num_tokens_per_tile
1190
+ ] for t in tile_range_for_decode]
1191
+
1192
+ return cross_attention_mask, kv_range_for_decode
1193
+
1194
+ def get_full_text_row_masked_out_mask(
1195
+ self,
1196
+ attn_metadata: AttentionMetadata,
1197
+ device: torch.device,
1198
+ ) -> torch.Tensor:
1199
full_text_row_masked_out_mask = torch.ones(
1200
(attn_metadata.num_prefill_tokens, 1), dtype=torch.bool)
1201
start_pos = 0
1048
- for seq_len, encoder_seq_len in zip(
1049
- attn_metadata.seq_lens_tensor.cpu(),
1050
- attn_metadata.encoder_seq_lens):
1202
+ for seq_len, encoder_seq_len in zip(attn_metadata.seq_lens,
1203
+ attn_metadata.encoder_seq_lens):
1204
if encoder_seq_len == 0:
1205
full_text_row_masked_out_mask[start_pos:start_pos +
1206
seq_len] = False
1207
start_pos += seq_len
1208
full_text_row_masked_out_mask = full_text_row_masked_out_mask.to(
1056
- cross_attention_states.device)
1057
-
1058
- return cross_attention_states, full_text_row_masked_out_mask
1209
+ device)
1210
+ return full_text_row_masked_out_mask
1211
1212
def forward(
1213
self,
@@ -1069,39 +1221,54 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1221
attn_metadata.num_decode_tokens > 0:
1222
raise ValueError("Chunk prefill not supported")
1223
image_inputs = self._parse_and_validate_image_input(**kwargs)
1224
+ cross_attention_states = None
1225
+ cross_attention_mask = None
1226
+ kv_range_for_decode = None
1227
+
1228
+ # For 1) text-only prefill and decode, 2) image-present decode.
1229
if image_inputs is None:
1073
- cross_attention_mask = None
1230
full_text_row_masked_out_mask = (
1231
attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to(
1232
input_ids.device)
1077
- cross_attention_states = None
1233
skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1234
+
1235
+ # For image-present prefill.
1236
else:
1080
- # NOTE: llama's reference implementation runs vision model on CPU
1081
- pixel_values = image_inputs['data']
1082
- aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1083
- aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1084
- cross_attention_states = self.vision_model(pixel_values,
1085
- aspect_ratio_ids,
1086
- aspect_ratio_mask)
1087
- cross_attention_states = self.multi_modal_projector(
1088
- cross_attention_states)
1089
-
1090
- bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1091
- cross_attention_states = cross_attention_states.view(
1092
- bsz, -1, image_token_dim)
1093
-
1094
- cross_attention_states, full_text_row_masked_out_mask = \
1095
- self.flat_encoder_result(cross_attention_states, attn_metadata)
1237
skip_cross_attention = False
1097
- # TODO: support multi-image by this mask
1098
- cross_attention_mask = None
1238
+
1239
+ # Get the actual number of encoder tokens for each sample.
1240
+ # Because attn_metadata.encoder_seq_lens only counts the last
1241
+ # group of images for each sample, which is used to cheat the
1242
+ # block manager to allocate blocks for those images only.
1243
+ # See input_processor_for_mllama() for more details.
1244
+ num_tiles_tensor = kwargs.pop("num_tiles")
1245
+ num_tiles = [t[0].tolist() for t in num_tiles_tensor]
1246
+ num_tokens_per_tile = (self.image_size // 14)**2 + 1
1247
+ actual_encoder_seq_lens = [
1248
+ sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
1249
+ ]
1250
+ for actual_len, last_group_len in zip(
1251
+ actual_encoder_seq_lens, attn_metadata.encoder_seq_lens):
1252
+ assert actual_len >= last_group_len
1253
+
1254
+ cross_attention_states = self.get_cross_attention_states(
1255
+ image_inputs, attn_metadata, actual_encoder_seq_lens)
1256
+
1257
+ full_text_row_masked_out_mask = \
1258
+ self.get_full_text_row_masked_out_mask(
1259
+ attn_metadata, input_ids.device)
1260
+
1261
+ cross_attention_mask, kv_range_for_decode = \
1262
+ self.get_cross_attention_mask(
1263
+ input_ids, attn_metadata, num_tiles,
1264
+ num_tokens_per_tile, cross_attention_states.dtype)
1265
1266
outputs = self.language_model(
1267
input_ids=input_ids,
1268
positions=positions,
1269
cross_attention_states=cross_attention_states,
1270
cross_attention_mask=cross_attention_mask,
1271
+ kv_range_for_decode=kv_range_for_decode,
1272
full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1273
kv_caches=kv_caches,
1274
attn_metadata=attn_metadata,
@@ -1140,3 +1307,76 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1307
weight_loader = getattr(param, "weight_loader",
1308
default_weight_loader)
1309
weight_loader(param, loaded_weight)
1310
+
1311
+
1312
+def skip_attention_mask(sparse_mask: List[List[int]]) -> bool:
1313
+ for mask in sparse_mask:
1314
+ # Skip text-only samples.
1315
+ if len(mask) == 0:
1316
+ continue
1317
+ # If the sample contains more than 1 images,
1318
+ # we can't skip mask.
1319
+ if len(mask) != 1:
1320
+ return False
1321
+ # If the sample contains only 1 image,
1322
+ # but the image is not the leading one,
1323
+ # we can't skip mask.
1324
+ if mask[0][0] != 0 or mask[0][1] != -1:
1325
+ return False
1326
+ return True
1327
+
1328
+
1329
+def convert_sparse_cross_attention_mask_to_dense(
1330
+ sparse_mask: List[List[List[int]]],
1331
+ num_tiles: List[List[int]],
1332
+ lengths: List[int],
1333
+) -> Tuple[np.ndarray, List[Tuple[int, int]]]:
1334
+ total_length = sum(lengths)
1335
+ total_tiles = sum([sum(tiles) for tiles in num_tiles])
1336
+ dense_mask = np.zeros(shape=(total_length, total_tiles), dtype=np.int64)
1337
+ # A list of ranges, range[i] = [start, end] means
1338
+ # if the i-th sample has N tiles in total, the tiles[start, end]
1339
+ # will be used for cross-attention decoding.
1340
+ tile_range_for_decode = []
1341
+
1342
+ seq_start = 0
1343
+ tile_start = 0
1344
+ for masks, tiles, length in zip(sparse_mask, num_tiles, lengths):
1345
+ ts, td = -1, 0
1346
+ for mask, tile in zip(masks, tiles):
1347
+ if len(mask) != 2:
1348
+ continue
1349
+ start, end = mask
1350
+ end = min(end, length)
1351
+ if end == -1:
1352
+ end = length
1353
+ if end == length:
1354
+ if ts == -1:
1355
+ ts = tile_start
1356
+ td += tile
1357
+ dense_mask[seq_start + start:seq_start + end,
1358
+ tile_start:tile_start + tile] = 1
1359
+ tile_start += tile
1360
+ tile_range_for_decode.append((ts, ts + td))
1361
+ seq_start += length
1362
+
1363
+ return dense_mask, tile_range_for_decode
1364
+
1365
+
1366
+def convert_dense_cross_attention_mask_to_tensor(
1367
+ cross_attention_token_mask: np.ndarray,
1368
+ num_tokens_per_tile: int,
1369
+ device: torch.device,
1370
+ dtype: torch.dtype,
1371
+) -> torch.Tensor:
1372
+ mask = torch.tensor(cross_attention_token_mask, dtype=dtype, device=device)
1373
+ mask = mask.repeat_interleave(num_tokens_per_tile, dim=1)
1374
+
1375
+ mask = 1.0 - mask
1376
+ mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(dtype).min)
1377
+
1378
+ ninf = torch.finfo(dtype).min
1379
+ full_text_mask = ((mask != ninf).any(dim=-1).type_as(mask)[..., None])
1380
+ mask *= full_text_mask
1381
+ # (num_prompt_tokens, num_encoder_tokens)
1382
+ return mask