@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0fe4fe86d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Make llama3.2 support multiple and interleaved images (#9095)

Xiang Xu committed Oct 14, 2024 at 15:24 UTC f0fe4fe86d45763cb5904ac256ac6241c5eb2fde
3 files changed +384 -42
examples/offline_inference_vision_language_multi_image.py
+23
@@ -234,12 +234,35 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
234 )
235
236
237 +def load_mllama(question, image_urls: List[str]) -> ModelRequestData:
238 + model_name = "meta-llama/Llama-3.2-11B-Vision-Instruct"
239 +
240 + # The configuration below has been confirmed to launch on a single L40 GPU.
241 + llm = LLM(
242 + model=model_name,
243 + max_model_len=4096,
244 + max_num_seqs=16,
245 + enforce_eager=True,
246 + limit_mm_per_prompt={"image": len(image_urls)},
247 + )
248 +
249 + prompt = f"<|image|><|image|><|begin_of_text|>{question}"
250 + return ModelRequestData(
251 + llm=llm,
252 + prompt=prompt,
253 + stop_token_ids=None,
254 + image_data=[fetch_image(url) for url in image_urls],
255 + chat_template=None,
256 + )
257 +
258 +
259 model_example_map = {
260 "phi3_v": load_phi3v,
261 "internvl_chat": load_internvl,
262 "NVLM_D": load_nvlm_d,
263 "qwen2_vl": load_qwen2_vl,
264 "qwen_vl_chat": load_qwenvl_chat,
265 + "mllama": load_mllama,
266 }
267
268
tests/models/encoder_decoder/vision_language/test_mllama.py
+82 -3
@@ -12,7 +12,7 @@ from ....conftest import (IMAGE_ASSETS, HfRunner, PromptImageInput, VllmRunner,
12 from ....utils import large_gpu_test
13 from ...utils import check_logprobs_close
14
15 -_LIMIT_IMAGE_PER_PROMPT = 1
15 +_LIMIT_IMAGE_PER_PROMPT = 3
16
17 HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
18 "stop_sign":
@@ -244,8 +244,9 @@ def _run_test(
244 @pytest.mark.parametrize("dtype", ["bfloat16"])
245 @pytest.mark.parametrize("max_tokens", [128])
246 @pytest.mark.parametrize("num_logprobs", [5])
247 -def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
248 - max_tokens, num_logprobs) -> None:
247 +def test_models_single_leading_image(hf_runner, vllm_runner, image_assets,
248 + model, sizes, dtype, max_tokens,
249 + num_logprobs) -> None:
250 run_test(
251 hf_runner,
252 vllm_runner,
@@ -257,3 +258,81 @@ def test_models(hf_runner, vllm_runner, image_assets, model, sizes, dtype,
258 num_logprobs=num_logprobs,
259 tensor_parallel_size=1,
260 )
261 +
262 +
263 +@large_gpu_test(min_gb=48)
264 +@pytest.mark.parametrize("model", models)
265 +@pytest.mark.parametrize("dtype", ["bfloat16"])
266 +@pytest.mark.parametrize("max_tokens", [128])
267 +@pytest.mark.parametrize("num_logprobs", [5])
268 +def test_models_multi_leading_images(hf_runner, vllm_runner, image_assets,
269 + model, dtype, max_tokens,
270 + num_logprobs) -> None:
271 +
272 + stop_sign = image_assets[0].pil_image
273 + cherry_blossom = image_assets[1].pil_image
274 +
275 + inputs = [(
276 + [
277 + "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
278 + "<|image|><|image|><|begin_of_text|>Describe 2 images.", # noqa: E501
279 + "<|image|><|image|><|image|><|begin_of_text|>Describe 3 images.", # noqa: E501
280 + ],
281 + [
282 + [stop_sign, cherry_blossom],
283 + # Images with different sizes.
284 + [
285 + stop_sign.resize((512, 512)),
286 + stop_sign,
287 + ],
288 + [
289 + stop_sign,
290 + stop_sign.resize((512, 1536)),
291 + cherry_blossom.resize((512, 1024)),
292 + ],
293 + ])]
294 +
295 + _run_test(
296 + hf_runner,
297 + vllm_runner,
298 + inputs,
299 + model,
300 + dtype=dtype,
301 + max_tokens=max_tokens,
302 + num_logprobs=num_logprobs,
303 + tensor_parallel_size=1,
304 + )
305 +
306 +
307 +@large_gpu_test(min_gb=48)
308 +@pytest.mark.parametrize("model", models)
309 +@pytest.mark.parametrize("dtype", ["bfloat16"])
310 +@pytest.mark.parametrize("max_tokens", [128])
311 +@pytest.mark.parametrize("num_logprobs", [5])
312 +def test_models_interleaved_images(hf_runner, vllm_runner, image_assets, model,
313 + dtype, max_tokens, num_logprobs) -> None:
314 +
315 + stop_sign = image_assets[0].pil_image
316 + cherry_blossom = image_assets[1].pil_image
317 +
318 + inputs = [(
319 + [
320 + "<|begin_of_text|>The content of the image <|image|> is", # noqa: E501
321 + "<|begin_of_text|>Between the first image <|image|> and the second image<|image|>, " # noqa: E501
322 + "which is a stop sign and which is a cherry blossom?", # noqa: E501
323 + ],
324 + [
325 + [stop_sign],
326 + [stop_sign, cherry_blossom],
327 + ])]
328 +
329 + _run_test(
330 + hf_runner,
331 + vllm_runner,
332 + inputs,
333 + model,
334 + dtype=dtype,
335 + max_tokens=max_tokens,
336 + num_logprobs=num_logprobs,
337 + tensor_parallel_size=1,
338 + )
vllm/model_executor/models/mllama.py
+279 -39
@@ -18,6 +18,7 @@ from array import array
18 from typing import (Iterable, List, Literal, Mapping, Optional, Tuple,
19 TypedDict, Union)
20
21 +import numpy as np
22 import torch
23 import torch.nn.functional as F
24 import torch.utils.checkpoint
@@ -28,9 +29,12 @@ from transformers.modeling_outputs import (BaseModelOutput,
29 CausalLMOutputWithPast)
30 from transformers.models.mllama.image_processing_mllama import (
31 get_optimal_tiled_canvas)
32 +from transformers.models.mllama.processing_mllama import (
33 + get_cross_attention_token_mask)
34
35 import vllm.distributed.parallel_state as ps
36 from vllm.attention import Attention, AttentionMetadata, AttentionType
37 +from vllm.attention.ops.paged_attn import PagedAttention
38 from vllm.config import CacheConfig, MultiModalConfig
39 from vllm.distributed import get_tensor_model_parallel_world_size
40 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
@@ -72,6 +76,16 @@ class MllamaImagePixelInputs(TypedDict):
76 # TODO: support LlamaImageEmbeddingInputs
77
78
79 +def _get_num_image_in_last_group(prompt_token_ids: List[int]) -> int:
80 + num_images = 0
81 + for token_id in prompt_token_ids[::-1]:
82 + if token_id == MLLAMA_IMAGE_TOKEN_ID:
83 + num_images += 1
84 + elif num_images > 0:
85 + break
86 + return num_images
87 +
88 +
89 def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
90 # move encoder_prompt to prompt
91 if llm_inputs.get("prompt") is None:
@@ -91,12 +105,16 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
105 llm_inputs["encoder_multi_modal_data"] = {}
106 return llm_inputs
107
94 - # get num_tiles
108 if isinstance(multi_modal_data['image'], Image.Image):
109 multi_modal_data['image'] = [multi_modal_data['image']]
110 + # Since only the last group of consecutive images
111 + # are attended by the decoded tokens, we only need to
112 + # get the number of tiles for those images.
113 + num_decode_images = _get_num_image_in_last_group(
114 + llm_inputs["prompt_token_ids"])
115 hf_config = ctx.model_config.hf_config
116 num_tiles = 0
99 - for image in multi_modal_data["image"]:
117 + for image in multi_modal_data["image"][::-1]:
118 width, height = image.size
119 tile_size = hf_config.vision_config.image_size
120 canvas_height, canvas_width = get_optimal_tiled_canvas(
@@ -108,8 +126,13 @@ def input_processor_for_mllama(ctx: InputContext, llm_inputs: LLMInputs):
126 num_tiles_height = canvas_height // tile_size
127 num_tiles_width = canvas_width // tile_size
128 num_tiles += num_tiles_height * num_tiles_width
129 + num_decode_images -= 1
130 + if num_decode_images == 0:
131 + break
132
112 - # set encoder prompt based on num_tiles
133 + # Set encoder prompt length based on the number of tiles.
134 + # This tells the block manager to allocate correct number
135 + # of slots for encoder tokens.
136 assert hf_config.vision_config.image_size % 14 == 0, \
137 "chunk size should be multiple of 14"
138 token_per_chunk = (hf_config.vision_config.image_size // 14)**2 + 1
@@ -675,6 +698,7 @@ class MllamaTextCrossAttention(nn.Module):
698 self,
699 hidden_states: torch.Tensor,
700 attention_mask: Optional[torch.Tensor],
701 + kv_range_for_decode: Optional[List[Tuple[int, int]]],
702 cross_attention_states: Optional[torch.Tensor],
703 kv_cache: torch.Tensor,
704 attn_metadata: AttentionMetadata,
@@ -697,15 +721,71 @@ class MllamaTextCrossAttention(nn.Module):
721 q = q.view(-1, self.num_local_heads, self.head_dim)
722 q = self.q_norm(q)
723
700 - output = self.attn(q,
701 - k,
702 - v,
703 - kv_cache,
704 - attn_metadata,
705 - attn_type=AttentionType.ENCODER_DECODER)
724 + if attention_mask is not None:
725 + output = self.attention_with_mask(q, k, v, kv_cache,
726 + attention_mask,
727 + kv_range_for_decode,
728 + attn_metadata)
729 + else:
730 + output = self.attn(q,
731 + k,
732 + v,
733 + kv_cache,
734 + attn_metadata,
735 + attn_type=AttentionType.ENCODER_DECODER)
736 out, _ = self.o_proj(output)
737 return out
738
739 + def attention_with_mask(
740 + self,
741 + q: torch.Tensor,
742 + k: torch.Tensor,
743 + v: torch.Tensor,
744 + kv_cache: torch.Tensor,
745 + attention_mask: torch.Tensor,
746 + kv_range_for_decode: List[Tuple[int, int]],
747 + attn_metadata: AttentionMetadata,
748 + ) -> torch.Tensor:
749 + # Skip writing kv-cache for the initial profiling run.
750 + if len(kv_cache.shape) == 3:
751 + key_cache, value_cache = PagedAttention.split_kv_cache(
752 + kv_cache, self.num_local_key_value_heads, self.head_dim)
753 + cached_k = torch.cat([k[s:e] for s, e in kv_range_for_decode])
754 + cached_v = torch.cat([v[s:e] for s, e in kv_range_for_decode])
755 + PagedAttention.write_to_paged_cache(
756 + cached_k, cached_v, key_cache, value_cache,
757 + attn_metadata.cross_slot_mapping, "auto", 1.0, 1.0)
758 + # We have to call torch.sdpa for prefill when using a
759 + # custom cross-attention mask. Because the mask is not a
760 + # standard causal mask, neither a block diagonal mask which
761 + # can be optimized by xformers.BlockDiagonalMask.
762 + # The mask is specially calculated for supporting multi
763 + # images and interleaved images.
764 + q_len = q.shape[0]
765 + kv_len = k.shape[0]
766 + q = q.transpose(0, 1).view(self.num_local_key_value_heads,
767 + self.num_key_value_groups, q_len,
768 + self.head_dim)
769 + k = k.transpose(0,
770 + 1)[:,
771 + None, :, :].expand(self.num_local_key_value_heads,
772 + self.num_key_value_groups,
773 + kv_len, self.head_dim)
774 + v = v.transpose(0,
775 + 1)[:,
776 + None, :, :].expand(self.num_local_key_value_heads,
777 + self.num_key_value_groups,
778 + kv_len, self.head_dim)
779 + attention_mask = attention_mask.view(1, 1, q_len, kv_len)
780 + output = F.scaled_dot_product_attention(q,
781 + k,
782 + v,
783 + attn_mask=attention_mask,
784 + is_causal=False)
785 + output = output.permute(2, 0, 1, 3).reshape(
786 + q_len, self.num_local_heads * self.head_dim)
787 + return output
788 +
789
790 class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
791 """Cross-attention transformer block with tanh-gated attention
@@ -741,6 +821,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
821 hidden_states: torch.Tensor,
822 cross_attention_states: torch.Tensor,
823 cross_attention_mask: torch.Tensor,
824 + kv_range_for_decode: Optional[List[Tuple[int, int]]],
825 full_text_row_masked_out_mask: torch.Tensor,
826 kv_cache: List[torch.Tensor],
827 attn_metadata: AttentionMetadata,
@@ -751,6 +832,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
832 hidden_states = self.cross_attn(
833 hidden_states=hidden_states,
834 attention_mask=cross_attention_mask,
835 + kv_range_for_decode=kv_range_for_decode,
836 cross_attention_states=cross_attention_states,
837 kv_cache=kv_cache,
838 attn_metadata=attn_metadata,
@@ -804,6 +886,7 @@ class MllamaTextModel(nn.Module):
886 positions: Optional[torch.LongTensor],
887 cross_attention_states: Optional[torch.LongTensor],
888 cross_attention_mask: Optional[torch.LongTensor],
889 + kv_range_for_decode: Optional[List[Tuple[int, int]]],
890 full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
891 torch.Tensor]],
892 kv_caches: List[torch.Tensor],
@@ -820,6 +903,7 @@ class MllamaTextModel(nn.Module):
903 hidden_states=hidden_states,
904 cross_attention_states=cross_attention_states,
905 cross_attention_mask=cross_attention_mask,
906 + kv_range_for_decode=kv_range_for_decode,
907 full_text_row_masked_out_mask=
908 full_text_row_masked_out_mask,
909 kv_cache=kv_caches[idx],
@@ -868,6 +952,7 @@ class MllamaForCausalLM(nn.Module):
952 positions: Optional[torch.LongTensor],
953 cross_attention_states: Optional[torch.LongTensor],
954 cross_attention_mask: Optional[torch.LongTensor],
955 + kv_range_for_decode: Optional[List[Tuple[int, int]]],
956 full_text_row_masked_out_mask: Optional[Tuple[torch.Tensor,
957 torch.Tensor]],
958 kv_caches: List[torch.Tensor],
@@ -879,6 +964,7 @@ class MllamaForCausalLM(nn.Module):
964 positions=positions,
965 cross_attention_states=cross_attention_states,
966 cross_attention_mask=cross_attention_mask,
967 + kv_range_for_decode=kv_range_for_decode,
968 full_text_row_masked_out_mask=full_text_row_masked_out_mask,
969 kv_caches=kv_caches,
970 attn_metadata=attn_metadata,
@@ -1026,36 +1112,102 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1112 raise AssertionError("This line should be unreachable.")
1113
1114 def flat_encoder_result(self, cross_attention_states: torch.Tensor,
1029 - attn_metadata: AttentionMetadata):
1115 + attn_metadata: AttentionMetadata,
1116 + actual_encoder_seq_lens: List[int]):
1117
1118 cross_attention_states_flat = torch.zeros(
1032 - sum(attn_metadata.encoder_seq_lens),
1119 + sum(actual_encoder_seq_lens),
1120 cross_attention_states.shape[-1],
1121 device=cross_attention_states.device,
1122 dtype=cross_attention_states.dtype)
1123 start_pos = 0
1037 - for seq_len, vision_token_in_batch in zip(
1038 - attn_metadata.encoder_seq_lens, cross_attention_states):
1124 + for seq_len, vision_token_in_batch in zip(actual_encoder_seq_lens,
1125 + cross_attention_states):
1126 end_pos = start_pos + seq_len
1127 cross_attention_states_flat[
1128 start_pos:end_pos] = vision_token_in_batch[:seq_len]
1129 start_pos = end_pos
1130 cross_attention_states = cross_attention_states_flat
1131 + return cross_attention_states
1132 +
1133 + def get_cross_attention_states(
1134 + self,
1135 + image_inputs: MllamaImagePixelInputs,
1136 + attn_metadata: AttentionMetadata,
1137 + actual_encoder_seq_lens: List[int],
1138 + ) -> Tuple[torch.Tensor]:
1139 + # NOTE: llama's reference implementation runs vision model on CPU
1140 + pixel_values = image_inputs['data']
1141 + aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1142 + aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1143 + cross_attention_states = self.vision_model(pixel_values,
1144 + aspect_ratio_ids,
1145 + aspect_ratio_mask)
1146 + cross_attention_states = self.multi_modal_projector(
1147 + cross_attention_states)
1148 +
1149 + bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1150 + cross_attention_states = cross_attention_states.view(
1151 + bsz, -1, image_token_dim)
1152 +
1153 + cross_attention_states = self.flat_encoder_result(
1154 + cross_attention_states, attn_metadata, actual_encoder_seq_lens)
1155 +
1156 + return cross_attention_states
1157 +
1158 + def get_cross_attention_mask(
1159 + self,
1160 + input_ids: torch.Tensor,
1161 + attn_metadata: AttentionMetadata,
1162 + num_tiles: List[List[int]],
1163 + num_tokens_per_tile: int,
1164 + dtype: torch.dtype,
1165 + ) -> Tuple[torch.Tensor, torch.Tensor]:
1166 + token_ids = input_ids.tolist()
1167 + start = 0
1168 + batch_token_ids = []
1169 + for seq_len in attn_metadata.seq_lens:
1170 + batch_token_ids.append(token_ids[start:start + seq_len])
1171 + start += seq_len
1172 + sparse_mask = [
1173 + get_cross_attention_token_mask(t, MLLAMA_IMAGE_TOKEN_ID)
1174 + for t in batch_token_ids
1175 + ]
1176
1177 + # Skip generating cross-attention mask if all samples
1178 + # are text-only or have only 1 leading image.
1179 + if skip_attention_mask(sparse_mask):
1180 + return None, None
1181 +
1182 + dense_mask, tile_range_for_decode = \
1183 + convert_sparse_cross_attention_mask_to_dense(
1184 + sparse_mask, num_tiles, attn_metadata.seq_lens)
1185 + cross_attention_mask = \
1186 + convert_dense_cross_attention_mask_to_tensor(
1187 + dense_mask, num_tokens_per_tile, input_ids.device, dtype)
1188 + kv_range_for_decode = [[
1189 + t[0] * num_tokens_per_tile, t[1] * num_tokens_per_tile
1190 + ] for t in tile_range_for_decode]
1191 +
1192 + return cross_attention_mask, kv_range_for_decode
1193 +
1194 + def get_full_text_row_masked_out_mask(
1195 + self,
1196 + attn_metadata: AttentionMetadata,
1197 + device: torch.device,
1198 + ) -> torch.Tensor:
1199 full_text_row_masked_out_mask = torch.ones(
1200 (attn_metadata.num_prefill_tokens, 1), dtype=torch.bool)
1201 start_pos = 0
1048 - for seq_len, encoder_seq_len in zip(
1049 - attn_metadata.seq_lens_tensor.cpu(),
1050 - attn_metadata.encoder_seq_lens):
1202 + for seq_len, encoder_seq_len in zip(attn_metadata.seq_lens,
1203 + attn_metadata.encoder_seq_lens):
1204 if encoder_seq_len == 0:
1205 full_text_row_masked_out_mask[start_pos:start_pos +
1206 seq_len] = False
1207 start_pos += seq_len
1208 full_text_row_masked_out_mask = full_text_row_masked_out_mask.to(
1056 - cross_attention_states.device)
1057 -
1058 - return cross_attention_states, full_text_row_masked_out_mask
1209 + device)
1210 + return full_text_row_masked_out_mask
1211
1212 def forward(
1213 self,
@@ -1069,39 +1221,54 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1221 attn_metadata.num_decode_tokens > 0:
1222 raise ValueError("Chunk prefill not supported")
1223 image_inputs = self._parse_and_validate_image_input(**kwargs)
1224 + cross_attention_states = None
1225 + cross_attention_mask = None
1226 + kv_range_for_decode = None
1227 +
1228 + # For 1) text-only prefill and decode, 2) image-present decode.
1229 if image_inputs is None:
1073 - cross_attention_mask = None
1230 full_text_row_masked_out_mask = (
1231 attn_metadata.encoder_seq_lens_tensor != 0).reshape(-1, 1).to(
1232 input_ids.device)
1077 - cross_attention_states = None
1233 skip_cross_attention = max(attn_metadata.encoder_seq_lens) == 0
1234 +
1235 + # For image-present prefill.
1236 else:
1080 - # NOTE: llama's reference implementation runs vision model on CPU
1081 - pixel_values = image_inputs['data']
1082 - aspect_ratio_ids = image_inputs['aspect_ratio_ids']
1083 - aspect_ratio_mask = image_inputs['aspect_ratio_mask']
1084 - cross_attention_states = self.vision_model(pixel_values,
1085 - aspect_ratio_ids,
1086 - aspect_ratio_mask)
1087 - cross_attention_states = self.multi_modal_projector(
1088 - cross_attention_states)
1089 -
1090 - bsz, _, _, _, image_token_dim = tuple(cross_attention_states.shape)
1091 - cross_attention_states = cross_attention_states.view(
1092 - bsz, -1, image_token_dim)
1093 -
1094 - cross_attention_states, full_text_row_masked_out_mask = \
1095 - self.flat_encoder_result(cross_attention_states, attn_metadata)
1237 skip_cross_attention = False
1097 - # TODO: support multi-image by this mask
1098 - cross_attention_mask = None
1238 +
1239 + # Get the actual number of encoder tokens for each sample.
1240 + # Because attn_metadata.encoder_seq_lens only counts the last
1241 + # group of images for each sample, which is used to cheat the
1242 + # block manager to allocate blocks for those images only.
1243 + # See input_processor_for_mllama() for more details.
1244 + num_tiles_tensor = kwargs.pop("num_tiles")
1245 + num_tiles = [t[0].tolist() for t in num_tiles_tensor]
1246 + num_tokens_per_tile = (self.image_size // 14)**2 + 1
1247 + actual_encoder_seq_lens = [
1248 + sum(num_tile) * num_tokens_per_tile for num_tile in num_tiles
1249 + ]
1250 + for actual_len, last_group_len in zip(
1251 + actual_encoder_seq_lens, attn_metadata.encoder_seq_lens):
1252 + assert actual_len >= last_group_len
1253 +
1254 + cross_attention_states = self.get_cross_attention_states(
1255 + image_inputs, attn_metadata, actual_encoder_seq_lens)
1256 +
1257 + full_text_row_masked_out_mask = \
1258 + self.get_full_text_row_masked_out_mask(
1259 + attn_metadata, input_ids.device)
1260 +
1261 + cross_attention_mask, kv_range_for_decode = \
1262 + self.get_cross_attention_mask(
1263 + input_ids, attn_metadata, num_tiles,
1264 + num_tokens_per_tile, cross_attention_states.dtype)
1265
1266 outputs = self.language_model(
1267 input_ids=input_ids,
1268 positions=positions,
1269 cross_attention_states=cross_attention_states,
1270 cross_attention_mask=cross_attention_mask,
1271 + kv_range_for_decode=kv_range_for_decode,
1272 full_text_row_masked_out_mask=full_text_row_masked_out_mask,
1273 kv_caches=kv_caches,
1274 attn_metadata=attn_metadata,
@@ -1140,3 +1307,76 @@ class MllamaForConditionalGeneration(nn.Module, SupportsMultiModal):
1307 weight_loader = getattr(param, "weight_loader",
1308 default_weight_loader)
1309 weight_loader(param, loaded_weight)
1310 +
1311 +
1312 +def skip_attention_mask(sparse_mask: List[List[int]]) -> bool:
1313 + for mask in sparse_mask:
1314 + # Skip text-only samples.
1315 + if len(mask) == 0:
1316 + continue
1317 + # If the sample contains more than 1 images,
1318 + # we can't skip mask.
1319 + if len(mask) != 1:
1320 + return False
1321 + # If the sample contains only 1 image,
1322 + # but the image is not the leading one,
1323 + # we can't skip mask.
1324 + if mask[0][0] != 0 or mask[0][1] != -1:
1325 + return False
1326 + return True
1327 +
1328 +
1329 +def convert_sparse_cross_attention_mask_to_dense(
1330 + sparse_mask: List[List[List[int]]],
1331 + num_tiles: List[List[int]],
1332 + lengths: List[int],
1333 +) -> Tuple[np.ndarray, List[Tuple[int, int]]]:
1334 + total_length = sum(lengths)
1335 + total_tiles = sum([sum(tiles) for tiles in num_tiles])
1336 + dense_mask = np.zeros(shape=(total_length, total_tiles), dtype=np.int64)
1337 + # A list of ranges, range[i] = [start, end] means
1338 + # if the i-th sample has N tiles in total, the tiles[start, end]
1339 + # will be used for cross-attention decoding.
1340 + tile_range_for_decode = []
1341 +
1342 + seq_start = 0
1343 + tile_start = 0
1344 + for masks, tiles, length in zip(sparse_mask, num_tiles, lengths):
1345 + ts, td = -1, 0
1346 + for mask, tile in zip(masks, tiles):
1347 + if len(mask) != 2:
1348 + continue
1349 + start, end = mask
1350 + end = min(end, length)
1351 + if end == -1:
1352 + end = length
1353 + if end == length:
1354 + if ts == -1:
1355 + ts = tile_start
1356 + td += tile
1357 + dense_mask[seq_start + start:seq_start + end,
1358 + tile_start:tile_start + tile] = 1
1359 + tile_start += tile
1360 + tile_range_for_decode.append((ts, ts + td))
1361 + seq_start += length
1362 +
1363 + return dense_mask, tile_range_for_decode
1364 +
1365 +
1366 +def convert_dense_cross_attention_mask_to_tensor(
1367 + cross_attention_token_mask: np.ndarray,
1368 + num_tokens_per_tile: int,
1369 + device: torch.device,
1370 + dtype: torch.dtype,
1371 +) -> torch.Tensor:
1372 + mask = torch.tensor(cross_attention_token_mask, dtype=dtype, device=device)
1373 + mask = mask.repeat_interleave(num_tokens_per_tile, dim=1)
1374 +
1375 + mask = 1.0 - mask
1376 + mask = mask.masked_fill(mask.to(torch.bool), torch.finfo(dtype).min)
1377 +
1378 + ninf = torch.finfo(dtype).min
1379 + full_text_mask = ((mask != ninf).any(dim=-1).type_as(mask)[..., None])
1380 + mask *= full_text_mask
1381 + # (num_prompt_tokens, num_encoder_tokens)
1382 + return mask