@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9d3ecf016
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[VLM] Merged multi-modal processor for Molmo (#12966)

Cyrus Leung committed Feb 13, 2025 at 20:34 UTC c9d3ecf016deb619a3bce6d59049224e8eb12364
9 files changed +750 -498
docs/source/models/supported_models.md
+1 -1
@@ -793,7 +793,7 @@ See [this page](#generative-models) for more information on how to use generativ
793 - * `MolmoForCausalLM`
794 * Molmo
795 * T + I
796 - * `allenai/Molmo-7B-D-0924`, `allenai/Molmo-72B-0924`, etc.
796 + * `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc.
797 * ✅︎
798 * ✅︎
799 * ✅︎
tests/models/decoder_only/language/test_models.py
+1 -1
@@ -27,7 +27,7 @@ from ...utils import check_logprobs_close
27 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
28 ),
29 pytest.param(
30 - "THUDM/chatglm3-6b", # ChatGLM (text-only)
30 + "THUDM/chatglm3-6b", # chatglm (text-only)
31 ),
32 pytest.param(
33 "meta-llama/Llama-3.2-1B-Instruct", # llama
tests/models/decoder_only/vision_language/test_models.py
+2 -3
@@ -404,11 +404,10 @@ VLM_TEST_SETTINGS = {
404 "molmo": VLMTestInfo(
405 models=["allenai/Molmo-7B-D-0924"],
406 test_type=(VLMTestType.IMAGE),
407 - prompt_formatter=lambda img_prompt:"User: " + img_prompt + " Assistant:", # noqa: E501
407 + prompt_formatter=identity,
408 max_model_len=4096,
409 max_num_seqs=2,
410 - image_size_factors=[(),(1.0, 1.0, 1.0)],
411 - patch_hf_runner=model_utils.mlomo_patch_hf_runner,
410 + patch_hf_runner=model_utils.molmo_patch_hf_runner,
411 postprocess_inputs=model_utils.molmo_post_processor,
412 ),
413 # Tests for phi3v currently live in another file because of a bug in
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+21 -77
@@ -6,7 +6,7 @@ typically specific to a small subset of models.
6 import re
7 import types
8 from pathlib import PosixPath
9 -from typing import Any, Callable, Dict, List, Optional, Tuple, Union
9 +from typing import Callable, List, Optional, Tuple, Union
10
11 import torch
12 from PIL.Image import Image
@@ -17,9 +17,7 @@ from vllm.sequence import SampleLogprobs
17 from vllm.transformers_utils.tokenizer import patch_padding_side
18 from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
20 -from .....conftest import (HfRunner, ImageAsset, PromptAudioInput,
21 - PromptImageInput, PromptVideoInput, _ImageAssets)
22 -from ....utils import TokensTextLogprobs
20 +from .....conftest import HfRunner, ImageAsset, _ImageAssets
21 from .types import RunnerOutput
22
23
@@ -522,74 +520,7 @@ def minicpmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
520 return hf_model
521
522
525 -def _generate_greedy_logprobs_limit(
526 - self,
527 - prompts: List[str],
528 - max_tokens: int,
529 - num_logprobs: int,
530 - images: Optional[PromptImageInput] = None,
531 - audios: Optional[PromptAudioInput] = None,
532 - videos: Optional[PromptVideoInput] = None,
533 - **kwargs: Any,
534 -) -> List[TokensTextLogprobs]:
535 - all_inputs = self.get_inputs(prompts,
536 - images=images,
537 - videos=videos,
538 - audios=audios)
539 -
540 - # Process in batches for inference.
541 - if len(all_inputs):
542 - input_ids_lst = []
543 - images_lst = []
544 - images_input_idx_lst = []
545 - imges_masks_lst = []
546 - for inputs in all_inputs:
547 - input_ids_lst.append(inputs["input_ids"])
548 - images_lst.append(inputs["images"])
549 - images_input_idx_lst.append(inputs["image_input_idx"])
550 - imges_masks_lst.append(inputs["image_masks"])
551 - batch_inputs = {}
552 - batch_inputs['input_ids'] = torch.cat(input_ids_lst, dim=0)
553 - batch_inputs['images'] = torch.cat(images_lst, dim=0)
554 - batch_inputs['image_input_idx'] = torch.cat(images_input_idx_lst,
555 - dim=0)
556 - batch_inputs['image_masks'] = torch.cat(imges_masks_lst, dim=0)
557 -
558 - outputs = self.model.generate_from_batch(
559 - batch=self.wrap_device(batch_inputs,
560 - device=self.model.device.type),
561 - generation_config=GenerationConfig(
562 - max_new_tokens=max_tokens,
563 - stop_strings="<|endoftext|>",
564 - do_sample=False,
565 - ),
566 - tokenizer=self.tokenizer,
567 - output_hidden_states=True,
568 - return_dict_in_generate=True,
569 - )
570 -
571 - all_logprobs: List[List[Dict[int, float]]] = []
572 - all_output_ids: List[List[int]] = []
573 - all_output_strs: List[str] = []
574 -
575 - for index in range(len(all_inputs)):
576 - (
577 - seq_logprobs_lst,
578 - output_len,
579 - ) = self._hidden_states_to_logprobs(outputs.hidden_states,
580 - num_logprobs)
581 - all_logprobs.append(seq_logprobs_lst)
582 - seq_ids = outputs.sequences[index]
583 - output_ids = seq_ids[-output_len:]
584 - all_output_ids.append(output_ids.tolist())
585 - all_output_strs.append(self.tokenizer.decode(output_ids))
586 - outputs = zip(all_output_ids, all_output_strs, all_logprobs)
587 - return [(output_ids, output_str, output_logprobs)
588 - for output_ids, output_str, output_logprobs in outputs]
589 -
590 -
591 -####### Molmo-specific HuggingFace runner patchers
592 -def mlomo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
523 +def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
524 """Patches and returns an instance of the HfRunner to use for Molmo."""
525 hf_processor = hf_model.processor
526
@@ -598,10 +529,23 @@ def mlomo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
529
530 hf_model.processor = _processor
531
601 - setattr( # noqa: B010
602 - hf_model,
603 - "generate_greedy_logprobs_limit",
604 - types.MethodType(_generate_greedy_logprobs_limit, hf_model),
605 - )
532 + def _generate(self, max_new_tokens=None, do_sample=None, **kwargs):
533 + batch = {
534 + k: kwargs.pop(k)
535 + for k in ("input_ids", "images", "image_input_idx", "image_masks")
536 + if k in kwargs
537 + }
538 +
539 + return self.generate_from_batch(
540 + batch,
541 + generation_config=GenerationConfig(
542 + max_new_tokens=max_new_tokens,
543 + stop_strings="<|endoftext|>",
544 + do_sample=do_sample,
545 + ),
546 + **kwargs,
547 + )
548 +
549 + hf_model.model.generate = types.MethodType(_generate, hf_model.model)
550
551 return hf_model
tests/models/multimodal/processing/test_common.py
+2
@@ -168,6 +168,8 @@ def _test_processing_correctness(
168 "mistral-community/pixtral-12b",
169 "openbmb/MiniCPM-o-2_6",
170 "openbmb/MiniCPM-V-2_6",
171 + "allenai/Molmo-7B-D-0924",
172 + "allenai/Molmo-7B-O-0924",
173 "nvidia/NVLM-D-72B",
174 "Qwen/Qwen-VL-Chat",
175 "Qwen/Qwen2-VL-2B-Instruct",
tests/models/registry.py
+1
@@ -256,6 +256,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
256 "MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-V-2_6",
257 trust_remote_code=True),
258 "MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
259 + extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
260 trust_remote_code=True),
261 "NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
262 trust_remote_code=True),
vllm/model_executor/models/molmo.py
+681 -342
@@ -1,18 +1,20 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import math
4 -import re
5 -from array import array
4 from dataclasses import dataclass
7 -from functools import lru_cache, partial
8 -from typing import Iterable, List, Mapping, Optional, Set, Tuple, TypedDict
5 +from functools import cached_property, partial
6 +from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
7 + Union, cast)
8
9 +import numpy as np
10 import torch
11 +import torch.nn as nn
12 +import torch.nn.functional as F
13 from einops import rearrange
12 -from PIL import Image
13 -from torch import nn
14 -from torch.nn import functional as F
15 -from transformers import PretrainedConfig
14 +from transformers import (BatchFeature, PretrainedConfig, ProcessorMixin,
15 + TensorType)
16 +from transformers.image_utils import ImageInput
17 +from transformers.tokenization_utils_base import TextInput
18
19 from vllm.attention import Attention, AttentionMetadata
20 from vllm.attention.layer import MultiHeadAttention
@@ -22,8 +24,6 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
24 get_tensor_model_parallel_world_size,
25 split_tensor_along_last_dim,
26 tensor_model_parallel_all_gather)
25 -from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26 - InputContext, token_inputs)
27 from vllm.model_executor import SamplingMetadata
28 from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
29 SiluAndMul)
@@ -40,15 +40,21 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40 ParallelLMHead, VocabParallelEmbedding)
41 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 from vllm.model_executor.models.module_mapping import MultiModelKeys
43 -from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
44 -from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
45 -from vllm.multimodal.utils import cached_get_tokenizer
46 -from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
47 - SequenceData)
48 -from vllm.transformers_utils.processor import get_processor
43 +from vllm.multimodal import MULTIMODAL_REGISTRY
44 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
45 + NestedTensors)
46 +from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47 + MultiModalDataItems)
48 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
49 + BaseProcessingInfo, PromptReplacement,
50 + PromptReplacementDetails)
51 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52 +from vllm.sequence import IntermediateTensors
53 +from vllm.utils import JSONTree, json_map_leaves
54
55 from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
51 -from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
56 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
57 + is_pp_missing_parameter,
58 make_empty_intermediate_tensors_factory, make_layers,
59 maybe_prefix, merge_multimodal_embeddings)
60
@@ -56,38 +62,39 @@ from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
62 VIT_LAYERS = [-2, -9]
63 NUM_PREFIX_TOKENS = 1
64 ADDITIONAL_VOCAB_SIZE = 128
59 -DEFAULT_IMAGE_PATCH_TOKEN_ID = 152066
60 -DEFAULT_IM_START_TOKEN_ID = 152067
61 -DEFAULT_IM_END_TOKEN_ID = 152064
62 -DEFAULT_IM_COL_TOKEN_ID = 152065
65 +IMAGE_PATCH_TOKEN = "<im_patch>"
66 +IM_COL_TOKEN = "<im_col>"
67 +IM_START_TOKEN = "<im_start>"
68 +IM_END_TOKEN = "<im_end>"
69 +POOLING_SIZE = 2
70
71
72 class MolmoImageInputs(TypedDict):
66 - images: torch.Tensor
67 - """Shape:
68 - `(batch_size, num_crops, num_patch, patch_dim)`
69 - """
73 + images: Union[torch.Tensor, List[torch.Tensor]]
74 + """Shape: `(batch_size, num_crops, num_patch, patch_dim)`"""
75
71 - image_input_idx: torch.Tensor
72 - """Shape:
73 - `(batch_size, num_crops, num_patch)`
74 - """
76 + image_masks: Optional[Union[torch.Tensor, List[torch.Tensor]]]
77 + """Shape: `(batch_size, num_crops, num_patch)`"""
78
76 - seq_len: torch.Tensor
77 - """Shape:
78 - `(batch_size, )`
79 + feat_is_patch: Union[torch.Tensor, List[torch.Tensor]]
80 """
81 + A boolean mask indicating which image features correspond
82 + to patch tokens.
83
81 - image_masks: Optional[torch.Tensor]
82 - """Shape:
83 - `(batch_size, num_crops, num_patch)`
84 + Shape: `(batch_size, num_crops, num_patch)`
85 """
86
86 - image_start_end: Tuple[int, int]
87 - """Starting and ending index of placeholder
88 - tokens
87 + embed_is_patch: Union[torch.Tensor, List[torch.Tensor]]
88 + """
89 + A boolean mask indicating which image embeddings correspond
90 + to patch tokens.
91 +
92 + Shape: `(batch_size, num_embeds)`
93 """
94
95 + num_crops: torch.Tensor
96 + """Shape: `(batch_size, num_images)`"""
97 +
98
99 @dataclass
100 class VisionBackboneConfig:
@@ -335,7 +342,7 @@ class VisionTransformer(nn.Module):
342
343 def forward(self,
344 x: torch.Tensor,
338 - patch_num: int = None) -> List[torch.Tensor]:
345 + patch_num: Optional[int] = None) -> List[torch.Tensor]:
346 """
347 : param x: (batch_size, num_patch, n_pixels)
348 """
@@ -465,7 +472,7 @@ class MolmoAttention(nn.Module):
472 return output
473
474
468 -class LanuageModelMLP(nn.Module):
475 +class LanguageModelMLP(nn.Module):
476 """Molmo's LLM mlp."""
477
478 def __init__(self,
@@ -559,7 +566,7 @@ class MolmoDecoderLayer(nn.Module):
566 prefix=f"{prefix}.self_attn")
567
568 # MLP block.
562 - self.mlp = LanuageModelMLP(config, quant_config=quant_config)
569 + self.mlp = LanguageModelMLP(config, quant_config=quant_config)
570
571 # LayerNorm
572 assert config.layer_norm_type == "rms"
@@ -638,8 +645,8 @@ class MolmoVisionBackbone(nn.Module):
645 self.vit_layers = VIT_LAYERS
646 self.image_num_patch = vision_config.image_num_patch
647 self.llm_patches_per_crop = (
641 - (self.image_num_patch[0] + 1) // 2,
642 - (self.image_num_patch[1] + 1) // 2,
648 + (self.image_num_patch[0] + 1) // POOLING_SIZE,
649 + (self.image_num_patch[1] + 1) // POOLING_SIZE,
650 )
651 self.image_vit = VisionTransformer(vision_config,
652 quant_config=quant_config)
@@ -723,19 +730,19 @@ class MolmoVisionBackbone(nn.Module):
730 image_features = image_features.reshape(
731 (batch_size, num_image) + self.image_num_patch + (-1, ), )
732
726 - if self.image_num_patch[0] % 2 == 1:
727 - # Pad so we can still pool 2x2 patches
733 + if (missing_w := self.image_num_patch[0] % POOLING_SIZE):
734 + # Padding for image pooling (see below)
735 image_features = F.pad(
736 image_features,
730 - (0, 0, 0, 1, 0, 1, 0, 0, 0, 0),
737 + (0, 0, 0, missing_w, 0, missing_w, 0, 0, 0, 0),
738 )
739
740 # image pooling
741 image_features = rearrange(
742 image_features,
743 'b n (h dh) (w dw) c -> (b n h w) (dh dw) c',
737 - dh=2,
738 - dw=2,
744 + dh=POOLING_SIZE,
745 + dw=POOLING_SIZE,
746 )
747
748 query = image_features.mean(-2, keepdim=True)
@@ -888,249 +895,513 @@ class MolmoModel(nn.Module):
895 return loaded_params
896
897
891 -cached_get_processor = lru_cache(get_processor)
898 +def _lowest_multiple(x: int, k: int) -> int:
899 + return (x // k) * k
900 +
901
902 +def get_num_patches(
903 + num_tiles: int,
904 + *,
905 + crop_patches: int,
906 + left_margin: int,
907 + right_margin: int,
908 + pooling_size: int,
909 +) -> int:
910 + if num_tiles == 1:
911 + return _lowest_multiple(crop_patches + pooling_size - 1, pooling_size)
912
894 -def get_num_patches(num_tiles: int, crop_patches: int, left_margin: int,
895 - right_margin: int, pooling_size: int) -> int:
913 crop_window_patches = crop_patches - (left_margin + right_margin)
897 - if num_tiles > 1:
898 - left_crop_window_patches = (crop_window_patches + left_margin +
899 - pooling_size -
900 - 1) // pooling_size * pooling_size
901 - middle_crop_window_patches = (crop_window_patches + pooling_size -
902 - 1) // pooling_size * pooling_size
903 - right_crop_window_patches = (crop_window_patches + right_margin +
904 - pooling_size -
905 - 1) // pooling_size * pooling_size
906 - return left_crop_window_patches + (
907 - num_tiles -
908 - 2) * middle_crop_window_patches + right_crop_window_patches
909 - else:
910 - single_crop_window_patches = (crop_patches + pooling_size -
911 - 1) // pooling_size * pooling_size
912 - return single_crop_window_patches
913 -
914 -
915 -def get_tokens(tiling_h: int, tiling_w: int, crop_patches: int,
916 - left_margin: int, right_margin: int, pooling_size: int) -> int:
917 - h = get_num_patches(tiling_h, crop_patches, left_margin, right_margin,
918 - pooling_size)
919 - w = get_num_patches(tiling_w, crop_patches, left_margin, right_margin,
920 - pooling_size)
921 - per_row = w // pooling_size + 1
922 - joint = per_row * (h // pooling_size) + 2
923 - image_token_length = (crop_patches + pooling_size - 1) // pooling_size
924 - resize = (image_token_length + 1) * image_token_length + 2
925 - return resize + joint
926 -
927 -
928 -def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
929 - right_margin: int, pooling_size: int) -> int:
930 - tilings = []
931 - for i in range(1, max_crops + 1):
932 - for j in range(1, max_crops + 1):
933 - if i * j <= max_crops:
934 - tilings.append((i, j))
935 - tokens = [
936 - get_tokens(tilings[i][0], tilings[i][1], crop_patches, left_margin,
937 - right_margin, pooling_size) for i in range(len(tilings))
938 - ]
939 - return max(tokens)
940 -
941 -
942 -def get_max_molmo_image_tokens(ctx: InputContext) -> int:
943 - processor = cached_get_processor(
944 - ctx.model_config.model,
945 - trust_remote_code=ctx.model_config.trust_remote_code,
946 - revision=ctx.model_config.code_revision)
947 - image_processor = processor.image_processor
948 - max_llm_image_tokens = get_max_tokens(
949 - image_processor.max_crops,
950 - image_processor.base_image_input_size[0] //
951 - image_processor.image_patch_size,
952 - image_processor.overlap_margins[0],
953 - image_processor.overlap_margins[1],
954 - 2,
914 +
915 + left_num = _lowest_multiple(
916 + crop_window_patches + left_margin + pooling_size - 1,
917 + pooling_size,
918 + )
919 + middle_num = _lowest_multiple(
920 + crop_window_patches + pooling_size - 1,
921 + pooling_size,
922 + )
923 + right_num = _lowest_multiple(
924 + crop_window_patches + right_margin + pooling_size - 1,
925 + pooling_size,
926 )
956 - return max_llm_image_tokens
927
928 + return left_num + (num_tiles - 2) * middle_num + right_num
929 +
930 +
931 +def get_patches_grid_size(
932 + *,
933 + tiling_h: int,
934 + tiling_w: int,
935 + crop_patches: int,
936 + left_margin: int,
937 + right_margin: int,
938 + pooling_size: int,
939 +) -> tuple[int, int]:
940 + nrows = get_num_patches(
941 + tiling_h,
942 + crop_patches=crop_patches,
943 + left_margin=left_margin,
944 + right_margin=right_margin,
945 + pooling_size=pooling_size,
946 + )
947 + ncols = get_num_patches(
948 + tiling_w,
949 + crop_patches=crop_patches,
950 + left_margin=left_margin,
951 + right_margin=right_margin,
952 + pooling_size=pooling_size,
953 + )
954
959 -# NOTE: preprocessing for the image data has been included in the
960 -# 'input_processor_for_molmo' function
961 -def image_input_mapper_for_molmo(
962 - ctx: InputContext,
963 - data: object,
964 -):
965 - if isinstance(data, list):
966 - assert len(data) == 1, "Molmo supports only one image per prompt."
967 - data = data[0]
968 -
969 - return MultiModalKwargs(data)
970 -
971 -
972 -def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
973 - mm_counts: Mapping[str, int]):
974 - processor = cached_get_processor(
975 - ctx.model_config.model,
976 - trust_remote_code=ctx.model_config.trust_remote_code,
977 - revision=ctx.model_config.code_revision)
978 - image_processor = processor.image_processor
979 -
980 - base_image_input_d = image_processor.image_patch_size
981 - left_margin, right_margin = image_processor.overlap_margins
982 - max_crops = image_processor.max_crops
983 -
984 - # Assume: prompt_token_ids always starts with bos_token_id followed image tokens # noqa: E501
985 - max_llm_image_tokens = get_max_molmo_image_tokens(ctx)
986 - if seq_len - max_llm_image_tokens - 1 < 0:
987 - raise RuntimeError(
988 - f"Molmo cannot process {max_crops} crops in a prompt, "
989 - "please increase max_model_len or reduce number of crops")
990 -
991 - # The vertical image has the maximum number of image tokens due to column tokens. # noqa: E501
992 - tiling = (max_crops, 1)
993 - total_margin_pixels = base_image_input_d * (right_margin + left_margin)
994 - crop_patches = image_processor.base_image_input_size[
995 - 0] // base_image_input_d
996 - crop_window_patches = crop_patches - (right_margin + left_margin)
997 - crop_window_size = crop_window_patches * base_image_input_d
998 -
999 - h = crop_window_size * tiling[0] + total_margin_pixels
1000 - w = crop_window_size * tiling[1] + total_margin_pixels
1001 -
1002 - dummy_image = Image.new("RGB", (w, h), color="red")
1003 -
1004 - out = processor.process("dummy prompt", dummy_image)
1005 -
1006 - token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
1007 - out["input_ids"][:1 + max_llm_image_tokens])
1008 - token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
1009 - [0]) * (seq_len - max_llm_image_tokens - 1)
1010 - dummy_seqdata = SequenceData(token_ids)
1011 - dummy_imgdata = {
1012 - "images": out["images"],
1013 - "image_input_idx": out["image_input_idx"],
1014 - }
1015 - if "image_masks" in out:
1016 - dummy_imgdata["image_masks"] = out["image_masks"]
1017 - dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
1018 - size = 0
1019 - offset = -1
1020 - for i in range(len(token_ids)):
1021 - if token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1022 - DEFAULT_IM_START_TOKEN_ID, DEFAULT_IM_END_TOKEN_ID,
1023 - DEFAULT_IM_COL_TOKEN_ID):
1024 - if offset < 0:
1025 - offset = i
1026 - size += 1
1027 - dummy_imgdata["image_start_end"] = (offset, offset + size)
1028 - return DummyData(seq_data=dummy_seqdata,
1029 - multi_modal_data={"image": dummy_imgdata},
1030 - multi_modal_placeholders={
1031 - "image":
1032 - [PlaceholderRange(offset=offset, length=size)]
1033 - })
1034 -
1035 -
1036 -def pad_images(
1037 - max_total_crops: int,
1038 - images: torch.Tensor,
1039 - image_input_idx: torch.Tensor,
1040 - image_masks: Optional[torch.Tensor] = None,
955 + return nrows, ncols
956 +
957 +
958 +def get_candidate_tilings(max_num: int) -> list[tuple[int, int]]:
959 + tilings = [(i, j) for i in range(1, max_num + 1)
960 + for j in range(1, max_num + 1) if i * j <= max_num]
961 + return sorted(tilings, key=lambda x: x[0] * x[1])
962 +
963 +
964 +def select_tiling(
965 + *,
966 + height: int,
967 + width: int,
968 + patch_size: int,
969 + max_num_patches: int,
970 ):
1042 - n = max_total_crops - images.shape[0]
1043 - images = F.pad(images, (0, 0, 0, 0, 0, n), value=-1)
1044 - image_input_idx = F.pad(image_input_idx, (0, 0, 0, n), value=-1)
1045 - if image_masks is not None:
1046 - image_masks = F.pad(image_masks, (0, 0, 0, n), value=-1)
1047 - return images, image_input_idx, image_masks
1048 -
1049 -
1050 -def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1051 - prompt = inputs.get("prompt")
1052 - multi_modal_data = inputs.get("multi_modal_data")
1053 - image = None if multi_modal_data is None else multi_modal_data.get("image")
1054 -
1055 - model_config = ctx.model_config
1056 - processor = cached_get_processor(
1057 - ctx.model_config.model,
1058 - trust_remote_code=model_config.trust_remote_code,
1059 - revision=ctx.model_config.code_revision)
1060 - tokenizer = cached_get_tokenizer(
1061 - model_config.tokenizer,
1062 - trust_remote_code=model_config.trust_remote_code)
1063 -
1064 - # NOTE: message formatting for raw text prompt is only applied for
1065 - # offline inference; for online serving, the prompt is always in
1066 - # instruction format and tokenized.
1067 - if prompt is not None and re.match(r"^User:[\s\S]*?(Assistant:)*$",
1068 - prompt):
1069 - out = processor.process(prompt, image, message_format="none")
1070 - elif prompt is not None:
1071 - out = processor.process(prompt, image)
971 + tilings = get_candidate_tilings(max_num_patches)
972 + candidate_tilings = np.array(tilings, dtype=np.int32)
973 + candidate_resolutions = candidate_tilings * patch_size
974 +
975 + original_size = np.array([height, width], dtype=np.float32)
976 + required_scale_d = candidate_resolutions.astype(np.float32) / original_size
977 + required_scale = required_scale_d.min(axis=-1, keepdims=True)
978 +
979 + if (required_scale < 1).all():
980 + ix = required_scale.argmax()
981 else:
1073 - out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
1074 -
1075 - # If there is no image, return directly.
1076 - if image is None:
1077 - new_prompt_token_ids = out["input_ids"].tolist()
1078 - prompt = inputs.get("prompt")
1079 - if prompt is None:
1080 - prompt = tokenizer.decode(new_prompt_token_ids)
1081 - return token_inputs(
1082 - prompt_token_ids=new_prompt_token_ids,
1083 - prompt=prompt,
982 + ix = np.where(required_scale < 1.0, 10e9, required_scale).argmin()
983 +
984 + return candidate_tilings[ix]
985 +
986 +
987 +class MolmoProcessorWrapper:
988 + """
989 + Wraps :class:`MolmoProcessor` so that it can be called directly.
990 +
991 + The original definition can be found here:
992 + https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/preprocessing_molmo.py
993 + """
994 +
995 + def __init__(self, processor: ProcessorMixin):
996 + super().__init__()
997 +
998 + self.processor = processor
999 +
1000 + @cached_property
1001 + def vocab(self) -> dict[str, int]:
1002 + return self.processor.tokenizer.vocab # type: ignore
1003 +
1004 + @cached_property
1005 + def max_crops(self) -> int:
1006 + image_processor = self.processor.image_processor # type: ignore
1007 +
1008 + max_crops = image_processor.max_crops
1009 + assert isinstance(max_crops, int)
1010 +
1011 + return max_crops
1012 +
1013 + @cached_property
1014 + def base_image_input_size(self) -> tuple[int, int]:
1015 + image_processor = self.processor.image_processor # type: ignore
1016 +
1017 + base_image_input_size = image_processor.base_image_input_size
1018 + if isinstance(base_image_input_size, int):
1019 + return base_image_input_size, base_image_input_size
1020 +
1021 + return tuple(base_image_input_size)
1022 +
1023 + @cached_property
1024 + def image_patch_size(self) -> int:
1025 + image_processor = self.processor.image_processor # type: ignore
1026 +
1027 + image_patch_size = image_processor.image_patch_size
1028 + assert isinstance(image_patch_size, int)
1029 +
1030 + return image_patch_size
1031 +
1032 + @cached_property
1033 + def overlap_margins(self) -> tuple[int, int]:
1034 + image_processor = self.processor.image_processor # type: ignore
1035 +
1036 + left_margin, right_margin = image_processor.overlap_margins
1037 + assert isinstance(left_margin, int)
1038 + assert isinstance(right_margin, int)
1039 +
1040 + return left_margin, right_margin
1041 +
1042 + @cached_property
1043 + def image_token_length_w(self) -> int:
1044 + image_processor = self.processor.image_processor # type: ignore
1045 +
1046 + image_token_length_w = image_processor.image_token_length_w
1047 + assert isinstance(image_token_length_w, int)
1048 +
1049 + return image_token_length_w
1050 +
1051 + @cached_property
1052 + def image_token_length_h(self) -> int:
1053 + image_processor = self.processor.image_processor # type: ignore
1054 +
1055 + image_token_length_h = image_processor.image_token_length_h
1056 + assert isinstance(image_token_length_h, int)
1057 +
1058 + return image_token_length_h
1059 +
1060 + @property
1061 + def message_format(self) -> Optional[str]:
1062 + return "role"
1063 +
1064 + @property
1065 + def always_start_with_space(self) -> bool:
1066 + return True
1067 +
1068 + @cached_property
1069 + def image_patch_id(self) -> int:
1070 + return self.vocab[IMAGE_PATCH_TOKEN]
1071 +
1072 + @cached_property
1073 + def im_col_id(self) -> int:
1074 + return self.vocab[IM_COL_TOKEN]
1075 +
1076 + @cached_property
1077 + def im_start_id(self) -> int:
1078 + return self.vocab[IM_START_TOKEN]
1079 +
1080 + @cached_property
1081 + def im_end_id(self) -> int:
1082 + return self.vocab[IM_END_TOKEN]
1083 +
1084 + @property
1085 + def pooling_size(self) -> int:
1086 + return POOLING_SIZE
1087 +
1088 + def select_tiling(
1089 + self,
1090 + *,
1091 + image_width: int,
1092 + image_height: int,
1093 + ) -> tuple[int, int]:
1094 + max_crops = self.max_crops
1095 + left_margin, right_margin = self.overlap_margins
1096 + base_image_input_size = self.base_image_input_size
1097 + base_image_input_d = self.image_patch_size
1098 +
1099 + total_margin_pixels = base_image_input_d * (right_margin + left_margin)
1100 + crop_patches = base_image_input_size[0] // base_image_input_d
1101 + crop_window_patches = crop_patches - (right_margin + left_margin)
1102 + crop_window_size = crop_window_patches * base_image_input_d
1103 + tiling_h, tiling_w = select_tiling(
1104 + height=image_height - total_margin_pixels,
1105 + width=image_width - total_margin_pixels,
1106 + patch_size=crop_window_size,
1107 + max_num_patches=max_crops,
1108 )
1109
1086 - image_processor = processor.image_processor
1087 - max_total_crops = 1 + image_processor.max_crops
1088 - images, image_input_idx, image_masks = pad_images(
1089 - max_total_crops,
1090 - out["images"],
1091 - out["image_input_idx"],
1092 - out.get("image_masks"),
1093 - )
1094 - image_data = dict(
1095 - images=images,
1096 - image_input_idx=image_input_idx,
1097 - )
1098 - if image_masks is not None:
1099 - image_data["image_masks"] = image_masks
1100 -
1101 - new_prompt_token_ids = out["input_ids"].tolist()
1102 - image_data["seq_len"] = torch.tensor(len(new_prompt_token_ids),
1103 - dtype=torch.long)
1104 -
1105 - multi_modal_data = dict(image=image_data)
1106 - size = 0
1107 - offset = -1
1108 - for i in range(len(new_prompt_token_ids)):
1109 - if new_prompt_token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1110 - DEFAULT_IM_START_TOKEN_ID,
1111 - DEFAULT_IM_END_TOKEN_ID,
1112 - DEFAULT_IM_COL_TOKEN_ID):
1113 - if offset < 0:
1114 - offset = i
1115 - size += 1
1116 - image_data["image_start_end"] = (offset, offset + size)
1117 - prompt = inputs.get("prompt")
1118 - if prompt is None:
1119 - prompt = tokenizer.decode(new_prompt_token_ids)
1120 - return token_inputs(
1121 - prompt_token_ids=new_prompt_token_ids,
1122 - prompt=prompt,
1123 - multi_modal_data=multi_modal_data,
1124 - multi_modal_placeholders={
1125 - "image": [PlaceholderRange(offset=offset, length=size)]
1126 - },
1127 - )
1110 + return tiling_w, tiling_h
1111 +
1112 + def get_patches_grid_size(
1113 + self,
1114 + *,
1115 + image_width: int,
1116 + image_height: int,
1117 + ) -> tuple[int, int]:
1118 + left_margin, right_margin = self.overlap_margins
1119 + base_image_input_size = self.base_image_input_size
1120 + base_image_input_d = self.image_patch_size
1121 + pooling_size = self.pooling_size
1122 +
1123 + crop_patches = base_image_input_size[0] // base_image_input_d
1124 + tiling_w, tiling_h = self.select_tiling(
1125 + image_height=image_height,
1126 + image_width=image_width,
1127 + )
1128 +
1129 + nrows, ncols = get_patches_grid_size(
1130 + tiling_h=tiling_h,
1131 + tiling_w=tiling_w,
1132 + crop_patches=crop_patches,
1133 + left_margin=left_margin,
1134 + right_margin=right_margin,
1135 + pooling_size=pooling_size,
1136 + )
1137 +
1138 + return ncols, nrows
1139 +
1140 + def __call__(
1141 + self,
1142 + text: Optional[Union[TextInput, list[TextInput]]] = None,
1143 + images: Optional[Union[ImageInput, list[ImageInput]]] = None,
1144 + return_tensors: Optional[Union[str, TensorType]] = None,
1145 + **kwargs,
1146 + ) -> BatchFeature:
1147 + outputs = self.processor.process( # type: ignore
1148 + text, images, **kwargs)
1149 +
1150 + if images is None:
1151 + images = []
1152 + if not isinstance(images, list):
1153 + images = [images]
1154 +
1155 + input_ids: torch.Tensor = outputs.pop("input_ids")
1156 + outputs["input_ids"] = input_ids.unsqueeze(0)
1157 +
1158 + image_input_idx = outputs.pop("image_input_idx", None)
1159 + if image_input_idx is not None:
1160 + input_is_patch = input_ids == self.image_patch_id
1161 + image_input_idx_flat: torch.Tensor = image_input_idx.view(-1)
1162 + image_valid_flat = image_input_idx_flat >= 0
1163 + feat_is_patch_flat = image_valid_flat.clone()
1164 + feat_is_patch_flat[image_valid_flat] = (
1165 + input_is_patch[image_input_idx_flat[image_valid_flat]])
1166 + feat_is_patch = feat_is_patch_flat.view(*image_input_idx.shape)
1167 +
1168 + input_is_embed = torch.isin(
1169 + input_ids,
1170 + torch.tensor([
1171 + self.image_patch_id,
1172 + self.im_col_id,
1173 + self.im_start_id,
1174 + self.im_end_id,
1175 + ]),
1176 + )
1177 + embed_ids = input_ids[input_is_embed]
1178 + embed_is_patch = embed_ids == self.image_patch_id
1179 + assert embed_is_patch.sum() == feat_is_patch.sum()
1180
1181 + tilings = [
1182 + self.select_tiling(
1183 + image_width=image.size[0],
1184 + image_height=image.size[1],
1185 + ) for image in images
1186 + ]
1187 + # For each image: tiling_h * tiling_w + extra
1188 + num_crops = torch.tensor(tilings).prod(-1) + 1
1189 + assert num_crops.sum() == len(feat_is_patch)
1190
1130 -@MULTIMODAL_REGISTRY.register_image_input_mapper(image_input_mapper_for_molmo)
1131 -@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1132 -@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1133 -@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1191 + outputs["feat_is_patch"] = feat_is_patch
1192 + outputs["embed_is_patch"] = embed_is_patch
1193 + outputs["num_crops"] = num_crops
1194 + outputs["img_patch_id"] = self.image_patch_id
1195 +
1196 + return BatchFeature(outputs, tensor_type=return_tensors)
1197 +
1198 +
1199 +class MolmoProcessingInfo(BaseProcessingInfo):
1200 +
1201 + def get_hf_processor(self) -> MolmoProcessorWrapper:
1202 + processor = self.ctx.get_hf_processor()
1203 + return MolmoProcessorWrapper(processor)
1204 +
1205 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
1206 + return {"image": 1}
1207 +
1208 + def get_mm_max_tokens_per_item(
1209 + self,
1210 + seq_len: int,
1211 + mm_counts: Mapping[str, int],
1212 + ) -> Mapping[str, int]:
1213 + return {"image": self.get_max_image_tokens()}
1214 +
1215 + def get_num_image_tokens(
1216 + self,
1217 + *,
1218 + image_width: int,
1219 + image_height: int,
1220 + processor: Optional[MolmoProcessorWrapper],
1221 + ) -> int:
1222 + if processor is None:
1223 + processor = self.get_hf_processor()
1224 +
1225 + ncols, nrows = processor.get_patches_grid_size(
1226 + image_width=image_width,
1227 + image_height=image_height,
1228 + )
1229 + pooling_size = processor.pooling_size
1230 +
1231 + base_image_input_size = processor.base_image_input_size
1232 + base_image_input_d = processor.image_patch_size
1233 +
1234 + crop_patches = base_image_input_size[0] // base_image_input_d
1235 +
1236 + per_row = ncols // pooling_size + 1
1237 + joint = per_row * (nrows // pooling_size) + 2
1238 + image_token_length = (crop_patches + pooling_size - 1) // pooling_size
1239 + resize = (image_token_length + 1) * image_token_length + 2
1240 +
1241 + return resize + joint
1242 +
1243 + def get_max_image_tokens(self) -> int:
1244 + target_width, target_height = self.get_image_size_with_most_features()
1245 +
1246 + return self.get_num_image_tokens(
1247 + image_width=target_width,
1248 + image_height=target_height,
1249 + processor=None,
1250 + )
1251 +
1252 + def get_image_size_with_most_features(self) -> ImageSize:
1253 + processor = self.get_hf_processor()
1254 +
1255 + tilings = get_candidate_tilings(processor.max_crops)
1256 + base_h, base_w = processor.base_image_input_size
1257 +
1258 + largest_feature_size, largest_feature_pinpoint = 0, None
1259 + for wr, hr in tilings:
1260 + width, height = base_w * wr, base_h * hr
1261 +
1262 + feat_size = self.get_num_image_tokens(
1263 + image_width=width,
1264 + image_height=height,
1265 + processor=processor,
1266 + )
1267 + if feat_size > largest_feature_size:
1268 + largest_feature_size = feat_size
1269 + largest_feature_pinpoint = ImageSize(width=width,
1270 + height=height)
1271 +
1272 + if largest_feature_size == 0 or largest_feature_pinpoint is None:
1273 + raise ValueError("Cannot have a largest feature size of 0!")
1274 +
1275 + return largest_feature_pinpoint
1276 +
1277 +
1278 +class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1279 +
1280 + def get_dummy_processor_inputs(
1281 + self,
1282 + seq_len: int,
1283 + mm_counts: Mapping[str, int],
1284 + ) -> ProcessorInputs:
1285 + target_width, target_height = \
1286 + self.info.get_image_size_with_most_features()
1287 + num_images = mm_counts.get("image", 0)
1288 +
1289 + mm_data = {
1290 + "image":
1291 + self._get_dummy_images(width=target_width,
1292 + height=target_height,
1293 + num_images=num_images)
1294 + }
1295 +
1296 + return ProcessorInputs(
1297 + prompt_text="",
1298 + mm_data=mm_data,
1299 + )
1300 +
1301 +
1302 +class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1303 +
1304 + def _apply_hf_processor_tokens_only(
1305 + self,
1306 + prompt_tokens: list[int],
1307 + ) -> list[int]:
1308 + processor = self.info.get_hf_processor()
1309 +
1310 + # Apply the chat template to the tokens
1311 + tokens = processor.processor.get_tokens_input( # type: ignore
1312 + self.info.get_tokenizer().decode(prompt_tokens),
1313 + message_format=processor.message_format,
1314 + always_start_with_space=processor.always_start_with_space,
1315 + )
1316 +
1317 + processed_data = self.info.ctx.call_hf_processor(
1318 + processor, # type: ignore
1319 + dict(tokens=tokens),
1320 + )
1321 + prompt_ids, = processed_data.pop("input_ids").tolist()
1322 +
1323 + return prompt_ids
1324 +
1325 + def _get_mm_fields_config(
1326 + self,
1327 + hf_inputs: BatchFeature,
1328 + hf_processor_mm_kwargs: Mapping[str, object],
1329 + ) -> Mapping[str, MultiModalFieldConfig]:
1330 + num_crops = hf_inputs.get("num_crops", torch.empty(0))
1331 + num_images = len(num_crops)
1332 +
1333 + return dict(
1334 + images=MultiModalFieldConfig.flat_from_sizes("image", num_crops),
1335 + image_masks=MultiModalFieldConfig.flat_from_sizes(
1336 + "image", num_crops),
1337 + feat_is_patch=MultiModalFieldConfig.flat_from_sizes(
1338 + "image", num_crops),
1339 + embed_is_patch=MultiModalFieldConfig.shared("image", num_images),
1340 + num_crops=MultiModalFieldConfig.batched("image"),
1341 + img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1342 + )
1343 +
1344 + def _get_prompt_replacements(
1345 + self,
1346 + mm_items: MultiModalDataItems,
1347 + hf_processor_mm_kwargs: Mapping[str, object],
1348 + out_mm_kwargs: MultiModalKwargs,
1349 + ) -> list[PromptReplacement]:
1350 + processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1351 + tokenizer = self.info.get_tokenizer()
1352 +
1353 + image_token_length_w = processor.image_token_length_w
1354 + image_token_length_h = processor.image_token_length_h
1355 + pooling_size = processor.pooling_size
1356 +
1357 + user_str = "User:"
1358 + if processor.always_start_with_space:
1359 + user_str = " " + user_str
1360 +
1361 + user_tokens = tokenizer.encode(user_str, add_special_tokens=False)
1362 +
1363 + img_patch_id = processor.image_patch_id
1364 + img_col_id = processor.im_col_id
1365 + img_start_id = processor.im_start_id
1366 + img_end_id = processor.im_end_id
1367 +
1368 + extra_row = [img_patch_id] * image_token_length_w + [img_col_id]
1369 + extra_joint = ([img_start_id] + extra_row * image_token_length_h +
1370 + [img_end_id])
1371 +
1372 + def get_replacement_molmo(item_idx: int):
1373 + images = mm_items.get_items("image", ImageProcessorItems)
1374 + image_size = images.get_image_size(item_idx)
1375 +
1376 + ncols, nrows = processor.get_patches_grid_size(
1377 + image_width=image_size.width,
1378 + image_height=image_size.height,
1379 + )
1380 +
1381 + joint_row = ([img_patch_id] * ((ncols + 1) // pooling_size) +
1382 + [img_col_id])
1383 + joint = ([img_start_id] + joint_row *
1384 + ((nrows + 1) // pooling_size) + [img_end_id])
1385 +
1386 + image_tokens = extra_joint + joint
1387 +
1388 + return PromptReplacementDetails(
1389 + full=image_tokens + user_tokens,
1390 + features=image_tokens,
1391 + )
1392 +
1393 + return [
1394 + PromptReplacement(
1395 + modality="image",
1396 + target=user_str,
1397 + replacement=get_replacement_molmo,
1398 + )
1399 + ]
1400 +
1401 +
1402 +@MULTIMODAL_REGISTRY.register_processor(MolmoMultiModalProcessor,
1403 + info=MolmoProcessingInfo,
1404 + dummy_inputs=MolmoDummyInputsBuilder)
1405 class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1406 SupportsLoRA):
1407 hf_to_vllm_mapper = WeightsMapper(
@@ -1202,6 +1473,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1473 quant_config)
1474 self.model = MolmoModel(vllm_config=vllm_config,
1475 prefix=maybe_prefix(prefix, "model"))
1476 + self.img_patch_id = None
1477
1478 if self.config.weight_tying:
1479 self.lm_head = self.model.transformer.wte
@@ -1224,85 +1496,143 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1496 **kwargs: object,
1497 ) -> Optional[MolmoImageInputs]:
1498 images = kwargs.pop("images", None)
1227 - image_masks = kwargs.pop("image_masks", None)
1228 - image_start_end = kwargs.pop("image_start_end", None)
1499 if images is None:
1500 return None
1501
1232 - image_input_idx = kwargs.pop("image_input_idx", None)
1233 - seq_len = kwargs.pop("seq_len", None)
1234 - if image_input_idx is None:
1235 - raise ValueError("image_input_idx is required for Molmo model.")
1236 - if seq_len is None:
1237 - raise ValueError("seq_len is required for Molmo model.")
1238 - if not isinstance(seq_len, torch.Tensor):
1239 - seq_len = torch.tensor(seq_len)
1502 + if not isinstance(images, (torch.Tensor, list)):
1503 + raise ValueError("Incorrect type of images. "
1504 + f"Got type: {type(images)}")
1505 +
1506 + image_masks = kwargs.pop("image_masks", None)
1507 + if not (image_masks is None or isinstance(image_masks,
1508 + (torch.Tensor, list))):
1509 + raise ValueError("Incorrect type of image_masks. "
1510 + f"Got type: {type(image_masks)}")
1511 +
1512 + feat_is_patch = kwargs.pop("feat_is_patch", None)
1513 + if not isinstance(feat_is_patch, (torch.Tensor, list)):
1514 + raise ValueError("Incorrect type of feat_is_patch. "
1515 + f"Got type: {type(feat_is_patch)}")
1516 +
1517 + embed_is_patch = kwargs.pop("embed_is_patch", None)
1518 + if not isinstance(embed_is_patch, (torch.Tensor, list)):
1519 + raise ValueError("Incorrect type of embed_is_patch. "
1520 + f"Got type: {type(embed_is_patch)}")
1521 +
1522 + num_crops = kwargs.pop("num_crops", None)
1523 + if not isinstance(num_crops, torch.Tensor):
1524 + raise ValueError("Incorrect type of num_crops. "
1525 + f"Got type: {type(num_crops)}")
1526 +
1527 + img_patch_id = kwargs.pop("img_patch_id", None)
1528 + if not isinstance(img_patch_id, torch.Tensor):
1529 + raise ValueError("Incorrect type of num_crops. "
1530 + f"Got type: {type(num_crops)}")
1531 + self.img_patch_id = img_patch_id.flatten().unique().item()
1532
1533 return MolmoImageInputs(
1534 images=images,
1243 - image_input_idx=image_input_idx,
1244 - seq_len=seq_len,
1535 image_masks=image_masks,
1246 - image_start_end=image_start_end,
1536 + feat_is_patch=feat_is_patch,
1537 + embed_is_patch=embed_is_patch,
1538 + num_crops=num_crops,
1539 )
1540
1541 def _process_image_input(
1542 self,
1543 image_input: MolmoImageInputs,
1252 - ) -> torch.Tensor:
1253 -
1254 - image_features = self.vision_backbone(
1255 - images=image_input["images"],
1256 - image_masks=image_input["image_masks"],
1257 - )
1544 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
1545 + if isinstance(image_input["images"], list):
1546 + # Call the vision backbone on the whole batch at once
1547 + images_flat = flatten_bn(image_input["images"], concat=True)
1548 + image_masks_flat = (None if (image_masks :=
1549 + image_input["image_masks"]) is None
1550 + else flatten_bn(image_masks, concat=True))
1551 +
1552 + image_features_flat = self.vision_backbone(
1553 + images=images_flat.unsqueeze(0),
1554 + image_masks=(None if image_masks_flat is None else
1555 + image_masks_flat.unsqueeze(0)),
1556 + ).squeeze(0)
1557 +
1558 + # Reconstruct the batch dimension
1559 + image_features = image_features_flat.split(
1560 + image_input["num_crops"].sum(-1).tolist())
1561 + else:
1562 + image_features = self.vision_backbone(
1563 + images=image_input["images"],
1564 + image_masks=image_input["image_masks"],
1565 + )
1566
1567 return image_features
1568
1569 + def _get_mm_embeds(
1570 + self,
1571 + features: torch.Tensor, # Shape: (num_crop, num_patch, d)
1572 + feat_is_patch: torch.Tensor, # Shape: (num_crop, num_patch)
1573 + num_crops: torch.Tensor, # Shape: (num_images,)
1574 + embed_is_patch: torch.Tensor, # Shape: (num_embeds,)
1575 + ) -> list[torch.Tensor]:
1576 + """
1577 + Scatter the patch features into a contiguous tensor that corresponds
1578 + to the embedding tokens defined by the multimodal processor.
1579 +
1580 + Note:
1581 + The original code only considers patch tokens as feature
1582 + tokens, but our processor considers all image-related tokens
1583 + as feature tokens because the feature tokens need to be
1584 + consecutive in `input_ids`.
1585 +
1586 + Example:
1587 + A simplified example for one item in the batch:
1588 +
1589 + .. code-block::
1590 +
1591 + Embedding tokens (from HF processor):
1592 + [<start> <patch> <patch> <col> <patch> <patch> <col> <end> ]
1593 +
1594 + embed_is_patch (from HF processor):
1595 + [ False True True False True True False False ]
1596 +
1597 + Encoder outputs (from model):
1598 + [ p1 p2 0 p3 p4 0 ]
1599 +
1600 + feat_is_patch (from HF processor):
1601 + [ True True False True True False ]
1602 +
1603 + The resulting embedding tensor is:
1604 + [ nan p1 p2 nan p3 p4 nan nan ]
1605 + """
1606 + num_crops_per_image = num_crops.tolist()
1607 + feats_per_image = features.split(num_crops_per_image)
1608 + f_is_patch_per_image = feat_is_patch.split(num_crops_per_image)
1609 +
1610 + _, _, embed_dim = features.shape
1611 + (num_embeds, ) = embed_is_patch.shape
1612 +
1613 + embeds_in_batch = list[torch.Tensor]()
1614 + for feats, f_is_patch in zip(feats_per_image, f_is_patch_per_image):
1615 + embeds = feats.new_full((num_embeds, embed_dim), torch.nan)
1616 + embeds[embed_is_patch] = feats[f_is_patch]
1617 + embeds_in_batch.append(embeds)
1618 +
1619 + return embeds_in_batch
1620 +
1621 def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
1622 image_input = self._parse_and_validate_image_input(**kwargs)
1623 if image_input is None:
1624 return None
1625 +
1626 image_features = self._process_image_input(image_input)
1266 - image_input_idx = image_input["image_input_idx"]
1267 - seq_len = image_input["seq_len"]
1268 - batch_size, num_image, num_patch = image_features.shape[:3]
1269 - assert image_input_idx.shape == (batch_size, num_image, num_patch)
1270 -
1271 - # insert the image feature into the embedding.
1272 - image_features = image_features.view(batch_size, num_image * num_patch,
1273 - -1)
1274 - image_input_idx = image_input_idx.view(batch_size,
1275 - num_image * num_patch)
1276 -
1277 - valid = image_input_idx >= 0
1278 - image_features = image_features * valid[:, :, None].to(
1279 - image_features.dtype)
1280 - image_features = image_features.view(
1281 - batch_size * num_image * num_patch, -1).contiguous()
1282 -
1283 - image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1284 - offset = torch.cat([seq_len.new_zeros(1),
1285 - seq_len.cumsum(dim=0)[:-1]],
1286 - dim=0)[:, None]
1287 - image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1288 - image_input_idx = image_input_idx.flatten()[:, None]
1289 - mat = image_input_idx == torch.arange(
1290 - seq_len.sum().item(), device=image_features.device)[None, :]
1291 - mat = mat.to(image_features.dtype)
1292 -
1293 - # Note: In this original implementation from AI2, the final
1294 - # vision_embeddings will be always be the same length
1295 - # of input embeddings.
1296 - vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1297 -
1298 - # Split by the sizes of the input sequences. For each full embedding,
1299 - # extract the actual vision embeddings to be merged.
1300 - vision_embeddings = list(vision_embeddings.split(seq_len.tolist()))
1301 - for i in range(len(vision_embeddings)):
1302 - start, end = image_input['image_start_end'][i]
1303 - vision_embeddings[i] = vision_embeddings[i][start:end]
1304 -
1305 - return vision_embeddings
1627 +
1628 + return [
1629 + self._get_mm_embeds(*args) for args in zip(
1630 + image_features,
1631 + image_input["feat_is_patch"],
1632 + image_input["num_crops"],
1633 + image_input["embed_is_patch"],
1634 + )
1635 + ]
1636
1637 def get_input_embeddings(
1638 self,
@@ -1311,11 +1641,20 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1641 ) -> torch.Tensor:
1642 inputs_embeds = self.model.get_input_embeddings(input_ids)
1643 if multimodal_embeddings is not None:
1644 + assert self.img_patch_id is not None
1645 +
1646 + # Extract the patch tokens scattered in _get_mm_embeds
1647 + patch_embeddings = json_map_leaves(
1648 + lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),
1649 + cast(JSONTree[torch.Tensor], multimodal_embeddings),
1650 + )
1651 +
1652 inputs_embeds = merge_multimodal_embeddings(
1315 - input_ids, inputs_embeds, multimodal_embeddings, [
1316 - DEFAULT_IMAGE_PATCH_TOKEN_ID, DEFAULT_IM_START_TOKEN_ID,
1317 - DEFAULT_IM_END_TOKEN_ID, DEFAULT_IM_COL_TOKEN_ID
1318 - ])
1653 + input_ids,
1654 + inputs_embeds,
1655 + cast(NestedTensors, patch_embeddings),
1656 + self.img_patch_id,
1657 + )
1658 return inputs_embeds
1659
1660 def forward(
vllm/multimodal/inputs.py
+40 -40
@@ -353,17 +353,17 @@ class MultiModalFieldConfig:
353
354 Example:
355
356 - .. code-block::
356 + .. code-block::
357
358 - Input:
359 - Data: [[AAAA]
360 - [BBBB]
361 - [CCCC]]
358 + Input:
359 + Data: [[AAAA]
360 + [BBBB]
361 + [CCCC]]
362
363 - Output:
364 - Element 1: [AAAA]
365 - Element 2: [BBBB]
366 - Element 3: [CCCC]
363 + Output:
364 + Element 1: [AAAA]
365 + Element 2: [BBBB]
366 + Element 3: [CCCC]
367 """
368 return MultiModalFieldConfig(
369 field=MultiModalBatchedField(),
@@ -384,18 +384,18 @@ class MultiModalFieldConfig:
384
385 Example:
386
387 - .. code-block::
388 -
389 - Given:
390 - slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
387 + .. code-block::
388 +
389 + Given:
390 + slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
391
392 - Input:
393 - Data: [AAABBBBCC]
392 + Input:
393 + Data: [AAABBBBCC]
394
395 - Output:
396 - Element 1: [AAA]
397 - Element 2: [BBBB]
398 - Element 3: [CC]
395 + Output:
396 + Element 1: [AAA]
397 + Element 2: [BBBB]
398 + Element 3: [CC]
399 """
400 return MultiModalFieldConfig(
401 field=MultiModalFlatField(slices=slices),
@@ -416,18 +416,18 @@ class MultiModalFieldConfig:
416
417 Example:
418
419 - .. code-block::
420 -
421 - Given:
422 - size_per_item: [3, 4, 2]
419 + .. code-block::
420 +
421 + Given:
422 + size_per_item: [3, 4, 2]
423
424 - Input:
425 - Data: [AAABBBBCC]
424 + Input:
425 + Data: [AAABBBBCC]
426
427 - Output:
428 - Element 1: [AAA]
429 - Element 2: [BBBB]
430 - Element 3: [CC]
427 + Output:
428 + Element 1: [AAA]
429 + Element 2: [BBBB]
430 + Element 3: [CC]
431
432 See also:
433 :func:`MultiModalFieldConfig.flat`
@@ -456,19 +456,19 @@ class MultiModalFieldConfig:
456
457 Example:
458
459 - .. code-block::
460 -
461 - Given:
462 - batch_size: 4
459 + .. code-block::
460 +
461 + Given:
462 + batch_size: 4
463
464 - Input:
465 - Data: [XYZ]
464 + Input:
465 + Data: [XYZ]
466
467 - Output:
468 - Element 1: [XYZ]
469 - Element 2: [XYZ]
470 - Element 3: [XYZ]
471 - Element 4: [XYZ]
467 + Output:
468 + Element 1: [XYZ]
469 + Element 2: [XYZ]
470 + Element 3: [XYZ]
471 + Element 4: [XYZ]
472 """
473 return MultiModalFieldConfig(
474 field=MultiModalSharedField(batch_size),
vllm/utils.py
+1 -34
@@ -33,8 +33,7 @@ from dataclasses import dataclass, field
33 from functools import cache, lru_cache, partial, wraps
34 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
35 Dict, Generator, Generic, Iterator, List, Literal,
36 - NamedTuple, Optional, Tuple, Type, TypeVar, Union,
37 - overload)
36 + NamedTuple, Optional, Tuple, Type, TypeVar, Union)
37 from uuid import uuid4
38
39 import cloudpickle
@@ -826,38 +825,6 @@ JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
825 """A nested JSON structure where the leaves need not be JSON-serializable."""
826
827
829 -@overload
830 -def json_map_leaves(
831 - func: Callable[[T], U],
832 - value: Dict[str, JSONTree[T]],
833 -) -> Dict[str, JSONTree[U]]:
834 - ...
835 -
836 -
837 -@overload
838 -def json_map_leaves(
839 - func: Callable[[T], U],
840 - value: List[JSONTree[T]],
841 -) -> List[JSONTree[U]]:
842 - ...
843 -
844 -
845 -@overload
846 -def json_map_leaves(
847 - func: Callable[[T], U],
848 - value: Tuple[JSONTree[T], ...],
849 -) -> Tuple[JSONTree[U], ...]:
850 - ...
851 -
852 -
853 -@overload
854 -def json_map_leaves(
855 - func: Callable[[T], U],
856 - value: JSONTree[T],
857 -) -> JSONTree[U]:
858 - ...
859 -
860 -
828 def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]:
829 if isinstance(value, dict):
830 return {k: json_map_leaves(func, v) for k, v in value.items()}