A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[VLM] Merged multi-modal processor for Molmo (#12966)
Cyrus Leung committed
Feb 13, 2025 at 20:34 UTC
c9d3ecf016deb619a3bce6d59049224e8eb12364
9 files changed
+750
-498
docs/source/models/supported_models.md
+1
-1
@@ -793,7 +793,7 @@ See [this page](#generative-models) for more information on how to use generativ
793
- * `MolmoForCausalLM`
794
* Molmo
795
* T + I
796
- * `allenai/Molmo-7B-D-0924`, `allenai/Molmo-72B-0924`, etc.
796
+ * `allenai/Molmo-7B-D-0924`, `allenai/Molmo-7B-O-0924`, etc.
797
* ✅︎
798
* ✅︎
799
* ✅︎
tests/models/decoder_only/language/test_models.py
+1
-1
@@ -27,7 +27,7 @@ from ...utils import check_logprobs_close
27
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
28
),
29
pytest.param(
30
- "THUDM/chatglm3-6b", # ChatGLM (text-only)
30
+ "THUDM/chatglm3-6b", # chatglm (text-only)
31
),
32
pytest.param(
33
"meta-llama/Llama-3.2-1B-Instruct", # llama
tests/models/decoder_only/vision_language/test_models.py
+2
-3
@@ -404,11 +404,10 @@ VLM_TEST_SETTINGS = {
404
"molmo": VLMTestInfo(
405
models=["allenai/Molmo-7B-D-0924"],
406
test_type=(VLMTestType.IMAGE),
407
- prompt_formatter=lambda img_prompt:"User: " + img_prompt + " Assistant:", # noqa: E501
407
+ prompt_formatter=identity,
408
max_model_len=4096,
409
max_num_seqs=2,
410
- image_size_factors=[(),(1.0, 1.0, 1.0)],
411
- patch_hf_runner=model_utils.mlomo_patch_hf_runner,
410
+ patch_hf_runner=model_utils.molmo_patch_hf_runner,
411
postprocess_inputs=model_utils.molmo_post_processor,
412
),
413
# Tests for phi3v currently live in another file because of a bug in
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+21
-77
@@ -6,7 +6,7 @@ typically specific to a small subset of models.
6
import re
7
import types
8
from pathlib import PosixPath
9
-from typing import Any, Callable, Dict, List, Optional, Tuple, Union
9
+from typing import Callable, List, Optional, Tuple, Union
10
11
import torch
12
from PIL.Image import Image
@@ -17,9 +17,7 @@ from vllm.sequence import SampleLogprobs
17
from vllm.transformers_utils.tokenizer import patch_padding_side
18
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE
19
20
-from .....conftest import (HfRunner, ImageAsset, PromptAudioInput,
21
- PromptImageInput, PromptVideoInput, _ImageAssets)
22
-from ....utils import TokensTextLogprobs
20
+from .....conftest import HfRunner, ImageAsset, _ImageAssets
21
from .types import RunnerOutput
22
23
@@ -522,74 +520,7 @@ def minicpmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
520
return hf_model
521
522
525
-def _generate_greedy_logprobs_limit(
526
- self,
527
- prompts: List[str],
528
- max_tokens: int,
529
- num_logprobs: int,
530
- images: Optional[PromptImageInput] = None,
531
- audios: Optional[PromptAudioInput] = None,
532
- videos: Optional[PromptVideoInput] = None,
533
- **kwargs: Any,
534
-) -> List[TokensTextLogprobs]:
535
- all_inputs = self.get_inputs(prompts,
536
- images=images,
537
- videos=videos,
538
- audios=audios)
539
-
540
- # Process in batches for inference.
541
- if len(all_inputs):
542
- input_ids_lst = []
543
- images_lst = []
544
- images_input_idx_lst = []
545
- imges_masks_lst = []
546
- for inputs in all_inputs:
547
- input_ids_lst.append(inputs["input_ids"])
548
- images_lst.append(inputs["images"])
549
- images_input_idx_lst.append(inputs["image_input_idx"])
550
- imges_masks_lst.append(inputs["image_masks"])
551
- batch_inputs = {}
552
- batch_inputs['input_ids'] = torch.cat(input_ids_lst, dim=0)
553
- batch_inputs['images'] = torch.cat(images_lst, dim=0)
554
- batch_inputs['image_input_idx'] = torch.cat(images_input_idx_lst,
555
- dim=0)
556
- batch_inputs['image_masks'] = torch.cat(imges_masks_lst, dim=0)
557
-
558
- outputs = self.model.generate_from_batch(
559
- batch=self.wrap_device(batch_inputs,
560
- device=self.model.device.type),
561
- generation_config=GenerationConfig(
562
- max_new_tokens=max_tokens,
563
- stop_strings="<|endoftext|>",
564
- do_sample=False,
565
- ),
566
- tokenizer=self.tokenizer,
567
- output_hidden_states=True,
568
- return_dict_in_generate=True,
569
- )
570
-
571
- all_logprobs: List[List[Dict[int, float]]] = []
572
- all_output_ids: List[List[int]] = []
573
- all_output_strs: List[str] = []
574
-
575
- for index in range(len(all_inputs)):
576
- (
577
- seq_logprobs_lst,
578
- output_len,
579
- ) = self._hidden_states_to_logprobs(outputs.hidden_states,
580
- num_logprobs)
581
- all_logprobs.append(seq_logprobs_lst)
582
- seq_ids = outputs.sequences[index]
583
- output_ids = seq_ids[-output_len:]
584
- all_output_ids.append(output_ids.tolist())
585
- all_output_strs.append(self.tokenizer.decode(output_ids))
586
- outputs = zip(all_output_ids, all_output_strs, all_logprobs)
587
- return [(output_ids, output_str, output_logprobs)
588
- for output_ids, output_str, output_logprobs in outputs]
589
-
590
-
591
-####### Molmo-specific HuggingFace runner patchers
592
-def mlomo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
523
+def molmo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
524
"""Patches and returns an instance of the HfRunner to use for Molmo."""
525
hf_processor = hf_model.processor
526
@@ -598,10 +529,23 @@ def mlomo_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
529
530
hf_model.processor = _processor
531
601
- setattr( # noqa: B010
602
- hf_model,
603
- "generate_greedy_logprobs_limit",
604
- types.MethodType(_generate_greedy_logprobs_limit, hf_model),
605
- )
532
+ def _generate(self, max_new_tokens=None, do_sample=None, **kwargs):
533
+ batch = {
534
+ k: kwargs.pop(k)
535
+ for k in ("input_ids", "images", "image_input_idx", "image_masks")
536
+ if k in kwargs
537
+ }
538
+
539
+ return self.generate_from_batch(
540
+ batch,
541
+ generation_config=GenerationConfig(
542
+ max_new_tokens=max_new_tokens,
543
+ stop_strings="<|endoftext|>",
544
+ do_sample=do_sample,
545
+ ),
546
+ **kwargs,
547
+ )
548
+
549
+ hf_model.model.generate = types.MethodType(_generate, hf_model.model)
550
551
return hf_model
tests/models/multimodal/processing/test_common.py
+2
@@ -168,6 +168,8 @@ def _test_processing_correctness(
168
"mistral-community/pixtral-12b",
169
"openbmb/MiniCPM-o-2_6",
170
"openbmb/MiniCPM-V-2_6",
171
+ "allenai/Molmo-7B-D-0924",
172
+ "allenai/Molmo-7B-O-0924",
173
"nvidia/NVLM-D-72B",
174
"Qwen/Qwen-VL-Chat",
175
"Qwen/Qwen2-VL-2B-Instruct",
tests/models/registry.py
+1
@@ -256,6 +256,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
256
"MiniCPMV": _HfExamplesInfo("openbmb/MiniCPM-V-2_6",
257
trust_remote_code=True),
258
"MolmoForCausalLM": _HfExamplesInfo("allenai/Molmo-7B-D-0924",
259
+ extras={"olmo": "allenai/Molmo-7B-O-0924"}, # noqa: E501
260
trust_remote_code=True),
261
"NVLM_D": _HfExamplesInfo("nvidia/NVLM-D-72B",
262
trust_remote_code=True),
vllm/model_executor/models/molmo.py
+681
-342
@@ -1,18 +1,20 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
import math
4
-import re
5
-from array import array
4
from dataclasses import dataclass
7
-from functools import lru_cache, partial
8
-from typing import Iterable, List, Mapping, Optional, Set, Tuple, TypedDict
5
+from functools import cached_property, partial
6
+from typing import (Iterable, List, Mapping, Optional, Set, Tuple, TypedDict,
7
+ Union, cast)
8
9
+import numpy as np
10
import torch
11
+import torch.nn as nn
12
+import torch.nn.functional as F
13
from einops import rearrange
12
-from PIL import Image
13
-from torch import nn
14
-from torch.nn import functional as F
15
-from transformers import PretrainedConfig
14
+from transformers import (BatchFeature, PretrainedConfig, ProcessorMixin,
15
+ TensorType)
16
+from transformers.image_utils import ImageInput
17
+from transformers.tokenization_utils_base import TextInput
18
19
from vllm.attention import Attention, AttentionMetadata
20
from vllm.attention.layer import MultiHeadAttention
@@ -22,8 +24,6 @@ from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
24
get_tensor_model_parallel_world_size,
25
split_tensor_along_last_dim,
26
tensor_model_parallel_all_gather)
25
-from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
26
- InputContext, token_inputs)
27
from vllm.model_executor import SamplingMetadata
28
from vllm.model_executor.layers.activation import (MulAndSilu, QuickGELU,
29
SiluAndMul)
@@ -40,15 +40,21 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
40
ParallelLMHead, VocabParallelEmbedding)
41
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
from vllm.model_executor.models.module_mapping import MultiModelKeys
43
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
44
-from vllm.multimodal.inputs import NestedTensors, PlaceholderRange
45
-from vllm.multimodal.utils import cached_get_tokenizer
46
-from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
47
- SequenceData)
48
-from vllm.transformers_utils.processor import get_processor
43
+from vllm.multimodal import MULTIMODAL_REGISTRY
44
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
45
+ NestedTensors)
46
+from vllm.multimodal.parse import (ImageProcessorItems, ImageSize,
47
+ MultiModalDataItems)
48
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
49
+ BaseProcessingInfo, PromptReplacement,
50
+ PromptReplacementDetails)
51
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
52
+from vllm.sequence import IntermediateTensors
53
+from vllm.utils import JSONTree, json_map_leaves
54
55
from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
51
-from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
56
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
57
+ is_pp_missing_parameter,
58
make_empty_intermediate_tensors_factory, make_layers,
59
maybe_prefix, merge_multimodal_embeddings)
60
@@ -56,38 +62,39 @@ from .utils import (AutoWeightsLoader, WeightsMapper, is_pp_missing_parameter,
62
VIT_LAYERS = [-2, -9]
63
NUM_PREFIX_TOKENS = 1
64
ADDITIONAL_VOCAB_SIZE = 128
59
-DEFAULT_IMAGE_PATCH_TOKEN_ID = 152066
60
-DEFAULT_IM_START_TOKEN_ID = 152067
61
-DEFAULT_IM_END_TOKEN_ID = 152064
62
-DEFAULT_IM_COL_TOKEN_ID = 152065
65
+IMAGE_PATCH_TOKEN = "<im_patch>"
66
+IM_COL_TOKEN = "<im_col>"
67
+IM_START_TOKEN = "<im_start>"
68
+IM_END_TOKEN = "<im_end>"
69
+POOLING_SIZE = 2
70
71
72
class MolmoImageInputs(TypedDict):
66
- images: torch.Tensor
67
- """Shape:
68
- `(batch_size, num_crops, num_patch, patch_dim)`
69
- """
73
+ images: Union[torch.Tensor, List[torch.Tensor]]
74
+ """Shape: `(batch_size, num_crops, num_patch, patch_dim)`"""
75
71
- image_input_idx: torch.Tensor
72
- """Shape:
73
- `(batch_size, num_crops, num_patch)`
74
- """
76
+ image_masks: Optional[Union[torch.Tensor, List[torch.Tensor]]]
77
+ """Shape: `(batch_size, num_crops, num_patch)`"""
78
76
- seq_len: torch.Tensor
77
- """Shape:
78
- `(batch_size, )`
79
+ feat_is_patch: Union[torch.Tensor, List[torch.Tensor]]
80
"""
81
+ A boolean mask indicating which image features correspond
82
+ to patch tokens.
83
81
- image_masks: Optional[torch.Tensor]
82
- """Shape:
83
- `(batch_size, num_crops, num_patch)`
84
+ Shape: `(batch_size, num_crops, num_patch)`
85
"""
86
86
- image_start_end: Tuple[int, int]
87
- """Starting and ending index of placeholder
88
- tokens
87
+ embed_is_patch: Union[torch.Tensor, List[torch.Tensor]]
88
+ """
89
+ A boolean mask indicating which image embeddings correspond
90
+ to patch tokens.
91
+
92
+ Shape: `(batch_size, num_embeds)`
93
"""
94
95
+ num_crops: torch.Tensor
96
+ """Shape: `(batch_size, num_images)`"""
97
+
98
99
@dataclass
100
class VisionBackboneConfig:
@@ -335,7 +342,7 @@ class VisionTransformer(nn.Module):
342
343
def forward(self,
344
x: torch.Tensor,
338
- patch_num: int = None) -> List[torch.Tensor]:
345
+ patch_num: Optional[int] = None) -> List[torch.Tensor]:
346
"""
347
: param x: (batch_size, num_patch, n_pixels)
348
"""
@@ -465,7 +472,7 @@ class MolmoAttention(nn.Module):
472
return output
473
474
468
-class LanuageModelMLP(nn.Module):
475
+class LanguageModelMLP(nn.Module):
476
"""Molmo's LLM mlp."""
477
478
def __init__(self,
@@ -559,7 +566,7 @@ class MolmoDecoderLayer(nn.Module):
566
prefix=f"{prefix}.self_attn")
567
568
# MLP block.
562
- self.mlp = LanuageModelMLP(config, quant_config=quant_config)
569
+ self.mlp = LanguageModelMLP(config, quant_config=quant_config)
570
571
# LayerNorm
572
assert config.layer_norm_type == "rms"
@@ -638,8 +645,8 @@ class MolmoVisionBackbone(nn.Module):
645
self.vit_layers = VIT_LAYERS
646
self.image_num_patch = vision_config.image_num_patch
647
self.llm_patches_per_crop = (
641
- (self.image_num_patch[0] + 1) // 2,
642
- (self.image_num_patch[1] + 1) // 2,
648
+ (self.image_num_patch[0] + 1) // POOLING_SIZE,
649
+ (self.image_num_patch[1] + 1) // POOLING_SIZE,
650
)
651
self.image_vit = VisionTransformer(vision_config,
652
quant_config=quant_config)
@@ -723,19 +730,19 @@ class MolmoVisionBackbone(nn.Module):
730
image_features = image_features.reshape(
731
(batch_size, num_image) + self.image_num_patch + (-1, ), )
732
726
- if self.image_num_patch[0] % 2 == 1:
727
- # Pad so we can still pool 2x2 patches
733
+ if (missing_w := self.image_num_patch[0] % POOLING_SIZE):
734
+ # Padding for image pooling (see below)
735
image_features = F.pad(
736
image_features,
730
- (0, 0, 0, 1, 0, 1, 0, 0, 0, 0),
737
+ (0, 0, 0, missing_w, 0, missing_w, 0, 0, 0, 0),
738
)
739
740
# image pooling
741
image_features = rearrange(
742
image_features,
743
'b n (h dh) (w dw) c -> (b n h w) (dh dw) c',
737
- dh=2,
738
- dw=2,
744
+ dh=POOLING_SIZE,
745
+ dw=POOLING_SIZE,
746
)
747
748
query = image_features.mean(-2, keepdim=True)
@@ -888,249 +895,513 @@ class MolmoModel(nn.Module):
895
return loaded_params
896
897
891
-cached_get_processor = lru_cache(get_processor)
898
+def _lowest_multiple(x: int, k: int) -> int:
899
+ return (x // k) * k
900
+
901
902
+def get_num_patches(
903
+ num_tiles: int,
904
+ *,
905
+ crop_patches: int,
906
+ left_margin: int,
907
+ right_margin: int,
908
+ pooling_size: int,
909
+) -> int:
910
+ if num_tiles == 1:
911
+ return _lowest_multiple(crop_patches + pooling_size - 1, pooling_size)
912
894
-def get_num_patches(num_tiles: int, crop_patches: int, left_margin: int,
895
- right_margin: int, pooling_size: int) -> int:
913
crop_window_patches = crop_patches - (left_margin + right_margin)
897
- if num_tiles > 1:
898
- left_crop_window_patches = (crop_window_patches + left_margin +
899
- pooling_size -
900
- 1) // pooling_size * pooling_size
901
- middle_crop_window_patches = (crop_window_patches + pooling_size -
902
- 1) // pooling_size * pooling_size
903
- right_crop_window_patches = (crop_window_patches + right_margin +
904
- pooling_size -
905
- 1) // pooling_size * pooling_size
906
- return left_crop_window_patches + (
907
- num_tiles -
908
- 2) * middle_crop_window_patches + right_crop_window_patches
909
- else:
910
- single_crop_window_patches = (crop_patches + pooling_size -
911
- 1) // pooling_size * pooling_size
912
- return single_crop_window_patches
913
-
914
-
915
-def get_tokens(tiling_h: int, tiling_w: int, crop_patches: int,
916
- left_margin: int, right_margin: int, pooling_size: int) -> int:
917
- h = get_num_patches(tiling_h, crop_patches, left_margin, right_margin,
918
- pooling_size)
919
- w = get_num_patches(tiling_w, crop_patches, left_margin, right_margin,
920
- pooling_size)
921
- per_row = w // pooling_size + 1
922
- joint = per_row * (h // pooling_size) + 2
923
- image_token_length = (crop_patches + pooling_size - 1) // pooling_size
924
- resize = (image_token_length + 1) * image_token_length + 2
925
- return resize + joint
926
-
927
-
928
-def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
929
- right_margin: int, pooling_size: int) -> int:
930
- tilings = []
931
- for i in range(1, max_crops + 1):
932
- for j in range(1, max_crops + 1):
933
- if i * j <= max_crops:
934
- tilings.append((i, j))
935
- tokens = [
936
- get_tokens(tilings[i][0], tilings[i][1], crop_patches, left_margin,
937
- right_margin, pooling_size) for i in range(len(tilings))
938
- ]
939
- return max(tokens)
940
-
941
-
942
-def get_max_molmo_image_tokens(ctx: InputContext) -> int:
943
- processor = cached_get_processor(
944
- ctx.model_config.model,
945
- trust_remote_code=ctx.model_config.trust_remote_code,
946
- revision=ctx.model_config.code_revision)
947
- image_processor = processor.image_processor
948
- max_llm_image_tokens = get_max_tokens(
949
- image_processor.max_crops,
950
- image_processor.base_image_input_size[0] //
951
- image_processor.image_patch_size,
952
- image_processor.overlap_margins[0],
953
- image_processor.overlap_margins[1],
954
- 2,
914
+
915
+ left_num = _lowest_multiple(
916
+ crop_window_patches + left_margin + pooling_size - 1,
917
+ pooling_size,
918
+ )
919
+ middle_num = _lowest_multiple(
920
+ crop_window_patches + pooling_size - 1,
921
+ pooling_size,
922
+ )
923
+ right_num = _lowest_multiple(
924
+ crop_window_patches + right_margin + pooling_size - 1,
925
+ pooling_size,
926
)
956
- return max_llm_image_tokens
927
928
+ return left_num + (num_tiles - 2) * middle_num + right_num
929
+
930
+
931
+def get_patches_grid_size(
932
+ *,
933
+ tiling_h: int,
934
+ tiling_w: int,
935
+ crop_patches: int,
936
+ left_margin: int,
937
+ right_margin: int,
938
+ pooling_size: int,
939
+) -> tuple[int, int]:
940
+ nrows = get_num_patches(
941
+ tiling_h,
942
+ crop_patches=crop_patches,
943
+ left_margin=left_margin,
944
+ right_margin=right_margin,
945
+ pooling_size=pooling_size,
946
+ )
947
+ ncols = get_num_patches(
948
+ tiling_w,
949
+ crop_patches=crop_patches,
950
+ left_margin=left_margin,
951
+ right_margin=right_margin,
952
+ pooling_size=pooling_size,
953
+ )
954
959
-# NOTE: preprocessing for the image data has been included in the
960
-# 'input_processor_for_molmo' function
961
-def image_input_mapper_for_molmo(
962
- ctx: InputContext,
963
- data: object,
964
-):
965
- if isinstance(data, list):
966
- assert len(data) == 1, "Molmo supports only one image per prompt."
967
- data = data[0]
968
-
969
- return MultiModalKwargs(data)
970
-
971
-
972
-def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
973
- mm_counts: Mapping[str, int]):
974
- processor = cached_get_processor(
975
- ctx.model_config.model,
976
- trust_remote_code=ctx.model_config.trust_remote_code,
977
- revision=ctx.model_config.code_revision)
978
- image_processor = processor.image_processor
979
-
980
- base_image_input_d = image_processor.image_patch_size
981
- left_margin, right_margin = image_processor.overlap_margins
982
- max_crops = image_processor.max_crops
983
-
984
- # Assume: prompt_token_ids always starts with bos_token_id followed image tokens # noqa: E501
985
- max_llm_image_tokens = get_max_molmo_image_tokens(ctx)
986
- if seq_len - max_llm_image_tokens - 1 < 0:
987
- raise RuntimeError(
988
- f"Molmo cannot process {max_crops} crops in a prompt, "
989
- "please increase max_model_len or reduce number of crops")
990
-
991
- # The vertical image has the maximum number of image tokens due to column tokens. # noqa: E501
992
- tiling = (max_crops, 1)
993
- total_margin_pixels = base_image_input_d * (right_margin + left_margin)
994
- crop_patches = image_processor.base_image_input_size[
995
- 0] // base_image_input_d
996
- crop_window_patches = crop_patches - (right_margin + left_margin)
997
- crop_window_size = crop_window_patches * base_image_input_d
998
-
999
- h = crop_window_size * tiling[0] + total_margin_pixels
1000
- w = crop_window_size * tiling[1] + total_margin_pixels
1001
-
1002
- dummy_image = Image.new("RGB", (w, h), color="red")
1003
-
1004
- out = processor.process("dummy prompt", dummy_image)
1005
-
1006
- token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
1007
- out["input_ids"][:1 + max_llm_image_tokens])
1008
- token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
1009
- [0]) * (seq_len - max_llm_image_tokens - 1)
1010
- dummy_seqdata = SequenceData(token_ids)
1011
- dummy_imgdata = {
1012
- "images": out["images"],
1013
- "image_input_idx": out["image_input_idx"],
1014
- }
1015
- if "image_masks" in out:
1016
- dummy_imgdata["image_masks"] = out["image_masks"]
1017
- dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
1018
- size = 0
1019
- offset = -1
1020
- for i in range(len(token_ids)):
1021
- if token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1022
- DEFAULT_IM_START_TOKEN_ID, DEFAULT_IM_END_TOKEN_ID,
1023
- DEFAULT_IM_COL_TOKEN_ID):
1024
- if offset < 0:
1025
- offset = i
1026
- size += 1
1027
- dummy_imgdata["image_start_end"] = (offset, offset + size)
1028
- return DummyData(seq_data=dummy_seqdata,
1029
- multi_modal_data={"image": dummy_imgdata},
1030
- multi_modal_placeholders={
1031
- "image":
1032
- [PlaceholderRange(offset=offset, length=size)]
1033
- })
1034
-
1035
-
1036
-def pad_images(
1037
- max_total_crops: int,
1038
- images: torch.Tensor,
1039
- image_input_idx: torch.Tensor,
1040
- image_masks: Optional[torch.Tensor] = None,
955
+ return nrows, ncols
956
+
957
+
958
+def get_candidate_tilings(max_num: int) -> list[tuple[int, int]]:
959
+ tilings = [(i, j) for i in range(1, max_num + 1)
960
+ for j in range(1, max_num + 1) if i * j <= max_num]
961
+ return sorted(tilings, key=lambda x: x[0] * x[1])
962
+
963
+
964
+def select_tiling(
965
+ *,
966
+ height: int,
967
+ width: int,
968
+ patch_size: int,
969
+ max_num_patches: int,
970
):
1042
- n = max_total_crops - images.shape[0]
1043
- images = F.pad(images, (0, 0, 0, 0, 0, n), value=-1)
1044
- image_input_idx = F.pad(image_input_idx, (0, 0, 0, n), value=-1)
1045
- if image_masks is not None:
1046
- image_masks = F.pad(image_masks, (0, 0, 0, n), value=-1)
1047
- return images, image_input_idx, image_masks
1048
-
1049
-
1050
-def input_processor_for_molmo(ctx: InputContext, inputs: DecoderOnlyInputs):
1051
- prompt = inputs.get("prompt")
1052
- multi_modal_data = inputs.get("multi_modal_data")
1053
- image = None if multi_modal_data is None else multi_modal_data.get("image")
1054
-
1055
- model_config = ctx.model_config
1056
- processor = cached_get_processor(
1057
- ctx.model_config.model,
1058
- trust_remote_code=model_config.trust_remote_code,
1059
- revision=ctx.model_config.code_revision)
1060
- tokenizer = cached_get_tokenizer(
1061
- model_config.tokenizer,
1062
- trust_remote_code=model_config.trust_remote_code)
1063
-
1064
- # NOTE: message formatting for raw text prompt is only applied for
1065
- # offline inference; for online serving, the prompt is always in
1066
- # instruction format and tokenized.
1067
- if prompt is not None and re.match(r"^User:[\s\S]*?(Assistant:)*$",
1068
- prompt):
1069
- out = processor.process(prompt, image, message_format="none")
1070
- elif prompt is not None:
1071
- out = processor.process(prompt, image)
971
+ tilings = get_candidate_tilings(max_num_patches)
972
+ candidate_tilings = np.array(tilings, dtype=np.int32)
973
+ candidate_resolutions = candidate_tilings * patch_size
974
+
975
+ original_size = np.array([height, width], dtype=np.float32)
976
+ required_scale_d = candidate_resolutions.astype(np.float32) / original_size
977
+ required_scale = required_scale_d.min(axis=-1, keepdims=True)
978
+
979
+ if (required_scale < 1).all():
980
+ ix = required_scale.argmax()
981
else:
1073
- out = processor.process(None, image, tokens=inputs["prompt_token_ids"])
1074
-
1075
- # If there is no image, return directly.
1076
- if image is None:
1077
- new_prompt_token_ids = out["input_ids"].tolist()
1078
- prompt = inputs.get("prompt")
1079
- if prompt is None:
1080
- prompt = tokenizer.decode(new_prompt_token_ids)
1081
- return token_inputs(
1082
- prompt_token_ids=new_prompt_token_ids,
1083
- prompt=prompt,
982
+ ix = np.where(required_scale < 1.0, 10e9, required_scale).argmin()
983
+
984
+ return candidate_tilings[ix]
985
+
986
+
987
+class MolmoProcessorWrapper:
988
+ """
989
+ Wraps :class:`MolmoProcessor` so that it can be called directly.
990
+
991
+ The original definition can be found here:
992
+ https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/preprocessing_molmo.py
993
+ """
994
+
995
+ def __init__(self, processor: ProcessorMixin):
996
+ super().__init__()
997
+
998
+ self.processor = processor
999
+
1000
+ @cached_property
1001
+ def vocab(self) -> dict[str, int]:
1002
+ return self.processor.tokenizer.vocab # type: ignore
1003
+
1004
+ @cached_property
1005
+ def max_crops(self) -> int:
1006
+ image_processor = self.processor.image_processor # type: ignore
1007
+
1008
+ max_crops = image_processor.max_crops
1009
+ assert isinstance(max_crops, int)
1010
+
1011
+ return max_crops
1012
+
1013
+ @cached_property
1014
+ def base_image_input_size(self) -> tuple[int, int]:
1015
+ image_processor = self.processor.image_processor # type: ignore
1016
+
1017
+ base_image_input_size = image_processor.base_image_input_size
1018
+ if isinstance(base_image_input_size, int):
1019
+ return base_image_input_size, base_image_input_size
1020
+
1021
+ return tuple(base_image_input_size)
1022
+
1023
+ @cached_property
1024
+ def image_patch_size(self) -> int:
1025
+ image_processor = self.processor.image_processor # type: ignore
1026
+
1027
+ image_patch_size = image_processor.image_patch_size
1028
+ assert isinstance(image_patch_size, int)
1029
+
1030
+ return image_patch_size
1031
+
1032
+ @cached_property
1033
+ def overlap_margins(self) -> tuple[int, int]:
1034
+ image_processor = self.processor.image_processor # type: ignore
1035
+
1036
+ left_margin, right_margin = image_processor.overlap_margins
1037
+ assert isinstance(left_margin, int)
1038
+ assert isinstance(right_margin, int)
1039
+
1040
+ return left_margin, right_margin
1041
+
1042
+ @cached_property
1043
+ def image_token_length_w(self) -> int:
1044
+ image_processor = self.processor.image_processor # type: ignore
1045
+
1046
+ image_token_length_w = image_processor.image_token_length_w
1047
+ assert isinstance(image_token_length_w, int)
1048
+
1049
+ return image_token_length_w
1050
+
1051
+ @cached_property
1052
+ def image_token_length_h(self) -> int:
1053
+ image_processor = self.processor.image_processor # type: ignore
1054
+
1055
+ image_token_length_h = image_processor.image_token_length_h
1056
+ assert isinstance(image_token_length_h, int)
1057
+
1058
+ return image_token_length_h
1059
+
1060
+ @property
1061
+ def message_format(self) -> Optional[str]:
1062
+ return "role"
1063
+
1064
+ @property
1065
+ def always_start_with_space(self) -> bool:
1066
+ return True
1067
+
1068
+ @cached_property
1069
+ def image_patch_id(self) -> int:
1070
+ return self.vocab[IMAGE_PATCH_TOKEN]
1071
+
1072
+ @cached_property
1073
+ def im_col_id(self) -> int:
1074
+ return self.vocab[IM_COL_TOKEN]
1075
+
1076
+ @cached_property
1077
+ def im_start_id(self) -> int:
1078
+ return self.vocab[IM_START_TOKEN]
1079
+
1080
+ @cached_property
1081
+ def im_end_id(self) -> int:
1082
+ return self.vocab[IM_END_TOKEN]
1083
+
1084
+ @property
1085
+ def pooling_size(self) -> int:
1086
+ return POOLING_SIZE
1087
+
1088
+ def select_tiling(
1089
+ self,
1090
+ *,
1091
+ image_width: int,
1092
+ image_height: int,
1093
+ ) -> tuple[int, int]:
1094
+ max_crops = self.max_crops
1095
+ left_margin, right_margin = self.overlap_margins
1096
+ base_image_input_size = self.base_image_input_size
1097
+ base_image_input_d = self.image_patch_size
1098
+
1099
+ total_margin_pixels = base_image_input_d * (right_margin + left_margin)
1100
+ crop_patches = base_image_input_size[0] // base_image_input_d
1101
+ crop_window_patches = crop_patches - (right_margin + left_margin)
1102
+ crop_window_size = crop_window_patches * base_image_input_d
1103
+ tiling_h, tiling_w = select_tiling(
1104
+ height=image_height - total_margin_pixels,
1105
+ width=image_width - total_margin_pixels,
1106
+ patch_size=crop_window_size,
1107
+ max_num_patches=max_crops,
1108
)
1109
1086
- image_processor = processor.image_processor
1087
- max_total_crops = 1 + image_processor.max_crops
1088
- images, image_input_idx, image_masks = pad_images(
1089
- max_total_crops,
1090
- out["images"],
1091
- out["image_input_idx"],
1092
- out.get("image_masks"),
1093
- )
1094
- image_data = dict(
1095
- images=images,
1096
- image_input_idx=image_input_idx,
1097
- )
1098
- if image_masks is not None:
1099
- image_data["image_masks"] = image_masks
1100
-
1101
- new_prompt_token_ids = out["input_ids"].tolist()
1102
- image_data["seq_len"] = torch.tensor(len(new_prompt_token_ids),
1103
- dtype=torch.long)
1104
-
1105
- multi_modal_data = dict(image=image_data)
1106
- size = 0
1107
- offset = -1
1108
- for i in range(len(new_prompt_token_ids)):
1109
- if new_prompt_token_ids[i] in (DEFAULT_IMAGE_PATCH_TOKEN_ID,
1110
- DEFAULT_IM_START_TOKEN_ID,
1111
- DEFAULT_IM_END_TOKEN_ID,
1112
- DEFAULT_IM_COL_TOKEN_ID):
1113
- if offset < 0:
1114
- offset = i
1115
- size += 1
1116
- image_data["image_start_end"] = (offset, offset + size)
1117
- prompt = inputs.get("prompt")
1118
- if prompt is None:
1119
- prompt = tokenizer.decode(new_prompt_token_ids)
1120
- return token_inputs(
1121
- prompt_token_ids=new_prompt_token_ids,
1122
- prompt=prompt,
1123
- multi_modal_data=multi_modal_data,
1124
- multi_modal_placeholders={
1125
- "image": [PlaceholderRange(offset=offset, length=size)]
1126
- },
1127
- )
1110
+ return tiling_w, tiling_h
1111
+
1112
+ def get_patches_grid_size(
1113
+ self,
1114
+ *,
1115
+ image_width: int,
1116
+ image_height: int,
1117
+ ) -> tuple[int, int]:
1118
+ left_margin, right_margin = self.overlap_margins
1119
+ base_image_input_size = self.base_image_input_size
1120
+ base_image_input_d = self.image_patch_size
1121
+ pooling_size = self.pooling_size
1122
+
1123
+ crop_patches = base_image_input_size[0] // base_image_input_d
1124
+ tiling_w, tiling_h = self.select_tiling(
1125
+ image_height=image_height,
1126
+ image_width=image_width,
1127
+ )
1128
+
1129
+ nrows, ncols = get_patches_grid_size(
1130
+ tiling_h=tiling_h,
1131
+ tiling_w=tiling_w,
1132
+ crop_patches=crop_patches,
1133
+ left_margin=left_margin,
1134
+ right_margin=right_margin,
1135
+ pooling_size=pooling_size,
1136
+ )
1137
+
1138
+ return ncols, nrows
1139
+
1140
+ def __call__(
1141
+ self,
1142
+ text: Optional[Union[TextInput, list[TextInput]]] = None,
1143
+ images: Optional[Union[ImageInput, list[ImageInput]]] = None,
1144
+ return_tensors: Optional[Union[str, TensorType]] = None,
1145
+ **kwargs,
1146
+ ) -> BatchFeature:
1147
+ outputs = self.processor.process( # type: ignore
1148
+ text, images, **kwargs)
1149
+
1150
+ if images is None:
1151
+ images = []
1152
+ if not isinstance(images, list):
1153
+ images = [images]
1154
+
1155
+ input_ids: torch.Tensor = outputs.pop("input_ids")
1156
+ outputs["input_ids"] = input_ids.unsqueeze(0)
1157
+
1158
+ image_input_idx = outputs.pop("image_input_idx", None)
1159
+ if image_input_idx is not None:
1160
+ input_is_patch = input_ids == self.image_patch_id
1161
+ image_input_idx_flat: torch.Tensor = image_input_idx.view(-1)
1162
+ image_valid_flat = image_input_idx_flat >= 0
1163
+ feat_is_patch_flat = image_valid_flat.clone()
1164
+ feat_is_patch_flat[image_valid_flat] = (
1165
+ input_is_patch[image_input_idx_flat[image_valid_flat]])
1166
+ feat_is_patch = feat_is_patch_flat.view(*image_input_idx.shape)
1167
+
1168
+ input_is_embed = torch.isin(
1169
+ input_ids,
1170
+ torch.tensor([
1171
+ self.image_patch_id,
1172
+ self.im_col_id,
1173
+ self.im_start_id,
1174
+ self.im_end_id,
1175
+ ]),
1176
+ )
1177
+ embed_ids = input_ids[input_is_embed]
1178
+ embed_is_patch = embed_ids == self.image_patch_id
1179
+ assert embed_is_patch.sum() == feat_is_patch.sum()
1180
1181
+ tilings = [
1182
+ self.select_tiling(
1183
+ image_width=image.size[0],
1184
+ image_height=image.size[1],
1185
+ ) for image in images
1186
+ ]
1187
+ # For each image: tiling_h * tiling_w + extra
1188
+ num_crops = torch.tensor(tilings).prod(-1) + 1
1189
+ assert num_crops.sum() == len(feat_is_patch)
1190
1130
-@MULTIMODAL_REGISTRY.register_image_input_mapper(image_input_mapper_for_molmo)
1131
-@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1132
-@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1133
-@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1191
+ outputs["feat_is_patch"] = feat_is_patch
1192
+ outputs["embed_is_patch"] = embed_is_patch
1193
+ outputs["num_crops"] = num_crops
1194
+ outputs["img_patch_id"] = self.image_patch_id
1195
+
1196
+ return BatchFeature(outputs, tensor_type=return_tensors)
1197
+
1198
+
1199
+class MolmoProcessingInfo(BaseProcessingInfo):
1200
+
1201
+ def get_hf_processor(self) -> MolmoProcessorWrapper:
1202
+ processor = self.ctx.get_hf_processor()
1203
+ return MolmoProcessorWrapper(processor)
1204
+
1205
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
1206
+ return {"image": 1}
1207
+
1208
+ def get_mm_max_tokens_per_item(
1209
+ self,
1210
+ seq_len: int,
1211
+ mm_counts: Mapping[str, int],
1212
+ ) -> Mapping[str, int]:
1213
+ return {"image": self.get_max_image_tokens()}
1214
+
1215
+ def get_num_image_tokens(
1216
+ self,
1217
+ *,
1218
+ image_width: int,
1219
+ image_height: int,
1220
+ processor: Optional[MolmoProcessorWrapper],
1221
+ ) -> int:
1222
+ if processor is None:
1223
+ processor = self.get_hf_processor()
1224
+
1225
+ ncols, nrows = processor.get_patches_grid_size(
1226
+ image_width=image_width,
1227
+ image_height=image_height,
1228
+ )
1229
+ pooling_size = processor.pooling_size
1230
+
1231
+ base_image_input_size = processor.base_image_input_size
1232
+ base_image_input_d = processor.image_patch_size
1233
+
1234
+ crop_patches = base_image_input_size[0] // base_image_input_d
1235
+
1236
+ per_row = ncols // pooling_size + 1
1237
+ joint = per_row * (nrows // pooling_size) + 2
1238
+ image_token_length = (crop_patches + pooling_size - 1) // pooling_size
1239
+ resize = (image_token_length + 1) * image_token_length + 2
1240
+
1241
+ return resize + joint
1242
+
1243
+ def get_max_image_tokens(self) -> int:
1244
+ target_width, target_height = self.get_image_size_with_most_features()
1245
+
1246
+ return self.get_num_image_tokens(
1247
+ image_width=target_width,
1248
+ image_height=target_height,
1249
+ processor=None,
1250
+ )
1251
+
1252
+ def get_image_size_with_most_features(self) -> ImageSize:
1253
+ processor = self.get_hf_processor()
1254
+
1255
+ tilings = get_candidate_tilings(processor.max_crops)
1256
+ base_h, base_w = processor.base_image_input_size
1257
+
1258
+ largest_feature_size, largest_feature_pinpoint = 0, None
1259
+ for wr, hr in tilings:
1260
+ width, height = base_w * wr, base_h * hr
1261
+
1262
+ feat_size = self.get_num_image_tokens(
1263
+ image_width=width,
1264
+ image_height=height,
1265
+ processor=processor,
1266
+ )
1267
+ if feat_size > largest_feature_size:
1268
+ largest_feature_size = feat_size
1269
+ largest_feature_pinpoint = ImageSize(width=width,
1270
+ height=height)
1271
+
1272
+ if largest_feature_size == 0 or largest_feature_pinpoint is None:
1273
+ raise ValueError("Cannot have a largest feature size of 0!")
1274
+
1275
+ return largest_feature_pinpoint
1276
+
1277
+
1278
+class MolmoDummyInputsBuilder(BaseDummyInputsBuilder[MolmoProcessingInfo]):
1279
+
1280
+ def get_dummy_processor_inputs(
1281
+ self,
1282
+ seq_len: int,
1283
+ mm_counts: Mapping[str, int],
1284
+ ) -> ProcessorInputs:
1285
+ target_width, target_height = \
1286
+ self.info.get_image_size_with_most_features()
1287
+ num_images = mm_counts.get("image", 0)
1288
+
1289
+ mm_data = {
1290
+ "image":
1291
+ self._get_dummy_images(width=target_width,
1292
+ height=target_height,
1293
+ num_images=num_images)
1294
+ }
1295
+
1296
+ return ProcessorInputs(
1297
+ prompt_text="",
1298
+ mm_data=mm_data,
1299
+ )
1300
+
1301
+
1302
+class MolmoMultiModalProcessor(BaseMultiModalProcessor[MolmoProcessingInfo]):
1303
+
1304
+ def _apply_hf_processor_tokens_only(
1305
+ self,
1306
+ prompt_tokens: list[int],
1307
+ ) -> list[int]:
1308
+ processor = self.info.get_hf_processor()
1309
+
1310
+ # Apply the chat template to the tokens
1311
+ tokens = processor.processor.get_tokens_input( # type: ignore
1312
+ self.info.get_tokenizer().decode(prompt_tokens),
1313
+ message_format=processor.message_format,
1314
+ always_start_with_space=processor.always_start_with_space,
1315
+ )
1316
+
1317
+ processed_data = self.info.ctx.call_hf_processor(
1318
+ processor, # type: ignore
1319
+ dict(tokens=tokens),
1320
+ )
1321
+ prompt_ids, = processed_data.pop("input_ids").tolist()
1322
+
1323
+ return prompt_ids
1324
+
1325
+ def _get_mm_fields_config(
1326
+ self,
1327
+ hf_inputs: BatchFeature,
1328
+ hf_processor_mm_kwargs: Mapping[str, object],
1329
+ ) -> Mapping[str, MultiModalFieldConfig]:
1330
+ num_crops = hf_inputs.get("num_crops", torch.empty(0))
1331
+ num_images = len(num_crops)
1332
+
1333
+ return dict(
1334
+ images=MultiModalFieldConfig.flat_from_sizes("image", num_crops),
1335
+ image_masks=MultiModalFieldConfig.flat_from_sizes(
1336
+ "image", num_crops),
1337
+ feat_is_patch=MultiModalFieldConfig.flat_from_sizes(
1338
+ "image", num_crops),
1339
+ embed_is_patch=MultiModalFieldConfig.shared("image", num_images),
1340
+ num_crops=MultiModalFieldConfig.batched("image"),
1341
+ img_patch_id=MultiModalFieldConfig.shared("image", num_images),
1342
+ )
1343
+
1344
+ def _get_prompt_replacements(
1345
+ self,
1346
+ mm_items: MultiModalDataItems,
1347
+ hf_processor_mm_kwargs: Mapping[str, object],
1348
+ out_mm_kwargs: MultiModalKwargs,
1349
+ ) -> list[PromptReplacement]:
1350
+ processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
1351
+ tokenizer = self.info.get_tokenizer()
1352
+
1353
+ image_token_length_w = processor.image_token_length_w
1354
+ image_token_length_h = processor.image_token_length_h
1355
+ pooling_size = processor.pooling_size
1356
+
1357
+ user_str = "User:"
1358
+ if processor.always_start_with_space:
1359
+ user_str = " " + user_str
1360
+
1361
+ user_tokens = tokenizer.encode(user_str, add_special_tokens=False)
1362
+
1363
+ img_patch_id = processor.image_patch_id
1364
+ img_col_id = processor.im_col_id
1365
+ img_start_id = processor.im_start_id
1366
+ img_end_id = processor.im_end_id
1367
+
1368
+ extra_row = [img_patch_id] * image_token_length_w + [img_col_id]
1369
+ extra_joint = ([img_start_id] + extra_row * image_token_length_h +
1370
+ [img_end_id])
1371
+
1372
+ def get_replacement_molmo(item_idx: int):
1373
+ images = mm_items.get_items("image", ImageProcessorItems)
1374
+ image_size = images.get_image_size(item_idx)
1375
+
1376
+ ncols, nrows = processor.get_patches_grid_size(
1377
+ image_width=image_size.width,
1378
+ image_height=image_size.height,
1379
+ )
1380
+
1381
+ joint_row = ([img_patch_id] * ((ncols + 1) // pooling_size) +
1382
+ [img_col_id])
1383
+ joint = ([img_start_id] + joint_row *
1384
+ ((nrows + 1) // pooling_size) + [img_end_id])
1385
+
1386
+ image_tokens = extra_joint + joint
1387
+
1388
+ return PromptReplacementDetails(
1389
+ full=image_tokens + user_tokens,
1390
+ features=image_tokens,
1391
+ )
1392
+
1393
+ return [
1394
+ PromptReplacement(
1395
+ modality="image",
1396
+ target=user_str,
1397
+ replacement=get_replacement_molmo,
1398
+ )
1399
+ ]
1400
+
1401
+
1402
+@MULTIMODAL_REGISTRY.register_processor(MolmoMultiModalProcessor,
1403
+ info=MolmoProcessingInfo,
1404
+ dummy_inputs=MolmoDummyInputsBuilder)
1405
class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1406
SupportsLoRA):
1407
hf_to_vllm_mapper = WeightsMapper(
@@ -1202,6 +1473,7 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1473
quant_config)
1474
self.model = MolmoModel(vllm_config=vllm_config,
1475
prefix=maybe_prefix(prefix, "model"))
1476
+ self.img_patch_id = None
1477
1478
if self.config.weight_tying:
1479
self.lm_head = self.model.transformer.wte
@@ -1224,85 +1496,143 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1496
**kwargs: object,
1497
) -> Optional[MolmoImageInputs]:
1498
images = kwargs.pop("images", None)
1227
- image_masks = kwargs.pop("image_masks", None)
1228
- image_start_end = kwargs.pop("image_start_end", None)
1499
if images is None:
1500
return None
1501
1232
- image_input_idx = kwargs.pop("image_input_idx", None)
1233
- seq_len = kwargs.pop("seq_len", None)
1234
- if image_input_idx is None:
1235
- raise ValueError("image_input_idx is required for Molmo model.")
1236
- if seq_len is None:
1237
- raise ValueError("seq_len is required for Molmo model.")
1238
- if not isinstance(seq_len, torch.Tensor):
1239
- seq_len = torch.tensor(seq_len)
1502
+ if not isinstance(images, (torch.Tensor, list)):
1503
+ raise ValueError("Incorrect type of images. "
1504
+ f"Got type: {type(images)}")
1505
+
1506
+ image_masks = kwargs.pop("image_masks", None)
1507
+ if not (image_masks is None or isinstance(image_masks,
1508
+ (torch.Tensor, list))):
1509
+ raise ValueError("Incorrect type of image_masks. "
1510
+ f"Got type: {type(image_masks)}")
1511
+
1512
+ feat_is_patch = kwargs.pop("feat_is_patch", None)
1513
+ if not isinstance(feat_is_patch, (torch.Tensor, list)):
1514
+ raise ValueError("Incorrect type of feat_is_patch. "
1515
+ f"Got type: {type(feat_is_patch)}")
1516
+
1517
+ embed_is_patch = kwargs.pop("embed_is_patch", None)
1518
+ if not isinstance(embed_is_patch, (torch.Tensor, list)):
1519
+ raise ValueError("Incorrect type of embed_is_patch. "
1520
+ f"Got type: {type(embed_is_patch)}")
1521
+
1522
+ num_crops = kwargs.pop("num_crops", None)
1523
+ if not isinstance(num_crops, torch.Tensor):
1524
+ raise ValueError("Incorrect type of num_crops. "
1525
+ f"Got type: {type(num_crops)}")
1526
+
1527
+ img_patch_id = kwargs.pop("img_patch_id", None)
1528
+ if not isinstance(img_patch_id, torch.Tensor):
1529
+ raise ValueError("Incorrect type of num_crops. "
1530
+ f"Got type: {type(num_crops)}")
1531
+ self.img_patch_id = img_patch_id.flatten().unique().item()
1532
1533
return MolmoImageInputs(
1534
images=images,
1243
- image_input_idx=image_input_idx,
1244
- seq_len=seq_len,
1535
image_masks=image_masks,
1246
- image_start_end=image_start_end,
1536
+ feat_is_patch=feat_is_patch,
1537
+ embed_is_patch=embed_is_patch,
1538
+ num_crops=num_crops,
1539
)
1540
1541
def _process_image_input(
1542
self,
1543
image_input: MolmoImageInputs,
1252
- ) -> torch.Tensor:
1253
-
1254
- image_features = self.vision_backbone(
1255
- images=image_input["images"],
1256
- image_masks=image_input["image_masks"],
1257
- )
1544
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
1545
+ if isinstance(image_input["images"], list):
1546
+ # Call the vision backbone on the whole batch at once
1547
+ images_flat = flatten_bn(image_input["images"], concat=True)
1548
+ image_masks_flat = (None if (image_masks :=
1549
+ image_input["image_masks"]) is None
1550
+ else flatten_bn(image_masks, concat=True))
1551
+
1552
+ image_features_flat = self.vision_backbone(
1553
+ images=images_flat.unsqueeze(0),
1554
+ image_masks=(None if image_masks_flat is None else
1555
+ image_masks_flat.unsqueeze(0)),
1556
+ ).squeeze(0)
1557
+
1558
+ # Reconstruct the batch dimension
1559
+ image_features = image_features_flat.split(
1560
+ image_input["num_crops"].sum(-1).tolist())
1561
+ else:
1562
+ image_features = self.vision_backbone(
1563
+ images=image_input["images"],
1564
+ image_masks=image_input["image_masks"],
1565
+ )
1566
1567
return image_features
1568
1569
+ def _get_mm_embeds(
1570
+ self,
1571
+ features: torch.Tensor, # Shape: (num_crop, num_patch, d)
1572
+ feat_is_patch: torch.Tensor, # Shape: (num_crop, num_patch)
1573
+ num_crops: torch.Tensor, # Shape: (num_images,)
1574
+ embed_is_patch: torch.Tensor, # Shape: (num_embeds,)
1575
+ ) -> list[torch.Tensor]:
1576
+ """
1577
+ Scatter the patch features into a contiguous tensor that corresponds
1578
+ to the embedding tokens defined by the multimodal processor.
1579
+
1580
+ Note:
1581
+ The original code only considers patch tokens as feature
1582
+ tokens, but our processor considers all image-related tokens
1583
+ as feature tokens because the feature tokens need to be
1584
+ consecutive in `input_ids`.
1585
+
1586
+ Example:
1587
+ A simplified example for one item in the batch:
1588
+
1589
+ .. code-block::
1590
+
1591
+ Embedding tokens (from HF processor):
1592
+ [<start> <patch> <patch> <col> <patch> <patch> <col> <end> ]
1593
+
1594
+ embed_is_patch (from HF processor):
1595
+ [ False True True False True True False False ]
1596
+
1597
+ Encoder outputs (from model):
1598
+ [ p1 p2 0 p3 p4 0 ]
1599
+
1600
+ feat_is_patch (from HF processor):
1601
+ [ True True False True True False ]
1602
+
1603
+ The resulting embedding tensor is:
1604
+ [ nan p1 p2 nan p3 p4 nan nan ]
1605
+ """
1606
+ num_crops_per_image = num_crops.tolist()
1607
+ feats_per_image = features.split(num_crops_per_image)
1608
+ f_is_patch_per_image = feat_is_patch.split(num_crops_per_image)
1609
+
1610
+ _, _, embed_dim = features.shape
1611
+ (num_embeds, ) = embed_is_patch.shape
1612
+
1613
+ embeds_in_batch = list[torch.Tensor]()
1614
+ for feats, f_is_patch in zip(feats_per_image, f_is_patch_per_image):
1615
+ embeds = feats.new_full((num_embeds, embed_dim), torch.nan)
1616
+ embeds[embed_is_patch] = feats[f_is_patch]
1617
+ embeds_in_batch.append(embeds)
1618
+
1619
+ return embeds_in_batch
1620
+
1621
def get_multimodal_embeddings(self, **kwargs) -> Optional[NestedTensors]:
1622
image_input = self._parse_and_validate_image_input(**kwargs)
1623
if image_input is None:
1624
return None
1625
+
1626
image_features = self._process_image_input(image_input)
1266
- image_input_idx = image_input["image_input_idx"]
1267
- seq_len = image_input["seq_len"]
1268
- batch_size, num_image, num_patch = image_features.shape[:3]
1269
- assert image_input_idx.shape == (batch_size, num_image, num_patch)
1270
-
1271
- # insert the image feature into the embedding.
1272
- image_features = image_features.view(batch_size, num_image * num_patch,
1273
- -1)
1274
- image_input_idx = image_input_idx.view(batch_size,
1275
- num_image * num_patch)
1276
-
1277
- valid = image_input_idx >= 0
1278
- image_features = image_features * valid[:, :, None].to(
1279
- image_features.dtype)
1280
- image_features = image_features.view(
1281
- batch_size * num_image * num_patch, -1).contiguous()
1282
-
1283
- image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1284
- offset = torch.cat([seq_len.new_zeros(1),
1285
- seq_len.cumsum(dim=0)[:-1]],
1286
- dim=0)[:, None]
1287
- image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1288
- image_input_idx = image_input_idx.flatten()[:, None]
1289
- mat = image_input_idx == torch.arange(
1290
- seq_len.sum().item(), device=image_features.device)[None, :]
1291
- mat = mat.to(image_features.dtype)
1292
-
1293
- # Note: In this original implementation from AI2, the final
1294
- # vision_embeddings will be always be the same length
1295
- # of input embeddings.
1296
- vision_embeddings = torch.einsum('nd,nm->md', image_features, mat)
1297
-
1298
- # Split by the sizes of the input sequences. For each full embedding,
1299
- # extract the actual vision embeddings to be merged.
1300
- vision_embeddings = list(vision_embeddings.split(seq_len.tolist()))
1301
- for i in range(len(vision_embeddings)):
1302
- start, end = image_input['image_start_end'][i]
1303
- vision_embeddings[i] = vision_embeddings[i][start:end]
1304
-
1305
- return vision_embeddings
1627
+
1628
+ return [
1629
+ self._get_mm_embeds(*args) for args in zip(
1630
+ image_features,
1631
+ image_input["feat_is_patch"],
1632
+ image_input["num_crops"],
1633
+ image_input["embed_is_patch"],
1634
+ )
1635
+ ]
1636
1637
def get_input_embeddings(
1638
self,
@@ -1311,11 +1641,20 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP,
1641
) -> torch.Tensor:
1642
inputs_embeds = self.model.get_input_embeddings(input_ids)
1643
if multimodal_embeddings is not None:
1644
+ assert self.img_patch_id is not None
1645
+
1646
+ # Extract the patch tokens scattered in _get_mm_embeds
1647
+ patch_embeddings = json_map_leaves(
1648
+ lambda x: x[~x.isnan()].view(-1, *x.shape[1:]),
1649
+ cast(JSONTree[torch.Tensor], multimodal_embeddings),
1650
+ )
1651
+
1652
inputs_embeds = merge_multimodal_embeddings(
1315
- input_ids, inputs_embeds, multimodal_embeddings, [
1316
- DEFAULT_IMAGE_PATCH_TOKEN_ID, DEFAULT_IM_START_TOKEN_ID,
1317
- DEFAULT_IM_END_TOKEN_ID, DEFAULT_IM_COL_TOKEN_ID
1318
- ])
1653
+ input_ids,
1654
+ inputs_embeds,
1655
+ cast(NestedTensors, patch_embeddings),
1656
+ self.img_patch_id,
1657
+ )
1658
return inputs_embeds
1659
1660
def forward(
vllm/multimodal/inputs.py
+40
-40
@@ -353,17 +353,17 @@ class MultiModalFieldConfig:
353
354
Example:
355
356
- .. code-block::
356
+ .. code-block::
357
358
- Input:
359
- Data: [[AAAA]
360
- [BBBB]
361
- [CCCC]]
358
+ Input:
359
+ Data: [[AAAA]
360
+ [BBBB]
361
+ [CCCC]]
362
363
- Output:
364
- Element 1: [AAAA]
365
- Element 2: [BBBB]
366
- Element 3: [CCCC]
363
+ Output:
364
+ Element 1: [AAAA]
365
+ Element 2: [BBBB]
366
+ Element 3: [CCCC]
367
"""
368
return MultiModalFieldConfig(
369
field=MultiModalBatchedField(),
@@ -384,18 +384,18 @@ class MultiModalFieldConfig:
384
385
Example:
386
387
- .. code-block::
388
-
389
- Given:
390
- slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
387
+ .. code-block::
388
+
389
+ Given:
390
+ slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
391
392
- Input:
393
- Data: [AAABBBBCC]
392
+ Input:
393
+ Data: [AAABBBBCC]
394
395
- Output:
396
- Element 1: [AAA]
397
- Element 2: [BBBB]
398
- Element 3: [CC]
395
+ Output:
396
+ Element 1: [AAA]
397
+ Element 2: [BBBB]
398
+ Element 3: [CC]
399
"""
400
return MultiModalFieldConfig(
401
field=MultiModalFlatField(slices=slices),
@@ -416,18 +416,18 @@ class MultiModalFieldConfig:
416
417
Example:
418
419
- .. code-block::
420
-
421
- Given:
422
- size_per_item: [3, 4, 2]
419
+ .. code-block::
420
+
421
+ Given:
422
+ size_per_item: [3, 4, 2]
423
424
- Input:
425
- Data: [AAABBBBCC]
424
+ Input:
425
+ Data: [AAABBBBCC]
426
427
- Output:
428
- Element 1: [AAA]
429
- Element 2: [BBBB]
430
- Element 3: [CC]
427
+ Output:
428
+ Element 1: [AAA]
429
+ Element 2: [BBBB]
430
+ Element 3: [CC]
431
432
See also:
433
:func:`MultiModalFieldConfig.flat`
@@ -456,19 +456,19 @@ class MultiModalFieldConfig:
456
457
Example:
458
459
- .. code-block::
460
-
461
- Given:
462
- batch_size: 4
459
+ .. code-block::
460
+
461
+ Given:
462
+ batch_size: 4
463
464
- Input:
465
- Data: [XYZ]
464
+ Input:
465
+ Data: [XYZ]
466
467
- Output:
468
- Element 1: [XYZ]
469
- Element 2: [XYZ]
470
- Element 3: [XYZ]
471
- Element 4: [XYZ]
467
+ Output:
468
+ Element 1: [XYZ]
469
+ Element 2: [XYZ]
470
+ Element 3: [XYZ]
471
+ Element 4: [XYZ]
472
"""
473
return MultiModalFieldConfig(
474
field=MultiModalSharedField(batch_size),
vllm/utils.py
+1
-34
@@ -33,8 +33,7 @@ from dataclasses import dataclass, field
33
from functools import cache, lru_cache, partial, wraps
34
from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
35
Dict, Generator, Generic, Iterator, List, Literal,
36
- NamedTuple, Optional, Tuple, Type, TypeVar, Union,
37
- overload)
36
+ NamedTuple, Optional, Tuple, Type, TypeVar, Union)
37
from uuid import uuid4
38
39
import cloudpickle
@@ -826,38 +825,6 @@ JSONTree = Union[Dict[str, "JSONTree[T]"], List["JSONTree[T]"],
825
"""A nested JSON structure where the leaves need not be JSON-serializable."""
826
827
829
-@overload
830
-def json_map_leaves(
831
- func: Callable[[T], U],
832
- value: Dict[str, JSONTree[T]],
833
-) -> Dict[str, JSONTree[U]]:
834
- ...
835
-
836
-
837
-@overload
838
-def json_map_leaves(
839
- func: Callable[[T], U],
840
- value: List[JSONTree[T]],
841
-) -> List[JSONTree[U]]:
842
- ...
843
-
844
-
845
-@overload
846
-def json_map_leaves(
847
- func: Callable[[T], U],
848
- value: Tuple[JSONTree[T], ...],
849
-) -> Tuple[JSONTree[U], ...]:
850
- ...
851
-
852
-
853
-@overload
854
-def json_map_leaves(
855
- func: Callable[[T], U],
856
- value: JSONTree[T],
857
-) -> JSONTree[U]:
858
- ...
859
-
860
-
828
def json_map_leaves(func: Callable[[T], U], value: JSONTree[T]) -> JSONTree[U]:
829
if isinstance(value, dict):
830
return {k: json_map_leaves(func, v) for k, v in value.items()}