A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Molmo vLLM Integration (#9016)
Co-authored-by: sanghol <sanghol@allenai.org> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
Reza Salehi committed
Oct 14, 2024 at 07:56 UTC
dfe43a207161051c10daaae064936f4a4d2a597c
7 files changed
+1319
-3
docs/source/models/supported_models.rst
+6
@@ -399,6 +399,12 @@ Text Generation
399
- :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
400
-
401
-
402
+ * - :code:`MolmoForCausalLM`
403
+ - Molmo
404
+ - Image
405
+ - :code:`allenai/Molmo-7B-D-0924`, :code:`allenai/Molmo-72B-0924`, etc.
406
+ -
407
+ - ✅︎
408
* - :code:`NVLM_D_Model`
409
- NVLM-D 1.0
410
- Image\ :sup:`E+`
examples/offline_inference_vision_language.py
+18
@@ -300,6 +300,23 @@ def run_mllama(question: str, modality: str):
300
return llm, prompt, stop_token_ids
301
302
303
+# Molmo
304
+def run_molmo(question, modality):
305
+ assert modality == "image"
306
+
307
+ model_name = "allenai/Molmo-7B-D-0924"
308
+
309
+ llm = LLM(
310
+ model=model_name,
311
+ trust_remote_code=True,
312
+ dtype="bfloat16",
313
+ )
314
+
315
+ prompt = question
316
+ stop_token_ids = None
317
+ return llm, prompt, stop_token_ids
318
+
319
+
320
# GLM-4v
321
def run_glm4v(question: str, modality: str):
322
assert modality == "image"
@@ -331,6 +348,7 @@ model_example_map = {
348
"qwen_vl": run_qwen_vl,
349
"qwen2_vl": run_qwen2_vl,
350
"mllama": run_mllama,
351
+ "molmo": run_molmo,
352
"glm4v": run_glm4v,
353
}
354
vllm/entrypoints/chat_utils.py
+2
@@ -163,6 +163,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
163
return "<|image|>"
164
if model_type == "qwen2_vl":
165
return "<|vision_start|><|image_pad|><|vision_end|>"
166
+ if model_type == "molmo":
167
+ return ""
168
169
raise TypeError(f"Unknown model type: {model_type}")
170
elif modality == "audio":
vllm/model_executor/models/__init__.py
+1
-1
@@ -20,4 +20,4 @@ __all__ = [
20
"supports_multimodal",
21
"SupportsPP",
22
"supports_pp",
23
-]
23
+]
\ No newline at end of file
vllm/model_executor/models/molmo.py
new
+1290
@@ -0,0 +1,1290 @@
1
+import logging
2
+import math
3
+import re
4
+from array import array
5
+from dataclasses import dataclass
6
+from functools import lru_cache, partial
7
+from typing import (Any, Iterable, List, Mapping, Optional, Tuple, TypedDict,
8
+ Union)
9
+
10
+import torch
11
+from einops import rearrange
12
+from PIL import Image
13
+from torch import nn
14
+from torch.nn import functional as F
15
+from transformers import PretrainedConfig
16
+
17
+import vllm.envs as envs
18
+from vllm.attention import Attention, AttentionMetadata
19
+from vllm.attention.selector import (_Backend, backend_name_to_enum,
20
+ get_global_forced_attn_backend)
21
+from vllm.config import CacheConfig, MultiModalConfig
22
+from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23
+ get_tensor_model_parallel_world_size,
24
+ split_tensor_along_last_dim,
25
+ tensor_model_parallel_all_gather)
26
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
27
+from vllm.model_executor import SamplingMetadata
28
+from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
29
+from vllm.model_executor.layers.layernorm import RMSNorm
30
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
+ MergedColumnParallelLinear,
32
+ QKVParallelLinear,
33
+ RowParallelLinear)
34
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
+from vllm.model_executor.layers.quantization.base_config import (
36
+ QuantizationConfig)
37
+from vllm.model_executor.layers.rotary_embedding import get_rope
38
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39
+from vllm.model_executor.layers.vocab_parallel_embedding import (
40
+ ParallelLMHead, VocabParallelEmbedding)
41
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42
+from vllm.model_executor.models.interfaces import SupportsMultiModal
43
+from vllm.model_executor.models.utils import make_layers
44
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
45
+from vllm.platforms import current_platform
46
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
47
+ SequenceData)
48
+from vllm.transformers_utils.processor import get_processor
49
+
50
+log = logging.getLogger(__name__)
51
+
52
+# TODO: hard-coded for now. Consider making it configurable.
53
+VIT_LAYERS = [-2, -9]
54
+NUM_PREFIX_TOKENS = 1
55
+ADDITIONAL_VOCAB_SIZE = 128
56
+
57
+
58
+class MolmoImageInputs(TypedDict):
59
+ images: torch.Tensor
60
+ """Shape:
61
+ `(batch_size, num_crops, num_patch, patch_dim)`
62
+ """
63
+
64
+ image_input_idx: torch.Tensor
65
+ """Shape:
66
+ `(batch_size, num_crops, num_patch)`
67
+ """
68
+
69
+ seq_len: torch.Tensor
70
+ """Shape:
71
+ `(batch_size, )`
72
+ """
73
+
74
+ image_masks: Optional[torch.Tensor]
75
+ """Shape:
76
+ `(batch_size, num_crops, num_patch)`
77
+ """
78
+
79
+
80
+@dataclass
81
+class VisionBackboneConfig:
82
+ image_default_input_size: Tuple[int, int] = (336, 336)
83
+ image_patch_size: int = 14
84
+ image_pos_patch_size: int = 14
85
+ image_emb_dim: int = 1024
86
+ image_num_heads: int = 16
87
+ image_num_key_value_heads: int = 16
88
+ image_num_layers: int = 23
89
+ image_mlp_dim: int = 4096
90
+ image_mlp_activations: str = "quick_gelu"
91
+ image_num_pos: int = 577
92
+ image_norm_eps: float = 1e-5
93
+
94
+ def __post_init__(self):
95
+ self.image_default_input_size = tuple(
96
+ self.image_default_input_size) # type: ignore[assignment]
97
+
98
+ @property
99
+ def image_num_patch(self):
100
+ h, w = self.image_default_input_size
101
+ return h // self.image_patch_size, w // self.image_patch_size
102
+
103
+
104
+class ViTMLP(nn.Module):
105
+ """MLP used in Vision Transformer."""
106
+
107
+ def __init__(
108
+ self,
109
+ config: VisionBackboneConfig,
110
+ quant_config: Optional[QuantizationConfig] = None,
111
+ ):
112
+ super().__init__()
113
+ self.w1 = ColumnParallelLinear(
114
+ config.image_emb_dim,
115
+ config.image_mlp_dim,
116
+ bias=True,
117
+ quant_config=quant_config,
118
+ )
119
+ # Activation function.
120
+ assert config.image_mlp_activations == "quick_gelu"
121
+ self.act = QuickGELU()
122
+ self.w2 = RowParallelLinear(
123
+ config.image_mlp_dim,
124
+ config.image_emb_dim,
125
+ bias=True,
126
+ quant_config=quant_config,
127
+ )
128
+
129
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
130
+ x, _ = self.w1(x)
131
+ x = self.act(x)
132
+ x, _ = self.w2(x)
133
+ return x
134
+
135
+
136
+class MultiHeadDotProductAttention(nn.Module):
137
+ """Multi-head attention used in Vision Transformer."""
138
+
139
+ def __init__(
140
+ self,
141
+ config: VisionBackboneConfig,
142
+ use_bias: bool = True,
143
+ nlayers: int = 1,
144
+ quant_config: Optional[QuantizationConfig] = None,
145
+ ):
146
+ super().__init__()
147
+
148
+ self.hidden_size = config.image_emb_dim
149
+ self.total_num_heads = config.image_num_heads
150
+ tp_size = get_tensor_model_parallel_world_size()
151
+
152
+ assert self.hidden_size % self.total_num_heads == 0
153
+ assert self.total_num_heads % tp_size == 0
154
+
155
+ self.num_heads = self.total_num_heads // tp_size
156
+ self.head_dim = self.hidden_size // self.total_num_heads
157
+
158
+ self.total_num_kv_heads = config.image_num_key_value_heads
159
+ if self.total_num_kv_heads >= tp_size:
160
+ assert self.total_num_kv_heads % tp_size == 0
161
+ else:
162
+ assert tp_size % self.total_num_kv_heads == 0
163
+
164
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
165
+
166
+ self.wq = ColumnParallelLinear(
167
+ nlayers * self.hidden_size,
168
+ self.total_num_heads * self.head_dim,
169
+ bias=use_bias,
170
+ quant_config=quant_config,
171
+ )
172
+ self.wk = ColumnParallelLinear(
173
+ nlayers * self.hidden_size,
174
+ self.total_num_kv_heads * self.head_dim,
175
+ bias=use_bias,
176
+ quant_config=quant_config,
177
+ )
178
+ self.wv = ColumnParallelLinear(
179
+ nlayers * self.hidden_size,
180
+ self.total_num_kv_heads * self.head_dim,
181
+ bias=use_bias,
182
+ quant_config=quant_config,
183
+ )
184
+ self.wo = RowParallelLinear(
185
+ self.total_num_heads * self.head_dim,
186
+ self.hidden_size,
187
+ bias=use_bias,
188
+ quant_config=quant_config,
189
+ )
190
+
191
+ # Detect attention implementation.
192
+ selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
193
+ if selected_backend is None:
194
+ backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
195
+ if backend_by_env_var is not None:
196
+ selected_backend = backend_name_to_enum(backend_by_env_var)
197
+ if selected_backend is None:
198
+ # For Volta and Turing GPUs, use xformers instead.
199
+ device_available = current_platform.get_device_capability()[0] >= 8
200
+ if device_available:
201
+ from transformers.utils import is_flash_attn_2_available
202
+ if is_flash_attn_2_available():
203
+ self._use_flash_attn = True
204
+ else:
205
+ log.warning(
206
+ "Current Molmo implementation has a bug with "
207
+ "`vllm-flash-attn` inside vision module, so we use "
208
+ "xformers backend instead. You can run `pip install "
209
+ "flash-attn to use flash-attention backend.")
210
+ self._use_flash_attn = False
211
+ else:
212
+ self._use_flash_attn = False
213
+ else:
214
+ if selected_backend == _Backend.FLASH_ATTN:
215
+ self._use_flash_attn = True
216
+ elif selected_backend == _Backend.XFORMERS:
217
+ self._use_flash_attn = False
218
+ else:
219
+ raise RuntimeError(
220
+ f"Molmo does not support {selected_backend} backend now.")
221
+
222
+ def forward(self,
223
+ inputs_q: torch.Tensor,
224
+ inputs_kv: Optional[torch.Tensor] = None) -> torch.Tensor:
225
+
226
+ if inputs_kv is not None:
227
+ inputs_k = inputs_kv
228
+ inputs_v = inputs_kv
229
+ else:
230
+ inputs_k = inputs_q
231
+ inputs_v = inputs_q
232
+
233
+ xq, _ = self.wq(inputs_q)
234
+ xk, _ = self.wk(inputs_k)
235
+ xv, _ = self.wv(inputs_v)
236
+ q_shape = xq.size()[:-1] + (self.num_heads, self.head_dim)
237
+ kv_shape = xk.size()[:-1] + (self.num_kv_heads, self.head_dim)
238
+ xq = xq.view(*q_shape)
239
+ xk = xk.view(*kv_shape)
240
+ xv = xv.view(*kv_shape)
241
+
242
+ if self._use_flash_attn:
243
+ from flash_attn import flash_attn_func
244
+ output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
245
+ else:
246
+ from xformers import ops as xops
247
+ output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
248
+
249
+ output = rearrange(output, "b s h d -> b s (h d)").contiguous()
250
+ output, _ = self.wo(output)
251
+
252
+ return output
253
+
254
+
255
+class ResidualAttentionBlock(nn.Module):
256
+ """Residual attention block used in Vision Transformer."""
257
+
258
+ def __init__(
259
+ self,
260
+ config: VisionBackboneConfig,
261
+ quant_config: Optional[QuantizationConfig] = None,
262
+ ):
263
+ super().__init__()
264
+ self.attention = MultiHeadDotProductAttention(
265
+ config, quant_config=quant_config)
266
+ self.feed_forward = ViTMLP(config, quant_config)
267
+ self.attention_norm = nn.LayerNorm(
268
+ config.image_emb_dim,
269
+ eps=config.image_norm_eps,
270
+ )
271
+ self.ffn_norm = nn.LayerNorm(
272
+ config.image_emb_dim,
273
+ eps=config.image_norm_eps,
274
+ )
275
+
276
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
277
+ x = x + self.attention(self.attention_norm(x))
278
+ x = x + self.feed_forward(self.ffn_norm(x))
279
+ return x
280
+
281
+
282
+class BlockCollection(nn.Module):
283
+ """Collection of residual attention blocks used in Vision Transformer."""
284
+
285
+ def __init__(
286
+ self,
287
+ config: VisionBackboneConfig,
288
+ quant_config: Optional[QuantizationConfig] = None,
289
+ ):
290
+ super().__init__()
291
+ self.resblocks = nn.ModuleList([
292
+ ResidualAttentionBlock(config, quant_config)
293
+ for _ in range(config.image_num_layers)
294
+ ])
295
+
296
+ def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
297
+ hidden_states = []
298
+ for r in self.resblocks:
299
+ x = r(x)
300
+ hidden_states.append(x)
301
+ return hidden_states
302
+
303
+
304
+def _expand_token(token: torch.Tensor, batch_size: int) -> torch.Tensor:
305
+ return token.view(1, 1, -1).expand(batch_size, -1, -1)
306
+
307
+
308
+class VisionTransformer(nn.Module):
309
+ """Vision Transformer used in Vision Backbone."""
310
+
311
+ def __init__(
312
+ self,
313
+ config: VisionBackboneConfig,
314
+ quant_config: Optional[QuantizationConfig] = None,
315
+ ):
316
+ super().__init__()
317
+ scale = config.image_emb_dim**-0.5
318
+ self.patch_num = config.image_num_patch
319
+ self.class_embedding = nn.Parameter(
320
+ torch.randn(config.image_emb_dim) * scale)
321
+ self.num_prefix_tokens: int = NUM_PREFIX_TOKENS
322
+ self.positional_embedding = nn.Parameter(
323
+ torch.randn(config.image_num_pos, config.image_emb_dim) * scale)
324
+ image_patch_size = config.image_patch_size
325
+ self.patch_embedding = nn.Linear(
326
+ image_patch_size * image_patch_size * 3,
327
+ config.image_emb_dim,
328
+ bias=False,
329
+ )
330
+ self.pre_ln = nn.LayerNorm(config.image_emb_dim,
331
+ eps=config.image_norm_eps)
332
+ self.transformer = BlockCollection(config, quant_config)
333
+
334
+ def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
335
+ cls_emb = self.positional_embedding[0:1]
336
+ pos_emb = self.positional_embedding[1:]
337
+
338
+ pos_emb = pos_emb.reshape(
339
+ (int(math.sqrt(pos_emb.shape[0])),
340
+ int(math.sqrt(pos_emb.shape[0])), pos_emb.shape[1]))
341
+
342
+ (patch_num_0, patch_num_1) = patch_num
343
+
344
+ if pos_emb.shape[0] != patch_num_0 or pos_emb.shape[1] != patch_num_1:
345
+ # from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
346
+ pos_emb = pos_emb.unsqueeze(0).permute(0, 3, 1, 2)
347
+ pos_emb = F.interpolate(
348
+ pos_emb,
349
+ size=(patch_num_0, patch_num_1),
350
+ mode="bicubic",
351
+ align_corners=False,
352
+ antialias=True,
353
+ )
354
+ pos_emb = pos_emb.permute(0, 2, 3, 1).squeeze(0)
355
+
356
+ pos_emb = pos_emb.reshape(-1, pos_emb.shape[-1])
357
+ x = x + torch.cat([cls_emb[None, :, :], pos_emb[None, :, :]],
358
+ dim=1).to(x.dtype)
359
+ return x
360
+
361
+ def forward(self,
362
+ x: torch.Tensor,
363
+ patch_num: int = None) -> List[torch.Tensor]:
364
+ """
365
+ : param x: (batch_size, num_patch, n_pixels)
366
+ """
367
+ if patch_num is None:
368
+ patch_num = self.patch_num
369
+ B, N, D = x.shape
370
+
371
+ x = self.patch_embedding(x)
372
+
373
+ # class embeddings and positional embeddings
374
+ x = torch.cat(
375
+ [_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x],
376
+ dim=1)
377
+ x = self.add_pos_emb(x, patch_num)
378
+
379
+ x = self.pre_ln(x)
380
+
381
+ hidden_states = self.transformer(x)
382
+ return hidden_states
383
+
384
+
385
+class MolmoAttention(nn.Module):
386
+ """Molmo's LLM attention."""
387
+
388
+ def __init__(
389
+ self,
390
+ config: PretrainedConfig,
391
+ cache_config: Optional[CacheConfig] = None,
392
+ quant_config: Optional[QuantizationConfig] = None,
393
+ ) -> None:
394
+ super().__init__()
395
+ self.hidden_size = config.hidden_size
396
+ self.tp_size = get_tensor_model_parallel_world_size()
397
+ self.total_num_heads = config.num_attention_heads
398
+
399
+ assert self.hidden_size % self.total_num_heads == 0
400
+ assert self.total_num_heads % self.tp_size == 0
401
+
402
+ self.num_heads = self.total_num_heads // self.tp_size
403
+ self.total_num_kv_heads = config.num_key_value_heads \
404
+ or self.total_num_heads
405
+ if self.total_num_kv_heads >= self.tp_size:
406
+ assert self.total_num_kv_heads % self.tp_size == 0
407
+ else:
408
+ assert self.tp_size % self.total_num_kv_heads == 0
409
+
410
+ self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size)
411
+ self.head_dim = self.hidden_size // self.total_num_heads
412
+ self.q_size = self.num_heads * self.head_dim
413
+ self.kv_size = self.num_kv_heads * self.head_dim
414
+ self.max_position_embeddings = config.max_position_embeddings
415
+ self.rope_theta = config.rope_theta
416
+
417
+ # Attention input projection. Projects x -> (q, k, v)
418
+ self.qkv_proj = QKVParallelLinear(
419
+ self.hidden_size,
420
+ self.head_dim,
421
+ self.total_num_heads,
422
+ self.total_num_kv_heads,
423
+ bias=config.qkv_bias,
424
+ quant_config=quant_config,
425
+ )
426
+
427
+ self.tp_rank: Optional[int] = None
428
+ self.k_norm: Optional[nn.Module] = None
429
+ self.q_norm: Optional[nn.Module] = None
430
+ if config.attention_layer_norm:
431
+ self.tp_rank = get_tensor_model_parallel_rank()
432
+ self.k_norm = RMSNorm(self.total_num_kv_heads * self.head_dim,
433
+ eps=config.layer_norm_eps)
434
+ self.q_norm = RMSNorm(config.hidden_size,
435
+ eps=config.layer_norm_eps)
436
+
437
+ # Rotary embeddings.
438
+ self.rotary_emb = get_rope(
439
+ self.head_dim,
440
+ rotary_dim=self.head_dim,
441
+ max_position=self.max_position_embeddings,
442
+ base=self.rope_theta,
443
+ )
444
+ self.scaling = self.head_dim**-0.5
445
+ self.attn = Attention(self.num_heads,
446
+ self.head_dim,
447
+ self.scaling,
448
+ num_kv_heads=self.num_kv_heads,
449
+ cache_config=cache_config,
450
+ quant_config=quant_config)
451
+
452
+ # Attention output projection.
453
+ self.o_proj = RowParallelLinear(
454
+ self.total_num_heads * self.head_dim,
455
+ self.hidden_size,
456
+ bias=False,
457
+ quant_config=quant_config,
458
+ )
459
+
460
+ def _apply_qk_norm(self, q: torch.Tensor,
461
+ k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
462
+ if self.tp_size > 1:
463
+ q = tensor_model_parallel_all_gather(q.contiguous())
464
+ k = tensor_model_parallel_all_gather(k.contiguous())
465
+ q = self.q_norm.forward_native(q)
466
+ k = self.k_norm.forward_native(k)
467
+ if self.tp_size > 1:
468
+ splitter = partial(split_tensor_along_last_dim,
469
+ num_partitions=self.tp_size)
470
+ q = splitter(q)[self.tp_rank]
471
+ k = splitter(k)[self.tp_rank]
472
+ return q, k
473
+
474
+ def forward(
475
+ self,
476
+ positions: torch.Tensor,
477
+ hidden_states: torch.Tensor,
478
+ kv_cache: torch.Tensor,
479
+ attn_metadata: AttentionMetadata,
480
+ ) -> torch.Tensor:
481
+ qkv, _ = self.qkv_proj(hidden_states)
482
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
483
+ if self.q_norm is not None and self.k_norm is not None:
484
+ q, k = self._apply_qk_norm(q, k)
485
+ q, k = self.rotary_emb(positions, q, k)
486
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
487
+ output, _ = self.o_proj(attn_output)
488
+ return output
489
+
490
+
491
+class MolmoMLP(nn.Module):
492
+ """Molmo's LLM mlp."""
493
+
494
+ def __init__(
495
+ self,
496
+ config: PretrainedConfig,
497
+ input_dim: Optional[int] = None,
498
+ quant_config: Optional[QuantizationConfig] = None,
499
+ ) -> None:
500
+ super().__init__()
501
+ self.hidden_size = config.hidden_size
502
+ self.intermediate_size = config.intermediate_size // 2
503
+
504
+ # Feed-forward input projection.
505
+ self.gate_up_proj = MergedColumnParallelLinear(
506
+ input_dim or self.hidden_size,
507
+ [self.intermediate_size] * 2,
508
+ bias=False,
509
+ quant_config=quant_config,
510
+ )
511
+
512
+ # Activation function.
513
+ self.act_fn = SiluAndMul()
514
+
515
+ # Feed-forward output projection.
516
+ self.down_proj = RowParallelLinear(
517
+ self.intermediate_size,
518
+ self.hidden_size,
519
+ bias=False,
520
+ quant_config=quant_config,
521
+ )
522
+
523
+ def forward(
524
+ self,
525
+ x: torch.Tensor,
526
+ ) -> torch.Tensor:
527
+ gate_up, _ = self.gate_up_proj(x)
528
+ x = self.act_fn(gate_up)
529
+ x, _ = self.down_proj(x)
530
+ return x
531
+
532
+
533
+class MolmoDecoderLayer(nn.Module):
534
+
535
+ def __init__(
536
+ self,
537
+ config: PretrainedConfig,
538
+ cache_config: Optional[CacheConfig] = None,
539
+ quant_config: Optional[QuantizationConfig] = None,
540
+ ) -> None:
541
+ super().__init__()
542
+ # Attention block.
543
+ self.self_attn = MolmoAttention(config, cache_config, quant_config)
544
+
545
+ # MLP block.
546
+ self.mlp = MolmoMLP(config, quant_config=quant_config)
547
+
548
+ # LayerNorm
549
+ assert config.layer_norm_type == "rms"
550
+ self.input_layernorm = RMSNorm(config.hidden_size,
551
+ eps=config.layer_norm_eps)
552
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
553
+ eps=config.layer_norm_eps)
554
+
555
+ def forward(
556
+ self,
557
+ positions: torch.Tensor,
558
+ hidden_states: torch.Tensor,
559
+ kv_cache: torch.Tensor,
560
+ attn_metadata: AttentionMetadata,
561
+ residual: Optional[torch.Tensor],
562
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
563
+ # Self Attention
564
+ if residual is None:
565
+ residual = hidden_states
566
+ hidden_states = self.input_layernorm(hidden_states)
567
+ else:
568
+ hidden_states, residual = self.input_layernorm(
569
+ hidden_states, residual)
570
+ hidden_states = self.self_attn(
571
+ positions=positions,
572
+ hidden_states=hidden_states,
573
+ kv_cache=kv_cache,
574
+ attn_metadata=attn_metadata,
575
+ )
576
+
577
+ hidden_states, residual = self.post_attention_layernorm(
578
+ hidden_states, residual)
579
+ hidden_states = self.mlp(hidden_states)
580
+ return hidden_states, residual
581
+
582
+
583
+class MolmoDecoderNormAfterLayer(MolmoDecoderLayer):
584
+
585
+ def forward(
586
+ self,
587
+ positions: torch.Tensor,
588
+ hidden_states: torch.Tensor,
589
+ kv_cache: torch.Tensor,
590
+ attn_metadata: AttentionMetadata,
591
+ residual: Optional[torch.Tensor],
592
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
593
+ # Self Attention
594
+ residual = hidden_states
595
+ hidden_states = self.self_attn(
596
+ positions=positions,
597
+ hidden_states=hidden_states,
598
+ kv_cache=kv_cache,
599
+ attn_metadata=attn_metadata,
600
+ )
601
+
602
+ hidden_states = self.input_layernorm(hidden_states)
603
+ hidden_states = hidden_states + residual
604
+ residual = hidden_states
605
+
606
+ hidden_states = self.mlp(hidden_states)
607
+ hidden_states = self.post_attention_layernorm(hidden_states)
608
+ hidden_states = hidden_states + residual
609
+ residual = None
610
+ return hidden_states, residual
611
+
612
+
613
+class MolmoVisionBackbone(nn.Module):
614
+
615
+ def __init__(
616
+ self,
617
+ config: PretrainedConfig,
618
+ vision_config: VisionBackboneConfig,
619
+ quant_config: Optional[QuantizationConfig] = None,
620
+ ) -> None:
621
+ super().__init__()
622
+ self.vit_layers = VIT_LAYERS
623
+ self.image_num_patch = vision_config.image_num_patch
624
+ self.llm_patches_per_crop = (
625
+ (self.image_num_patch[0] + 1) // 2,
626
+ (self.image_num_patch[1] + 1) // 2,
627
+ )
628
+ self.image_vit = VisionTransformer(vision_config,
629
+ quant_config=quant_config)
630
+ self.num_prefix_tokens = self.image_vit.num_prefix_tokens
631
+ assert self.num_prefix_tokens in {
632
+ 0, 1
633
+ }, "Only 0 or 1 prefix tokens are supported"
634
+ self.image_pooling_2d = MultiHeadDotProductAttention(
635
+ vision_config,
636
+ nlayers=len(self.vit_layers),
637
+ quant_config=quant_config)
638
+ self.image_projector = MolmoMLP(
639
+ config,
640
+ input_dim=vision_config.image_emb_dim,
641
+ quant_config=quant_config,
642
+ )
643
+
644
+ image_dim = vision_config.image_emb_dim * len(self.vit_layers)
645
+ self.pad_embed = nn.Parameter(torch.zeros((2, image_dim)))
646
+
647
+ @property
648
+ def dtype(self) -> torch.dtype:
649
+ return self.image_vit.patch_embedding.weight.dtype
650
+
651
+ @property
652
+ def device(self) -> torch.device:
653
+ return self.image_vit.patch_embedding.weight.device
654
+
655
+ def encode_image(self, images: torch.Tensor) -> torch.Tensor:
656
+ """
657
+ : param images: (batch_size, num_crops, num_patch, n_pixels)
658
+ """
659
+ B, T, N, D = images.shape
660
+
661
+ mask = ~torch.all(
662
+ images.view(B * T, N, D) == -1, dim=(1, 2), keepdim=True)
663
+
664
+ images = images.view(B * T, N, D)
665
+ image_features = self.image_vit(images)
666
+
667
+ if self.vit_layers is not None:
668
+ features = []
669
+ for layer in self.vit_layers:
670
+ features.append(image_features[layer])
671
+ image_features = torch.cat(features, dim=-1)
672
+ else:
673
+ image_features = image_features[-1]
674
+
675
+ if self.num_prefix_tokens > 0:
676
+ image_features = image_features[:, 1:]
677
+
678
+ image_features = image_features * mask
679
+ image_features = image_features.view(B, T, N, -1)
680
+
681
+ return image_features
682
+
683
+ def forward(
684
+ self, images: torch.Tensor, image_masks: torch.Tensor
685
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
686
+
687
+ # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim) # noqa: E501
688
+ batch_size, num_image = images.shape[:2]
689
+ images = images.to(device=self.device, dtype=self.dtype)
690
+ image_features = self.encode_image(images)
691
+
692
+ og_dtype = image_features.dtype
693
+ assert image_masks is not None
694
+ pad_embed = self.pad_embed[:, None, None, None, :]
695
+ all_pad = image_masks == 0
696
+ partial_pad = torch.logical_and(
697
+ image_masks < 1,
698
+ torch.logical_not(all_pad)).to(dtype=torch.float32)
699
+ all_pad = all_pad.to(dtype=torch.float32)
700
+ image_features = image_features + pad_embed[0] * torch.unsqueeze(
701
+ all_pad, -1)
702
+ image_features = image_features + pad_embed[1] * torch.unsqueeze(
703
+ partial_pad, -1)
704
+
705
+ image_features = image_features.to(og_dtype)
706
+
707
+ image_features = image_features.reshape(
708
+ (batch_size, num_image) + self.image_num_patch + (-1, ), )
709
+
710
+ if self.image_num_patch[0] % 2 == 1:
711
+ # Pad so we can still pool 2x2 patches
712
+ image_features = F.pad(
713
+ image_features,
714
+ (0, 0, 0, 1, 0, 1, 0, 0, 0, 0),
715
+ )
716
+
717
+ # image pooling
718
+ image_features = rearrange(
719
+ image_features,
720
+ 'b n (h dh) (w dw) c -> (b n h w) (dh dw) c',
721
+ dh=2,
722
+ dw=2,
723
+ )
724
+
725
+ query = image_features.mean(-2, keepdim=True)
726
+ image_features = self.image_pooling_2d(query, image_features)
727
+
728
+ h, w = self.llm_patches_per_crop
729
+ image_features = image_features.view(batch_size, num_image, h * w, -1)
730
+
731
+ image_features = self.image_projector(image_features)
732
+
733
+ # image_features: (batch_size, num_image, num_patch, d_model)
734
+ return image_features
735
+
736
+
737
+class MolmoModel(nn.Module):
738
+
739
+ def __init__(
740
+ self,
741
+ config: PretrainedConfig,
742
+ cache_config: Optional[CacheConfig] = None,
743
+ quant_config: Optional[QuantizationConfig] = None,
744
+ prefix: str = "",
745
+ ) -> None:
746
+ super().__init__()
747
+ self.config = config
748
+
749
+ self.embedding_size = config.embedding_size or config.vocab_size
750
+ self.embedding_size += ADDITIONAL_VOCAB_SIZE
751
+ self.embed_tokens = VocabParallelEmbedding(
752
+ self.embedding_size,
753
+ config.hidden_size,
754
+ quant_config=quant_config,
755
+ )
756
+
757
+ decoder_layer = MolmoDecoderNormAfterLayer if config.norm_after \
758
+ else MolmoDecoderLayer
759
+ self.start_layer, self.end_layer, self.layers = make_layers(
760
+ config.num_hidden_layers,
761
+ lambda prefix: decoder_layer(config, cache_config, quant_config),
762
+ prefix=f"{prefix}.layers",
763
+ )
764
+
765
+ assert config.layer_norm_type == "rms"
766
+ self.norm = RMSNorm(config.hidden_size, config.layer_norm_eps)
767
+
768
+ def forward(
769
+ self,
770
+ input_ids: torch.Tensor,
771
+ positions: torch.Tensor,
772
+ kv_caches: List[torch.Tensor],
773
+ attn_metadata: AttentionMetadata,
774
+ intermediate_tensors: Optional[IntermediateTensors] = None,
775
+ inputs_embeds: Optional[torch.Tensor] = None,
776
+ ) -> torch.Tensor:
777
+ if get_pp_group().is_first_rank:
778
+ if inputs_embeds is not None:
779
+ hidden_states = inputs_embeds
780
+ else:
781
+ hidden_states = self.embed_tokens(input_ids)
782
+ residual = None
783
+ else:
784
+ assert intermediate_tensors is not None
785
+ hidden_states = intermediate_tensors["hidden_states"]
786
+ residual = intermediate_tensors["residual"]
787
+
788
+ # Apply blocks one-by-one.
789
+ for i in range(self.start_layer, self.end_layer):
790
+ layer = self.layers[i]
791
+ hidden_states, residual = layer(
792
+ positions,
793
+ hidden_states,
794
+ kv_caches[i - self.start_layer],
795
+ attn_metadata,
796
+ residual,
797
+ )
798
+ if not get_pp_group().is_last_rank:
799
+ return IntermediateTensors({
800
+ "hidden_states": hidden_states,
801
+ "residual": residual
802
+ })
803
+ if residual is not None:
804
+ hidden_states, _ = self.norm(hidden_states, residual)
805
+ else:
806
+ hidden_states = self.norm(hidden_states)
807
+ return hidden_states
808
+
809
+
810
+cached_get_processor = lru_cache(get_processor)
811
+
812
+
813
+def get_num_patches(num_tiles: int, crop_patches: int, left_margin: int,
814
+ right_margin: int, pooling_size: int) -> int:
815
+ crop_window_patches = crop_patches - (left_margin + right_margin)
816
+ if num_tiles > 1:
817
+ left_crop_window_patches = (crop_window_patches + left_margin +
818
+ pooling_size -
819
+ 1) // pooling_size * pooling_size
820
+ middle_crop_window_patches = (crop_window_patches + pooling_size -
821
+ 1) // pooling_size * pooling_size
822
+ right_crop_window_patches = (crop_window_patches + right_margin +
823
+ pooling_size -
824
+ 1) // pooling_size * pooling_size
825
+ return left_crop_window_patches + (
826
+ num_tiles -
827
+ 2) * middle_crop_window_patches + right_crop_window_patches
828
+ else:
829
+ single_crop_window_patches = (crop_patches + pooling_size -
830
+ 1) // pooling_size * pooling_size
831
+ return single_crop_window_patches
832
+
833
+
834
+def get_tokens(tiling_h: int, tiling_w: int, crop_patches: int,
835
+ left_margin: int, right_margin: int, pooling_size: int) -> int:
836
+ h = get_num_patches(tiling_h, crop_patches, left_margin, right_margin,
837
+ pooling_size)
838
+ w = get_num_patches(tiling_w, crop_patches, left_margin, right_margin,
839
+ pooling_size)
840
+ per_row = w // pooling_size + 1
841
+ joint = per_row * (h // pooling_size) + 2
842
+ image_token_length = (crop_patches + pooling_size - 1) // pooling_size
843
+ resize = (image_token_length + 1) * image_token_length + 2
844
+ return resize + joint
845
+
846
+
847
+def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
848
+ right_margin: int, pooling_size: int) -> int:
849
+ tilings = []
850
+ for i in range(1, max_crops + 1):
851
+ for j in range(1, max_crops + 1):
852
+ if i * j <= max_crops:
853
+ tilings.append((i, j))
854
+ tokens = [
855
+ get_tokens(tilings[i][0], tilings[i][1], crop_patches, left_margin,
856
+ right_margin, pooling_size) for i in range(len(tilings))
857
+ ]
858
+ return max(tokens)
859
+
860
+
861
+def get_max_molmo_image_tokens(ctx: InputContext) -> int:
862
+ processor = cached_get_processor(ctx.model_config.model,
863
+ trust_remote_code=True,
864
+ revision=ctx.model_config.code_revision)
865
+ image_processor = processor.image_processor
866
+ max_llm_image_tokens = get_max_tokens(
867
+ image_processor.max_crops,
868
+ image_processor.base_image_input_size[0] //
869
+ image_processor.image_patch_size,
870
+ image_processor.overlap_margins[0],
871
+ image_processor.overlap_margins[1],
872
+ 2,
873
+ )
874
+ return max_llm_image_tokens
875
+
876
+
877
+# NOTE: preprocessing for the image data has been included in the
878
+# 'input_processor_for_molmo' function
879
+def image_input_mapper_for_molmo(
880
+ ctx: InputContext,
881
+ data: object,
882
+):
883
+ return MultiModalInputs(data)
884
+
885
+
886
+def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
887
+ mm_counts: Mapping[str, int]):
888
+ processor = cached_get_processor(ctx.model_config.model,
889
+ trust_remote_code=True,
890
+ revision=ctx.model_config.code_revision)
891
+ image_processor = processor.image_processor
892
+
893
+ base_image_input_d = image_processor.image_patch_size
894
+ left_margin, right_margin = image_processor.overlap_margins
895
+ max_crops = image_processor.max_crops
896
+
897
+ # Assume: prompt_token_ids always starts with bos_token_id followed image tokens # noqa: E501
898
+ max_llm_image_tokens = get_max_molmo_image_tokens(ctx)
899
+ if seq_len - max_llm_image_tokens - 1 < 0:
900
+ raise RuntimeError(
901
+ f"Molmo cannot process {max_crops} crops in a prompt, "
902
+ "please increase max_model_len or reduce number of crops")
903
+
904
+ # The vertical image has the maximum number of image tokens due to column tokens. # noqa: E501
905
+ tiling = (max_crops, 1)
906
+ total_margin_pixels = base_image_input_d * (right_margin + left_margin)
907
+ crop_patches = image_processor.base_image_input_size[
908
+ 0] // base_image_input_d
909
+ crop_window_patches = crop_patches - (right_margin + left_margin)
910
+ crop_window_size = crop_window_patches * base_image_input_d
911
+
912
+ h = crop_window_size * tiling[0] + total_margin_pixels
913
+ w = crop_window_size * tiling[1] + total_margin_pixels
914
+
915
+ dummy_image = Image.new("RGB", (w, h), color="red")
916
+
917
+ out = processor.process("dummy prompt", dummy_image)
918
+
919
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
920
+ out["input_ids"][:1 + max_llm_image_tokens])
921
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
922
+ [0]) * (seq_len - max_llm_image_tokens - 1)
923
+ dummy_seqdata = SequenceData(token_ids)
924
+ dummy_imgdata = {
925
+ "images": out["images"],
926
+ "image_input_idx": out["image_input_idx"],
927
+ }
928
+ if "image_masks" in out:
929
+ dummy_imgdata["image_masks"] = out["image_masks"]
930
+ dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
931
+ return dummy_seqdata, {"image": dummy_imgdata}
932
+
933
+
934
+def pad_images(
935
+ max_total_crops: int,
936
+ images: torch.Tensor,
937
+ image_input_idx: torch.Tensor,
938
+ image_masks: Optional[torch.Tensor] = None,
939
+):
940
+ n = max_total_crops - images.shape[0]
941
+ images = F.pad(images, (0, 0, 0, 0, 0, n), value=-1)
942
+ image_input_idx = F.pad(image_input_idx, (0, 0, 0, n), value=-1)
943
+ if image_masks is not None:
944
+ image_masks = F.pad(image_masks, (0, 0, 0, n), value=-1)
945
+ return images, image_input_idx, image_masks
946
+
947
+
948
+def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
949
+ prompt = llm_inputs["prompt"]
950
+ multi_modal_data = llm_inputs.get("multi_modal_data")
951
+ image = multi_modal_data.get("image")
952
+ processor = cached_get_processor(ctx.model_config.model,
953
+ trust_remote_code=True,
954
+ revision=ctx.model_config.code_revision)
955
+
956
+ # NOTE: message formatting for raw text prompt is only applied for
957
+ # offline inference; for online inference, the prompt is always in
958
+ # instruction format and tokenized.
959
+ if prompt is not None and re.match(r"^User:[\s\S]*?(Assistant:)*$",
960
+ prompt):
961
+ out = processor.process(prompt, image, message_format="none")
962
+ elif prompt is not None:
963
+ out = processor.process(prompt, image)
964
+ else:
965
+ out = processor.process(None,
966
+ image,
967
+ tokens=llm_inputs["prompt_token_ids"])
968
+
969
+ image_processor = processor.image_processor
970
+ max_total_crops = 1 + image_processor.max_crops
971
+ if image is not None:
972
+ images, image_input_idx, image_masks = pad_images(
973
+ max_total_crops,
974
+ out["images"],
975
+ out["image_input_idx"],
976
+ out.get("image_masks"),
977
+ )
978
+ else:
979
+ base_image_input_size = image_processor.base_image_input_size
980
+ image_patch_size = image_processor.image_patch_size
981
+ image_num_patch = (
982
+ base_image_input_size[0] // image_patch_size,
983
+ base_image_input_size[1] // image_patch_size,
984
+ )
985
+ n_pixels = image_patch_size * image_patch_size * 3
986
+ n_patches = image_num_patch[0] * image_num_patch[1]
987
+
988
+ image_length_w = image_processor.image_token_length_w
989
+ image_length_h = image_processor.image_token_length_h
990
+ tokens_per_image = image_length_w * image_length_h
991
+ images = torch.full(
992
+ (max_total_crops, n_patches, n_pixels),
993
+ -1,
994
+ dtype=torch.float32,
995
+ )
996
+ image_input_idx = torch.full(
997
+ (max_total_crops, tokens_per_image),
998
+ -1,
999
+ dtype=torch.int32,
1000
+ )
1001
+ if image_processor.image_padding_mask:
1002
+ image_masks = torch.full(
1003
+ (max_total_crops, n_patches),
1004
+ -1,
1005
+ dtype=torch.float32,
1006
+ )
1007
+
1008
+ image_data = dict(
1009
+ images=images,
1010
+ image_input_idx=image_input_idx,
1011
+ )
1012
+ if image_masks is not None:
1013
+ image_data["image_masks"] = image_masks
1014
+
1015
+ image_data["seq_len"] = torch.tensor(len(out["input_ids"]),
1016
+ dtype=torch.long)
1017
+
1018
+ multi_modal_data = dict(image=image_data)
1019
+
1020
+ return LLMInputs(
1021
+ prompt_token_ids=out["input_ids"],
1022
+ prompt=llm_inputs["prompt"],
1023
+ multi_modal_data=multi_modal_data,
1024
+ )
1025
+
1026
+
1027
+@MULTIMODAL_REGISTRY.register_image_input_mapper(image_input_mapper_for_molmo)
1028
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1029
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1030
+@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1031
+class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1032
+
1033
+ def __init__(
1034
+ self,
1035
+ config: PretrainedConfig,
1036
+ multimodal_config: Optional[MultiModalConfig] = None,
1037
+ cache_config: Optional[CacheConfig] = None,
1038
+ quant_config: Optional[Mapping[str, Any]] = None,
1039
+ ) -> None:
1040
+ super().__init__()
1041
+
1042
+ self.config = config
1043
+ self.multimodal_config = multimodal_config
1044
+
1045
+ vision_config = VisionBackboneConfig()
1046
+ self.vision_backbone = MolmoVisionBackbone(config, vision_config,
1047
+ quant_config)
1048
+ self.model = MolmoModel(config, cache_config, quant_config)
1049
+
1050
+ if self.config.weight_tying:
1051
+ self.lm_head = self.model.transformer.wte
1052
+ else:
1053
+ self.lm_head = ParallelLMHead(
1054
+ config.embedding_size or config.vocab_size,
1055
+ config.hidden_size,
1056
+ quant_config=quant_config,
1057
+ )
1058
+
1059
+ self.logits_processor = LogitsProcessor(config.embedding_size
1060
+ or config.vocab_size)
1061
+ self.sampler = Sampler()
1062
+
1063
+ def _parse_and_validate_image_input(
1064
+ self,
1065
+ **kwargs: object,
1066
+ ) -> Optional[MolmoImageInputs]:
1067
+ images = kwargs.pop("images", None)
1068
+ image_masks = kwargs.pop("image_masks", None)
1069
+ if images is None:
1070
+ return None
1071
+
1072
+ image_input_idx = kwargs.pop("image_input_idx", None)
1073
+ seq_len = kwargs.pop("seq_len", None)
1074
+ if image_input_idx is None:
1075
+ raise ValueError("image_input_idx is required for Molmo model.")
1076
+ if seq_len is None:
1077
+ raise ValueError("seq_len is required for Molmo model.")
1078
+ if not isinstance(seq_len, torch.Tensor):
1079
+ seq_len = torch.tensor(seq_len)
1080
+
1081
+ return MolmoImageInputs(
1082
+ images=images,
1083
+ image_input_idx=image_input_idx,
1084
+ seq_len=seq_len,
1085
+ image_masks=image_masks,
1086
+ )
1087
+
1088
+ def _process_image_input(
1089
+ self,
1090
+ image_input: MolmoImageInputs,
1091
+ ) -> torch.Tensor:
1092
+
1093
+ image_features = self.vision_backbone(
1094
+ images=image_input["images"],
1095
+ image_masks=image_input["image_masks"],
1096
+ )
1097
+
1098
+ return image_features
1099
+
1100
+ def _merge_multimodal_embeddings(
1101
+ self,
1102
+ inputs_embeds: torch.Tensor,
1103
+ image_features: torch.Tensor,
1104
+ image_input_idx: torch.Tensor,
1105
+ seq_len: Union[torch.Tensor, List[torch.Tensor]],
1106
+ ) -> torch.Tensor:
1107
+ batch_size, num_image, num_patch = image_features.shape[:3]
1108
+ assert image_input_idx.shape == (batch_size, num_image, num_patch)
1109
+
1110
+ image_features = image_features.to(inputs_embeds.device)
1111
+ seq_len = seq_len.to(inputs_embeds.device)
1112
+
1113
+ # insert the image feature into the embedding.
1114
+ image_features = image_features.view(batch_size, num_image * num_patch,
1115
+ -1)
1116
+ image_input_idx = image_input_idx.view(batch_size,
1117
+ num_image * num_patch)
1118
+
1119
+ valid = image_input_idx >= 0
1120
+ image_features = image_features * valid[:, :, None].to(
1121
+ image_features.dtype)
1122
+ image_features = image_features.view(
1123
+ batch_size * num_image * num_patch, -1).contiguous()
1124
+
1125
+ image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1126
+ offset = torch.cat(
1127
+ [seq_len.new_zeros(
1128
+ (1)), seq_len.cumsum(dim=0)[:-1]], dim=0)[:, None]
1129
+ image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1130
+ image_input_idx = image_input_idx.flatten()[:, None]
1131
+ mat = image_input_idx == torch.arange(
1132
+ seq_len.sum().item(), device=inputs_embeds.device)[None, :]
1133
+ mat = mat.to(image_features.dtype)
1134
+
1135
+ inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md',
1136
+ image_features, mat)
1137
+
1138
+ return inputs_embeds
1139
+
1140
+ def forward(
1141
+ self,
1142
+ input_ids: torch.LongTensor,
1143
+ positions: torch.LongTensor,
1144
+ kv_caches: List[torch.Tensor],
1145
+ attn_metadata: AttentionMetadata,
1146
+ **kwargs: object,
1147
+ ) -> SamplerOutput:
1148
+
1149
+ image_input = self._parse_and_validate_image_input(**kwargs)
1150
+
1151
+ if image_input is not None:
1152
+ inputs_embeds = self.model.embed_tokens(input_ids)
1153
+ image_features = self._process_image_input(image_input)
1154
+
1155
+ inputs_embeds = self._merge_multimodal_embeddings(
1156
+ inputs_embeds,
1157
+ image_features,
1158
+ image_input["image_input_idx"],
1159
+ image_input["seq_len"],
1160
+ )
1161
+
1162
+ input_ids = None
1163
+ else:
1164
+ inputs_embeds = None
1165
+
1166
+ hidden_states = self.model(
1167
+ input_ids=input_ids,
1168
+ positions=positions,
1169
+ kv_caches=kv_caches,
1170
+ attn_metadata=attn_metadata,
1171
+ inputs_embeds=inputs_embeds,
1172
+ )
1173
+
1174
+ return hidden_states
1175
+
1176
+ def compute_logits(self, hidden_states: torch.Tensor,
1177
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
1178
+ logits = self.logits_processor(self.lm_head, hidden_states,
1179
+ sampling_metadata)
1180
+ return logits
1181
+
1182
+ def sample(
1183
+ self,
1184
+ logits: torch.Tensor,
1185
+ sampling_metadata: SamplingMetadata,
1186
+ ) -> Optional[SamplerOutput]:
1187
+ next_tokens = self.sampler(logits, sampling_metadata)
1188
+ return next_tokens
1189
+
1190
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1191
+
1192
+ params_mapping = [
1193
+ ("model.transformer.ln_f.weight", "model.norm.weight"),
1194
+ ("attn_out", "self_attn.o_proj"),
1195
+ ("att_proj", "self_attn.qkv_proj"),
1196
+ ("q_norm", "self_attn.q_norm"),
1197
+ ("k_norm", "self_attn.k_norm"),
1198
+ ("attn_norm", "input_layernorm"),
1199
+ ("ff_norm", "post_attention_layernorm"),
1200
+ ]
1201
+
1202
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
1203
+
1204
+ embedding_weight = dict()
1205
+ projector_weight = dict()
1206
+ for name, loaded_weight in weights:
1207
+ if "rotary_emb.inv_freq" in name:
1208
+ continue
1209
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
1210
+ continue
1211
+
1212
+ if "wte.embedding" in name:
1213
+ embedding_weight["embedding"] = loaded_weight
1214
+ continue
1215
+
1216
+ if "wte.new_embedding" in name:
1217
+ embedding_weight["new_embedding"] = loaded_weight
1218
+ continue
1219
+
1220
+ if "vision_backbone" in name:
1221
+ if name.startswith("model"):
1222
+ name = name[len("model."):]
1223
+ if 'image_projector' in name:
1224
+ if 'w1' in name:
1225
+ projector_weight['gate_proj'] = loaded_weight
1226
+ elif 'w3' in name:
1227
+ projector_weight['up_proj'] = loaded_weight
1228
+ elif 'w2' in name:
1229
+ projector_weight['down_proj'] = loaded_weight
1230
+ else:
1231
+ raise ValueError(
1232
+ f"Unexpected projector weight: {name}")
1233
+ continue
1234
+ else:
1235
+ if "transformer.blocks" in name:
1236
+ name = name.replace("transformer.blocks", "layers")
1237
+
1238
+ if "ff_proj" in name:
1239
+ name = name.replace("ff_proj", "mlp.gate_up_proj")
1240
+ assert 'weight' in name
1241
+ up_weight, gate_weight = loaded_weight.chunk(2, dim=0)
1242
+ loaded_weight = torch.cat([gate_weight, up_weight], dim=0)
1243
+
1244
+ elif "ff_out" in name:
1245
+ if "layers" in name:
1246
+ name = name.replace("ff_out", "mlp.down_proj")
1247
+ else:
1248
+ # lm head
1249
+ name = name.replace("model.transformer.ff_out",
1250
+ "lm_head")
1251
+
1252
+ else:
1253
+ for (param_name, weight_name) in params_mapping:
1254
+ if param_name in name:
1255
+ name = name.replace(param_name, weight_name)
1256
+ break
1257
+
1258
+ try:
1259
+ # Skip loading extra bias for GPTQ models.
1260
+ if name.endswith(".bias") and name not in params_dict:
1261
+ continue
1262
+ param = params_dict[name]
1263
+ except KeyError:
1264
+ raise ValueError(f"Unexpected weight: {name}") from None
1265
+
1266
+ weight_loader = getattr(param, "weight_loader",
1267
+ default_weight_loader)
1268
+ weight_loader(param, loaded_weight)
1269
+
1270
+ gate_up_proj_weight = torch.cat(
1271
+ [projector_weight["gate_proj"], projector_weight["up_proj"]],
1272
+ dim=0)
1273
+ name = "vision_backbone.image_projector.gate_up_proj.weight"
1274
+ param = params_dict[name]
1275
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
1276
+ weight_loader(param, gate_up_proj_weight)
1277
+
1278
+ down_proj_weight = projector_weight["down_proj"]
1279
+ name = "vision_backbone.image_projector.down_proj.weight"
1280
+ param = params_dict[name]
1281
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
1282
+ weight_loader(param, down_proj_weight)
1283
+
1284
+ embedding_weight = torch.cat(
1285
+ [embedding_weight["embedding"], embedding_weight["new_embedding"]],
1286
+ dim=0)
1287
+ name = "model.embed_tokens.weight"
1288
+ param = params_dict[name]
1289
+ weight_loader = getattr(param, "weight_loader", default_weight_loader)
1290
+ weight_loader(param, embedding_weight)
vllm/model_executor/models/qwen2_vl.py
+1
-2
@@ -1167,8 +1167,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1167
continue
1168
param = params_dict[name]
1169
except KeyError:
1170
- print(params_dict.keys())
1171
- raise
1170
+ raise ValueError(f"Unexpected weight: {name}") from None
1171
1172
weight_loader = getattr(param, "weight_loader",
1173
default_weight_loader)
vllm/model_executor/models/registry.py
+1
@@ -104,6 +104,7 @@ _MULTIMODAL_MODELS = {
104
"LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
105
"LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
106
"MiniCPMV": ("minicpmv", "MiniCPMV"),
107
+ "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
108
"NVLM_D": ("nvlm_d", "NVLM_D_Model"),
109
"PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
110
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),