@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dfe43a2071
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Molmo vLLM Integration (#9016)

Co-authored-by: sanghol <sanghol@allenai.org> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>

Reza Salehi committed Oct 14, 2024 at 07:56 UTC dfe43a207161051c10daaae064936f4a4d2a597c
7 files changed +1319 -3
docs/source/models/supported_models.rst
+6
@@ -399,6 +399,12 @@ Text Generation
399 - :code:`meta-llama/Llama-3.2-90B-Vision-Instruct`, :code:`meta-llama/Llama-3.2-11B-Vision`, etc.
400 -
401 -
402 + * - :code:`MolmoForCausalLM`
403 + - Molmo
404 + - Image
405 + - :code:`allenai/Molmo-7B-D-0924`, :code:`allenai/Molmo-72B-0924`, etc.
406 + -
407 + - ✅︎
408 * - :code:`NVLM_D_Model`
409 - NVLM-D 1.0
410 - Image\ :sup:`E+`
examples/offline_inference_vision_language.py
+18
@@ -300,6 +300,23 @@ def run_mllama(question: str, modality: str):
300 return llm, prompt, stop_token_ids
301
302
303 +# Molmo
304 +def run_molmo(question, modality):
305 + assert modality == "image"
306 +
307 + model_name = "allenai/Molmo-7B-D-0924"
308 +
309 + llm = LLM(
310 + model=model_name,
311 + trust_remote_code=True,
312 + dtype="bfloat16",
313 + )
314 +
315 + prompt = question
316 + stop_token_ids = None
317 + return llm, prompt, stop_token_ids
318 +
319 +
320 # GLM-4v
321 def run_glm4v(question: str, modality: str):
322 assert modality == "image"
@@ -331,6 +348,7 @@ model_example_map = {
348 "qwen_vl": run_qwen_vl,
349 "qwen2_vl": run_qwen2_vl,
350 "mllama": run_mllama,
351 + "molmo": run_molmo,
352 "glm4v": run_glm4v,
353 }
354
vllm/entrypoints/chat_utils.py
+2
@@ -163,6 +163,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
163 return "<|image|>"
164 if model_type == "qwen2_vl":
165 return "<|vision_start|><|image_pad|><|vision_end|>"
166 + if model_type == "molmo":
167 + return ""
168
169 raise TypeError(f"Unknown model type: {model_type}")
170 elif modality == "audio":
vllm/model_executor/models/__init__.py
+1 -1
@@ -20,4 +20,4 @@ __all__ = [
20 "supports_multimodal",
21 "SupportsPP",
22 "supports_pp",
23 -]
23 +]
\ No newline at end of file
vllm/model_executor/models/molmo.py new
+1290
@@ -0,0 +1,1290 @@
1 +import logging
2 +import math
3 +import re
4 +from array import array
5 +from dataclasses import dataclass
6 +from functools import lru_cache, partial
7 +from typing import (Any, Iterable, List, Mapping, Optional, Tuple, TypedDict,
8 + Union)
9 +
10 +import torch
11 +from einops import rearrange
12 +from PIL import Image
13 +from torch import nn
14 +from torch.nn import functional as F
15 +from transformers import PretrainedConfig
16 +
17 +import vllm.envs as envs
18 +from vllm.attention import Attention, AttentionMetadata
19 +from vllm.attention.selector import (_Backend, backend_name_to_enum,
20 + get_global_forced_attn_backend)
21 +from vllm.config import CacheConfig, MultiModalConfig
22 +from vllm.distributed import (get_pp_group, get_tensor_model_parallel_rank,
23 + get_tensor_model_parallel_world_size,
24 + split_tensor_along_last_dim,
25 + tensor_model_parallel_all_gather)
26 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
27 +from vllm.model_executor import SamplingMetadata
28 +from vllm.model_executor.layers.activation import QuickGELU, SiluAndMul
29 +from vllm.model_executor.layers.layernorm import RMSNorm
30 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 + MergedColumnParallelLinear,
32 + QKVParallelLinear,
33 + RowParallelLinear)
34 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 +from vllm.model_executor.layers.quantization.base_config import (
36 + QuantizationConfig)
37 +from vllm.model_executor.layers.rotary_embedding import get_rope
38 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
39 +from vllm.model_executor.layers.vocab_parallel_embedding import (
40 + ParallelLMHead, VocabParallelEmbedding)
41 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
42 +from vllm.model_executor.models.interfaces import SupportsMultiModal
43 +from vllm.model_executor.models.utils import make_layers
44 +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalInputs
45 +from vllm.platforms import current_platform
46 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
47 + SequenceData)
48 +from vllm.transformers_utils.processor import get_processor
49 +
50 +log = logging.getLogger(__name__)
51 +
52 +# TODO: hard-coded for now. Consider making it configurable.
53 +VIT_LAYERS = [-2, -9]
54 +NUM_PREFIX_TOKENS = 1
55 +ADDITIONAL_VOCAB_SIZE = 128
56 +
57 +
58 +class MolmoImageInputs(TypedDict):
59 + images: torch.Tensor
60 + """Shape:
61 + `(batch_size, num_crops, num_patch, patch_dim)`
62 + """
63 +
64 + image_input_idx: torch.Tensor
65 + """Shape:
66 + `(batch_size, num_crops, num_patch)`
67 + """
68 +
69 + seq_len: torch.Tensor
70 + """Shape:
71 + `(batch_size, )`
72 + """
73 +
74 + image_masks: Optional[torch.Tensor]
75 + """Shape:
76 + `(batch_size, num_crops, num_patch)`
77 + """
78 +
79 +
80 +@dataclass
81 +class VisionBackboneConfig:
82 + image_default_input_size: Tuple[int, int] = (336, 336)
83 + image_patch_size: int = 14
84 + image_pos_patch_size: int = 14
85 + image_emb_dim: int = 1024
86 + image_num_heads: int = 16
87 + image_num_key_value_heads: int = 16
88 + image_num_layers: int = 23
89 + image_mlp_dim: int = 4096
90 + image_mlp_activations: str = "quick_gelu"
91 + image_num_pos: int = 577
92 + image_norm_eps: float = 1e-5
93 +
94 + def __post_init__(self):
95 + self.image_default_input_size = tuple(
96 + self.image_default_input_size) # type: ignore[assignment]
97 +
98 + @property
99 + def image_num_patch(self):
100 + h, w = self.image_default_input_size
101 + return h // self.image_patch_size, w // self.image_patch_size
102 +
103 +
104 +class ViTMLP(nn.Module):
105 + """MLP used in Vision Transformer."""
106 +
107 + def __init__(
108 + self,
109 + config: VisionBackboneConfig,
110 + quant_config: Optional[QuantizationConfig] = None,
111 + ):
112 + super().__init__()
113 + self.w1 = ColumnParallelLinear(
114 + config.image_emb_dim,
115 + config.image_mlp_dim,
116 + bias=True,
117 + quant_config=quant_config,
118 + )
119 + # Activation function.
120 + assert config.image_mlp_activations == "quick_gelu"
121 + self.act = QuickGELU()
122 + self.w2 = RowParallelLinear(
123 + config.image_mlp_dim,
124 + config.image_emb_dim,
125 + bias=True,
126 + quant_config=quant_config,
127 + )
128 +
129 + def forward(self, x: torch.Tensor) -> torch.Tensor:
130 + x, _ = self.w1(x)
131 + x = self.act(x)
132 + x, _ = self.w2(x)
133 + return x
134 +
135 +
136 +class MultiHeadDotProductAttention(nn.Module):
137 + """Multi-head attention used in Vision Transformer."""
138 +
139 + def __init__(
140 + self,
141 + config: VisionBackboneConfig,
142 + use_bias: bool = True,
143 + nlayers: int = 1,
144 + quant_config: Optional[QuantizationConfig] = None,
145 + ):
146 + super().__init__()
147 +
148 + self.hidden_size = config.image_emb_dim
149 + self.total_num_heads = config.image_num_heads
150 + tp_size = get_tensor_model_parallel_world_size()
151 +
152 + assert self.hidden_size % self.total_num_heads == 0
153 + assert self.total_num_heads % tp_size == 0
154 +
155 + self.num_heads = self.total_num_heads // tp_size
156 + self.head_dim = self.hidden_size // self.total_num_heads
157 +
158 + self.total_num_kv_heads = config.image_num_key_value_heads
159 + if self.total_num_kv_heads >= tp_size:
160 + assert self.total_num_kv_heads % tp_size == 0
161 + else:
162 + assert tp_size % self.total_num_kv_heads == 0
163 +
164 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
165 +
166 + self.wq = ColumnParallelLinear(
167 + nlayers * self.hidden_size,
168 + self.total_num_heads * self.head_dim,
169 + bias=use_bias,
170 + quant_config=quant_config,
171 + )
172 + self.wk = ColumnParallelLinear(
173 + nlayers * self.hidden_size,
174 + self.total_num_kv_heads * self.head_dim,
175 + bias=use_bias,
176 + quant_config=quant_config,
177 + )
178 + self.wv = ColumnParallelLinear(
179 + nlayers * self.hidden_size,
180 + self.total_num_kv_heads * self.head_dim,
181 + bias=use_bias,
182 + quant_config=quant_config,
183 + )
184 + self.wo = RowParallelLinear(
185 + self.total_num_heads * self.head_dim,
186 + self.hidden_size,
187 + bias=use_bias,
188 + quant_config=quant_config,
189 + )
190 +
191 + # Detect attention implementation.
192 + selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
193 + if selected_backend is None:
194 + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
195 + if backend_by_env_var is not None:
196 + selected_backend = backend_name_to_enum(backend_by_env_var)
197 + if selected_backend is None:
198 + # For Volta and Turing GPUs, use xformers instead.
199 + device_available = current_platform.get_device_capability()[0] >= 8
200 + if device_available:
201 + from transformers.utils import is_flash_attn_2_available
202 + if is_flash_attn_2_available():
203 + self._use_flash_attn = True
204 + else:
205 + log.warning(
206 + "Current Molmo implementation has a bug with "
207 + "`vllm-flash-attn` inside vision module, so we use "
208 + "xformers backend instead. You can run `pip install "
209 + "flash-attn to use flash-attention backend.")
210 + self._use_flash_attn = False
211 + else:
212 + self._use_flash_attn = False
213 + else:
214 + if selected_backend == _Backend.FLASH_ATTN:
215 + self._use_flash_attn = True
216 + elif selected_backend == _Backend.XFORMERS:
217 + self._use_flash_attn = False
218 + else:
219 + raise RuntimeError(
220 + f"Molmo does not support {selected_backend} backend now.")
221 +
222 + def forward(self,
223 + inputs_q: torch.Tensor,
224 + inputs_kv: Optional[torch.Tensor] = None) -> torch.Tensor:
225 +
226 + if inputs_kv is not None:
227 + inputs_k = inputs_kv
228 + inputs_v = inputs_kv
229 + else:
230 + inputs_k = inputs_q
231 + inputs_v = inputs_q
232 +
233 + xq, _ = self.wq(inputs_q)
234 + xk, _ = self.wk(inputs_k)
235 + xv, _ = self.wv(inputs_v)
236 + q_shape = xq.size()[:-1] + (self.num_heads, self.head_dim)
237 + kv_shape = xk.size()[:-1] + (self.num_kv_heads, self.head_dim)
238 + xq = xq.view(*q_shape)
239 + xk = xk.view(*kv_shape)
240 + xv = xv.view(*kv_shape)
241 +
242 + if self._use_flash_attn:
243 + from flash_attn import flash_attn_func
244 + output = flash_attn_func(xq, xk, xv, dropout_p=0.0, causal=False)
245 + else:
246 + from xformers import ops as xops
247 + output = xops.memory_efficient_attention_forward(xq, xk, xv, p=0)
248 +
249 + output = rearrange(output, "b s h d -> b s (h d)").contiguous()
250 + output, _ = self.wo(output)
251 +
252 + return output
253 +
254 +
255 +class ResidualAttentionBlock(nn.Module):
256 + """Residual attention block used in Vision Transformer."""
257 +
258 + def __init__(
259 + self,
260 + config: VisionBackboneConfig,
261 + quant_config: Optional[QuantizationConfig] = None,
262 + ):
263 + super().__init__()
264 + self.attention = MultiHeadDotProductAttention(
265 + config, quant_config=quant_config)
266 + self.feed_forward = ViTMLP(config, quant_config)
267 + self.attention_norm = nn.LayerNorm(
268 + config.image_emb_dim,
269 + eps=config.image_norm_eps,
270 + )
271 + self.ffn_norm = nn.LayerNorm(
272 + config.image_emb_dim,
273 + eps=config.image_norm_eps,
274 + )
275 +
276 + def forward(self, x: torch.Tensor) -> torch.Tensor:
277 + x = x + self.attention(self.attention_norm(x))
278 + x = x + self.feed_forward(self.ffn_norm(x))
279 + return x
280 +
281 +
282 +class BlockCollection(nn.Module):
283 + """Collection of residual attention blocks used in Vision Transformer."""
284 +
285 + def __init__(
286 + self,
287 + config: VisionBackboneConfig,
288 + quant_config: Optional[QuantizationConfig] = None,
289 + ):
290 + super().__init__()
291 + self.resblocks = nn.ModuleList([
292 + ResidualAttentionBlock(config, quant_config)
293 + for _ in range(config.image_num_layers)
294 + ])
295 +
296 + def forward(self, x: torch.Tensor) -> List[torch.Tensor]:
297 + hidden_states = []
298 + for r in self.resblocks:
299 + x = r(x)
300 + hidden_states.append(x)
301 + return hidden_states
302 +
303 +
304 +def _expand_token(token: torch.Tensor, batch_size: int) -> torch.Tensor:
305 + return token.view(1, 1, -1).expand(batch_size, -1, -1)
306 +
307 +
308 +class VisionTransformer(nn.Module):
309 + """Vision Transformer used in Vision Backbone."""
310 +
311 + def __init__(
312 + self,
313 + config: VisionBackboneConfig,
314 + quant_config: Optional[QuantizationConfig] = None,
315 + ):
316 + super().__init__()
317 + scale = config.image_emb_dim**-0.5
318 + self.patch_num = config.image_num_patch
319 + self.class_embedding = nn.Parameter(
320 + torch.randn(config.image_emb_dim) * scale)
321 + self.num_prefix_tokens: int = NUM_PREFIX_TOKENS
322 + self.positional_embedding = nn.Parameter(
323 + torch.randn(config.image_num_pos, config.image_emb_dim) * scale)
324 + image_patch_size = config.image_patch_size
325 + self.patch_embedding = nn.Linear(
326 + image_patch_size * image_patch_size * 3,
327 + config.image_emb_dim,
328 + bias=False,
329 + )
330 + self.pre_ln = nn.LayerNorm(config.image_emb_dim,
331 + eps=config.image_norm_eps)
332 + self.transformer = BlockCollection(config, quant_config)
333 +
334 + def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor:
335 + cls_emb = self.positional_embedding[0:1]
336 + pos_emb = self.positional_embedding[1:]
337 +
338 + pos_emb = pos_emb.reshape(
339 + (int(math.sqrt(pos_emb.shape[0])),
340 + int(math.sqrt(pos_emb.shape[0])), pos_emb.shape[1]))
341 +
342 + (patch_num_0, patch_num_1) = patch_num
343 +
344 + if pos_emb.shape[0] != patch_num_0 or pos_emb.shape[1] != patch_num_1:
345 + # from https://github.com/facebookresearch/mae/blob/main/util/pos_embed.py
346 + pos_emb = pos_emb.unsqueeze(0).permute(0, 3, 1, 2)
347 + pos_emb = F.interpolate(
348 + pos_emb,
349 + size=(patch_num_0, patch_num_1),
350 + mode="bicubic",
351 + align_corners=False,
352 + antialias=True,
353 + )
354 + pos_emb = pos_emb.permute(0, 2, 3, 1).squeeze(0)
355 +
356 + pos_emb = pos_emb.reshape(-1, pos_emb.shape[-1])
357 + x = x + torch.cat([cls_emb[None, :, :], pos_emb[None, :, :]],
358 + dim=1).to(x.dtype)
359 + return x
360 +
361 + def forward(self,
362 + x: torch.Tensor,
363 + patch_num: int = None) -> List[torch.Tensor]:
364 + """
365 + : param x: (batch_size, num_patch, n_pixels)
366 + """
367 + if patch_num is None:
368 + patch_num = self.patch_num
369 + B, N, D = x.shape
370 +
371 + x = self.patch_embedding(x)
372 +
373 + # class embeddings and positional embeddings
374 + x = torch.cat(
375 + [_expand_token(self.class_embedding, x.shape[0]).to(x.dtype), x],
376 + dim=1)
377 + x = self.add_pos_emb(x, patch_num)
378 +
379 + x = self.pre_ln(x)
380 +
381 + hidden_states = self.transformer(x)
382 + return hidden_states
383 +
384 +
385 +class MolmoAttention(nn.Module):
386 + """Molmo's LLM attention."""
387 +
388 + def __init__(
389 + self,
390 + config: PretrainedConfig,
391 + cache_config: Optional[CacheConfig] = None,
392 + quant_config: Optional[QuantizationConfig] = None,
393 + ) -> None:
394 + super().__init__()
395 + self.hidden_size = config.hidden_size
396 + self.tp_size = get_tensor_model_parallel_world_size()
397 + self.total_num_heads = config.num_attention_heads
398 +
399 + assert self.hidden_size % self.total_num_heads == 0
400 + assert self.total_num_heads % self.tp_size == 0
401 +
402 + self.num_heads = self.total_num_heads // self.tp_size
403 + self.total_num_kv_heads = config.num_key_value_heads \
404 + or self.total_num_heads
405 + if self.total_num_kv_heads >= self.tp_size:
406 + assert self.total_num_kv_heads % self.tp_size == 0
407 + else:
408 + assert self.tp_size % self.total_num_kv_heads == 0
409 +
410 + self.num_kv_heads = max(1, self.total_num_kv_heads // self.tp_size)
411 + self.head_dim = self.hidden_size // self.total_num_heads
412 + self.q_size = self.num_heads * self.head_dim
413 + self.kv_size = self.num_kv_heads * self.head_dim
414 + self.max_position_embeddings = config.max_position_embeddings
415 + self.rope_theta = config.rope_theta
416 +
417 + # Attention input projection. Projects x -> (q, k, v)
418 + self.qkv_proj = QKVParallelLinear(
419 + self.hidden_size,
420 + self.head_dim,
421 + self.total_num_heads,
422 + self.total_num_kv_heads,
423 + bias=config.qkv_bias,
424 + quant_config=quant_config,
425 + )
426 +
427 + self.tp_rank: Optional[int] = None
428 + self.k_norm: Optional[nn.Module] = None
429 + self.q_norm: Optional[nn.Module] = None
430 + if config.attention_layer_norm:
431 + self.tp_rank = get_tensor_model_parallel_rank()
432 + self.k_norm = RMSNorm(self.total_num_kv_heads * self.head_dim,
433 + eps=config.layer_norm_eps)
434 + self.q_norm = RMSNorm(config.hidden_size,
435 + eps=config.layer_norm_eps)
436 +
437 + # Rotary embeddings.
438 + self.rotary_emb = get_rope(
439 + self.head_dim,
440 + rotary_dim=self.head_dim,
441 + max_position=self.max_position_embeddings,
442 + base=self.rope_theta,
443 + )
444 + self.scaling = self.head_dim**-0.5
445 + self.attn = Attention(self.num_heads,
446 + self.head_dim,
447 + self.scaling,
448 + num_kv_heads=self.num_kv_heads,
449 + cache_config=cache_config,
450 + quant_config=quant_config)
451 +
452 + # Attention output projection.
453 + self.o_proj = RowParallelLinear(
454 + self.total_num_heads * self.head_dim,
455 + self.hidden_size,
456 + bias=False,
457 + quant_config=quant_config,
458 + )
459 +
460 + def _apply_qk_norm(self, q: torch.Tensor,
461 + k: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
462 + if self.tp_size > 1:
463 + q = tensor_model_parallel_all_gather(q.contiguous())
464 + k = tensor_model_parallel_all_gather(k.contiguous())
465 + q = self.q_norm.forward_native(q)
466 + k = self.k_norm.forward_native(k)
467 + if self.tp_size > 1:
468 + splitter = partial(split_tensor_along_last_dim,
469 + num_partitions=self.tp_size)
470 + q = splitter(q)[self.tp_rank]
471 + k = splitter(k)[self.tp_rank]
472 + return q, k
473 +
474 + def forward(
475 + self,
476 + positions: torch.Tensor,
477 + hidden_states: torch.Tensor,
478 + kv_cache: torch.Tensor,
479 + attn_metadata: AttentionMetadata,
480 + ) -> torch.Tensor:
481 + qkv, _ = self.qkv_proj(hidden_states)
482 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
483 + if self.q_norm is not None and self.k_norm is not None:
484 + q, k = self._apply_qk_norm(q, k)
485 + q, k = self.rotary_emb(positions, q, k)
486 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
487 + output, _ = self.o_proj(attn_output)
488 + return output
489 +
490 +
491 +class MolmoMLP(nn.Module):
492 + """Molmo's LLM mlp."""
493 +
494 + def __init__(
495 + self,
496 + config: PretrainedConfig,
497 + input_dim: Optional[int] = None,
498 + quant_config: Optional[QuantizationConfig] = None,
499 + ) -> None:
500 + super().__init__()
501 + self.hidden_size = config.hidden_size
502 + self.intermediate_size = config.intermediate_size // 2
503 +
504 + # Feed-forward input projection.
505 + self.gate_up_proj = MergedColumnParallelLinear(
506 + input_dim or self.hidden_size,
507 + [self.intermediate_size] * 2,
508 + bias=False,
509 + quant_config=quant_config,
510 + )
511 +
512 + # Activation function.
513 + self.act_fn = SiluAndMul()
514 +
515 + # Feed-forward output projection.
516 + self.down_proj = RowParallelLinear(
517 + self.intermediate_size,
518 + self.hidden_size,
519 + bias=False,
520 + quant_config=quant_config,
521 + )
522 +
523 + def forward(
524 + self,
525 + x: torch.Tensor,
526 + ) -> torch.Tensor:
527 + gate_up, _ = self.gate_up_proj(x)
528 + x = self.act_fn(gate_up)
529 + x, _ = self.down_proj(x)
530 + return x
531 +
532 +
533 +class MolmoDecoderLayer(nn.Module):
534 +
535 + def __init__(
536 + self,
537 + config: PretrainedConfig,
538 + cache_config: Optional[CacheConfig] = None,
539 + quant_config: Optional[QuantizationConfig] = None,
540 + ) -> None:
541 + super().__init__()
542 + # Attention block.
543 + self.self_attn = MolmoAttention(config, cache_config, quant_config)
544 +
545 + # MLP block.
546 + self.mlp = MolmoMLP(config, quant_config=quant_config)
547 +
548 + # LayerNorm
549 + assert config.layer_norm_type == "rms"
550 + self.input_layernorm = RMSNorm(config.hidden_size,
551 + eps=config.layer_norm_eps)
552 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
553 + eps=config.layer_norm_eps)
554 +
555 + def forward(
556 + self,
557 + positions: torch.Tensor,
558 + hidden_states: torch.Tensor,
559 + kv_cache: torch.Tensor,
560 + attn_metadata: AttentionMetadata,
561 + residual: Optional[torch.Tensor],
562 + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
563 + # Self Attention
564 + if residual is None:
565 + residual = hidden_states
566 + hidden_states = self.input_layernorm(hidden_states)
567 + else:
568 + hidden_states, residual = self.input_layernorm(
569 + hidden_states, residual)
570 + hidden_states = self.self_attn(
571 + positions=positions,
572 + hidden_states=hidden_states,
573 + kv_cache=kv_cache,
574 + attn_metadata=attn_metadata,
575 + )
576 +
577 + hidden_states, residual = self.post_attention_layernorm(
578 + hidden_states, residual)
579 + hidden_states = self.mlp(hidden_states)
580 + return hidden_states, residual
581 +
582 +
583 +class MolmoDecoderNormAfterLayer(MolmoDecoderLayer):
584 +
585 + def forward(
586 + self,
587 + positions: torch.Tensor,
588 + hidden_states: torch.Tensor,
589 + kv_cache: torch.Tensor,
590 + attn_metadata: AttentionMetadata,
591 + residual: Optional[torch.Tensor],
592 + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
593 + # Self Attention
594 + residual = hidden_states
595 + hidden_states = self.self_attn(
596 + positions=positions,
597 + hidden_states=hidden_states,
598 + kv_cache=kv_cache,
599 + attn_metadata=attn_metadata,
600 + )
601 +
602 + hidden_states = self.input_layernorm(hidden_states)
603 + hidden_states = hidden_states + residual
604 + residual = hidden_states
605 +
606 + hidden_states = self.mlp(hidden_states)
607 + hidden_states = self.post_attention_layernorm(hidden_states)
608 + hidden_states = hidden_states + residual
609 + residual = None
610 + return hidden_states, residual
611 +
612 +
613 +class MolmoVisionBackbone(nn.Module):
614 +
615 + def __init__(
616 + self,
617 + config: PretrainedConfig,
618 + vision_config: VisionBackboneConfig,
619 + quant_config: Optional[QuantizationConfig] = None,
620 + ) -> None:
621 + super().__init__()
622 + self.vit_layers = VIT_LAYERS
623 + self.image_num_patch = vision_config.image_num_patch
624 + self.llm_patches_per_crop = (
625 + (self.image_num_patch[0] + 1) // 2,
626 + (self.image_num_patch[1] + 1) // 2,
627 + )
628 + self.image_vit = VisionTransformer(vision_config,
629 + quant_config=quant_config)
630 + self.num_prefix_tokens = self.image_vit.num_prefix_tokens
631 + assert self.num_prefix_tokens in {
632 + 0, 1
633 + }, "Only 0 or 1 prefix tokens are supported"
634 + self.image_pooling_2d = MultiHeadDotProductAttention(
635 + vision_config,
636 + nlayers=len(self.vit_layers),
637 + quant_config=quant_config)
638 + self.image_projector = MolmoMLP(
639 + config,
640 + input_dim=vision_config.image_emb_dim,
641 + quant_config=quant_config,
642 + )
643 +
644 + image_dim = vision_config.image_emb_dim * len(self.vit_layers)
645 + self.pad_embed = nn.Parameter(torch.zeros((2, image_dim)))
646 +
647 + @property
648 + def dtype(self) -> torch.dtype:
649 + return self.image_vit.patch_embedding.weight.dtype
650 +
651 + @property
652 + def device(self) -> torch.device:
653 + return self.image_vit.patch_embedding.weight.device
654 +
655 + def encode_image(self, images: torch.Tensor) -> torch.Tensor:
656 + """
657 + : param images: (batch_size, num_crops, num_patch, n_pixels)
658 + """
659 + B, T, N, D = images.shape
660 +
661 + mask = ~torch.all(
662 + images.view(B * T, N, D) == -1, dim=(1, 2), keepdim=True)
663 +
664 + images = images.view(B * T, N, D)
665 + image_features = self.image_vit(images)
666 +
667 + if self.vit_layers is not None:
668 + features = []
669 + for layer in self.vit_layers:
670 + features.append(image_features[layer])
671 + image_features = torch.cat(features, dim=-1)
672 + else:
673 + image_features = image_features[-1]
674 +
675 + if self.num_prefix_tokens > 0:
676 + image_features = image_features[:, 1:]
677 +
678 + image_features = image_features * mask
679 + image_features = image_features.view(B, T, N, -1)
680 +
681 + return image_features
682 +
683 + def forward(
684 + self, images: torch.Tensor, image_masks: torch.Tensor
685 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
686 +
687 + # image_features: (batch_size, num_crops(=num_image), num_patch, nximage_emb_dim) # noqa: E501
688 + batch_size, num_image = images.shape[:2]
689 + images = images.to(device=self.device, dtype=self.dtype)
690 + image_features = self.encode_image(images)
691 +
692 + og_dtype = image_features.dtype
693 + assert image_masks is not None
694 + pad_embed = self.pad_embed[:, None, None, None, :]
695 + all_pad = image_masks == 0
696 + partial_pad = torch.logical_and(
697 + image_masks < 1,
698 + torch.logical_not(all_pad)).to(dtype=torch.float32)
699 + all_pad = all_pad.to(dtype=torch.float32)
700 + image_features = image_features + pad_embed[0] * torch.unsqueeze(
701 + all_pad, -1)
702 + image_features = image_features + pad_embed[1] * torch.unsqueeze(
703 + partial_pad, -1)
704 +
705 + image_features = image_features.to(og_dtype)
706 +
707 + image_features = image_features.reshape(
708 + (batch_size, num_image) + self.image_num_patch + (-1, ), )
709 +
710 + if self.image_num_patch[0] % 2 == 1:
711 + # Pad so we can still pool 2x2 patches
712 + image_features = F.pad(
713 + image_features,
714 + (0, 0, 0, 1, 0, 1, 0, 0, 0, 0),
715 + )
716 +
717 + # image pooling
718 + image_features = rearrange(
719 + image_features,
720 + 'b n (h dh) (w dw) c -> (b n h w) (dh dw) c',
721 + dh=2,
722 + dw=2,
723 + )
724 +
725 + query = image_features.mean(-2, keepdim=True)
726 + image_features = self.image_pooling_2d(query, image_features)
727 +
728 + h, w = self.llm_patches_per_crop
729 + image_features = image_features.view(batch_size, num_image, h * w, -1)
730 +
731 + image_features = self.image_projector(image_features)
732 +
733 + # image_features: (batch_size, num_image, num_patch, d_model)
734 + return image_features
735 +
736 +
737 +class MolmoModel(nn.Module):
738 +
739 + def __init__(
740 + self,
741 + config: PretrainedConfig,
742 + cache_config: Optional[CacheConfig] = None,
743 + quant_config: Optional[QuantizationConfig] = None,
744 + prefix: str = "",
745 + ) -> None:
746 + super().__init__()
747 + self.config = config
748 +
749 + self.embedding_size = config.embedding_size or config.vocab_size
750 + self.embedding_size += ADDITIONAL_VOCAB_SIZE
751 + self.embed_tokens = VocabParallelEmbedding(
752 + self.embedding_size,
753 + config.hidden_size,
754 + quant_config=quant_config,
755 + )
756 +
757 + decoder_layer = MolmoDecoderNormAfterLayer if config.norm_after \
758 + else MolmoDecoderLayer
759 + self.start_layer, self.end_layer, self.layers = make_layers(
760 + config.num_hidden_layers,
761 + lambda prefix: decoder_layer(config, cache_config, quant_config),
762 + prefix=f"{prefix}.layers",
763 + )
764 +
765 + assert config.layer_norm_type == "rms"
766 + self.norm = RMSNorm(config.hidden_size, config.layer_norm_eps)
767 +
768 + def forward(
769 + self,
770 + input_ids: torch.Tensor,
771 + positions: torch.Tensor,
772 + kv_caches: List[torch.Tensor],
773 + attn_metadata: AttentionMetadata,
774 + intermediate_tensors: Optional[IntermediateTensors] = None,
775 + inputs_embeds: Optional[torch.Tensor] = None,
776 + ) -> torch.Tensor:
777 + if get_pp_group().is_first_rank:
778 + if inputs_embeds is not None:
779 + hidden_states = inputs_embeds
780 + else:
781 + hidden_states = self.embed_tokens(input_ids)
782 + residual = None
783 + else:
784 + assert intermediate_tensors is not None
785 + hidden_states = intermediate_tensors["hidden_states"]
786 + residual = intermediate_tensors["residual"]
787 +
788 + # Apply blocks one-by-one.
789 + for i in range(self.start_layer, self.end_layer):
790 + layer = self.layers[i]
791 + hidden_states, residual = layer(
792 + positions,
793 + hidden_states,
794 + kv_caches[i - self.start_layer],
795 + attn_metadata,
796 + residual,
797 + )
798 + if not get_pp_group().is_last_rank:
799 + return IntermediateTensors({
800 + "hidden_states": hidden_states,
801 + "residual": residual
802 + })
803 + if residual is not None:
804 + hidden_states, _ = self.norm(hidden_states, residual)
805 + else:
806 + hidden_states = self.norm(hidden_states)
807 + return hidden_states
808 +
809 +
810 +cached_get_processor = lru_cache(get_processor)
811 +
812 +
813 +def get_num_patches(num_tiles: int, crop_patches: int, left_margin: int,
814 + right_margin: int, pooling_size: int) -> int:
815 + crop_window_patches = crop_patches - (left_margin + right_margin)
816 + if num_tiles > 1:
817 + left_crop_window_patches = (crop_window_patches + left_margin +
818 + pooling_size -
819 + 1) // pooling_size * pooling_size
820 + middle_crop_window_patches = (crop_window_patches + pooling_size -
821 + 1) // pooling_size * pooling_size
822 + right_crop_window_patches = (crop_window_patches + right_margin +
823 + pooling_size -
824 + 1) // pooling_size * pooling_size
825 + return left_crop_window_patches + (
826 + num_tiles -
827 + 2) * middle_crop_window_patches + right_crop_window_patches
828 + else:
829 + single_crop_window_patches = (crop_patches + pooling_size -
830 + 1) // pooling_size * pooling_size
831 + return single_crop_window_patches
832 +
833 +
834 +def get_tokens(tiling_h: int, tiling_w: int, crop_patches: int,
835 + left_margin: int, right_margin: int, pooling_size: int) -> int:
836 + h = get_num_patches(tiling_h, crop_patches, left_margin, right_margin,
837 + pooling_size)
838 + w = get_num_patches(tiling_w, crop_patches, left_margin, right_margin,
839 + pooling_size)
840 + per_row = w // pooling_size + 1
841 + joint = per_row * (h // pooling_size) + 2
842 + image_token_length = (crop_patches + pooling_size - 1) // pooling_size
843 + resize = (image_token_length + 1) * image_token_length + 2
844 + return resize + joint
845 +
846 +
847 +def get_max_tokens(max_crops: int, crop_patches: int, left_margin: int,
848 + right_margin: int, pooling_size: int) -> int:
849 + tilings = []
850 + for i in range(1, max_crops + 1):
851 + for j in range(1, max_crops + 1):
852 + if i * j <= max_crops:
853 + tilings.append((i, j))
854 + tokens = [
855 + get_tokens(tilings[i][0], tilings[i][1], crop_patches, left_margin,
856 + right_margin, pooling_size) for i in range(len(tilings))
857 + ]
858 + return max(tokens)
859 +
860 +
861 +def get_max_molmo_image_tokens(ctx: InputContext) -> int:
862 + processor = cached_get_processor(ctx.model_config.model,
863 + trust_remote_code=True,
864 + revision=ctx.model_config.code_revision)
865 + image_processor = processor.image_processor
866 + max_llm_image_tokens = get_max_tokens(
867 + image_processor.max_crops,
868 + image_processor.base_image_input_size[0] //
869 + image_processor.image_patch_size,
870 + image_processor.overlap_margins[0],
871 + image_processor.overlap_margins[1],
872 + 2,
873 + )
874 + return max_llm_image_tokens
875 +
876 +
877 +# NOTE: preprocessing for the image data has been included in the
878 +# 'input_processor_for_molmo' function
879 +def image_input_mapper_for_molmo(
880 + ctx: InputContext,
881 + data: object,
882 +):
883 + return MultiModalInputs(data)
884 +
885 +
886 +def dummy_data_for_molmo(ctx: InputContext, seq_len: int,
887 + mm_counts: Mapping[str, int]):
888 + processor = cached_get_processor(ctx.model_config.model,
889 + trust_remote_code=True,
890 + revision=ctx.model_config.code_revision)
891 + image_processor = processor.image_processor
892 +
893 + base_image_input_d = image_processor.image_patch_size
894 + left_margin, right_margin = image_processor.overlap_margins
895 + max_crops = image_processor.max_crops
896 +
897 + # Assume: prompt_token_ids always starts with bos_token_id followed image tokens # noqa: E501
898 + max_llm_image_tokens = get_max_molmo_image_tokens(ctx)
899 + if seq_len - max_llm_image_tokens - 1 < 0:
900 + raise RuntimeError(
901 + f"Molmo cannot process {max_crops} crops in a prompt, "
902 + "please increase max_model_len or reduce number of crops")
903 +
904 + # The vertical image has the maximum number of image tokens due to column tokens. # noqa: E501
905 + tiling = (max_crops, 1)
906 + total_margin_pixels = base_image_input_d * (right_margin + left_margin)
907 + crop_patches = image_processor.base_image_input_size[
908 + 0] // base_image_input_d
909 + crop_window_patches = crop_patches - (right_margin + left_margin)
910 + crop_window_size = crop_window_patches * base_image_input_d
911 +
912 + h = crop_window_size * tiling[0] + total_margin_pixels
913 + w = crop_window_size * tiling[1] + total_margin_pixels
914 +
915 + dummy_image = Image.new("RGB", (w, h), color="red")
916 +
917 + out = processor.process("dummy prompt", dummy_image)
918 +
919 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
920 + out["input_ids"][:1 + max_llm_image_tokens])
921 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
922 + [0]) * (seq_len - max_llm_image_tokens - 1)
923 + dummy_seqdata = SequenceData(token_ids)
924 + dummy_imgdata = {
925 + "images": out["images"],
926 + "image_input_idx": out["image_input_idx"],
927 + }
928 + if "image_masks" in out:
929 + dummy_imgdata["image_masks"] = out["image_masks"]
930 + dummy_imgdata["seq_len"] = torch.tensor(seq_len, dtype=torch.long)
931 + return dummy_seqdata, {"image": dummy_imgdata}
932 +
933 +
934 +def pad_images(
935 + max_total_crops: int,
936 + images: torch.Tensor,
937 + image_input_idx: torch.Tensor,
938 + image_masks: Optional[torch.Tensor] = None,
939 +):
940 + n = max_total_crops - images.shape[0]
941 + images = F.pad(images, (0, 0, 0, 0, 0, n), value=-1)
942 + image_input_idx = F.pad(image_input_idx, (0, 0, 0, n), value=-1)
943 + if image_masks is not None:
944 + image_masks = F.pad(image_masks, (0, 0, 0, n), value=-1)
945 + return images, image_input_idx, image_masks
946 +
947 +
948 +def input_processor_for_molmo(ctx: InputContext, llm_inputs: LLMInputs):
949 + prompt = llm_inputs["prompt"]
950 + multi_modal_data = llm_inputs.get("multi_modal_data")
951 + image = multi_modal_data.get("image")
952 + processor = cached_get_processor(ctx.model_config.model,
953 + trust_remote_code=True,
954 + revision=ctx.model_config.code_revision)
955 +
956 + # NOTE: message formatting for raw text prompt is only applied for
957 + # offline inference; for online inference, the prompt is always in
958 + # instruction format and tokenized.
959 + if prompt is not None and re.match(r"^User:[\s\S]*?(Assistant:)*$",
960 + prompt):
961 + out = processor.process(prompt, image, message_format="none")
962 + elif prompt is not None:
963 + out = processor.process(prompt, image)
964 + else:
965 + out = processor.process(None,
966 + image,
967 + tokens=llm_inputs["prompt_token_ids"])
968 +
969 + image_processor = processor.image_processor
970 + max_total_crops = 1 + image_processor.max_crops
971 + if image is not None:
972 + images, image_input_idx, image_masks = pad_images(
973 + max_total_crops,
974 + out["images"],
975 + out["image_input_idx"],
976 + out.get("image_masks"),
977 + )
978 + else:
979 + base_image_input_size = image_processor.base_image_input_size
980 + image_patch_size = image_processor.image_patch_size
981 + image_num_patch = (
982 + base_image_input_size[0] // image_patch_size,
983 + base_image_input_size[1] // image_patch_size,
984 + )
985 + n_pixels = image_patch_size * image_patch_size * 3
986 + n_patches = image_num_patch[0] * image_num_patch[1]
987 +
988 + image_length_w = image_processor.image_token_length_w
989 + image_length_h = image_processor.image_token_length_h
990 + tokens_per_image = image_length_w * image_length_h
991 + images = torch.full(
992 + (max_total_crops, n_patches, n_pixels),
993 + -1,
994 + dtype=torch.float32,
995 + )
996 + image_input_idx = torch.full(
997 + (max_total_crops, tokens_per_image),
998 + -1,
999 + dtype=torch.int32,
1000 + )
1001 + if image_processor.image_padding_mask:
1002 + image_masks = torch.full(
1003 + (max_total_crops, n_patches),
1004 + -1,
1005 + dtype=torch.float32,
1006 + )
1007 +
1008 + image_data = dict(
1009 + images=images,
1010 + image_input_idx=image_input_idx,
1011 + )
1012 + if image_masks is not None:
1013 + image_data["image_masks"] = image_masks
1014 +
1015 + image_data["seq_len"] = torch.tensor(len(out["input_ids"]),
1016 + dtype=torch.long)
1017 +
1018 + multi_modal_data = dict(image=image_data)
1019 +
1020 + return LLMInputs(
1021 + prompt_token_ids=out["input_ids"],
1022 + prompt=llm_inputs["prompt"],
1023 + multi_modal_data=multi_modal_data,
1024 + )
1025 +
1026 +
1027 +@MULTIMODAL_REGISTRY.register_image_input_mapper(image_input_mapper_for_molmo)
1028 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_molmo_image_tokens)
1029 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_molmo)
1030 +@INPUT_REGISTRY.register_input_processor(input_processor_for_molmo)
1031 +class MolmoForCausalLM(nn.Module, SupportsMultiModal):
1032 +
1033 + def __init__(
1034 + self,
1035 + config: PretrainedConfig,
1036 + multimodal_config: Optional[MultiModalConfig] = None,
1037 + cache_config: Optional[CacheConfig] = None,
1038 + quant_config: Optional[Mapping[str, Any]] = None,
1039 + ) -> None:
1040 + super().__init__()
1041 +
1042 + self.config = config
1043 + self.multimodal_config = multimodal_config
1044 +
1045 + vision_config = VisionBackboneConfig()
1046 + self.vision_backbone = MolmoVisionBackbone(config, vision_config,
1047 + quant_config)
1048 + self.model = MolmoModel(config, cache_config, quant_config)
1049 +
1050 + if self.config.weight_tying:
1051 + self.lm_head = self.model.transformer.wte
1052 + else:
1053 + self.lm_head = ParallelLMHead(
1054 + config.embedding_size or config.vocab_size,
1055 + config.hidden_size,
1056 + quant_config=quant_config,
1057 + )
1058 +
1059 + self.logits_processor = LogitsProcessor(config.embedding_size
1060 + or config.vocab_size)
1061 + self.sampler = Sampler()
1062 +
1063 + def _parse_and_validate_image_input(
1064 + self,
1065 + **kwargs: object,
1066 + ) -> Optional[MolmoImageInputs]:
1067 + images = kwargs.pop("images", None)
1068 + image_masks = kwargs.pop("image_masks", None)
1069 + if images is None:
1070 + return None
1071 +
1072 + image_input_idx = kwargs.pop("image_input_idx", None)
1073 + seq_len = kwargs.pop("seq_len", None)
1074 + if image_input_idx is None:
1075 + raise ValueError("image_input_idx is required for Molmo model.")
1076 + if seq_len is None:
1077 + raise ValueError("seq_len is required for Molmo model.")
1078 + if not isinstance(seq_len, torch.Tensor):
1079 + seq_len = torch.tensor(seq_len)
1080 +
1081 + return MolmoImageInputs(
1082 + images=images,
1083 + image_input_idx=image_input_idx,
1084 + seq_len=seq_len,
1085 + image_masks=image_masks,
1086 + )
1087 +
1088 + def _process_image_input(
1089 + self,
1090 + image_input: MolmoImageInputs,
1091 + ) -> torch.Tensor:
1092 +
1093 + image_features = self.vision_backbone(
1094 + images=image_input["images"],
1095 + image_masks=image_input["image_masks"],
1096 + )
1097 +
1098 + return image_features
1099 +
1100 + def _merge_multimodal_embeddings(
1101 + self,
1102 + inputs_embeds: torch.Tensor,
1103 + image_features: torch.Tensor,
1104 + image_input_idx: torch.Tensor,
1105 + seq_len: Union[torch.Tensor, List[torch.Tensor]],
1106 + ) -> torch.Tensor:
1107 + batch_size, num_image, num_patch = image_features.shape[:3]
1108 + assert image_input_idx.shape == (batch_size, num_image, num_patch)
1109 +
1110 + image_features = image_features.to(inputs_embeds.device)
1111 + seq_len = seq_len.to(inputs_embeds.device)
1112 +
1113 + # insert the image feature into the embedding.
1114 + image_features = image_features.view(batch_size, num_image * num_patch,
1115 + -1)
1116 + image_input_idx = image_input_idx.view(batch_size,
1117 + num_image * num_patch)
1118 +
1119 + valid = image_input_idx >= 0
1120 + image_features = image_features * valid[:, :, None].to(
1121 + image_features.dtype)
1122 + image_features = image_features.view(
1123 + batch_size * num_image * num_patch, -1).contiguous()
1124 +
1125 + image_input_idx = image_input_idx * valid.to(image_input_idx.dtype)
1126 + offset = torch.cat(
1127 + [seq_len.new_zeros(
1128 + (1)), seq_len.cumsum(dim=0)[:-1]], dim=0)[:, None]
1129 + image_input_idx = image_input_idx + offset.to(image_input_idx.dtype)
1130 + image_input_idx = image_input_idx.flatten()[:, None]
1131 + mat = image_input_idx == torch.arange(
1132 + seq_len.sum().item(), device=inputs_embeds.device)[None, :]
1133 + mat = mat.to(image_features.dtype)
1134 +
1135 + inputs_embeds = inputs_embeds + torch.einsum('nd,nm->md',
1136 + image_features, mat)
1137 +
1138 + return inputs_embeds
1139 +
1140 + def forward(
1141 + self,
1142 + input_ids: torch.LongTensor,
1143 + positions: torch.LongTensor,
1144 + kv_caches: List[torch.Tensor],
1145 + attn_metadata: AttentionMetadata,
1146 + **kwargs: object,
1147 + ) -> SamplerOutput:
1148 +
1149 + image_input = self._parse_and_validate_image_input(**kwargs)
1150 +
1151 + if image_input is not None:
1152 + inputs_embeds = self.model.embed_tokens(input_ids)
1153 + image_features = self._process_image_input(image_input)
1154 +
1155 + inputs_embeds = self._merge_multimodal_embeddings(
1156 + inputs_embeds,
1157 + image_features,
1158 + image_input["image_input_idx"],
1159 + image_input["seq_len"],
1160 + )
1161 +
1162 + input_ids = None
1163 + else:
1164 + inputs_embeds = None
1165 +
1166 + hidden_states = self.model(
1167 + input_ids=input_ids,
1168 + positions=positions,
1169 + kv_caches=kv_caches,
1170 + attn_metadata=attn_metadata,
1171 + inputs_embeds=inputs_embeds,
1172 + )
1173 +
1174 + return hidden_states
1175 +
1176 + def compute_logits(self, hidden_states: torch.Tensor,
1177 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
1178 + logits = self.logits_processor(self.lm_head, hidden_states,
1179 + sampling_metadata)
1180 + return logits
1181 +
1182 + def sample(
1183 + self,
1184 + logits: torch.Tensor,
1185 + sampling_metadata: SamplingMetadata,
1186 + ) -> Optional[SamplerOutput]:
1187 + next_tokens = self.sampler(logits, sampling_metadata)
1188 + return next_tokens
1189 +
1190 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1191 +
1192 + params_mapping = [
1193 + ("model.transformer.ln_f.weight", "model.norm.weight"),
1194 + ("attn_out", "self_attn.o_proj"),
1195 + ("att_proj", "self_attn.qkv_proj"),
1196 + ("q_norm", "self_attn.q_norm"),
1197 + ("k_norm", "self_attn.k_norm"),
1198 + ("attn_norm", "input_layernorm"),
1199 + ("ff_norm", "post_attention_layernorm"),
1200 + ]
1201 +
1202 + params_dict = dict(self.named_parameters(remove_duplicate=False))
1203 +
1204 + embedding_weight = dict()
1205 + projector_weight = dict()
1206 + for name, loaded_weight in weights:
1207 + if "rotary_emb.inv_freq" in name:
1208 + continue
1209 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
1210 + continue
1211 +
1212 + if "wte.embedding" in name:
1213 + embedding_weight["embedding"] = loaded_weight
1214 + continue
1215 +
1216 + if "wte.new_embedding" in name:
1217 + embedding_weight["new_embedding"] = loaded_weight
1218 + continue
1219 +
1220 + if "vision_backbone" in name:
1221 + if name.startswith("model"):
1222 + name = name[len("model."):]
1223 + if 'image_projector' in name:
1224 + if 'w1' in name:
1225 + projector_weight['gate_proj'] = loaded_weight
1226 + elif 'w3' in name:
1227 + projector_weight['up_proj'] = loaded_weight
1228 + elif 'w2' in name:
1229 + projector_weight['down_proj'] = loaded_weight
1230 + else:
1231 + raise ValueError(
1232 + f"Unexpected projector weight: {name}")
1233 + continue
1234 + else:
1235 + if "transformer.blocks" in name:
1236 + name = name.replace("transformer.blocks", "layers")
1237 +
1238 + if "ff_proj" in name:
1239 + name = name.replace("ff_proj", "mlp.gate_up_proj")
1240 + assert 'weight' in name
1241 + up_weight, gate_weight = loaded_weight.chunk(2, dim=0)
1242 + loaded_weight = torch.cat([gate_weight, up_weight], dim=0)
1243 +
1244 + elif "ff_out" in name:
1245 + if "layers" in name:
1246 + name = name.replace("ff_out", "mlp.down_proj")
1247 + else:
1248 + # lm head
1249 + name = name.replace("model.transformer.ff_out",
1250 + "lm_head")
1251 +
1252 + else:
1253 + for (param_name, weight_name) in params_mapping:
1254 + if param_name in name:
1255 + name = name.replace(param_name, weight_name)
1256 + break
1257 +
1258 + try:
1259 + # Skip loading extra bias for GPTQ models.
1260 + if name.endswith(".bias") and name not in params_dict:
1261 + continue
1262 + param = params_dict[name]
1263 + except KeyError:
1264 + raise ValueError(f"Unexpected weight: {name}") from None
1265 +
1266 + weight_loader = getattr(param, "weight_loader",
1267 + default_weight_loader)
1268 + weight_loader(param, loaded_weight)
1269 +
1270 + gate_up_proj_weight = torch.cat(
1271 + [projector_weight["gate_proj"], projector_weight["up_proj"]],
1272 + dim=0)
1273 + name = "vision_backbone.image_projector.gate_up_proj.weight"
1274 + param = params_dict[name]
1275 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
1276 + weight_loader(param, gate_up_proj_weight)
1277 +
1278 + down_proj_weight = projector_weight["down_proj"]
1279 + name = "vision_backbone.image_projector.down_proj.weight"
1280 + param = params_dict[name]
1281 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
1282 + weight_loader(param, down_proj_weight)
1283 +
1284 + embedding_weight = torch.cat(
1285 + [embedding_weight["embedding"], embedding_weight["new_embedding"]],
1286 + dim=0)
1287 + name = "model.embed_tokens.weight"
1288 + param = params_dict[name]
1289 + weight_loader = getattr(param, "weight_loader", default_weight_loader)
1290 + weight_loader(param, embedding_weight)
vllm/model_executor/models/qwen2_vl.py
+1 -2
@@ -1167,8 +1167,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1167 continue
1168 param = params_dict[name]
1169 except KeyError:
1170 - print(params_dict.keys())
1171 - raise
1170 + raise ValueError(f"Unexpected weight: {name}") from None
1171
1172 weight_loader = getattr(param, "weight_loader",
1173 default_weight_loader)
vllm/model_executor/models/registry.py
+1
@@ -104,6 +104,7 @@ _MULTIMODAL_MODELS = {
104 "LlavaNextVideoForConditionalGeneration": ("llava_next_video", "LlavaNextVideoForConditionalGeneration"), # noqa: E501
105 "LlavaOnevisionForConditionalGeneration": ("llava_onevision", "LlavaOnevisionForConditionalGeneration"), # noqa: E501
106 "MiniCPMV": ("minicpmv", "MiniCPMV"),
107 + "MolmoForCausalLM": ("molmo", "MolmoForCausalLM"),
108 "NVLM_D": ("nvlm_d", "NVLM_D_Model"),
109 "PaliGemmaForConditionalGeneration": ("paligemma", "PaliGemmaForConditionalGeneration"), # noqa: E501
110 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),