| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """DeepSeek-V4.1 vision variant. |
| 4 | |
| 5 | Thin multimodal wrapper around the text-only ``DeepseekV41LLMForCausalLM``: |
| 6 | |
| 7 | - ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE |
| 8 | positions; three learned vectors (``image_start`` / ``image_newline`` / |
| 9 | ``image_end``) fill the delimiter positions. Every image-span position |
| 10 | carries ``image_token_id`` (129264) in ``input_ids``; the per-position |
| 11 | roles come from the processor's ``types`` tensor (see |
| 12 | ``common/mm_preprocess.py``, and ``common/vision.py`` for the tower). |
| 13 | - Merged embeddings enter the text model via ``inputs_embeds``, i.e. before |
| 14 | its hyper-connection stream expansion. Raw ``input_ids`` still flow into |
| 15 | the model so the MoE router can apply ``bias_vl`` to image tokens |
| 16 | (``requires_raw_input_tokens``). |
| 17 | """ |
| 18 | |
| 19 | from collections.abc import Iterable |
| 20 | from typing import Annotated |
| 21 | |
| 22 | import torch |
| 23 | from torch import nn |
| 24 | |
| 25 | from vllm.distributed import get_tensor_model_parallel_world_size |
| 26 | from vllm.model_executor.models.interfaces import ( |
| 27 | MultiModalEmbeddings, |
| 28 | SupportsEagle3, |
| 29 | SupportsEncoderCudaGraph, |
| 30 | SupportsMultiModal, |
| 31 | SupportsPP, |
| 32 | ) |
| 33 | from vllm.model_executor.models.utils import ( |
| 34 | AutoWeightsLoader, |
| 35 | WeightsMapper, |
| 36 | maybe_prefix, |
| 37 | ) |
| 38 | from vllm.model_executor.models.vision import is_vit_use_data_parallel |
| 39 | from vllm.models.deepseek_v4.common.vision import ( |
| 40 | DeepseekV4Aligner, |
| 41 | DeepseekV4ViT, |
| 42 | run_dp_sharded_vision_tower, |
| 43 | ) |
| 44 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 45 | from vllm.utils.tensor_schema import TensorSchema, TensorShape |
| 46 | |
| 47 | from ..common.mm_preprocess import ( |
| 48 | IMAGE_PLACEHOLDER, |
| 49 | DeepseekV4VLDummyInputsBuilder, |
| 50 | DeepseekV4VLMultiModalProcessor, |
| 51 | DeepseekV4VLProcessingInfo, |
| 52 | ) |
| 53 | from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin |
| 54 | from ..decoder_replay_layers import DecoderReplayLayers |
| 55 | from .model import ( |
| 56 | DeepseekV41LLMForCausalLM, |
| 57 | _linear_scale_param_name, |
| 58 | _make_deepseek_v4_weights_mapper, |
| 59 | ) |
| 60 | |
| 61 | |
| 62 | class DeepseekV4VLImagePixelInputs(TensorSchema): |
| 63 | """ViT patch inputs for one batched set of images. |
| 64 | |
| 65 | Dimensions: |
| 66 | - np: Total ViT patches across images (sum of n_vit_h * n_vit_w) |
| 67 | - c: Number of image channels (3) |
| 68 | - p: ViT patch size |
| 69 | - ni: Number of images |
| 70 | - ns: Total image-span positions (sum of n_llm_h * (n_llm_w + 1) + 2) |
| 71 | """ |
| 72 | |
| 73 | patches: Annotated[ |
| 74 | torch.Tensor, TensorShape("np", 3, "p", "p", dynamic_dims={"np"}) |
| 75 | ] |
| 76 | # [n_vit_h, n_vit_w] per image |
| 77 | vit_grid: Annotated[torch.Tensor, TensorShape("ni", 2)] |
| 78 | # [n_llm_h, n_llm_w] per image |
| 79 | llm_grid: Annotated[torch.Tensor, TensorShape("ni", 2)] |
| 80 | types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})] |
| 81 | |
| 82 | |
| 83 | def _make_deepseek_v4_vl_weights_mapper( |
| 84 | expert_dtype: str, linear_scale_name: str |
| 85 | ) -> WeightsMapper: |
| 86 | """Text-checkpoint mapping rules re-rooted under ``language_model.``.""" |
| 87 | base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name) |
| 88 | orig_to_new_prefix: dict[str, str | None] = { |
| 89 | "layers.": "language_model.model.layers.", |
| 90 | "embed.": "language_model.model.embed.", |
| 91 | "norm.": "language_model.model.norm.", |
| 92 | "hc_head": "language_model.model.hc_head", |
| 93 | "mtp.": "language_model.model.mtp.", |
| 94 | } |
| 95 | return WeightsMapper( |
| 96 | orig_to_new_prefix=orig_to_new_prefix, |
| 97 | orig_to_new_regex=base.orig_to_new_regex, |
| 98 | orig_to_new_suffix={ |
| 99 | "head.weight": "language_model.lm_head.weight", |
| 100 | "embed.weight": "embed_tokens.weight", |
| 101 | ".ffn.gate.bias": ".ffn.gate.e_score_correction_bias", |
| 102 | }, |
| 103 | orig_to_new_substr={ |
| 104 | ".shared_experts.w2": ".shared_experts.down_proj", |
| 105 | # The MTP/DSpark draft heads are not supported for the vision |
| 106 | # variant; drop their weights. |
| 107 | "mtp.": None, |
| 108 | }, |
| 109 | ) |
| 110 | |
| 111 | |
| 112 | @MULTIMODAL_REGISTRY.register_processor( |
| 113 | DeepseekV4VLMultiModalProcessor, |
| 114 | info=DeepseekV4VLProcessingInfo, |
| 115 | dummy_inputs=DeepseekV4VLDummyInputsBuilder, |
| 116 | ) |
| 117 | class DeepseekV41ForCausalLM( |
| 118 | nn.Module, |
| 119 | DeepseekV4VLEncoderCudaGraphMixin, |
| 120 | SupportsMultiModal, |
| 121 | SupportsEncoderCudaGraph, |
| 122 | SupportsPP, |
| 123 | SupportsEagle3, |
| 124 | ): |
| 125 | """Multimodal entry point for DeepSeek-V4.1 checkpoints with a vision tower. |
| 126 | |
| 127 | ``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters) |
| 128 | delegates through ``language_model`` via the protocol defaults. |
| 129 | ``SupportsEncoderCudaGraph`` (ViT CUDA graph capture/replay) is |
| 130 | implemented by ``DeepseekV4VLEncoderCudaGraphMixin``. |
| 131 | """ |
| 132 | |
| 133 | supports_encoder_tp_data = True |
| 134 | |
| 135 | # The MoE router needs raw token ids to detect image-span tokens |
| 136 | # (all carrying image_token_id, see common/mm_preprocess.py) and apply |
| 137 | # bias_vl. |
| 138 | requires_raw_input_tokens = True |
| 139 | |
| 140 | @classmethod |
| 141 | def get_placeholder_str(cls, modality: str, i: int) -> str | None: |
| 142 | if modality == "image": |
| 143 | return IMAGE_PLACEHOLDER |
| 144 | raise ValueError(f"Unsupported modality: {modality!r}") |
| 145 | |
| 146 | def __init__(self, *, vllm_config, prefix: str = "") -> None: |
| 147 | super().__init__() |
| 148 | model_config = vllm_config.model_config |
| 149 | config = model_config.hf_config |
| 150 | self.config = config |
| 151 | self.multimodal_config = model_config.multimodal_config |
| 152 | assert self.multimodal_config is not None |
| 153 | |
| 154 | # The tower is always built; _mark_tower_model stubs it out |
| 155 | # (StageMissingLayer, weights skipped) when the image limit is 0. |
| 156 | with self._mark_tower_model(vllm_config, {"image"}): |
| 157 | self.use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads) |
| 158 | self.vision = DeepseekV4ViT(config) |
| 159 | self.aligner = DeepseekV4Aligner(config) |
| 160 | self.image_start = nn.Parameter( |
| 161 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 162 | ) |
| 163 | self.image_end = nn.Parameter( |
| 164 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 165 | ) |
| 166 | self.image_newline = nn.Parameter( |
| 167 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 168 | ) |
| 169 | self.vision.to(dtype=model_config.dtype) |
| 170 | self.aligner.to(dtype=model_config.dtype) |
| 171 | |
| 172 | with self._mark_language_model(vllm_config): |
| 173 | self.language_model = DeepseekV41LLMForCausalLM( |
| 174 | vllm_config=vllm_config, |
| 175 | prefix=maybe_prefix(prefix, "language_model"), |
| 176 | ) |
| 177 | # The outer mapper (see load_weights) fully resolves HF names into |
| 178 | # this wrapper's namespace before AutoWeightsLoader strips the |
| 179 | # "language_model." prefix and delegates to the child's load_weights, |
| 180 | # so the child's own mapper must be a no-op. Its suffix rules are not |
| 181 | # idempotent (e.g. "lm_head.weight".endswith("head.weight") would |
| 182 | # re-fire "head.weight" -> "lm_head.weight"). |
| 183 | self.language_model.hf_to_vllm_mapper = WeightsMapper() |
| 184 | self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] |
| 185 | self.language_model.make_empty_intermediate_tensors |
| 186 | ) |
| 187 | |
| 188 | expert_dtype = getattr(config, "expert_dtype", "fp4") |
| 189 | self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper( |
| 190 | expert_dtype, _linear_scale_param_name(vllm_config, expert_dtype) |
| 191 | ) |
| 192 | |
| 193 | def _parse_and_validate_image_input( |
| 194 | self, **kwargs: object |
| 195 | ) -> DeepseekV4VLImagePixelInputs | None: |
| 196 | patches = kwargs.pop("patches", None) |
| 197 | if patches is None: |
| 198 | return None |
| 199 | return DeepseekV4VLImagePixelInputs( |
| 200 | patches=patches, |
| 201 | vit_grid=kwargs.pop("vit_grid"), |
| 202 | llm_grid=kwargs.pop("llm_grid"), |
| 203 | types=kwargs.pop("types"), |
| 204 | resolve_bindings={"p": self.config.vision_patch_size}, |
| 205 | ) |
| 206 | |
| 207 | def _process_image_input( |
| 208 | self, |
| 209 | image_input: DeepseekV4VLImagePixelInputs, |
| 210 | ) -> tuple[torch.Tensor, ...]: |
| 211 | patches = image_input.patches.to(self.aligner.w1.weight.dtype) |
| 212 | vit_grid = image_input.vit_grid.tolist() |
| 213 | |
| 214 | image_embeds_list: list[torch.Tensor] |
| 215 | if self.use_data_parallel and get_tensor_model_parallel_world_size() > 1: |
| 216 | # Data-parallel ViT: shard images across TP ranks and all-gather |
| 217 | # the per-image embeddings (weights are replicated on every rank). |
| 218 | image_embeds_list = run_dp_sharded_vision_tower( |
| 219 | self.vision, self.aligner, patches, vit_grid |
| 220 | ) |
| 221 | else: |
| 222 | image_embeds_list = [] |
| 223 | vit_offset = 0 |
| 224 | for n_vit_h, n_vit_w in vit_grid: |
| 225 | n_vit = n_vit_h * n_vit_w |
| 226 | image_embeds_list.append( |
| 227 | self._encode_image( |
| 228 | patches[vit_offset : vit_offset + n_vit], n_vit_h, n_vit_w |
| 229 | ) |
| 230 | ) |
| 231 | vit_offset += n_vit |
| 232 | |
| 233 | embeds: list[torch.Tensor] = [] |
| 234 | span_offset = 0 |
| 235 | for image_embeds, (n_llm_h, n_llm_w) in zip( |
| 236 | image_embeds_list, image_input.llm_grid.tolist(), strict=True |
| 237 | ): |
| 238 | span_len = n_llm_h * (n_llm_w + 1) + 2 |
| 239 | embeds.append( |
| 240 | self._build_image_span( |
| 241 | image_embeds, |
| 242 | image_input.types[span_offset : span_offset + span_len], |
| 243 | ) |
| 244 | ) |
| 245 | span_offset += span_len |
| 246 | return tuple(embeds) |
| 247 | |
| 248 | def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: |
| 249 | image_input = self._parse_and_validate_image_input(**kwargs) |
| 250 | if image_input is None: |
| 251 | return [] |
| 252 | return self._process_image_input(image_input) |
| 253 | |
| 254 | def embed_input_ids( |
| 255 | self, |
| 256 | input_ids: torch.Tensor, |
| 257 | multimodal_embeddings: MultiModalEmbeddings | None = None, |
| 258 | *, |
| 259 | is_multimodal: torch.Tensor | None = None, |
| 260 | ) -> torch.Tensor: |
| 261 | from vllm.model_executor.models.utils import _merge_multimodal_embeddings |
| 262 | |
| 263 | inputs_embeds = self.language_model.embed_input_ids(input_ids) |
| 264 | |
| 265 | if multimodal_embeddings is None or len(multimodal_embeddings) == 0: |
| 266 | return inputs_embeds |
| 267 | |
| 268 | assert is_multimodal is not None |
| 269 | return _merge_multimodal_embeddings( |
| 270 | inputs_embeds=inputs_embeds, |
| 271 | multimodal_embeddings=multimodal_embeddings, |
| 272 | is_multimodal=is_multimodal, |
| 273 | ) |
| 274 | |
| 275 | @staticmethod |
| 276 | def get_model_state_cls(): |
| 277 | from .model_state import DeepseekV41ModelState |
| 278 | |
| 279 | return DeepseekV41ModelState |
| 280 | |
| 281 | @property |
| 282 | def token_lookback_depth(self) -> int: |
| 283 | return self.language_model.token_lookback_depth |
| 284 | |
| 285 | @property |
| 286 | def decoder_replay_layers(self) -> DecoderReplayLayers | None: |
| 287 | return self.language_model.decoder_replay_layers |
| 288 | |
| 289 | def forward( |
| 290 | self, |
| 291 | input_ids: torch.Tensor, |
| 292 | positions: torch.Tensor, |
| 293 | intermediate_tensors=None, |
| 294 | inputs_embeds: torch.Tensor | None = None, |
| 295 | lookback_token_ids: torch.Tensor | None = None, |
| 296 | **kwargs, |
| 297 | ) -> torch.Tensor: |
| 298 | return self.language_model( |
| 299 | input_ids, |
| 300 | positions, |
| 301 | intermediate_tensors, |
| 302 | inputs_embeds, |
| 303 | lookback_token_ids=lookback_token_ids, |
| 304 | ) |
| 305 | |
| 306 | def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: |
| 307 | return self.language_model.compute_logits(hidden_states) |
| 308 | |
| 309 | def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor: |
| 310 | return self.language_model.compute_logits_local(hidden_states) |
| 311 | |
| 312 | def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: |
| 313 | return self.language_model.get_expert_mapping() |
| 314 | |
| 315 | def get_mtp_target_hidden_states(self) -> torch.Tensor | None: |
| 316 | """Pre-hc_head residual stream buffer for the MTP/DSpark draft model.""" |
| 317 | return self.language_model.get_mtp_target_hidden_states() |
| 318 | |
| 319 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 320 | # Map HF names into this wrapper's namespace up front and sort, so |
| 321 | # the "language_model." group reaches the child loader as one |
| 322 | # contiguous block (AutoWeightsLoader delegates per contiguous group, |
| 323 | # and the child's load_weights finalizes fused expert weights, which |
| 324 | # must not run on a partially loaded model). |
| 325 | mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0]) |
| 326 | loader = AutoWeightsLoader(self) |
| 327 | loaded_params = loader.load_weights(mapped) |
| 328 | # The child's load_weights already ran its post-load finalization. |
| 329 | self._weights_finalized = True |
| 330 | return loaded_params |
| 331 | |
| 332 | def process_weights_after_loading(self) -> None: |
| 333 | # Model-level post-load hook (called by the loader after any load |
| 334 | # format). Under DummyModelLoader the child's load_weights — and |
| 335 | # hence its finalize step — is bypassed, so run it here instead. |
| 336 | if getattr(self, "_weights_finalized", False): |
| 337 | return |
| 338 | self.language_model.process_weights_after_loading() |