| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """DeepSeek-V4.1 vision variant. |
| 4 | |
| 5 | Thin multimodal wrapper around the text-only ``DeepseekV41LLMForCausalLM``: |
| 6 | |
| 7 | - ``vision`` ViT + ``aligner`` produce per-image embeddings for the IMAGE |
| 8 | positions; three learned vectors (``image_start`` / ``image_newline`` / |
| 9 | ``image_end``) fill the delimiter positions. Every image-span position |
| 10 | carries ``image_token_id`` (129264) in ``input_ids``; the per-position |
| 11 | roles come from the processor's ``types`` tensor (see |
| 12 | ``common/mm_preprocess.py``, and ``common/vision.py`` for the tower). |
| 13 | - Merged embeddings enter the text model via ``inputs_embeds``, i.e. before |
| 14 | its hyper-connection stream expansion. Raw ``input_ids`` still flow into |
| 15 | the model so the MoE router can apply ``bias_vl`` to image tokens |
| 16 | (``requires_raw_input_tokens``). |
| 17 | """ |
| 18 | |
| 19 | from collections.abc import Iterable |
| 20 | from typing import Annotated |
| 21 | |
| 22 | import torch |
| 23 | from torch import nn |
| 24 | |
| 25 | from vllm.distributed import get_tensor_model_parallel_world_size |
| 26 | from vllm.model_executor.models.interfaces import ( |
| 27 | MultiModalEmbeddings, |
| 28 | SupportsEagle3, |
| 29 | SupportsEncoderCudaGraph, |
| 30 | SupportsMultiModal, |
| 31 | SupportsPP, |
| 32 | ) |
| 33 | from vllm.model_executor.models.utils import ( |
| 34 | AutoWeightsLoader, |
| 35 | WeightsMapper, |
| 36 | maybe_prefix, |
| 37 | ) |
| 38 | from vllm.model_executor.models.vision import is_vit_use_data_parallel |
| 39 | from vllm.models.deepseek_v4.common.vision import ( |
| 40 | DeepseekV4Aligner, |
| 41 | DeepseekV4ViT, |
| 42 | run_dp_sharded_vision_tower, |
| 43 | ) |
| 44 | from vllm.multimodal import MULTIMODAL_REGISTRY |
| 45 | from vllm.utils.tensor_schema import TensorSchema, TensorShape |
| 46 | |
| 47 | from ..common.mm_preprocess import ( |
| 48 | IMAGE_PLACEHOLDER, |
| 49 | DeepseekV4VLDummyInputsBuilder, |
| 50 | DeepseekV4VLMultiModalProcessor, |
| 51 | DeepseekV4VLProcessingInfo, |
| 52 | ) |
| 53 | from ..common.vl_cudagraph import DeepseekV4VLEncoderCudaGraphMixin |
| 54 | from .model import ( |
| 55 | DeepseekV41LLMForCausalLM, |
| 56 | _linear_scale_param_name, |
| 57 | _make_deepseek_v4_weights_mapper, |
| 58 | ) |
| 59 | |
| 60 | |
| 61 | class DeepseekV4VLImagePixelInputs(TensorSchema): |
| 62 | """ViT patch inputs for one batched set of images. |
| 63 | |
| 64 | Dimensions: |
| 65 | - np: Total ViT patches across images (sum of n_vit_h * n_vit_w) |
| 66 | - c: Number of image channels (3) |
| 67 | - p: ViT patch size |
| 68 | - ni: Number of images |
| 69 | - ns: Total image-span positions (sum of n_llm_h * (n_llm_w + 1) + 2) |
| 70 | """ |
| 71 | |
| 72 | patches: Annotated[ |
| 73 | torch.Tensor, TensorShape("np", 3, "p", "p", dynamic_dims={"np"}) |
| 74 | ] |
| 75 | # [n_vit_h, n_vit_w] per image |
| 76 | vit_grid: Annotated[torch.Tensor, TensorShape("ni", 2)] |
| 77 | # [n_llm_h, n_llm_w] per image |
| 78 | llm_grid: Annotated[torch.Tensor, TensorShape("ni", 2)] |
| 79 | types: Annotated[torch.Tensor, TensorShape("ns", dynamic_dims={"ns"})] |
| 80 | |
| 81 | |
| 82 | _VL_PREFIX_MAPPING: dict[str, str | None] = { |
| 83 | "layers.": "language_model.model.layers.", |
| 84 | "embed.": "language_model.model.embed.", |
| 85 | "norm.": "language_model.model.norm.", |
| 86 | "hc_head": "language_model.model.hc_head", |
| 87 | "mtp.": "language_model.model.mtp.", |
| 88 | } |
| 89 | |
| 90 | |
| 91 | def _make_deepseek_v4_vl_weights_mapper( |
| 92 | expert_dtype: str, linear_scale_name: str |
| 93 | ) -> WeightsMapper: |
| 94 | """Text-checkpoint mapping rules re-rooted under ``language_model.``.""" |
| 95 | base = _make_deepseek_v4_weights_mapper(expert_dtype, linear_scale_name) |
| 96 | return WeightsMapper( |
| 97 | orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), |
| 98 | orig_to_new_regex=base.orig_to_new_regex, |
| 99 | orig_to_new_suffix={ |
| 100 | "head.weight": "language_model.lm_head.weight", |
| 101 | "embed.weight": "embed_tokens.weight", |
| 102 | ".ffn.gate.bias": ".ffn.gate.e_score_correction_bias", |
| 103 | }, |
| 104 | orig_to_new_substr={ |
| 105 | ".shared_experts.w2": ".shared_experts.down_proj", |
| 106 | # The MTP/DSpark draft heads are not supported for the vision |
| 107 | # variant; drop their weights. |
| 108 | "mtp.": None, |
| 109 | }, |
| 110 | ) |
| 111 | |
| 112 | |
| 113 | @MULTIMODAL_REGISTRY.register_processor( |
| 114 | DeepseekV4VLMultiModalProcessor, |
| 115 | info=DeepseekV4VLProcessingInfo, |
| 116 | dummy_inputs=DeepseekV4VLDummyInputsBuilder, |
| 117 | ) |
| 118 | class DeepseekV41ForCausalLM( |
| 119 | nn.Module, |
| 120 | DeepseekV4VLEncoderCudaGraphMixin, |
| 121 | SupportsMultiModal, |
| 122 | SupportsEncoderCudaGraph, |
| 123 | SupportsPP, |
| 124 | SupportsEagle3, |
| 125 | ): |
| 126 | """Multimodal entry point for DeepSeek-V4.1 checkpoints with a vision tower. |
| 127 | |
| 128 | ``SupportsEagle3`` (aux hidden-state plumbing for MTP/DSpark drafters) |
| 129 | delegates through ``language_model`` via the protocol defaults. |
| 130 | ``SupportsEncoderCudaGraph`` (ViT CUDA graph capture/replay) is |
| 131 | implemented by ``DeepseekV4VLEncoderCudaGraphMixin``. |
| 132 | """ |
| 133 | |
| 134 | supports_encoder_tp_data = True |
| 135 | |
| 136 | # Both of these are read off the *class* by |
| 137 | # ``configure_quant_config``/``SupportsQuant``, before ``__init__`` runs, |
| 138 | # to translate a quantization config's checkpoint-style module names into |
| 139 | # runtime prefixes. ``__init__`` later overwrites the instance |
| 140 | # ``hf_to_vllm_mapper`` with the full weight-loading mapper; only the |
| 141 | # prefix rules matter here. Without the class attributes, per-layer specs |
| 142 | # naming ``layers.N.attn.wq_a`` never match this wrapper's |
| 143 | # ``language_model.model.layers.N.attn.wq_a`` and silently fall back to |
| 144 | # the global spec. |
| 145 | packed_modules_mapping = DeepseekV41LLMForCausalLM.packed_modules_mapping |
| 146 | |
| 147 | hf_to_vllm_mapper = WeightsMapper( |
| 148 | orig_to_new_prefix=dict(_VL_PREFIX_MAPPING), |
| 149 | orig_to_new_substr={".shared_experts.w2": ".shared_experts.down_proj"}, |
| 150 | ) |
| 151 | |
| 152 | # The MoE router needs raw token ids to detect image-span tokens |
| 153 | # (all carrying image_token_id, see common/mm_preprocess.py) and apply |
| 154 | # bias_vl. |
| 155 | requires_raw_input_tokens = True |
| 156 | |
| 157 | @classmethod |
| 158 | def get_placeholder_str(cls, modality: str, i: int) -> str | None: |
| 159 | if modality == "image": |
| 160 | return IMAGE_PLACEHOLDER |
| 161 | raise ValueError(f"Unsupported modality: {modality!r}") |
| 162 | |
| 163 | def __init__(self, *, vllm_config, prefix: str = "") -> None: |
| 164 | super().__init__() |
| 165 | model_config = vllm_config.model_config |
| 166 | config = model_config.hf_config |
| 167 | self.config = config |
| 168 | self.multimodal_config = model_config.multimodal_config |
| 169 | assert self.multimodal_config is not None |
| 170 | |
| 171 | # The tower is always built; _mark_tower_model stubs it out |
| 172 | # (StageMissingLayer, weights skipped) when the image limit is 0. |
| 173 | with self._mark_tower_model(vllm_config, {"image"}): |
| 174 | self.use_data_parallel = is_vit_use_data_parallel(config.vision_n_heads) |
| 175 | self.vision = DeepseekV4ViT(config) |
| 176 | self.aligner = DeepseekV4Aligner(config) |
| 177 | self.image_start = nn.Parameter( |
| 178 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 179 | ) |
| 180 | self.image_end = nn.Parameter( |
| 181 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 182 | ) |
| 183 | self.image_newline = nn.Parameter( |
| 184 | torch.empty(config.hidden_size, dtype=torch.float32) |
| 185 | ) |
| 186 | self.vision.to(dtype=model_config.dtype) |
| 187 | self.aligner.to(dtype=model_config.dtype) |
| 188 | |
| 189 | with self._mark_language_model(vllm_config): |
| 190 | self.language_model = DeepseekV41LLMForCausalLM( |
| 191 | vllm_config=vllm_config, |
| 192 | prefix=maybe_prefix(prefix, "language_model"), |
| 193 | ) |
| 194 | # The outer mapper (see load_weights) fully resolves HF names into |
| 195 | # this wrapper's namespace before AutoWeightsLoader strips the |
| 196 | # "language_model." prefix and delegates to the child's load_weights, |
| 197 | # so the child's own mapper must be a no-op. Its suffix rules are not |
| 198 | # idempotent (e.g. "lm_head.weight".endswith("head.weight") would |
| 199 | # re-fire "head.weight" -> "lm_head.weight"). |
| 200 | self.language_model.hf_to_vllm_mapper = WeightsMapper() |
| 201 | self.make_empty_intermediate_tensors = ( # type: ignore[method-assign] |
| 202 | self.language_model.make_empty_intermediate_tensors |
| 203 | ) |
| 204 | |
| 205 | expert_dtype = getattr(config, "expert_dtype", "fp4") |
| 206 | self.hf_to_vllm_mapper = _make_deepseek_v4_vl_weights_mapper( |
| 207 | expert_dtype, _linear_scale_param_name(vllm_config, expert_dtype) |
| 208 | ) |
| 209 | |
| 210 | def _parse_and_validate_image_input( |
| 211 | self, **kwargs: object |
| 212 | ) -> DeepseekV4VLImagePixelInputs | None: |
| 213 | patches = kwargs.pop("patches", None) |
| 214 | if patches is None: |
| 215 | return None |
| 216 | return DeepseekV4VLImagePixelInputs( |
| 217 | patches=patches, |
| 218 | vit_grid=kwargs.pop("vit_grid"), |
| 219 | llm_grid=kwargs.pop("llm_grid"), |
| 220 | types=kwargs.pop("types"), |
| 221 | resolve_bindings={"p": self.config.vision_patch_size}, |
| 222 | ) |
| 223 | |
| 224 | def _process_image_input( |
| 225 | self, |
| 226 | image_input: DeepseekV4VLImagePixelInputs, |
| 227 | ) -> tuple[torch.Tensor, ...]: |
| 228 | patches = image_input.patches.to(self.aligner.w1.weight.dtype) |
| 229 | vit_grid = image_input.vit_grid.tolist() |
| 230 | |
| 231 | image_embeds_list: list[torch.Tensor] |
| 232 | if self.use_data_parallel and get_tensor_model_parallel_world_size() > 1: |
| 233 | # Data-parallel ViT: shard images across TP ranks and all-gather |
| 234 | # the per-image embeddings (weights are replicated on every rank). |
| 235 | image_embeds_list = run_dp_sharded_vision_tower( |
| 236 | self.vision, self.aligner, patches, vit_grid |
| 237 | ) |
| 238 | else: |
| 239 | image_embeds_list = [] |
| 240 | vit_offset = 0 |
| 241 | for n_vit_h, n_vit_w in vit_grid: |
| 242 | n_vit = n_vit_h * n_vit_w |
| 243 | image_embeds_list.append( |
| 244 | self._encode_image( |
| 245 | patches[vit_offset : vit_offset + n_vit], n_vit_h, n_vit_w |
| 246 | ) |
| 247 | ) |
| 248 | vit_offset += n_vit |
| 249 | |
| 250 | embeds: list[torch.Tensor] = [] |
| 251 | span_offset = 0 |
| 252 | for image_embeds, (n_llm_h, n_llm_w) in zip( |
| 253 | image_embeds_list, image_input.llm_grid.tolist(), strict=True |
| 254 | ): |
| 255 | span_len = n_llm_h * (n_llm_w + 1) + 2 |
| 256 | embeds.append( |
| 257 | self._build_image_span( |
| 258 | image_embeds, |
| 259 | image_input.types[span_offset : span_offset + span_len], |
| 260 | ) |
| 261 | ) |
| 262 | span_offset += span_len |
| 263 | return tuple(embeds) |
| 264 | |
| 265 | def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings: |
| 266 | image_input = self._parse_and_validate_image_input(**kwargs) |
| 267 | if image_input is None: |
| 268 | return [] |
| 269 | return self._process_image_input(image_input) |
| 270 | |
| 271 | def embed_input_ids( |
| 272 | self, |
| 273 | input_ids: torch.Tensor, |
| 274 | multimodal_embeddings: MultiModalEmbeddings | None = None, |
| 275 | *, |
| 276 | is_multimodal: torch.Tensor | None = None, |
| 277 | ) -> torch.Tensor: |
| 278 | from vllm.model_executor.models.utils import _merge_multimodal_embeddings |
| 279 | |
| 280 | inputs_embeds = self.language_model.embed_input_ids(input_ids) |
| 281 | |
| 282 | if multimodal_embeddings is None or len(multimodal_embeddings) == 0: |
| 283 | return inputs_embeds |
| 284 | |
| 285 | assert is_multimodal is not None |
| 286 | return _merge_multimodal_embeddings( |
| 287 | inputs_embeds=inputs_embeds, |
| 288 | multimodal_embeddings=multimodal_embeddings, |
| 289 | is_multimodal=is_multimodal, |
| 290 | ) |
| 291 | |
| 292 | @staticmethod |
| 293 | def get_model_state_cls(): |
| 294 | from ..nvidia.model_state import DeepseekV41ModelState |
| 295 | |
| 296 | return DeepseekV41ModelState |
| 297 | |
| 298 | @property |
| 299 | def token_lookback_depth(self) -> int: |
| 300 | return self.language_model.token_lookback_depth |
| 301 | |
| 302 | def forward( |
| 303 | self, |
| 304 | input_ids: torch.Tensor, |
| 305 | positions: torch.Tensor, |
| 306 | intermediate_tensors=None, |
| 307 | inputs_embeds: torch.Tensor | None = None, |
| 308 | lookback_token_ids: torch.Tensor | None = None, |
| 309 | **kwargs, |
| 310 | ) -> torch.Tensor: |
| 311 | return self.language_model( |
| 312 | input_ids, |
| 313 | positions, |
| 314 | intermediate_tensors, |
| 315 | inputs_embeds, |
| 316 | lookback_token_ids=lookback_token_ids, |
| 317 | ) |
| 318 | |
| 319 | def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None: |
| 320 | return self.language_model.compute_logits(hidden_states) |
| 321 | |
| 322 | def compute_logits_local(self, hidden_states: torch.Tensor) -> torch.Tensor: |
| 323 | return self.language_model.compute_logits_local(hidden_states) |
| 324 | |
| 325 | def get_expert_mapping(self) -> list[tuple[str, str, int, str]]: |
| 326 | return self.language_model.get_expert_mapping() |
| 327 | |
| 328 | def get_mtp_target_hidden_states(self) -> torch.Tensor | None: |
| 329 | """Pre-hc_head residual stream buffer for the MTP/DSpark draft model.""" |
| 330 | return self.language_model.get_mtp_target_hidden_states() |
| 331 | |
| 332 | def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: |
| 333 | # Map HF names into this wrapper's namespace up front and sort, so |
| 334 | # the "language_model." group reaches the child loader as one |
| 335 | # contiguous block (AutoWeightsLoader delegates per contiguous group, |
| 336 | # and the child's load_weights finalizes fused expert weights, which |
| 337 | # must not run on a partially loaded model). |
| 338 | mapped = sorted(self.hf_to_vllm_mapper.apply(weights), key=lambda x: x[0]) |
| 339 | loader = AutoWeightsLoader(self) |
| 340 | loaded_params = loader.load_weights(mapped) |
| 341 | # The child's load_weights already ran its post-load finalization. |
| 342 | self._weights_finalized = True |
| 343 | return loaded_params |
| 344 | |
| 345 | def process_weights_after_loading(self) -> None: |
| 346 | # Model-level post-load hook (called by the loader after any load |
| 347 | # format). Under DummyModelLoader the child's load_weights — and |
| 348 | # hence its finalize step — is bypassed, so run it here instead. |
| 349 | if getattr(self, "_weights_finalized", False): |
| 350 | return |
| 351 | self.language_model.process_weights_after_loading() |