| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | from typing import Any |
| 4 | |
| 5 | from transformers import PreTrainedConfig |
| 6 | |
| 7 | |
| 8 | class DeepseekV4Config(PreTrainedConfig): |
| 9 | model_type = "deepseek_v4" |
| 10 | |
| 11 | def __init__( |
| 12 | self, |
| 13 | max_position_embeddings: int = 1048576, |
| 14 | rope_scaling: dict[str, Any] | None = None, |
| 15 | rope_parameters: dict[str, Any] | None = None, |
| 16 | rope_theta: float = 10000.0, |
| 17 | vision_n_layers: int = 0, |
| 18 | vision_dim: int = 1024, |
| 19 | vision_n_heads: int = 16, |
| 20 | vision_inter_dim: int = 2816, |
| 21 | vision_patch_size: int = 14, |
| 22 | vision_rope_theta: float = 10000.0, |
| 23 | vision_downsample_ratio: int = 3, |
| 24 | vision_max_n_token: int = 384, |
| 25 | vision_min_pixels: int = 147456, |
| 26 | vision_max_wh_ratio: float = 8, |
| 27 | **kwargs, |
| 28 | ): |
| 29 | self.max_position_embeddings = max_position_embeddings |
| 30 | self.rope_scaling = rope_scaling |
| 31 | self.rope_theta = rope_theta |
| 32 | self.rope_parameters = rope_scaling or rope_parameters |
| 33 | # Vision tower config; vision_n_layers == 0 means text-only. |
| 34 | self.vision_n_layers = vision_n_layers |
| 35 | self.vision_dim = vision_dim |
| 36 | self.vision_n_heads = vision_n_heads |
| 37 | self.vision_inter_dim = vision_inter_dim |
| 38 | self.vision_patch_size = vision_patch_size |
| 39 | self.vision_rope_theta = vision_rope_theta |
| 40 | self.vision_downsample_ratio = vision_downsample_ratio |
| 41 | self.vision_max_n_token = vision_max_n_token |
| 42 | self.vision_min_pixels = vision_min_pixels |
| 43 | self.vision_max_wh_ratio = vision_max_wh_ratio |
| 44 | # The sparse-SWA index kernels widen the window within image spans |
| 45 | # in-kernel, so mm-prefix ranges longer than sliding_window must be |
| 46 | # kept (they are only consumed by those kernels). |
| 47 | self.mm_prefix_clamp_sliding_window = vision_n_layers > 0 |
| 48 | # The visibility span covers the sentinel block [IMAGE_START, |
| 49 | # IMAGE_END]; the mm placeholder additionally carries a leading |
| 50 | # compressor-alignment pad (see mm_preprocess.COMPRESS_PAD_TO). |
| 51 | self.mm_prefix_span_leading_pad_modulus = 4 if vision_n_layers > 0 else 0 |
| 52 | super().__init__(**kwargs) |