@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f967e51f38
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Initialize support for Deepseek-VL2 models (#11578)

Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Isotr0py committed Jan 12, 2025 at 16:17 UTC f967e51f386404c7ead21d3c59ddc195cf946975
17 files changed +1050 -9
.buildkite/test-pipeline.yaml
+1
@@ -52,6 +52,7 @@ steps:
52 - tests/worker
53 - tests/standalone_tests/lazy_torch_compile.py
54 commands:
55 + - pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git # Used by multimoda processing test
56 - python3 standalone_tests/lazy_torch_compile.py
57 - pytest -v -s mq_llm_engine # MQLLMEngine
58 - pytest -v -s async_engine # AsyncLLMEngine
docs/source/models/supported_models.md
+19 -1
@@ -610,6 +610,13 @@ See [this page](#generative-models) for more information on how to use generativ
610 -
611 - ✅︎
612 - ✅︎
613 +* - `DeepseekVLV2ForCausalLM`
614 + - DeepSeek-VL2
615 + - T + I<sup>+</sup>
616 + - `deepseek-ai/deepseek-vl2-tiny`(WIP), `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
617 + -
618 + - ✅︎
619 + - ✅︎
620 * - `FuyuForCausalLM`
621 - Fuyu
622 - T + I
@@ -755,8 +762,19 @@ See [this page](#generative-models) for more information on how to use generativ
762 <sup>E</sup> Pre-computed embeddings can be inputted for this modality.
763 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.
764
765 +````{note}
766 +The `deepseek-ai/deepseek-vl2-tiny` is not supported yet.
767 +
768 +To use `DeepSeek-VL2` series models, you need to install a fork version `deepseek_vl2` package:
769 +```shell
770 +pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git
771 +```
772 +
773 +Besides, to run `DeepSeek-VL2` series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
774 +````
775 +
776 ```{note}
759 -To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
777 +To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
778 ```
779
780 ```{note}
examples/offline_inference/vision_language.py
+18
@@ -66,6 +66,23 @@ def run_chameleon(question: str, modality: str):
66 return llm, prompt, stop_token_ids
67
68
69 +# Deepseek-VL2
70 +def run_deepseek_vl2(question: str, modality: str):
71 + assert modality == "image"
72 +
73 + model_name = "deepseek-ai/deepseek-vl2-small"
74 +
75 + llm = LLM(model=model_name,
76 + max_model_len=4096,
77 + max_num_seqs=2,
78 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
79 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]})
80 +
81 + prompt = f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
82 + stop_token_ids = None
83 + return llm, prompt, stop_token_ids
84 +
85 +
86 # Fuyu
87 def run_fuyu(question: str, modality: str):
88 assert modality == "image"
@@ -498,6 +515,7 @@ model_example_map = {
515 "aria": run_aria,
516 "blip-2": run_blip2,
517 "chameleon": run_chameleon,
518 + "deepseek_vl_v2": run_deepseek_vl2,
519 "fuyu": run_fuyu,
520 "glm4v": run_glm4v,
521 "h2ovl_chat": run_h2ovl,
examples/offline_inference/vision_language_multi_image.py
+23
@@ -54,6 +54,28 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
54 )
55
56
57 +def load_deepseek_vl2(question: str, image_urls: List[str]):
58 + model_name = "deepseek-ai/deepseek-vl2-small"
59 +
60 + llm = LLM(model=model_name,
61 + max_model_len=4096,
62 + max_num_seqs=2,
63 + hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
64 + limit_mm_per_prompt={"image": len(image_urls)})
65 +
66 + placeholder = "".join(f"image_{i}:<image>\n"
67 + for i, _ in enumerate(image_urls, start=1))
68 + prompt = f"<|User|>: {placeholder}{question}\n\n<|Assistant|>:"
69 +
70 + return ModelRequestData(
71 + llm=llm,
72 + prompt=prompt,
73 + stop_token_ids=None,
74 + image_data=[fetch_image(url) for url in image_urls],
75 + chat_template=None,
76 + )
77 +
78 +
79 def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
80 model_name = "h2oai/h2ovl-mississippi-2b"
81
@@ -372,6 +394,7 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
394
395 model_example_map = {
396 "aria": load_aria,
397 + "deepseek_vl2": load_deepseek_vl2,
398 "h2ovl_chat": load_h2onvl,
399 "idefics3": load_idefics3,
400 "internvl_chat": load_internvl,
tests/models/decoder_only/vision_language/test_models.py
+27
@@ -188,6 +188,33 @@ VLM_TEST_SETTINGS = {
188 max_tokens=8,
189 dtype="bfloat16",
190 ),
191 + "deepseek_vl_v2": VLMTestInfo(
192 + models=["deepseek-ai/deepseek-vl2-small"],
193 + test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
194 + dtype="bfloat16",
195 + prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
196 + max_model_len=4096,
197 + max_num_seqs=2,
198 + single_image_prompts=IMAGE_ASSETS.prompts({
199 + "stop_sign": "<image>\nWhat's the color of the stop sign and car?",
200 + "cherry_blossom": "<image>\nWhat's the color of the tower?",
201 + }),
202 + multi_image_prompt="image_1:<image>\nimage_2:<image>\nDescribe the two images shortly.", # noqa: E501
203 + vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
204 + image_size_factors=[(0.10, 0.15)],
205 + patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
206 + postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
207 + hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
208 + stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], # noqa: E501
209 + num_logprobs=5,
210 + marks=[
211 + pytest.mark.skipif(
212 + not is_flash_attn_2_available(),
213 + reason="Model needs flash-attn for numeric convergence.",
214 + ),
215 + large_gpu_mark(min_gb=48),
216 + ],
217 + ),
218 "fuyu": VLMTestInfo(
219 models=["adept/fuyu-8b"],
220 test_type=VLMTestType.IMAGE,
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+36
@@ -183,6 +183,14 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput,
183
184
185 ####### Post-processors for HF outputs
186 +def deepseekvl2_trunc_hf_output(hf_output: RunnerOutput,
187 + model: str) -> RunnerOutput:
188 + output_ids, output_str, out_logprobs = hf_output
189 + if output_str.endswith("<|end▁of▁sentence|>"):
190 + output_str = output_str.split("<|end▁of▁sentence|>")[0]
191 + return output_ids, output_str, out_logprobs
192 +
193 +
194 def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
195 model: str) -> RunnerOutput:
196 output_ids, output_str, out_logprobs = hf_output
@@ -261,6 +269,34 @@ def qwen_prompt_path_encoder(
269
270
271 ####### Model-specific HuggingFace runner patchers
272 +def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
273 + """Patches and returns an instance of the HfRunner to use for GLM4."""
274 + hf_processor = hf_model.processor
275 +
276 + def processor(*args, text="", images=None, **kwargs):
277 + if isinstance(images, Image):
278 + images = [images]
279 + # inputs is a custom class instead of dict or BatchFeature
280 + inputs = hf_processor(
281 + *args,
282 + prompt=text,
283 + images=images,
284 + **kwargs,
285 + )
286 + inputs = {
287 + k: inputs[k]
288 + for k in inputs.keys() # noqa
289 + if k not in ("seq_lens", "sft_format")
290 + }
291 + inputs = BatchEncoding(data=inputs, tensor_type="pt")
292 + return inputs
293 +
294 + hf_model.processor = processor
295 + hf_model.model.get_output_embeddings = lambda: \
296 + hf_model.model.language.model.embed_tokens
297 + return hf_model
298 +
299 +
300 def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
301 """Patches and returns an instance of the HfRunner to use for GLM4."""
302 hf_processor = hf_model.processor
tests/models/registry.py
+2
@@ -179,6 +179,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
179 trust_remote_code=True),
180 "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
181 is_available_online=False),
182 + # TODO(Isotr0py): Use deepseek-vl2-tiny for test after it's supported
183 + "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-small"), # noqa: E501
184 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
185 "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
186 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
tests/models/test_initialization.py
+3
@@ -26,6 +26,9 @@ def test_can_initialize(model_arch):
26
27 # Avoid OOM
28 def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
29 + if hf_config.model_type == "deepseek_vl_v2":
30 + hf_config.update({"architectures": ["DeepseekVLV2ForCausalLM"]})
31 +
32 if hasattr(hf_config, "text_config"):
33 text_config: PretrainedConfig = hf_config.text_config
34 else:
vllm/entrypoints/chat_utils.py
+2 -2
@@ -403,8 +403,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
403 if model_type.startswith("llava"):
404 return self._cached_token_str(self._tokenizer,
405 hf_config.image_token_index)
406 - if model_type in ("chameleon", "internvl_chat", "NVLM_D",
407 - "h2ovl_chat"):
406 + if model_type in ("chameleon", "deepseek_vl_v2", "internvl_chat",
407 + "NVLM_D", "h2ovl_chat"):
408 return "<image>"
409 if model_type == "mllama":
410 return "<|image|>"
vllm/model_executor/models/deepseek_v2.py
+17 -1
@@ -243,7 +243,11 @@ class DeepseekV2Attention(nn.Module):
243 bias=False,
244 quant_config=quant_config,
245 prefix=f"{prefix}.o_proj")
246 - rope_scaling["rope_type"] = 'deepseek_yarn'
246 + if rope_scaling:
247 + rope_scaling["rope_type"] = 'deepseek_yarn'
248 + self.use_normal_rope = False
249 + else:
250 + self.use_normal_rope = True
251 self.rotary_emb = get_rope(qk_rope_head_dim,
252 rotary_dim=qk_rope_head_dim,
253 max_position=max_position_embeddings,
@@ -298,7 +302,18 @@ class DeepseekV2Attention(nn.Module):
302 self.qk_nope_head_dim + self.v_head_dim)
303 k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
304 k_pe = latent_cache[:, :, self.kv_lora_rank:]
305 +
306 + if self.use_normal_rope:
307 + seq_len = positions.size(0)
308 + ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
309 + q_pe = q_pe.reshape(seq_len, -1)
310 + k_pe = k_pe.reshape(seq_len, -1)
311 +
312 q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
313 +
314 + if self.use_normal_rope:
315 + q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
316 +
317 q[..., self.qk_nope_head_dim:] = q_pe
318 k = torch.empty_like(q)
319 k[..., :self.qk_nope_head_dim] = k_nope
@@ -355,6 +370,7 @@ class DeepseekV2DecoderLayer(nn.Module):
370 quant_config=quant_config,
371 prefix=f"{prefix}.self_attn",
372 )
373 +
374 if (config.n_routed_experts is not None
375 and layer_idx >= config.first_k_dense_replace
376 and layer_idx % config.moe_layer_freq == 0):
vllm/model_executor/models/deepseek_v3.py
+18 -2
@@ -251,7 +251,11 @@ class DeepseekV3Attention(nn.Module):
251 bias=False,
252 quant_config=quant_config,
253 prefix=f"{prefix}.o_proj")
254 - rope_scaling["rope_type"] = 'deepseek_yarn'
254 + if rope_scaling:
255 + rope_scaling["rope_type"] = 'deepseek_yarn'
256 + self.use_normal_rope = False
257 + else:
258 + self.use_normal_rope = True
259 self.rotary_emb = get_rope(qk_rope_head_dim,
260 rotary_dim=qk_rope_head_dim,
261 max_position=max_position_embeddings,
@@ -306,7 +310,18 @@ class DeepseekV3Attention(nn.Module):
310 self.qk_nope_head_dim + self.v_head_dim)
311 k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
312 k_pe = latent_cache[:, :, self.kv_lora_rank:]
313 +
314 + if self.use_normal_rope:
315 + seq_len = positions.size(0)
316 + ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
317 + q_pe = q_pe.reshape(seq_len, -1)
318 + k_pe = k_pe.reshape(seq_len, -1)
319 +
320 q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
321 +
322 + if self.use_normal_rope:
323 + q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
324 +
325 q[..., self.qk_nope_head_dim:] = q_pe
326 k = torch.empty_like(q)
327 k[..., :self.qk_nope_head_dim] = k_nope
@@ -583,7 +598,8 @@ class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
598 continue
599
600 # TODO(simon): support nextn predict layers
586 - if self.config.num_nextn_predict_layers > 0:
601 + if hasattr(self.config, "num_nextn_predict_layers"
602 + ) and self.config.num_nextn_predict_layers > 0:
603 assert self.config.num_nextn_predict_layers == 1
604 layer_idx = self.config.num_hidden_layers
605 if name.startswith(f"model.layers.{layer_idx}"):
vllm/model_executor/models/deepseek_vl2.py new
+662
@@ -0,0 +1,662 @@
1 +# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
2 +"""Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
3 +import math
4 +from functools import cached_property, partial
5 +from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
6 + TypedDict, Union)
7 +
8 +import torch
9 +import torch.nn as nn
10 +import torch.nn.functional as F
11 +from einops import rearrange, repeat
12 +from transformers import AutoProcessor, BatchFeature, ProcessorMixin
13 +
14 +from vllm.attention import AttentionMetadata
15 +from vllm.config import VllmConfig
16 +from vllm.logger import init_logger
17 +from vllm.model_executor import SamplingMetadata
18 +from vllm.model_executor.layers.quantization import QuantizationConfig
19 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20 +from vllm.model_executor.model_loader.utils import set_default_torch_dtype
21 +from vllm.multimodal import MULTIMODAL_REGISTRY
22 +from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
23 + NestedTensors)
24 +from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
25 + ImageSize, MultiModalDataItems)
26 +from vllm.multimodal.processing import (BaseMultiModalProcessor,
27 + BaseProcessingInfo, PromptReplacement)
28 +from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
29 +from vllm.multimodal.utils import cached_get_tokenizer
30 +from vllm.sequence import IntermediateTensors
31 +from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
32 + MlpProjectorConfig,
33 + VisionEncoderConfig)
34 +from vllm.utils import is_list_of
35 +
36 +from .interfaces import SupportsMultiModal, SupportsPP
37 +from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
38 + init_vllm_registered_model, maybe_prefix,
39 + merge_multimodal_embeddings)
40 +
41 +logger = init_logger(__name__)
42 +
43 +# The image token id may be various
44 +_IMAGE_TOKEN = "<image>"
45 +
46 +
47 +class DeepseekVL2ImagePixelInputs(TypedDict):
48 + type: Literal["pixel_values"]
49 + data: Union[torch.Tensor, List[torch.Tensor]]
50 + """
51 + Shape: `(batch_size * num_images, num_channels, height, width)`
52 + """
53 + images_spatial_crop: torch.Tensor
54 + """
55 + Shape: `(batch_size * num_images, 2)`
56 + """
57 +
58 +
59 +class DeepseekVL2VImageEmbeddingInputs(TypedDict):
60 + type: Literal["image_embeds"]
61 + data: Union[torch.Tensor, List[torch.Tensor]]
62 + """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
63 +
64 + `hidden_size` must match the hidden size of language model backbone.
65 + """
66 +
67 +
68 +DeepseekVL2ImageInputs = Union[DeepseekVL2ImagePixelInputs,
69 + DeepseekVL2VImageEmbeddingInputs]
70 +
71 +
72 +class MlpProjector(nn.Module):
73 +
74 + def __init__(self, cfg: MlpProjectorConfig):
75 +
76 + super().__init__()
77 +
78 + self.cfg = cfg
79 + assert not cfg.token_pooling, (
80 + "Token pooling is not supported currently.")
81 +
82 + if cfg.projector_type == "downsample_mlp_gelu":
83 + mlp_depth = cfg.depth
84 + mlp_ratio = cfg.mlp_ratio
85 + modules = [
86 + nn.Linear(
87 + cfg.input_dim * cfg.downsample_ratio *
88 + cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
89 + ]
90 + for _ in range(1, mlp_depth - 1):
91 + modules.append(nn.GELU())
92 + modules.append(
93 + nn.Linear(cfg.n_embed * mlp_ratio,
94 + cfg.n_embed * mlp_ratio))
95 + modules.append(nn.GELU())
96 + modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
97 + modules = nn.Sequential(*modules)
98 +
99 + else:
100 + raise NotImplementedError(
101 + f"Unsupported projector type: {cfg.projector_type}")
102 +
103 + self.layers = modules
104 +
105 + def forward(self, x):
106 + bs, hw, input_dim = x.shape
107 + h = w = int((hw)**0.5)
108 + """compute padding"""
109 + if h % self.cfg.downsample_ratio:
110 + pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
111 + else:
112 + pad = 0
113 + x = x.reshape(bs, h, w, input_dim)
114 + if pad > 0:
115 + x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
116 + """4 to 1 concat"""
117 + x = x.permute(0, 3, 1, 2) # B, C, H, W
118 + x = F.unfold(x,
119 + kernel_size=self.cfg.downsample_ratio,
120 + stride=self.cfg.downsample_ratio,
121 + padding=0) # B, C*4, HW // 4
122 + x = x.permute(0, 2, 1)
123 +
124 + return self.layers(x)
125 +
126 +
127 +class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
128 +
129 + def get_hf_config(self):
130 + return self.ctx.get_hf_config(DeepseekVLV2Config)
131 +
132 + def get_hf_processor(self) -> ProcessorMixin:
133 + # TODO(Isotr0py): we should get rid of dependency on deepseek_vl2
134 + # in the future, because it's flasky and lack of maintenance.
135 + try:
136 + from deepseek_vl2.models.processing_deepseek_vl_v2 import (
137 + DeepseekVLV2Processor, select_best_resolution)
138 + AutoProcessor.register("DeepseekVLV2Processor",
139 + DeepseekVLV2Processor)
140 + except ModuleNotFoundError as exc:
141 + raise ModuleNotFoundError(
142 + "You need to `pip install "
143 + "git+https://github.com/deepseek-ai/DeepSeek-VL2.git` "
144 + "to use this model") from exc
145 +
146 + processor = self.ctx.get_hf_processor(DeepseekVLV2Processor)
147 + processor.select_best_resolution = partial(
148 + select_best_resolution,
149 + candidate_resolutions=processor.candidate_resolutions)
150 + return processor
151 +
152 + def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
153 + return {"image": None}
154 +
155 + def get_num_image_tokens(self, *, image_width: int,
156 + image_height: int) -> int:
157 + hf_processor = self.get_hf_processor()
158 + image_size = hf_processor.image_size
159 + patch_size = hf_processor.patch_size
160 + downsample_ratio = hf_processor.downsample_ratio
161 +
162 + best_width, best_height = hf_processor.select_best_resolution(
163 + (image_width, image_height))
164 +
165 + num_width_tiles, num_height_tiles = (best_width // image_size,
166 + best_height // image_size)
167 + h = w = math.ceil((image_size // patch_size) / downsample_ratio)
168 +
169 + global_views_tokens = h * (w + 1)
170 + local_views_tokens = (num_height_tiles * h) * (num_width_tiles * w + 1)
171 + return global_views_tokens + local_views_tokens + 1
172 +
173 + def get_image_size_with_most_features(self) -> ImageSize:
174 + hf_config = self.get_hf_config()
175 + candidate_resolutions = hf_config.candidate_resolutions
176 + height, width = max(candidate_resolutions,
177 + key=lambda x: self.get_num_image_tokens(
178 + image_width=x[1], image_height=x[0]))
179 + return ImageSize(width=width, height=height)
180 +
181 + def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
182 + max_image_size = self.get_image_size_with_most_features()
183 + max_image_tokens = self.get_num_image_tokens(
184 + image_height=max_image_size.height,
185 + image_width=max_image_size.width)
186 +
187 + return {"image": max_image_tokens}
188 +
189 +
190 +class DeepseekVL2DummyInputsBuilder(
191 + BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]):
192 +
193 + def get_dummy_processor_inputs(
194 + self,
195 + seq_len: int,
196 + mm_counts: Mapping[str, int],
197 + ) -> ProcessorInputs:
198 + num_images = mm_counts.get("image", 0)
199 + hf_processor = self.info.get_hf_processor()
200 + image_token: str = hf_processor.image_token
201 +
202 + max_image_size = self.info.get_image_size_with_most_features()
203 +
204 + mm_data = {
205 + "image":
206 + self._get_dummy_images(width=max_image_size.width,
207 + height=max_image_size.height,
208 + num_images=num_images)
209 + }
210 +
211 + return ProcessorInputs(
212 + prompt_text=image_token * num_images,
213 + mm_data=mm_data,
214 + )
215 +
216 +
217 +class DeepseekVL2MultiModalProcessor(
218 + BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]):
219 +
220 + def _call_hf_processor(
221 + self,
222 + prompt: str,
223 + mm_data: Mapping[str, object],
224 + mm_kwargs: Mapping[str, object],
225 + ) -> BatchFeature:
226 + if mm_data:
227 + outputs = self.info.ctx.call_hf_processor(
228 + self.info.get_hf_processor(**mm_kwargs),
229 + dict(prompt=prompt, **mm_data),
230 + mm_kwargs,
231 + )
232 +
233 + # Deepseek-vl2 processor don't return BatchFeature,
234 + # we need to manually create it
235 + processed_outputs = dict(input_ids=outputs["input_ids"])
236 + processed_outputs = BatchFeature(data=dict(processed_outputs),
237 + tensor_type="pt")
238 +
239 + # Remove batch dimension from processor outputs,
240 + # because we will try batch to create NestedTensors
241 + target_dtype = self.info.ctx.model_config.dtype
242 + pixel_values = outputs["images"].to(target_dtype).squeeze(0)
243 + images_spatial_crop = outputs["images_spatial_crop"].squeeze(0)
244 + patches_per_image = [
245 + x.prod().item() + 1 for x in images_spatial_crop
246 + ]
247 +
248 + # Rename `images` -> `pixel_values` to avoid confusion
249 + processed_outputs["pixel_values"] = list(
250 + pixel_values.split(patches_per_image))
251 + processed_outputs["images_spatial_crop"] = images_spatial_crop
252 + else:
253 + tokenizer = self.info.get_tokenizer()
254 + processed_outputs = tokenizer(prompt,
255 + add_special_tokens=True,
256 + return_tensors="pt")
257 +
258 + return processed_outputs
259 +
260 + def _get_mm_fields_config(
261 + self,
262 + hf_inputs: BatchFeature,
263 + hf_processor_mm_kwargs: Mapping[str, object],
264 + ) -> Mapping[str, MultiModalFieldConfig]:
265 + return dict(
266 + pixel_values=MultiModalFieldConfig.batched("image"),
267 + images_spatial_crop=MultiModalFieldConfig.batched("image"),
268 + image_embeds=MultiModalFieldConfig.batched("image"),
269 + )
270 +
271 + def _get_prompt_replacements(
272 + self,
273 + mm_items: MultiModalDataItems,
274 + hf_processor_mm_kwargs: Mapping[str, object],
275 + out_mm_kwargs: MultiModalKwargs,
276 + ) -> list[PromptReplacement]:
277 + hf_processor = self.info.get_hf_processor()
278 + image_token_id: int = hf_processor.image_token_id
279 +
280 + def get_replacement_deepseek_vl2(item_idx: int):
281 + images = mm_items.get_items(
282 + "image", (ImageEmbeddingItems, ImageProcessorItems))
283 +
284 + if isinstance(images, ImageEmbeddingItems):
285 + num_image_tokens = images.get_feature_size(item_idx)
286 + else:
287 + image_size = images.get_image_size(item_idx)
288 +
289 + num_image_tokens = self.info.get_num_image_tokens(
290 + image_width=image_size.width,
291 + image_height=image_size.height,
292 + )
293 + return [image_token_id] * num_image_tokens
294 +
295 + return [
296 + PromptReplacement(
297 + modality="image",
298 + target=[image_token_id],
299 + replacement=get_replacement_deepseek_vl2,
300 + )
301 + ]
302 +
303 +
304 +@MULTIMODAL_REGISTRY.register_processor(
305 + DeepseekVL2MultiModalProcessor,
306 + info=DeepseekVL2ProcessingInfo,
307 + dummy_inputs=DeepseekVL2DummyInputsBuilder)
308 +class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
309 +
310 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
311 + "language.": "language_model.",
312 + })
313 +
314 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
315 + super().__init__()
316 + config: DeepseekVLV2Config = vllm_config.model_config.hf_config
317 + quant_config = vllm_config.quant_config
318 + multimodal_config = vllm_config.model_config.multimodal_config
319 +
320 + self.config = config
321 + self.multimodal_config = multimodal_config
322 +
323 + self.vision_config = config.vision_config
324 + self.projector_config = config.projector_config
325 + self.text_config = config.text_config
326 +
327 + model_config = vllm_config.model_config
328 + tokenizer = cached_get_tokenizer(
329 + model_config.tokenizer,
330 + tokenizer_mode=model_config.tokenizer_mode,
331 + tokenizer_revision=model_config.tokenizer_revision,
332 + trust_remote_code=model_config.trust_remote_code,
333 + )
334 + self.image_token_id = tokenizer.vocab.get(_IMAGE_TOKEN)
335 +
336 + self.vision = self._init_vision_module(self.vision_config,
337 + quant_config,
338 + maybe_prefix(prefix, "vision"))
339 +
340 + self.projector = MlpProjector(self.projector_config)
341 + self.tile_tag = config.tile_tag
342 + self.global_view_pos = config.global_view_pos
343 +
344 + # special token for image token sequence format
345 + embed_std = 1 / torch.sqrt(
346 + torch.tensor(self.projector_config.n_embed, dtype=torch.float32))
347 + if self.tile_tag == "2D":
348 + # <|view_separator|>, <|\n|>
349 + self.image_newline = nn.Parameter(
350 + torch.randn(self.projector_config.n_embed) * embed_std)
351 + # This is a typo in original implementation
352 + self.view_seperator = nn.Parameter(
353 + torch.randn(self.projector_config.n_embed) * embed_std)
354 + else:
355 + raise ValueError(
356 + f"Only 2D tile_tag is supported currently, got: {self.tile_tag}"
357 + )
358 +
359 + self.language_model = init_vllm_registered_model(
360 + vllm_config=vllm_config,
361 + hf_config=self.text_config,
362 + prefix=maybe_prefix(prefix, "language"),
363 + architectures=["DeepseekV3ForCausalLM"]
364 + if self.text_config.topk_method == "noaux_tc" else
365 + ["DeepseekV2ForCausalLM"],
366 + )
367 +
368 + self.make_empty_intermediate_tensors = (
369 + self.language_model.make_empty_intermediate_tensors)
370 +
371 + def _init_vision_module(
372 + self,
373 + vision_config: VisionEncoderConfig,
374 + quant_config: Optional[QuantizationConfig],
375 + prefix: str = "",
376 + ) -> nn.Module:
377 + # TODO: refactor vision model through timm wrapper from transformers
378 + try:
379 + import timm
380 + except ImportError:
381 + raise ImportError("Please install timm") from ImportError
382 +
383 + with set_default_torch_dtype(torch.float16):
384 + model = timm.create_model(
385 + "vit_so400m_patch14_siglip_384.webli",
386 + pretrained=False,
387 + num_classes=0,
388 + dynamic_img_size=True,
389 + dynamic_img_pad=True,
390 + )
391 +
392 + model = model.to(dtype=torch.get_default_dtype())
393 + return model
394 +
395 + @cached_property
396 + def sampler(self):
397 + if hasattr(self.language_model, "sampler"):
398 + return self.language_model.sampler
399 +
400 + return get_sampler()
401 +
402 + def _validate_pixel_values(
403 + self, data: Union[torch.Tensor, List[torch.Tensor]]
404 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
405 +
406 + h = w = self.vision_config.image_size
407 + expected_dims = (3, h, w)
408 +
409 + def _validate_shape(d: torch.Tensor):
410 + actual_dims = tuple(d.shape[1:])
411 +
412 + if actual_dims != expected_dims:
413 + expected_expr = ("num_patches", *map(str, expected_dims))
414 + raise ValueError(
415 + "The expected shape of pixel values per image per batch "
416 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
417 +
418 + for d in data:
419 + _validate_shape(d)
420 +
421 + return data
422 +
423 + def _validate_images_spatial_crop(
424 + self, data: Union[torch.Tensor, List[torch.Tensor]]
425 + ) -> Union[torch.Tensor, List[torch.Tensor]]:
426 + expected_dims = 2
427 +
428 + def _validate_shape(d: torch.Tensor):
429 + actual_dims = d.size(-1)
430 +
431 + if actual_dims != expected_dims:
432 + expected_expr = str(expected_dims)
433 + raise ValueError(
434 + f"The expected shape of image sizes per image per batch "
435 + f"is {expected_expr}. You supplied {tuple(d.shape)}.")
436 +
437 + for d in data:
438 + _validate_shape(d)
439 +
440 + return data
441 +
442 + def _parse_and_validate_image_input(
443 + self, **kwargs: object) -> Optional[DeepseekVL2ImageInputs]:
444 + pixel_values = kwargs.pop("pixel_values", None)
445 + images_spatial_crop = kwargs.pop("images_spatial_crop", None)
446 + image_embeds = kwargs.pop("image_embeds", None)
447 +
448 + if pixel_values is None and image_embeds is None:
449 + return None
450 +
451 + if pixel_values is not None:
452 + if not isinstance(pixel_values, (torch.Tensor, list)):
453 + raise ValueError("Incorrect type of pixel values. "
454 + f"Got type: {type(pixel_values)}")
455 +
456 + if not isinstance(images_spatial_crop, (torch.Tensor, list)):
457 + raise ValueError("Incorrect type of image sizes. "
458 + f"Got type: {type(images_spatial_crop)}")
459 +
460 + return DeepseekVL2ImagePixelInputs(
461 + type="pixel_values",
462 + data=self._validate_pixel_values(flatten_bn(pixel_values)),
463 + images_spatial_crop=self._validate_images_spatial_crop(
464 + flatten_bn(images_spatial_crop, concat=True)))
465 +
466 + if image_embeds is not None:
467 + if not isinstance(image_embeds, torch.Tensor):
468 + raise ValueError("Incorrect type of image embeddings. "
469 + f"Got type: {type(image_embeds)}")
470 +
471 + return DeepseekVL2VImageEmbeddingInputs(
472 + type="image_embeds",
473 + data=flatten_bn(image_embeds),
474 + )
475 +
476 + raise AssertionError("This line should be unreachable.")
477 +
478 + def _pixel_values_to_embedding(
479 + self,
480 + pixel_values: NestedTensors,
481 + images_spatial_crop: torch.Tensor,
482 + ) -> NestedTensors:
483 + # Pixel_values: n_image * batch_size * [patch_per_img, 3, height, width]
484 + total_tiles = [x for x in pixel_values]
485 +
486 + # [batch_all_tiles, 3, height, width]
487 + total_tiles = torch.cat(total_tiles, dim=0)
488 +
489 + # [batch_all_tiles, vit_seq_len, c]
490 + images_feature = self.vision.forward_features(total_tiles)
491 +
492 + # [batch_all_tiles, hw, D]
493 + images_embeds = self.projector(images_feature)
494 +
495 + _, hw, n_dim = images_embeds.shape
496 + h = w = int(hw**0.5)
497 +
498 + # 根据self.tile_tag & self.global_view_pos填充image token sequence
499 + tile_index = 0
500 + vision_embeddings = []
501 + for jdx in range(images_spatial_crop.size(0)):
502 + # extra global & local features
503 + num_width_tiles, num_height_tiles = images_spatial_crop[jdx]
504 + if num_width_tiles == 0 or num_height_tiles == 0:
505 + break
506 + num_tiles_in_image = num_width_tiles * num_height_tiles
507 +
508 + # [hw, D]
509 + global_features = images_embeds[tile_index]
510 +
511 + # [num_height_tiles * num_width_tiles, hw, D]
512 + local_features = images_embeds[tile_index + 1:tile_index + 1 +
513 + num_tiles_in_image]
514 + tile_index += num_tiles_in_image + 1
515 +
516 + # format global and local features
517 + # ----------------- global view add newline -----------------
518 + # [hw, D] -> [h, w, D]
519 + global_features = global_features.view(h, w, n_dim)
520 +
521 + # [D] -> [h, 1, D]
522 + new_lines_in_global = repeat(self.image_newline, "d -> h 1 d", h=h)
523 +
524 + # cat([h, w, D], [h, 1, D], dim=1) -> [h, w + 1, D]
525 + global_features = torch.cat([global_features, new_lines_in_global],
526 + dim=1)
527 +
528 + # [h, w + 1, D] -> [h * (w + 1), D]
529 + global_features = global_features.view(-1, n_dim)
530 +
531 + # ----------------- local view add newline -----------------
532 + # [num_height_tiles * num_width_tiles, h * w, D] ->
533 + # [num_height_tiles * h, num_width_tiles * w, D]
534 + local_features = rearrange(local_features,
535 + "(th tw) (h w) d -> (th h) (tw w) d",
536 + th=num_height_tiles,
537 + tw=num_width_tiles,
538 + h=h,
539 + w=w)
540 +
541 + # [D] -> [num_height_tiles * h, 1, D]
542 + new_lines_in_local = repeat(self.image_newline,
543 + "d -> (th h) 1 d",
544 + th=num_height_tiles,
545 + h=h)
546 +
547 + # [num_height_tiles * h, num_width_tiles * w + 1, D]
548 + local_features = torch.cat([local_features, new_lines_in_local],
549 + dim=1)
550 +
551 + # [num_height_tiles * h, num_width_tiles * w + 1, D]
552 + # --> [(num_height_tiles * h) * (num_width_tiles * w + 1), D]
553 + local_features = local_features.view(-1, n_dim)
554 +
555 + # merge global and local tiles
556 + if self.global_view_pos == "head":
557 + global_local_features = torch.cat([
558 + global_features,
559 + self.view_seperator[None, :],
560 + local_features,
561 + ])
562 + else:
563 + global_local_features = torch.cat([
564 + local_features,
565 + self.view_seperator[None, :],
566 + global_features,
567 + ])
568 +
569 + vision_embeddings.append(global_local_features)
570 + return vision_embeddings
571 +
572 + def _process_image_input(
573 + self, image_input: DeepseekVL2ImageInputs) -> torch.Tensor:
574 + if image_input["type"] == "image_embeds":
575 + image_data = image_input["data"]
576 + if is_list_of(image_data, torch.Tensor):
577 + # it's already a list of tensors
578 + return image_data
579 + if len(image_data.shape) == 3:
580 + # 3D tensor
581 + return list(torch.unbind(image_data, dim=0))
582 + raise ValueError(
583 + "We expect batched 2D tensors;"
584 + "this can be either a list of 2D tensors or a single 3D tensor."
585 + )
586 +
587 + pixel_values = image_input["data"]
588 + images_spatial_crop = image_input["images_spatial_crop"]
589 +
590 + return self._pixel_values_to_embedding(
591 + pixel_values=pixel_values, images_spatial_crop=images_spatial_crop)
592 +
593 + def get_multimodal_embeddings(self, **kwargs: object) -> torch.Tensor:
594 + image_input = self._parse_and_validate_image_input(**kwargs)
595 + if image_input is None:
596 + return None
597 + vision_embeddings = self._process_image_input(image_input)
598 + return vision_embeddings
599 +
600 + def get_input_embeddings(
601 + self,
602 + input_ids: torch.Tensor,
603 + multimodal_embeddings: Optional[NestedTensors] = None,
604 + ) -> torch.Tensor:
605 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
606 + if multimodal_embeddings is not None:
607 + inputs_embeds = merge_multimodal_embeddings(
608 + input_ids, inputs_embeds, multimodal_embeddings,
609 + self.image_token_id)
610 + return inputs_embeds
611 +
612 + def forward(self,
613 + input_ids: torch.Tensor,
614 + positions: torch.Tensor,
615 + kv_caches: List[torch.Tensor],
616 + attn_metadata: AttentionMetadata,
617 + intermediate_tensors: Optional[IntermediateTensors] = None,
618 + inputs_embeds: Optional[torch.Tensor] = None,
619 + **kwargs: object):
620 +
621 + if intermediate_tensors is not None:
622 + inputs_embeds = None
623 +
624 + # NOTE: In v1, inputs_embeds is always generated at model runner, this
625 + # condition is for v0 compatibility
626 + elif inputs_embeds is None:
627 + vision_embeddings = self.get_multimodal_embeddings(**kwargs)
628 + inputs_embeds = self.get_input_embeddings(input_ids,
629 + vision_embeddings)
630 + input_ids = None
631 +
632 + hidden_states = self.language_model(input_ids,
633 + positions,
634 + kv_caches,
635 + attn_metadata,
636 + intermediate_tensors,
637 + inputs_embeds=inputs_embeds)
638 +
639 + return hidden_states
640 +
641 + def compute_logits(
642 + self,
643 + hidden_states: torch.Tensor,
644 + sampling_metadata: SamplingMetadata,
645 + ) -> Optional[torch.Tensor]:
646 + return self.language_model.compute_logits(hidden_states,
647 + sampling_metadata)
648 +
649 + def sample(
650 + self,
651 + logits: torch.Tensor,
652 + sampling_metadata: SamplingMetadata,
653 + ) -> Optional[SamplerOutput]:
654 + return self.language_model.sample(logits, sampling_metadata)
655 +
656 + def load_weights(self, weights: Iterable[Tuple[str,
657 + torch.Tensor]]) -> Set[str]:
658 +
659 + loader = AutoWeightsLoader(self)
660 + autoloaded_weights = loader.load_weights(weights,
661 + mapper=self.hf_to_vllm_mapper)
662 + return autoloaded_weights
vllm/model_executor/models/minicpmv.py
+1 -1
@@ -657,7 +657,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
657 quant_config: Optional[QuantizationConfig],
658 prefix: str = "",
659 ) -> nn.Module:
660 - # TODO: refactor this vision model
660 + # TODO: refactor vision model through timm wrapper from transformers
661 try:
662 import timm
663 except ImportError:
vllm/model_executor/models/registry.py
+1
@@ -149,6 +149,7 @@ _MULTIMODAL_MODELS = {
149 "ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
150 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
151 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
152 + "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
153 "FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
154 "H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
155 "InternVLChatModel": ("internvl", "InternVLChatModel"),
vllm/transformers_utils/config.py
+4 -2
@@ -23,8 +23,9 @@ from vllm.logger import init_logger
23 # yapf conflicts with isort for this block
24 # yapf: disable
25 from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26 - DbrxConfig, EAGLEConfig,
27 - ExaoneConfig, H2OVLChatConfig,
26 + DbrxConfig, DeepseekVLV2Config,
27 + EAGLEConfig, ExaoneConfig,
28 + H2OVLChatConfig,
29 InternVLChatConfig, JAISConfig,
30 MedusaConfig, MllamaConfig,
31 MLPSpeculatorConfig, MPTConfig,
@@ -54,6 +55,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
55 "chatglm": ChatGLMConfig,
56 "cohere2": Cohere2Config,
57 "dbrx": DbrxConfig,
58 + "deepseek_vl_v2": DeepseekVLV2Config,
59 "mpt": MPTConfig,
60 "RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
61 "RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2 from vllm.transformers_utils.configs.cohere2 import Cohere2Config
3 from vllm.transformers_utils.configs.dbrx import DbrxConfig
4 +from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
5 from vllm.transformers_utils.configs.eagle import EAGLEConfig
6 from vllm.transformers_utils.configs.exaone import ExaoneConfig
7 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -25,6 +26,7 @@ __all__ = [
26 "ChatGLMConfig",
27 "Cohere2Config",
28 "DbrxConfig",
29 + "DeepseekVLV2Config",
30 "MPTConfig",
31 "RWConfig",
32 "H2OVLChatConfig",
vllm/transformers_utils/configs/deepseek_vl2.py new
+214
@@ -0,0 +1,214 @@
1 +# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268
2 +from typing import Tuple
3 +
4 +from transformers.configuration_utils import PretrainedConfig
5 +
6 +
7 +class VisionEncoderConfig(PretrainedConfig):
8 + model_type: str = "vision"
9 +
10 + model_name: str = "vit_so400m_patch14_siglip_384.webli"
11 + image_size: int = 384
12 + patch_size: int = 16
13 + width: int = 1024
14 + layers: int = 24
15 + heads: int = 16
16 + mlp_ratio: int = 4
17 + global_pool: str = "map"
18 + ignore_head: bool = True
19 + class_token: bool = False
20 + num_classes: int = 0
21 + use_checkpoint: bool = False
22 + weight_init: str = "skip"
23 + deterministic: bool = False
24 + num_recomputing_layers: int = 0
25 +
26 + def __init__(self,
27 + model_name: str = "vit_so400m_patch14_siglip_384.webli",
28 + image_size: int = 384,
29 + patch_size: int = 16,
30 + width: int = 1024,
31 + layers: int = 24,
32 + heads: int = 16,
33 + mlp_ratio: int = 4,
34 + global_pool: str = "map",
35 + ignore_head: bool = True,
36 + class_token: bool = False,
37 + num_classes: int = 0,
38 + use_checkpoint: bool = False,
39 + **kwargs):
40 + self.model_name = model_name
41 + self.image_size = image_size
42 + self.patch_size = patch_size
43 + self.width = width
44 + self.layers = layers
45 + self.heads = heads
46 + self.mlp_ratio = mlp_ratio
47 + self.global_pool = global_pool
48 + self.ignore_head = ignore_head
49 + self.class_token = class_token
50 + self.num_classes = num_classes
51 + self.use_checkpoint = use_checkpoint
52 +
53 + super().__init__(**kwargs)
54 +
55 +
56 +class MlpProjectorConfig(PretrainedConfig):
57 + model_type = "mlp_projector"
58 + projector_type: str = "downsample_mlp_gelu"
59 + input_dim: int = 1152
60 + n_embed: int = 2048
61 + depth: int = 2
62 + mlp_ratio: int = 1
63 + downsample_ratio: int = 2
64 + token_pooling: bool = False
65 +
66 + def __init__(self,
67 + projector_type: str = "downsample_mlp_gelu",
68 + input_dim: int = 1152,
69 + n_embed: int = 2048,
70 + depth: int = 2,
71 + mlp_ratio: int = 1,
72 + downsample_ratio: int = 2,
73 + **kwargs):
74 + self.projector_type = projector_type
75 + self.input_dim = input_dim
76 + self.n_embed = n_embed
77 + self.depth = depth
78 + self.mlp_ratio = mlp_ratio
79 + self.downsample_ratio = downsample_ratio
80 +
81 + super().__init__(**kwargs)
82 +
83 +
84 +class DeepseekV2Config(PretrainedConfig):
85 +
86 + model_type = "deepseek_v2"
87 + keys_to_ignore_at_inference = ["past_key_values"]
88 +
89 + def __init__(
90 + self,
91 + vocab_size=102400,
92 + hidden_size=4096,
93 + intermediate_size=11008,
94 + moe_intermediate_size=1407,
95 + num_hidden_layers=30,
96 + num_attention_heads=32,
97 + num_key_value_heads=32,
98 + n_shared_experts=None,
99 + n_routed_experts=None,
100 + ep_size=1,
101 + routed_scaling_factor=1.0,
102 + kv_lora_rank=512,
103 + q_lora_rank=1536,
104 + qk_rope_head_dim=64,
105 + v_head_dim=128,
106 + qk_nope_head_dim=128,
107 + topk_method='gready',
108 + n_group=None,
109 + topk_group=None,
110 + num_experts_per_tok=None,
111 + moe_layer_freq=1,
112 + first_k_dense_replace=0,
113 + norm_topk_prob=False,
114 + scoring_func='softmax',
115 + aux_loss_alpha=0.001,
116 + seq_aux=True,
117 + hidden_act="silu",
118 + max_position_embeddings=2048,
119 + initializer_range=0.02,
120 + rms_norm_eps=1e-6,
121 + use_cache=True,
122 + pad_token_id=None,
123 + bos_token_id=100000,
124 + eos_token_id=100001,
125 + pretraining_tp=1,
126 + tie_word_embeddings=False,
127 + rope_theta=10000.0,
128 + rope_scaling=None,
129 + attention_bias=False,
130 + attention_dropout=0.0,
131 + use_mla=True,
132 + **kwargs,
133 + ):
134 + self.vocab_size = vocab_size
135 + self.max_position_embeddings = max_position_embeddings
136 + self.hidden_size = hidden_size
137 + self.intermediate_size = intermediate_size
138 + self.moe_intermediate_size = moe_intermediate_size
139 + self.num_hidden_layers = num_hidden_layers
140 + self.num_attention_heads = num_attention_heads
141 + self.n_shared_experts = n_shared_experts
142 + self.n_routed_experts = n_routed_experts
143 + self.ep_size = ep_size
144 + self.routed_scaling_factor = routed_scaling_factor
145 + self.kv_lora_rank = kv_lora_rank
146 + self.q_lora_rank = q_lora_rank
147 + self.qk_rope_head_dim = qk_rope_head_dim
148 + self.v_head_dim = v_head_dim
149 + self.qk_nope_head_dim = qk_nope_head_dim
150 + self.topk_method = topk_method
151 + self.n_group = n_group
152 + self.topk_group = topk_group
153 + self.num_experts_per_tok = num_experts_per_tok
154 + self.moe_layer_freq = moe_layer_freq
155 + self.first_k_dense_replace = first_k_dense_replace
156 + self.norm_topk_prob = norm_topk_prob
157 + self.scoring_func = scoring_func
158 + self.aux_loss_alpha = aux_loss_alpha
159 + self.seq_aux = seq_aux
160 + # for backward compatibility
161 + if num_key_value_heads is None:
162 + num_key_value_heads = num_attention_heads
163 +
164 + self.num_key_value_heads = num_key_value_heads
165 + self.hidden_act = hidden_act
166 + self.initializer_range = initializer_range
167 + self.rms_norm_eps = float(rms_norm_eps)
168 + self.pretraining_tp = pretraining_tp
169 + self.use_cache = use_cache
170 + self.rope_theta = rope_theta
171 + self.rope_scaling = rope_scaling
172 + self.attention_bias = attention_bias
173 + self.attention_dropout = attention_dropout
174 + self.use_mla = use_mla
175 +
176 + super().__init__(
177 + pad_token_id=pad_token_id,
178 + bos_token_id=bos_token_id,
179 + eos_token_id=eos_token_id,
180 + tie_word_embeddings=tie_word_embeddings,
181 + **kwargs,
182 + )
183 +
184 +
185 +class DeepseekVLV2Config(PretrainedConfig):
186 + model_type = "deepseek_vl_v2"
187 + vision_config: VisionEncoderConfig
188 + projector_config: MlpProjectorConfig
189 +
190 + tile_tag: str = "2D"
191 + global_view_pos: str = "head"
192 + candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384), )
193 +
194 + def __init__(self,
195 + tile_tag: str = "tile_tag",
196 + global_view_pos: str = "head",
197 + candidate_resolutions: Tuple[Tuple[int,
198 + int]] = ((384, 384), ),
199 + **kwargs):
200 + super().__init__(**kwargs)
201 +
202 + vision_config = kwargs.get("vision_config", {})
203 + self.vision_config = VisionEncoderConfig(**vision_config)
204 +
205 + projector_config = kwargs.get("projector_config", {})
206 + self.projector_config = MlpProjectorConfig(**projector_config)
207 +
208 + language_config = kwargs.get("language_config", {})
209 + self.text_config = DeepseekV2Config(**language_config)
210 +
211 + self.tile_tag = tile_tag
212 + self.global_view_pos = global_view_pos
213 + self.candidate_resolutions = candidate_resolutions
214 + self.vocab_size = self.text_config.vocab_size