A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model] Initialize support for Deepseek-VL2 models (#11578)
Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Isotr0py committed
Jan 12, 2025 at 16:17 UTC
f967e51f386404c7ead21d3c59ddc195cf946975
17 files changed
+1050
-9
.buildkite/test-pipeline.yaml
+1
@@ -52,6 +52,7 @@ steps:
52
- tests/worker
53
- tests/standalone_tests/lazy_torch_compile.py
54
commands:
55
+ - pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git # Used by multimoda processing test
56
- python3 standalone_tests/lazy_torch_compile.py
57
- pytest -v -s mq_llm_engine # MQLLMEngine
58
- pytest -v -s async_engine # AsyncLLMEngine
docs/source/models/supported_models.md
+19
-1
@@ -610,6 +610,13 @@ See [this page](#generative-models) for more information on how to use generativ
610
-
611
- ✅︎
612
- ✅︎
613
+* - `DeepseekVLV2ForCausalLM`
614
+ - DeepSeek-VL2
615
+ - T + I<sup>+</sup>
616
+ - `deepseek-ai/deepseek-vl2-tiny`(WIP), `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
617
+ -
618
+ - ✅︎
619
+ - ✅︎
620
* - `FuyuForCausalLM`
621
- Fuyu
622
- T + I
@@ -755,8 +762,19 @@ See [this page](#generative-models) for more information on how to use generativ
762
<sup>E</sup> Pre-computed embeddings can be inputted for this modality.
763
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
764
765
+````{note}
766
+The `deepseek-ai/deepseek-vl2-tiny` is not supported yet.
767
+
768
+To use `DeepSeek-VL2` series models, you need to install a fork version `deepseek_vl2` package:
769
+```shell
770
+pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git
771
+```
772
+
773
+Besides, to run `DeepSeek-VL2` series models, you have to pass `--hf_overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'` when running vLLM.
774
+````
775
+
776
```{note}
759
-To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
777
+To use `TIGER-Lab/Mantis-8B-siglip-llama3`, you have to pass `--hf_overrides '{"architectures": ["MantisForConditionalGeneration"]}'` when running vLLM.
778
```
779
780
```{note}
examples/offline_inference/vision_language.py
+18
@@ -66,6 +66,23 @@ def run_chameleon(question: str, modality: str):
66
return llm, prompt, stop_token_ids
67
68
69
+# Deepseek-VL2
70
+def run_deepseek_vl2(question: str, modality: str):
71
+ assert modality == "image"
72
+
73
+ model_name = "deepseek-ai/deepseek-vl2-small"
74
+
75
+ llm = LLM(model=model_name,
76
+ max_model_len=4096,
77
+ max_num_seqs=2,
78
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
79
+ hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]})
80
+
81
+ prompt = f"<|User|>: <image>\n{question}\n\n<|Assistant|>:"
82
+ stop_token_ids = None
83
+ return llm, prompt, stop_token_ids
84
+
85
+
86
# Fuyu
87
def run_fuyu(question: str, modality: str):
88
assert modality == "image"
@@ -498,6 +515,7 @@ model_example_map = {
515
"aria": run_aria,
516
"blip-2": run_blip2,
517
"chameleon": run_chameleon,
518
+ "deepseek_vl_v2": run_deepseek_vl2,
519
"fuyu": run_fuyu,
520
"glm4v": run_glm4v,
521
"h2ovl_chat": run_h2ovl,
examples/offline_inference/vision_language_multi_image.py
+23
@@ -54,6 +54,28 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
54
)
55
56
57
+def load_deepseek_vl2(question: str, image_urls: List[str]):
58
+ model_name = "deepseek-ai/deepseek-vl2-small"
59
+
60
+ llm = LLM(model=model_name,
61
+ max_model_len=4096,
62
+ max_num_seqs=2,
63
+ hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
64
+ limit_mm_per_prompt={"image": len(image_urls)})
65
+
66
+ placeholder = "".join(f"image_{i}:<image>\n"
67
+ for i, _ in enumerate(image_urls, start=1))
68
+ prompt = f"<|User|>: {placeholder}{question}\n\n<|Assistant|>:"
69
+
70
+ return ModelRequestData(
71
+ llm=llm,
72
+ prompt=prompt,
73
+ stop_token_ids=None,
74
+ image_data=[fetch_image(url) for url in image_urls],
75
+ chat_template=None,
76
+ )
77
+
78
+
79
def load_h2onvl(question: str, image_urls: List[str]) -> ModelRequestData:
80
model_name = "h2oai/h2ovl-mississippi-2b"
81
@@ -372,6 +394,7 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
394
395
model_example_map = {
396
"aria": load_aria,
397
+ "deepseek_vl2": load_deepseek_vl2,
398
"h2ovl_chat": load_h2onvl,
399
"idefics3": load_idefics3,
400
"internvl_chat": load_internvl,
tests/models/decoder_only/vision_language/test_models.py
+27
@@ -188,6 +188,33 @@ VLM_TEST_SETTINGS = {
188
max_tokens=8,
189
dtype="bfloat16",
190
),
191
+ "deepseek_vl_v2": VLMTestInfo(
192
+ models=["deepseek-ai/deepseek-vl2-small"],
193
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
194
+ dtype="bfloat16",
195
+ prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
196
+ max_model_len=4096,
197
+ max_num_seqs=2,
198
+ single_image_prompts=IMAGE_ASSETS.prompts({
199
+ "stop_sign": "<image>\nWhat's the color of the stop sign and car?",
200
+ "cherry_blossom": "<image>\nWhat's the color of the tower?",
201
+ }),
202
+ multi_image_prompt="image_1:<image>\nimage_2:<image>\nDescribe the two images shortly.", # noqa: E501
203
+ vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
204
+ image_size_factors=[(0.10, 0.15)],
205
+ patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
206
+ postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
207
+ hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
208
+ stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], # noqa: E501
209
+ num_logprobs=5,
210
+ marks=[
211
+ pytest.mark.skipif(
212
+ not is_flash_attn_2_available(),
213
+ reason="Model needs flash-attn for numeric convergence.",
214
+ ),
215
+ large_gpu_mark(min_gb=48),
216
+ ],
217
+ ),
218
"fuyu": VLMTestInfo(
219
models=["adept/fuyu-8b"],
220
test_type=VLMTestType.IMAGE,
tests/models/decoder_only/vision_language/vlm_utils/model_utils.py
+36
@@ -183,6 +183,14 @@ def paligemma_vllm_to_hf_output(vllm_output: RunnerOutput,
183
184
185
####### Post-processors for HF outputs
186
+def deepseekvl2_trunc_hf_output(hf_output: RunnerOutput,
187
+ model: str) -> RunnerOutput:
188
+ output_ids, output_str, out_logprobs = hf_output
189
+ if output_str.endswith("<|end▁of▁sentence|>"):
190
+ output_str = output_str.split("<|end▁of▁sentence|>")[0]
191
+ return output_ids, output_str, out_logprobs
192
+
193
+
194
def minicpmv_trunc_hf_output(hf_output: RunnerOutput,
195
model: str) -> RunnerOutput:
196
output_ids, output_str, out_logprobs = hf_output
@@ -261,6 +269,34 @@ def qwen_prompt_path_encoder(
269
270
271
####### Model-specific HuggingFace runner patchers
272
+def deepseekvl2_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
273
+ """Patches and returns an instance of the HfRunner to use for GLM4."""
274
+ hf_processor = hf_model.processor
275
+
276
+ def processor(*args, text="", images=None, **kwargs):
277
+ if isinstance(images, Image):
278
+ images = [images]
279
+ # inputs is a custom class instead of dict or BatchFeature
280
+ inputs = hf_processor(
281
+ *args,
282
+ prompt=text,
283
+ images=images,
284
+ **kwargs,
285
+ )
286
+ inputs = {
287
+ k: inputs[k]
288
+ for k in inputs.keys() # noqa
289
+ if k not in ("seq_lens", "sft_format")
290
+ }
291
+ inputs = BatchEncoding(data=inputs, tensor_type="pt")
292
+ return inputs
293
+
294
+ hf_model.processor = processor
295
+ hf_model.model.get_output_embeddings = lambda: \
296
+ hf_model.model.language.model.embed_tokens
297
+ return hf_model
298
+
299
+
300
def glm_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
301
"""Patches and returns an instance of the HfRunner to use for GLM4."""
302
hf_processor = hf_model.processor
tests/models/registry.py
+2
@@ -179,6 +179,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
179
trust_remote_code=True),
180
"ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
181
is_available_online=False),
182
+ # TODO(Isotr0py): Use deepseek-vl2-tiny for test after it's supported
183
+ "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-small"), # noqa: E501
184
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
185
"H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
186
"InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
tests/models/test_initialization.py
+3
@@ -26,6 +26,9 @@ def test_can_initialize(model_arch):
26
27
# Avoid OOM
28
def hf_overrides(hf_config: PretrainedConfig) -> PretrainedConfig:
29
+ if hf_config.model_type == "deepseek_vl_v2":
30
+ hf_config.update({"architectures": ["DeepseekVLV2ForCausalLM"]})
31
+
32
if hasattr(hf_config, "text_config"):
33
text_config: PretrainedConfig = hf_config.text_config
34
else:
vllm/entrypoints/chat_utils.py
+2
-2
@@ -403,8 +403,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
403
if model_type.startswith("llava"):
404
return self._cached_token_str(self._tokenizer,
405
hf_config.image_token_index)
406
- if model_type in ("chameleon", "internvl_chat", "NVLM_D",
407
- "h2ovl_chat"):
406
+ if model_type in ("chameleon", "deepseek_vl_v2", "internvl_chat",
407
+ "NVLM_D", "h2ovl_chat"):
408
return "<image>"
409
if model_type == "mllama":
410
return "<|image|>"
vllm/model_executor/models/deepseek_v2.py
+17
-1
@@ -243,7 +243,11 @@ class DeepseekV2Attention(nn.Module):
243
bias=False,
244
quant_config=quant_config,
245
prefix=f"{prefix}.o_proj")
246
- rope_scaling["rope_type"] = 'deepseek_yarn'
246
+ if rope_scaling:
247
+ rope_scaling["rope_type"] = 'deepseek_yarn'
248
+ self.use_normal_rope = False
249
+ else:
250
+ self.use_normal_rope = True
251
self.rotary_emb = get_rope(qk_rope_head_dim,
252
rotary_dim=qk_rope_head_dim,
253
max_position=max_position_embeddings,
@@ -298,7 +302,18 @@ class DeepseekV2Attention(nn.Module):
302
self.qk_nope_head_dim + self.v_head_dim)
303
k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
304
k_pe = latent_cache[:, :, self.kv_lora_rank:]
305
+
306
+ if self.use_normal_rope:
307
+ seq_len = positions.size(0)
308
+ ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
309
+ q_pe = q_pe.reshape(seq_len, -1)
310
+ k_pe = k_pe.reshape(seq_len, -1)
311
+
312
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
313
+
314
+ if self.use_normal_rope:
315
+ q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
316
+
317
q[..., self.qk_nope_head_dim:] = q_pe
318
k = torch.empty_like(q)
319
k[..., :self.qk_nope_head_dim] = k_nope
@@ -355,6 +370,7 @@ class DeepseekV2DecoderLayer(nn.Module):
370
quant_config=quant_config,
371
prefix=f"{prefix}.self_attn",
372
)
373
+
374
if (config.n_routed_experts is not None
375
and layer_idx >= config.first_k_dense_replace
376
and layer_idx % config.moe_layer_freq == 0):
vllm/model_executor/models/deepseek_v3.py
+18
-2
@@ -251,7 +251,11 @@ class DeepseekV3Attention(nn.Module):
251
bias=False,
252
quant_config=quant_config,
253
prefix=f"{prefix}.o_proj")
254
- rope_scaling["rope_type"] = 'deepseek_yarn'
254
+ if rope_scaling:
255
+ rope_scaling["rope_type"] = 'deepseek_yarn'
256
+ self.use_normal_rope = False
257
+ else:
258
+ self.use_normal_rope = True
259
self.rotary_emb = get_rope(qk_rope_head_dim,
260
rotary_dim=qk_rope_head_dim,
261
max_position=max_position_embeddings,
@@ -306,7 +310,18 @@ class DeepseekV3Attention(nn.Module):
310
self.qk_nope_head_dim + self.v_head_dim)
311
k_nope, v = kv.split([self.qk_nope_head_dim, self.v_head_dim], dim=-1)
312
k_pe = latent_cache[:, :, self.kv_lora_rank:]
313
+
314
+ if self.use_normal_rope:
315
+ seq_len = positions.size(0)
316
+ ori_q_pe_shape, ori_k_pe_shape = q_pe.shape, k_pe.shape
317
+ q_pe = q_pe.reshape(seq_len, -1)
318
+ k_pe = k_pe.reshape(seq_len, -1)
319
+
320
q_pe, k_pe = self.rotary_emb(positions, q_pe, k_pe)
321
+
322
+ if self.use_normal_rope:
323
+ q_pe, k_pe = q_pe.view(ori_q_pe_shape), k_pe.view(ori_k_pe_shape)
324
+
325
q[..., self.qk_nope_head_dim:] = q_pe
326
k = torch.empty_like(q)
327
k[..., :self.qk_nope_head_dim] = k_nope
@@ -583,7 +598,8 @@ class DeepseekV3ForCausalLM(nn.Module, SupportsPP):
598
continue
599
600
# TODO(simon): support nextn predict layers
586
- if self.config.num_nextn_predict_layers > 0:
601
+ if hasattr(self.config, "num_nextn_predict_layers"
602
+ ) and self.config.num_nextn_predict_layers > 0:
603
assert self.config.num_nextn_predict_layers == 1
604
layer_idx = self.config.num_hidden_layers
605
if name.startswith(f"model.layers.{layer_idx}"):
vllm/model_executor/models/deepseek_vl2.py
new
+662
@@ -0,0 +1,662 @@
1
+# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py
2
+"""Inference-only Deepseek-VL2 model compatible with HuggingFace weights."""
3
+import math
4
+from functools import cached_property, partial
5
+from typing import (Iterable, List, Literal, Mapping, Optional, Set, Tuple,
6
+ TypedDict, Union)
7
+
8
+import torch
9
+import torch.nn as nn
10
+import torch.nn.functional as F
11
+from einops import rearrange, repeat
12
+from transformers import AutoProcessor, BatchFeature, ProcessorMixin
13
+
14
+from vllm.attention import AttentionMetadata
15
+from vllm.config import VllmConfig
16
+from vllm.logger import init_logger
17
+from vllm.model_executor import SamplingMetadata
18
+from vllm.model_executor.layers.quantization import QuantizationConfig
19
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
20
+from vllm.model_executor.model_loader.utils import set_default_torch_dtype
21
+from vllm.multimodal import MULTIMODAL_REGISTRY
22
+from vllm.multimodal.inputs import (MultiModalFieldConfig, MultiModalKwargs,
23
+ NestedTensors)
24
+from vllm.multimodal.parse import (ImageEmbeddingItems, ImageProcessorItems,
25
+ ImageSize, MultiModalDataItems)
26
+from vllm.multimodal.processing import (BaseMultiModalProcessor,
27
+ BaseProcessingInfo, PromptReplacement)
28
+from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs
29
+from vllm.multimodal.utils import cached_get_tokenizer
30
+from vllm.sequence import IntermediateTensors
31
+from vllm.transformers_utils.configs.deepseek_vl2 import (DeepseekVLV2Config,
32
+ MlpProjectorConfig,
33
+ VisionEncoderConfig)
34
+from vllm.utils import is_list_of
35
+
36
+from .interfaces import SupportsMultiModal, SupportsPP
37
+from .utils import (AutoWeightsLoader, WeightsMapper, flatten_bn,
38
+ init_vllm_registered_model, maybe_prefix,
39
+ merge_multimodal_embeddings)
40
+
41
+logger = init_logger(__name__)
42
+
43
+# The image token id may be various
44
+_IMAGE_TOKEN = "<image>"
45
+
46
+
47
+class DeepseekVL2ImagePixelInputs(TypedDict):
48
+ type: Literal["pixel_values"]
49
+ data: Union[torch.Tensor, List[torch.Tensor]]
50
+ """
51
+ Shape: `(batch_size * num_images, num_channels, height, width)`
52
+ """
53
+ images_spatial_crop: torch.Tensor
54
+ """
55
+ Shape: `(batch_size * num_images, 2)`
56
+ """
57
+
58
+
59
+class DeepseekVL2VImageEmbeddingInputs(TypedDict):
60
+ type: Literal["image_embeds"]
61
+ data: Union[torch.Tensor, List[torch.Tensor]]
62
+ """Shape: `(batch_size * num_images, image_feature_size, hidden_size)`
63
+
64
+ `hidden_size` must match the hidden size of language model backbone.
65
+ """
66
+
67
+
68
+DeepseekVL2ImageInputs = Union[DeepseekVL2ImagePixelInputs,
69
+ DeepseekVL2VImageEmbeddingInputs]
70
+
71
+
72
+class MlpProjector(nn.Module):
73
+
74
+ def __init__(self, cfg: MlpProjectorConfig):
75
+
76
+ super().__init__()
77
+
78
+ self.cfg = cfg
79
+ assert not cfg.token_pooling, (
80
+ "Token pooling is not supported currently.")
81
+
82
+ if cfg.projector_type == "downsample_mlp_gelu":
83
+ mlp_depth = cfg.depth
84
+ mlp_ratio = cfg.mlp_ratio
85
+ modules = [
86
+ nn.Linear(
87
+ cfg.input_dim * cfg.downsample_ratio *
88
+ cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
89
+ ]
90
+ for _ in range(1, mlp_depth - 1):
91
+ modules.append(nn.GELU())
92
+ modules.append(
93
+ nn.Linear(cfg.n_embed * mlp_ratio,
94
+ cfg.n_embed * mlp_ratio))
95
+ modules.append(nn.GELU())
96
+ modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
97
+ modules = nn.Sequential(*modules)
98
+
99
+ else:
100
+ raise NotImplementedError(
101
+ f"Unsupported projector type: {cfg.projector_type}")
102
+
103
+ self.layers = modules
104
+
105
+ def forward(self, x):
106
+ bs, hw, input_dim = x.shape
107
+ h = w = int((hw)**0.5)
108
+ """compute padding"""
109
+ if h % self.cfg.downsample_ratio:
110
+ pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
111
+ else:
112
+ pad = 0
113
+ x = x.reshape(bs, h, w, input_dim)
114
+ if pad > 0:
115
+ x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
116
+ """4 to 1 concat"""
117
+ x = x.permute(0, 3, 1, 2) # B, C, H, W
118
+ x = F.unfold(x,
119
+ kernel_size=self.cfg.downsample_ratio,
120
+ stride=self.cfg.downsample_ratio,
121
+ padding=0) # B, C*4, HW // 4
122
+ x = x.permute(0, 2, 1)
123
+
124
+ return self.layers(x)
125
+
126
+
127
+class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
128
+
129
+ def get_hf_config(self):
130
+ return self.ctx.get_hf_config(DeepseekVLV2Config)
131
+
132
+ def get_hf_processor(self) -> ProcessorMixin:
133
+ # TODO(Isotr0py): we should get rid of dependency on deepseek_vl2
134
+ # in the future, because it's flasky and lack of maintenance.
135
+ try:
136
+ from deepseek_vl2.models.processing_deepseek_vl_v2 import (
137
+ DeepseekVLV2Processor, select_best_resolution)
138
+ AutoProcessor.register("DeepseekVLV2Processor",
139
+ DeepseekVLV2Processor)
140
+ except ModuleNotFoundError as exc:
141
+ raise ModuleNotFoundError(
142
+ "You need to `pip install "
143
+ "git+https://github.com/deepseek-ai/DeepSeek-VL2.git` "
144
+ "to use this model") from exc
145
+
146
+ processor = self.ctx.get_hf_processor(DeepseekVLV2Processor)
147
+ processor.select_best_resolution = partial(
148
+ select_best_resolution,
149
+ candidate_resolutions=processor.candidate_resolutions)
150
+ return processor
151
+
152
+ def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
153
+ return {"image": None}
154
+
155
+ def get_num_image_tokens(self, *, image_width: int,
156
+ image_height: int) -> int:
157
+ hf_processor = self.get_hf_processor()
158
+ image_size = hf_processor.image_size
159
+ patch_size = hf_processor.patch_size
160
+ downsample_ratio = hf_processor.downsample_ratio
161
+
162
+ best_width, best_height = hf_processor.select_best_resolution(
163
+ (image_width, image_height))
164
+
165
+ num_width_tiles, num_height_tiles = (best_width // image_size,
166
+ best_height // image_size)
167
+ h = w = math.ceil((image_size // patch_size) / downsample_ratio)
168
+
169
+ global_views_tokens = h * (w + 1)
170
+ local_views_tokens = (num_height_tiles * h) * (num_width_tiles * w + 1)
171
+ return global_views_tokens + local_views_tokens + 1
172
+
173
+ def get_image_size_with_most_features(self) -> ImageSize:
174
+ hf_config = self.get_hf_config()
175
+ candidate_resolutions = hf_config.candidate_resolutions
176
+ height, width = max(candidate_resolutions,
177
+ key=lambda x: self.get_num_image_tokens(
178
+ image_width=x[1], image_height=x[0]))
179
+ return ImageSize(width=width, height=height)
180
+
181
+ def get_mm_max_tokens_per_item(self, seq_len: int) -> Mapping[str, int]:
182
+ max_image_size = self.get_image_size_with_most_features()
183
+ max_image_tokens = self.get_num_image_tokens(
184
+ image_height=max_image_size.height,
185
+ image_width=max_image_size.width)
186
+
187
+ return {"image": max_image_tokens}
188
+
189
+
190
+class DeepseekVL2DummyInputsBuilder(
191
+ BaseDummyInputsBuilder[DeepseekVL2ProcessingInfo]):
192
+
193
+ def get_dummy_processor_inputs(
194
+ self,
195
+ seq_len: int,
196
+ mm_counts: Mapping[str, int],
197
+ ) -> ProcessorInputs:
198
+ num_images = mm_counts.get("image", 0)
199
+ hf_processor = self.info.get_hf_processor()
200
+ image_token: str = hf_processor.image_token
201
+
202
+ max_image_size = self.info.get_image_size_with_most_features()
203
+
204
+ mm_data = {
205
+ "image":
206
+ self._get_dummy_images(width=max_image_size.width,
207
+ height=max_image_size.height,
208
+ num_images=num_images)
209
+ }
210
+
211
+ return ProcessorInputs(
212
+ prompt_text=image_token * num_images,
213
+ mm_data=mm_data,
214
+ )
215
+
216
+
217
+class DeepseekVL2MultiModalProcessor(
218
+ BaseMultiModalProcessor[DeepseekVL2ProcessingInfo]):
219
+
220
+ def _call_hf_processor(
221
+ self,
222
+ prompt: str,
223
+ mm_data: Mapping[str, object],
224
+ mm_kwargs: Mapping[str, object],
225
+ ) -> BatchFeature:
226
+ if mm_data:
227
+ outputs = self.info.ctx.call_hf_processor(
228
+ self.info.get_hf_processor(**mm_kwargs),
229
+ dict(prompt=prompt, **mm_data),
230
+ mm_kwargs,
231
+ )
232
+
233
+ # Deepseek-vl2 processor don't return BatchFeature,
234
+ # we need to manually create it
235
+ processed_outputs = dict(input_ids=outputs["input_ids"])
236
+ processed_outputs = BatchFeature(data=dict(processed_outputs),
237
+ tensor_type="pt")
238
+
239
+ # Remove batch dimension from processor outputs,
240
+ # because we will try batch to create NestedTensors
241
+ target_dtype = self.info.ctx.model_config.dtype
242
+ pixel_values = outputs["images"].to(target_dtype).squeeze(0)
243
+ images_spatial_crop = outputs["images_spatial_crop"].squeeze(0)
244
+ patches_per_image = [
245
+ x.prod().item() + 1 for x in images_spatial_crop
246
+ ]
247
+
248
+ # Rename `images` -> `pixel_values` to avoid confusion
249
+ processed_outputs["pixel_values"] = list(
250
+ pixel_values.split(patches_per_image))
251
+ processed_outputs["images_spatial_crop"] = images_spatial_crop
252
+ else:
253
+ tokenizer = self.info.get_tokenizer()
254
+ processed_outputs = tokenizer(prompt,
255
+ add_special_tokens=True,
256
+ return_tensors="pt")
257
+
258
+ return processed_outputs
259
+
260
+ def _get_mm_fields_config(
261
+ self,
262
+ hf_inputs: BatchFeature,
263
+ hf_processor_mm_kwargs: Mapping[str, object],
264
+ ) -> Mapping[str, MultiModalFieldConfig]:
265
+ return dict(
266
+ pixel_values=MultiModalFieldConfig.batched("image"),
267
+ images_spatial_crop=MultiModalFieldConfig.batched("image"),
268
+ image_embeds=MultiModalFieldConfig.batched("image"),
269
+ )
270
+
271
+ def _get_prompt_replacements(
272
+ self,
273
+ mm_items: MultiModalDataItems,
274
+ hf_processor_mm_kwargs: Mapping[str, object],
275
+ out_mm_kwargs: MultiModalKwargs,
276
+ ) -> list[PromptReplacement]:
277
+ hf_processor = self.info.get_hf_processor()
278
+ image_token_id: int = hf_processor.image_token_id
279
+
280
+ def get_replacement_deepseek_vl2(item_idx: int):
281
+ images = mm_items.get_items(
282
+ "image", (ImageEmbeddingItems, ImageProcessorItems))
283
+
284
+ if isinstance(images, ImageEmbeddingItems):
285
+ num_image_tokens = images.get_feature_size(item_idx)
286
+ else:
287
+ image_size = images.get_image_size(item_idx)
288
+
289
+ num_image_tokens = self.info.get_num_image_tokens(
290
+ image_width=image_size.width,
291
+ image_height=image_size.height,
292
+ )
293
+ return [image_token_id] * num_image_tokens
294
+
295
+ return [
296
+ PromptReplacement(
297
+ modality="image",
298
+ target=[image_token_id],
299
+ replacement=get_replacement_deepseek_vl2,
300
+ )
301
+ ]
302
+
303
+
304
+@MULTIMODAL_REGISTRY.register_processor(
305
+ DeepseekVL2MultiModalProcessor,
306
+ info=DeepseekVL2ProcessingInfo,
307
+ dummy_inputs=DeepseekVL2DummyInputsBuilder)
308
+class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
309
+
310
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
311
+ "language.": "language_model.",
312
+ })
313
+
314
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
315
+ super().__init__()
316
+ config: DeepseekVLV2Config = vllm_config.model_config.hf_config
317
+ quant_config = vllm_config.quant_config
318
+ multimodal_config = vllm_config.model_config.multimodal_config
319
+
320
+ self.config = config
321
+ self.multimodal_config = multimodal_config
322
+
323
+ self.vision_config = config.vision_config
324
+ self.projector_config = config.projector_config
325
+ self.text_config = config.text_config
326
+
327
+ model_config = vllm_config.model_config
328
+ tokenizer = cached_get_tokenizer(
329
+ model_config.tokenizer,
330
+ tokenizer_mode=model_config.tokenizer_mode,
331
+ tokenizer_revision=model_config.tokenizer_revision,
332
+ trust_remote_code=model_config.trust_remote_code,
333
+ )
334
+ self.image_token_id = tokenizer.vocab.get(_IMAGE_TOKEN)
335
+
336
+ self.vision = self._init_vision_module(self.vision_config,
337
+ quant_config,
338
+ maybe_prefix(prefix, "vision"))
339
+
340
+ self.projector = MlpProjector(self.projector_config)
341
+ self.tile_tag = config.tile_tag
342
+ self.global_view_pos = config.global_view_pos
343
+
344
+ # special token for image token sequence format
345
+ embed_std = 1 / torch.sqrt(
346
+ torch.tensor(self.projector_config.n_embed, dtype=torch.float32))
347
+ if self.tile_tag == "2D":
348
+ # <|view_separator|>, <|\n|>
349
+ self.image_newline = nn.Parameter(
350
+ torch.randn(self.projector_config.n_embed) * embed_std)
351
+ # This is a typo in original implementation
352
+ self.view_seperator = nn.Parameter(
353
+ torch.randn(self.projector_config.n_embed) * embed_std)
354
+ else:
355
+ raise ValueError(
356
+ f"Only 2D tile_tag is supported currently, got: {self.tile_tag}"
357
+ )
358
+
359
+ self.language_model = init_vllm_registered_model(
360
+ vllm_config=vllm_config,
361
+ hf_config=self.text_config,
362
+ prefix=maybe_prefix(prefix, "language"),
363
+ architectures=["DeepseekV3ForCausalLM"]
364
+ if self.text_config.topk_method == "noaux_tc" else
365
+ ["DeepseekV2ForCausalLM"],
366
+ )
367
+
368
+ self.make_empty_intermediate_tensors = (
369
+ self.language_model.make_empty_intermediate_tensors)
370
+
371
+ def _init_vision_module(
372
+ self,
373
+ vision_config: VisionEncoderConfig,
374
+ quant_config: Optional[QuantizationConfig],
375
+ prefix: str = "",
376
+ ) -> nn.Module:
377
+ # TODO: refactor vision model through timm wrapper from transformers
378
+ try:
379
+ import timm
380
+ except ImportError:
381
+ raise ImportError("Please install timm") from ImportError
382
+
383
+ with set_default_torch_dtype(torch.float16):
384
+ model = timm.create_model(
385
+ "vit_so400m_patch14_siglip_384.webli",
386
+ pretrained=False,
387
+ num_classes=0,
388
+ dynamic_img_size=True,
389
+ dynamic_img_pad=True,
390
+ )
391
+
392
+ model = model.to(dtype=torch.get_default_dtype())
393
+ return model
394
+
395
+ @cached_property
396
+ def sampler(self):
397
+ if hasattr(self.language_model, "sampler"):
398
+ return self.language_model.sampler
399
+
400
+ return get_sampler()
401
+
402
+ def _validate_pixel_values(
403
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
404
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
405
+
406
+ h = w = self.vision_config.image_size
407
+ expected_dims = (3, h, w)
408
+
409
+ def _validate_shape(d: torch.Tensor):
410
+ actual_dims = tuple(d.shape[1:])
411
+
412
+ if actual_dims != expected_dims:
413
+ expected_expr = ("num_patches", *map(str, expected_dims))
414
+ raise ValueError(
415
+ "The expected shape of pixel values per image per batch "
416
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
417
+
418
+ for d in data:
419
+ _validate_shape(d)
420
+
421
+ return data
422
+
423
+ def _validate_images_spatial_crop(
424
+ self, data: Union[torch.Tensor, List[torch.Tensor]]
425
+ ) -> Union[torch.Tensor, List[torch.Tensor]]:
426
+ expected_dims = 2
427
+
428
+ def _validate_shape(d: torch.Tensor):
429
+ actual_dims = d.size(-1)
430
+
431
+ if actual_dims != expected_dims:
432
+ expected_expr = str(expected_dims)
433
+ raise ValueError(
434
+ f"The expected shape of image sizes per image per batch "
435
+ f"is {expected_expr}. You supplied {tuple(d.shape)}.")
436
+
437
+ for d in data:
438
+ _validate_shape(d)
439
+
440
+ return data
441
+
442
+ def _parse_and_validate_image_input(
443
+ self, **kwargs: object) -> Optional[DeepseekVL2ImageInputs]:
444
+ pixel_values = kwargs.pop("pixel_values", None)
445
+ images_spatial_crop = kwargs.pop("images_spatial_crop", None)
446
+ image_embeds = kwargs.pop("image_embeds", None)
447
+
448
+ if pixel_values is None and image_embeds is None:
449
+ return None
450
+
451
+ if pixel_values is not None:
452
+ if not isinstance(pixel_values, (torch.Tensor, list)):
453
+ raise ValueError("Incorrect type of pixel values. "
454
+ f"Got type: {type(pixel_values)}")
455
+
456
+ if not isinstance(images_spatial_crop, (torch.Tensor, list)):
457
+ raise ValueError("Incorrect type of image sizes. "
458
+ f"Got type: {type(images_spatial_crop)}")
459
+
460
+ return DeepseekVL2ImagePixelInputs(
461
+ type="pixel_values",
462
+ data=self._validate_pixel_values(flatten_bn(pixel_values)),
463
+ images_spatial_crop=self._validate_images_spatial_crop(
464
+ flatten_bn(images_spatial_crop, concat=True)))
465
+
466
+ if image_embeds is not None:
467
+ if not isinstance(image_embeds, torch.Tensor):
468
+ raise ValueError("Incorrect type of image embeddings. "
469
+ f"Got type: {type(image_embeds)}")
470
+
471
+ return DeepseekVL2VImageEmbeddingInputs(
472
+ type="image_embeds",
473
+ data=flatten_bn(image_embeds),
474
+ )
475
+
476
+ raise AssertionError("This line should be unreachable.")
477
+
478
+ def _pixel_values_to_embedding(
479
+ self,
480
+ pixel_values: NestedTensors,
481
+ images_spatial_crop: torch.Tensor,
482
+ ) -> NestedTensors:
483
+ # Pixel_values: n_image * batch_size * [patch_per_img, 3, height, width]
484
+ total_tiles = [x for x in pixel_values]
485
+
486
+ # [batch_all_tiles, 3, height, width]
487
+ total_tiles = torch.cat(total_tiles, dim=0)
488
+
489
+ # [batch_all_tiles, vit_seq_len, c]
490
+ images_feature = self.vision.forward_features(total_tiles)
491
+
492
+ # [batch_all_tiles, hw, D]
493
+ images_embeds = self.projector(images_feature)
494
+
495
+ _, hw, n_dim = images_embeds.shape
496
+ h = w = int(hw**0.5)
497
+
498
+ # 根据self.tile_tag & self.global_view_pos填充image token sequence
499
+ tile_index = 0
500
+ vision_embeddings = []
501
+ for jdx in range(images_spatial_crop.size(0)):
502
+ # extra global & local features
503
+ num_width_tiles, num_height_tiles = images_spatial_crop[jdx]
504
+ if num_width_tiles == 0 or num_height_tiles == 0:
505
+ break
506
+ num_tiles_in_image = num_width_tiles * num_height_tiles
507
+
508
+ # [hw, D]
509
+ global_features = images_embeds[tile_index]
510
+
511
+ # [num_height_tiles * num_width_tiles, hw, D]
512
+ local_features = images_embeds[tile_index + 1:tile_index + 1 +
513
+ num_tiles_in_image]
514
+ tile_index += num_tiles_in_image + 1
515
+
516
+ # format global and local features
517
+ # ----------------- global view add newline -----------------
518
+ # [hw, D] -> [h, w, D]
519
+ global_features = global_features.view(h, w, n_dim)
520
+
521
+ # [D] -> [h, 1, D]
522
+ new_lines_in_global = repeat(self.image_newline, "d -> h 1 d", h=h)
523
+
524
+ # cat([h, w, D], [h, 1, D], dim=1) -> [h, w + 1, D]
525
+ global_features = torch.cat([global_features, new_lines_in_global],
526
+ dim=1)
527
+
528
+ # [h, w + 1, D] -> [h * (w + 1), D]
529
+ global_features = global_features.view(-1, n_dim)
530
+
531
+ # ----------------- local view add newline -----------------
532
+ # [num_height_tiles * num_width_tiles, h * w, D] ->
533
+ # [num_height_tiles * h, num_width_tiles * w, D]
534
+ local_features = rearrange(local_features,
535
+ "(th tw) (h w) d -> (th h) (tw w) d",
536
+ th=num_height_tiles,
537
+ tw=num_width_tiles,
538
+ h=h,
539
+ w=w)
540
+
541
+ # [D] -> [num_height_tiles * h, 1, D]
542
+ new_lines_in_local = repeat(self.image_newline,
543
+ "d -> (th h) 1 d",
544
+ th=num_height_tiles,
545
+ h=h)
546
+
547
+ # [num_height_tiles * h, num_width_tiles * w + 1, D]
548
+ local_features = torch.cat([local_features, new_lines_in_local],
549
+ dim=1)
550
+
551
+ # [num_height_tiles * h, num_width_tiles * w + 1, D]
552
+ # --> [(num_height_tiles * h) * (num_width_tiles * w + 1), D]
553
+ local_features = local_features.view(-1, n_dim)
554
+
555
+ # merge global and local tiles
556
+ if self.global_view_pos == "head":
557
+ global_local_features = torch.cat([
558
+ global_features,
559
+ self.view_seperator[None, :],
560
+ local_features,
561
+ ])
562
+ else:
563
+ global_local_features = torch.cat([
564
+ local_features,
565
+ self.view_seperator[None, :],
566
+ global_features,
567
+ ])
568
+
569
+ vision_embeddings.append(global_local_features)
570
+ return vision_embeddings
571
+
572
+ def _process_image_input(
573
+ self, image_input: DeepseekVL2ImageInputs) -> torch.Tensor:
574
+ if image_input["type"] == "image_embeds":
575
+ image_data = image_input["data"]
576
+ if is_list_of(image_data, torch.Tensor):
577
+ # it's already a list of tensors
578
+ return image_data
579
+ if len(image_data.shape) == 3:
580
+ # 3D tensor
581
+ return list(torch.unbind(image_data, dim=0))
582
+ raise ValueError(
583
+ "We expect batched 2D tensors;"
584
+ "this can be either a list of 2D tensors or a single 3D tensor."
585
+ )
586
+
587
+ pixel_values = image_input["data"]
588
+ images_spatial_crop = image_input["images_spatial_crop"]
589
+
590
+ return self._pixel_values_to_embedding(
591
+ pixel_values=pixel_values, images_spatial_crop=images_spatial_crop)
592
+
593
+ def get_multimodal_embeddings(self, **kwargs: object) -> torch.Tensor:
594
+ image_input = self._parse_and_validate_image_input(**kwargs)
595
+ if image_input is None:
596
+ return None
597
+ vision_embeddings = self._process_image_input(image_input)
598
+ return vision_embeddings
599
+
600
+ def get_input_embeddings(
601
+ self,
602
+ input_ids: torch.Tensor,
603
+ multimodal_embeddings: Optional[NestedTensors] = None,
604
+ ) -> torch.Tensor:
605
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
606
+ if multimodal_embeddings is not None:
607
+ inputs_embeds = merge_multimodal_embeddings(
608
+ input_ids, inputs_embeds, multimodal_embeddings,
609
+ self.image_token_id)
610
+ return inputs_embeds
611
+
612
+ def forward(self,
613
+ input_ids: torch.Tensor,
614
+ positions: torch.Tensor,
615
+ kv_caches: List[torch.Tensor],
616
+ attn_metadata: AttentionMetadata,
617
+ intermediate_tensors: Optional[IntermediateTensors] = None,
618
+ inputs_embeds: Optional[torch.Tensor] = None,
619
+ **kwargs: object):
620
+
621
+ if intermediate_tensors is not None:
622
+ inputs_embeds = None
623
+
624
+ # NOTE: In v1, inputs_embeds is always generated at model runner, this
625
+ # condition is for v0 compatibility
626
+ elif inputs_embeds is None:
627
+ vision_embeddings = self.get_multimodal_embeddings(**kwargs)
628
+ inputs_embeds = self.get_input_embeddings(input_ids,
629
+ vision_embeddings)
630
+ input_ids = None
631
+
632
+ hidden_states = self.language_model(input_ids,
633
+ positions,
634
+ kv_caches,
635
+ attn_metadata,
636
+ intermediate_tensors,
637
+ inputs_embeds=inputs_embeds)
638
+
639
+ return hidden_states
640
+
641
+ def compute_logits(
642
+ self,
643
+ hidden_states: torch.Tensor,
644
+ sampling_metadata: SamplingMetadata,
645
+ ) -> Optional[torch.Tensor]:
646
+ return self.language_model.compute_logits(hidden_states,
647
+ sampling_metadata)
648
+
649
+ def sample(
650
+ self,
651
+ logits: torch.Tensor,
652
+ sampling_metadata: SamplingMetadata,
653
+ ) -> Optional[SamplerOutput]:
654
+ return self.language_model.sample(logits, sampling_metadata)
655
+
656
+ def load_weights(self, weights: Iterable[Tuple[str,
657
+ torch.Tensor]]) -> Set[str]:
658
+
659
+ loader = AutoWeightsLoader(self)
660
+ autoloaded_weights = loader.load_weights(weights,
661
+ mapper=self.hf_to_vllm_mapper)
662
+ return autoloaded_weights
vllm/model_executor/models/minicpmv.py
+1
-1
@@ -657,7 +657,7 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
657
quant_config: Optional[QuantizationConfig],
658
prefix: str = "",
659
) -> nn.Module:
660
- # TODO: refactor this vision model
660
+ # TODO: refactor vision model through timm wrapper from transformers
661
try:
662
import timm
663
except ImportError:
vllm/model_executor/models/registry.py
+1
@@ -149,6 +149,7 @@ _MULTIMODAL_MODELS = {
149
"ChameleonForConditionalGeneration": ("chameleon", "ChameleonForConditionalGeneration"), # noqa: E501
150
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
151
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
152
+ "DeepseekVLV2ForCausalLM": ("deepseek_vl2", "DeepseekVLV2ForCausalLM"),
153
"FuyuForCausalLM": ("fuyu", "FuyuForCausalLM"),
154
"H2OVLChatModel": ("h2ovl", "H2OVLChatModel"),
155
"InternVLChatModel": ("internvl", "InternVLChatModel"),
vllm/transformers_utils/config.py
+4
-2
@@ -23,8 +23,9 @@ from vllm.logger import init_logger
23
# yapf conflicts with isort for this block
24
# yapf: disable
25
from vllm.transformers_utils.configs import (ChatGLMConfig, Cohere2Config,
26
- DbrxConfig, EAGLEConfig,
27
- ExaoneConfig, H2OVLChatConfig,
26
+ DbrxConfig, DeepseekVLV2Config,
27
+ EAGLEConfig, ExaoneConfig,
28
+ H2OVLChatConfig,
29
InternVLChatConfig, JAISConfig,
30
MedusaConfig, MllamaConfig,
31
MLPSpeculatorConfig, MPTConfig,
@@ -54,6 +55,7 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
55
"chatglm": ChatGLMConfig,
56
"cohere2": Cohere2Config,
57
"dbrx": DbrxConfig,
58
+ "deepseek_vl_v2": DeepseekVLV2Config,
59
"mpt": MPTConfig,
60
"RefinedWeb": RWConfig, # For tiiuae/falcon-40b(-instruct)
61
"RefinedWebModel": RWConfig, # For tiiuae/falcon-7b(-instruct)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
2
from vllm.transformers_utils.configs.cohere2 import Cohere2Config
3
from vllm.transformers_utils.configs.dbrx import DbrxConfig
4
+from vllm.transformers_utils.configs.deepseek_vl2 import DeepseekVLV2Config
5
from vllm.transformers_utils.configs.eagle import EAGLEConfig
6
from vllm.transformers_utils.configs.exaone import ExaoneConfig
7
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
@@ -25,6 +26,7 @@ __all__ = [
26
"ChatGLMConfig",
27
"Cohere2Config",
28
"DbrxConfig",
29
+ "DeepseekVLV2Config",
30
"MPTConfig",
31
"RWConfig",
32
"H2OVLChatConfig",
vllm/transformers_utils/configs/deepseek_vl2.py
new
+214
@@ -0,0 +1,214 @@
1
+# adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/faf18023f24b962b32d9f0a2d89e402a8d383a78/deepseek_vl2/models/modeling_deepseek_vl_v2.py#L115-L268
2
+from typing import Tuple
3
+
4
+from transformers.configuration_utils import PretrainedConfig
5
+
6
+
7
+class VisionEncoderConfig(PretrainedConfig):
8
+ model_type: str = "vision"
9
+
10
+ model_name: str = "vit_so400m_patch14_siglip_384.webli"
11
+ image_size: int = 384
12
+ patch_size: int = 16
13
+ width: int = 1024
14
+ layers: int = 24
15
+ heads: int = 16
16
+ mlp_ratio: int = 4
17
+ global_pool: str = "map"
18
+ ignore_head: bool = True
19
+ class_token: bool = False
20
+ num_classes: int = 0
21
+ use_checkpoint: bool = False
22
+ weight_init: str = "skip"
23
+ deterministic: bool = False
24
+ num_recomputing_layers: int = 0
25
+
26
+ def __init__(self,
27
+ model_name: str = "vit_so400m_patch14_siglip_384.webli",
28
+ image_size: int = 384,
29
+ patch_size: int = 16,
30
+ width: int = 1024,
31
+ layers: int = 24,
32
+ heads: int = 16,
33
+ mlp_ratio: int = 4,
34
+ global_pool: str = "map",
35
+ ignore_head: bool = True,
36
+ class_token: bool = False,
37
+ num_classes: int = 0,
38
+ use_checkpoint: bool = False,
39
+ **kwargs):
40
+ self.model_name = model_name
41
+ self.image_size = image_size
42
+ self.patch_size = patch_size
43
+ self.width = width
44
+ self.layers = layers
45
+ self.heads = heads
46
+ self.mlp_ratio = mlp_ratio
47
+ self.global_pool = global_pool
48
+ self.ignore_head = ignore_head
49
+ self.class_token = class_token
50
+ self.num_classes = num_classes
51
+ self.use_checkpoint = use_checkpoint
52
+
53
+ super().__init__(**kwargs)
54
+
55
+
56
+class MlpProjectorConfig(PretrainedConfig):
57
+ model_type = "mlp_projector"
58
+ projector_type: str = "downsample_mlp_gelu"
59
+ input_dim: int = 1152
60
+ n_embed: int = 2048
61
+ depth: int = 2
62
+ mlp_ratio: int = 1
63
+ downsample_ratio: int = 2
64
+ token_pooling: bool = False
65
+
66
+ def __init__(self,
67
+ projector_type: str = "downsample_mlp_gelu",
68
+ input_dim: int = 1152,
69
+ n_embed: int = 2048,
70
+ depth: int = 2,
71
+ mlp_ratio: int = 1,
72
+ downsample_ratio: int = 2,
73
+ **kwargs):
74
+ self.projector_type = projector_type
75
+ self.input_dim = input_dim
76
+ self.n_embed = n_embed
77
+ self.depth = depth
78
+ self.mlp_ratio = mlp_ratio
79
+ self.downsample_ratio = downsample_ratio
80
+
81
+ super().__init__(**kwargs)
82
+
83
+
84
+class DeepseekV2Config(PretrainedConfig):
85
+
86
+ model_type = "deepseek_v2"
87
+ keys_to_ignore_at_inference = ["past_key_values"]
88
+
89
+ def __init__(
90
+ self,
91
+ vocab_size=102400,
92
+ hidden_size=4096,
93
+ intermediate_size=11008,
94
+ moe_intermediate_size=1407,
95
+ num_hidden_layers=30,
96
+ num_attention_heads=32,
97
+ num_key_value_heads=32,
98
+ n_shared_experts=None,
99
+ n_routed_experts=None,
100
+ ep_size=1,
101
+ routed_scaling_factor=1.0,
102
+ kv_lora_rank=512,
103
+ q_lora_rank=1536,
104
+ qk_rope_head_dim=64,
105
+ v_head_dim=128,
106
+ qk_nope_head_dim=128,
107
+ topk_method='gready',
108
+ n_group=None,
109
+ topk_group=None,
110
+ num_experts_per_tok=None,
111
+ moe_layer_freq=1,
112
+ first_k_dense_replace=0,
113
+ norm_topk_prob=False,
114
+ scoring_func='softmax',
115
+ aux_loss_alpha=0.001,
116
+ seq_aux=True,
117
+ hidden_act="silu",
118
+ max_position_embeddings=2048,
119
+ initializer_range=0.02,
120
+ rms_norm_eps=1e-6,
121
+ use_cache=True,
122
+ pad_token_id=None,
123
+ bos_token_id=100000,
124
+ eos_token_id=100001,
125
+ pretraining_tp=1,
126
+ tie_word_embeddings=False,
127
+ rope_theta=10000.0,
128
+ rope_scaling=None,
129
+ attention_bias=False,
130
+ attention_dropout=0.0,
131
+ use_mla=True,
132
+ **kwargs,
133
+ ):
134
+ self.vocab_size = vocab_size
135
+ self.max_position_embeddings = max_position_embeddings
136
+ self.hidden_size = hidden_size
137
+ self.intermediate_size = intermediate_size
138
+ self.moe_intermediate_size = moe_intermediate_size
139
+ self.num_hidden_layers = num_hidden_layers
140
+ self.num_attention_heads = num_attention_heads
141
+ self.n_shared_experts = n_shared_experts
142
+ self.n_routed_experts = n_routed_experts
143
+ self.ep_size = ep_size
144
+ self.routed_scaling_factor = routed_scaling_factor
145
+ self.kv_lora_rank = kv_lora_rank
146
+ self.q_lora_rank = q_lora_rank
147
+ self.qk_rope_head_dim = qk_rope_head_dim
148
+ self.v_head_dim = v_head_dim
149
+ self.qk_nope_head_dim = qk_nope_head_dim
150
+ self.topk_method = topk_method
151
+ self.n_group = n_group
152
+ self.topk_group = topk_group
153
+ self.num_experts_per_tok = num_experts_per_tok
154
+ self.moe_layer_freq = moe_layer_freq
155
+ self.first_k_dense_replace = first_k_dense_replace
156
+ self.norm_topk_prob = norm_topk_prob
157
+ self.scoring_func = scoring_func
158
+ self.aux_loss_alpha = aux_loss_alpha
159
+ self.seq_aux = seq_aux
160
+ # for backward compatibility
161
+ if num_key_value_heads is None:
162
+ num_key_value_heads = num_attention_heads
163
+
164
+ self.num_key_value_heads = num_key_value_heads
165
+ self.hidden_act = hidden_act
166
+ self.initializer_range = initializer_range
167
+ self.rms_norm_eps = float(rms_norm_eps)
168
+ self.pretraining_tp = pretraining_tp
169
+ self.use_cache = use_cache
170
+ self.rope_theta = rope_theta
171
+ self.rope_scaling = rope_scaling
172
+ self.attention_bias = attention_bias
173
+ self.attention_dropout = attention_dropout
174
+ self.use_mla = use_mla
175
+
176
+ super().__init__(
177
+ pad_token_id=pad_token_id,
178
+ bos_token_id=bos_token_id,
179
+ eos_token_id=eos_token_id,
180
+ tie_word_embeddings=tie_word_embeddings,
181
+ **kwargs,
182
+ )
183
+
184
+
185
+class DeepseekVLV2Config(PretrainedConfig):
186
+ model_type = "deepseek_vl_v2"
187
+ vision_config: VisionEncoderConfig
188
+ projector_config: MlpProjectorConfig
189
+
190
+ tile_tag: str = "2D"
191
+ global_view_pos: str = "head"
192
+ candidate_resolutions: Tuple[Tuple[int, int]] = ((384, 384), )
193
+
194
+ def __init__(self,
195
+ tile_tag: str = "tile_tag",
196
+ global_view_pos: str = "head",
197
+ candidate_resolutions: Tuple[Tuple[int,
198
+ int]] = ((384, 384), ),
199
+ **kwargs):
200
+ super().__init__(**kwargs)
201
+
202
+ vision_config = kwargs.get("vision_config", {})
203
+ self.vision_config = VisionEncoderConfig(**vision_config)
204
+
205
+ projector_config = kwargs.get("projector_config", {})
206
+ self.projector_config = MlpProjectorConfig(**projector_config)
207
+
208
+ language_config = kwargs.get("language_config", {})
209
+ self.text_config = DeepseekV2Config(**language_config)
210
+
211
+ self.tile_tag = tile_tag
212
+ self.global_view_pos = global_view_pos
213
+ self.candidate_resolutions = candidate_resolutions
214
+ self.vocab_size = self.text_config.vocab_size