A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model] Add support for deepseek-vl2-tiny model (#12068)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Jan 17, 2025 at 01:14 UTC
62b06ba23deaca5a0e7602cd2e3a85aeec57f306
6 files changed
+22
-21
docs/source/models/supported_models.md
+2
-3
@@ -618,7 +618,7 @@ See [this page](#generative-models) for more information on how to use generativ
618
* - `DeepseekVLV2ForCausalLM`
619
- DeepSeek-VL2
620
- T + I<sup>+</sup>
621
- - `deepseek-ai/deepseek-vl2-tiny`(WIP), `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
621
+ - `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
622
-
623
- ✅︎
624
- ✅︎
@@ -768,9 +768,8 @@ See [this page](#generative-models) for more information on how to use generativ
768
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
769
770
````{note}
771
-The `deepseek-ai/deepseek-vl2-tiny` is not supported yet.
772
-
771
To use `DeepSeek-VL2` series models, you need to install a fork version `deepseek_vl2` package:
772
+
773
```shell
774
pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git
775
```
examples/offline_inference/vision_language.py
+1
-1
@@ -70,7 +70,7 @@ def run_chameleon(question: str, modality: str):
70
def run_deepseek_vl2(question: str, modality: str):
71
assert modality == "image"
72
73
- model_name = "deepseek-ai/deepseek-vl2-small"
73
+ model_name = "deepseek-ai/deepseek-vl2-tiny"
74
75
llm = LLM(model=model_name,
76
max_model_len=4096,
examples/offline_inference/vision_language_multi_image.py
+1
-1
@@ -55,7 +55,7 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
55
56
57
def load_deepseek_vl2(question: str, image_urls: List[str]):
58
- model_name = "deepseek-ai/deepseek-vl2-small"
58
+ model_name = "deepseek-ai/deepseek-vl2-tiny"
59
60
llm = LLM(model=model_name,
61
max_model_len=4096,
tests/models/decoder_only/vision_language/test_models.py
+9
-11
@@ -9,6 +9,7 @@ from typing import Type
9
10
import pytest
11
from transformers import AutoModelForVision2Seq
12
+from transformers import __version__ as TRANSFORMERS_VERSION
13
from transformers.utils import is_flash_attn_2_available
14
15
from vllm.platforms import current_platform
@@ -189,30 +190,27 @@ VLM_TEST_SETTINGS = {
190
dtype="bfloat16",
191
),
192
"deepseek_vl_v2": VLMTestInfo(
192
- models=["deepseek-ai/deepseek-vl2-small"],
193
+ models=["deepseek-ai/deepseek-vl2-tiny"],
194
test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
194
- dtype="bfloat16",
195
prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
196
max_model_len=4096,
197
max_num_seqs=2,
198
single_image_prompts=IMAGE_ASSETS.prompts({
199
- "stop_sign": "<image>\nWhat's the color of the stop sign and car?",
200
- "cherry_blossom": "<image>\nWhat's the color of the tower?",
199
+ "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
200
+ "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
201
}),
202
- multi_image_prompt="image_1:<image>\nimage_2:<image>\nDescribe the two images shortly.", # noqa: E501
202
+ multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
203
vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
204
- image_size_factors=[(0.10, 0.15)],
204
patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
205
postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
206
hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
207
stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], # noqa: E501
209
- num_logprobs=5,
208
+ image_size_factors=[(), (1.0, ), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
209
marks=[
210
pytest.mark.skipif(
212
- not is_flash_attn_2_available(),
213
- reason="Model needs flash-attn for numeric convergence.",
214
- ),
215
- large_gpu_mark(min_gb=48),
211
+ TRANSFORMERS_VERSION >= "4.48.0",
212
+ reason="HF model is not compatible with transformers>=4.48.0",
213
+ )
214
],
215
),
216
"fuyu": VLMTestInfo(
tests/models/registry.py
+1
-2
@@ -181,8 +181,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
181
trust_remote_code=True),
182
"ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
183
is_available_online=False),
184
- # TODO(Isotr0py): Use deepseek-vl2-tiny for test after it's supported
185
- "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-small"), # noqa: E501
184
+ "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny"), # noqa: E501
185
"FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
186
"H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
187
"InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
vllm/model_executor/models/deepseek_vl2.py
+8
-3
@@ -356,13 +356,18 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
356
f"Only 2D tile_tag is supported currently, got: {self.tile_tag}"
357
)
358
359
+ if self.text_config.topk_method == "noaux_tc":
360
+ architectures = ["DeepseekV3ForCausalLM"]
361
+ elif not self.text_config.use_mla:
362
+ architectures = ["DeepseekForCausalLM"]
363
+ else:
364
+ architectures = ["DeepseekV2ForCausalLM"]
365
+
366
self.language_model = init_vllm_registered_model(
367
vllm_config=vllm_config,
368
hf_config=self.text_config,
369
prefix=maybe_prefix(prefix, "language"),
363
- architectures=["DeepseekV3ForCausalLM"]
364
- if self.text_config.topk_method == "noaux_tc" else
365
- ["DeepseekV2ForCausalLM"],
370
+ architectures=architectures,
371
)
372
373
self.make_empty_intermediate_tensors = (