@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 62b06ba23d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model] Add support for deepseek-vl2-tiny model (#12068)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Jan 17, 2025 at 01:14 UTC 62b06ba23deaca5a0e7602cd2e3a85aeec57f306
6 files changed +22 -21
docs/source/models/supported_models.md
+2 -3
@@ -618,7 +618,7 @@ See [this page](#generative-models) for more information on how to use generativ
618 * - `DeepseekVLV2ForCausalLM`
619 - DeepSeek-VL2
620 - T + I<sup>+</sup>
621 - - `deepseek-ai/deepseek-vl2-tiny`(WIP), `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
621 + - `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2` etc. (see note)
622 -
623 - ✅︎
624 - ✅︎
@@ -768,9 +768,8 @@ See [this page](#generative-models) for more information on how to use generativ
768 <sup>+</sup> Multiple items can be inputted per text prompt for this modality.
769
770 ````{note}
771 -The `deepseek-ai/deepseek-vl2-tiny` is not supported yet.
772 -
771 To use `DeepSeek-VL2` series models, you need to install a fork version `deepseek_vl2` package:
772 +
773 ```shell
774 pip install git+https://github.com/Isotr0py/DeepSeek-VL2.git
775 ```
examples/offline_inference/vision_language.py
+1 -1
@@ -70,7 +70,7 @@ def run_chameleon(question: str, modality: str):
70 def run_deepseek_vl2(question: str, modality: str):
71 assert modality == "image"
72
73 - model_name = "deepseek-ai/deepseek-vl2-small"
73 + model_name = "deepseek-ai/deepseek-vl2-tiny"
74
75 llm = LLM(model=model_name,
76 max_model_len=4096,
examples/offline_inference/vision_language_multi_image.py
+1 -1
@@ -55,7 +55,7 @@ def load_aria(question, image_urls: List[str]) -> ModelRequestData:
55
56
57 def load_deepseek_vl2(question: str, image_urls: List[str]):
58 - model_name = "deepseek-ai/deepseek-vl2-small"
58 + model_name = "deepseek-ai/deepseek-vl2-tiny"
59
60 llm = LLM(model=model_name,
61 max_model_len=4096,
tests/models/decoder_only/vision_language/test_models.py
+9 -11
@@ -9,6 +9,7 @@ from typing import Type
9
10 import pytest
11 from transformers import AutoModelForVision2Seq
12 +from transformers import __version__ as TRANSFORMERS_VERSION
13 from transformers.utils import is_flash_attn_2_available
14
15 from vllm.platforms import current_platform
@@ -189,30 +190,27 @@ VLM_TEST_SETTINGS = {
190 dtype="bfloat16",
191 ),
192 "deepseek_vl_v2": VLMTestInfo(
192 - models=["deepseek-ai/deepseek-vl2-small"],
193 + models=["deepseek-ai/deepseek-vl2-tiny"],
194 test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
194 - dtype="bfloat16",
195 prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
196 max_model_len=4096,
197 max_num_seqs=2,
198 single_image_prompts=IMAGE_ASSETS.prompts({
199 - "stop_sign": "<image>\nWhat's the color of the stop sign and car?",
200 - "cherry_blossom": "<image>\nWhat's the color of the tower?",
199 + "stop_sign": "<image>\nWhat's the content in the center of the image?", # noqa: E501
200 + "cherry_blossom": "<image>\nPlease infer the season with reason in details.", # noqa: E501
201 }),
202 - multi_image_prompt="image_1:<image>\nimage_2:<image>\nDescribe the two images shortly.", # noqa: E501
202 + multi_image_prompt="image_1:<image>\nimage_2:<image>\nWhich image can we see the car and the tower?", # noqa: E501
203 vllm_runner_kwargs={"hf_overrides": {"architectures": ["DeepseekVLV2ForCausalLM"]}}, # noqa: E501
204 - image_size_factors=[(0.10, 0.15)],
204 patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
205 postprocess_inputs=model_utils.cast_dtype_post_processor("images"),
206 hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
207 stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"], # noqa: E501
209 - num_logprobs=5,
208 + image_size_factors=[(), (1.0, ), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
209 marks=[
210 pytest.mark.skipif(
212 - not is_flash_attn_2_available(),
213 - reason="Model needs flash-attn for numeric convergence.",
214 - ),
215 - large_gpu_mark(min_gb=48),
211 + TRANSFORMERS_VERSION >= "4.48.0",
212 + reason="HF model is not compatible with transformers>=4.48.0",
213 + )
214 ],
215 ),
216 "fuyu": VLMTestInfo(
tests/models/registry.py
+1 -2
@@ -181,8 +181,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
181 trust_remote_code=True),
182 "ChatGLMForConditionalGeneration": _HfExamplesInfo("chatglm2-6b",
183 is_available_online=False),
184 - # TODO(Isotr0py): Use deepseek-vl2-tiny for test after it's supported
185 - "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-small"), # noqa: E501
184 + "DeepseekVLV2ForCausalLM": _HfExamplesInfo("deepseek-ai/deepseek-vl2-tiny"), # noqa: E501
185 "FuyuForCausalLM": _HfExamplesInfo("adept/fuyu-8b"),
186 "H2OVLChatModel": _HfExamplesInfo("h2oai/h2ovl-mississippi-800m"),
187 "InternVLChatModel": _HfExamplesInfo("OpenGVLab/InternVL2-1B",
vllm/model_executor/models/deepseek_vl2.py
+8 -3
@@ -356,13 +356,18 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
356 f"Only 2D tile_tag is supported currently, got: {self.tile_tag}"
357 )
358
359 + if self.text_config.topk_method == "noaux_tc":
360 + architectures = ["DeepseekV3ForCausalLM"]
361 + elif not self.text_config.use_mla:
362 + architectures = ["DeepseekForCausalLM"]
363 + else:
364 + architectures = ["DeepseekV2ForCausalLM"]
365 +
366 self.language_model = init_vllm_registered_model(
367 vllm_config=vllm_config,
368 hf_config=self.text_config,
369 prefix=maybe_prefix(prefix, "language"),
363 - architectures=["DeepseekV3ForCausalLM"]
364 - if self.text_config.topk_method == "noaux_tc" else
365 - ["DeepseekV2ForCausalLM"],
370 + architectures=architectures,
371 )
372
373 self.make_empty_intermediate_tensors = (