A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 19:00 UTC
Remove requirement to use `--hf-overrides` for `DeepseekVLV2ForCausalLM` (#35203)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Feb 25, 2026 at 06:00 UTC
f7967577f5563b45b1ad5b6e0fae5b639af17e28
5 files changed
+9
-56
docs/models/supported_models.md
+3
-5
@@ -682,7 +682,7 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
682
| `Blip2ForConditionalGeneration` | BLIP-2 | T + I<sup>E</sup> | `Salesforce/blip2-opt-2.7b`, `Salesforce/blip2-opt-6.7b`, etc. | ✅︎ | ✅︎ |
683
| `ChameleonForConditionalGeneration` | Chameleon | T + I | `facebook/chameleon-7b`, etc. | | ✅︎ |
684
| `Cohere2VisionForConditionalGeneration` | Command A Vision | T + I<sup>+</sup> | `CohereLabs/command-a-vision-07-2025`, etc. | | ✅︎ |
685
-| `DeepseekVLV2ForCausalLM`<sup>^</sup> | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
685
+| `DeepseekVLV2ForCausalLM` | DeepSeek-VL2 | T + I<sup>+</sup> | `deepseek-ai/deepseek-vl2-tiny`, `deepseek-ai/deepseek-vl2-small`, `deepseek-ai/deepseek-vl2`, etc. | | ✅︎ |
686
| `DeepseekOCRForCausalLM` | DeepSeek-OCR | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR`, etc. | ✅︎ | ✅︎ |
687
| `DeepseekOCR2ForCausalLM` | DeepSeek-OCR-2 | T + I<sup>+</sup> | `deepseek-ai/DeepSeek-OCR-2`, etc. | ✅︎ | ✅︎ |
688
| `Eagle2_5_VLForConditionalGeneration` | Eagle2.5-VL | T + I<sup>E+</sup> | `nvidia/Eagle2.5-8B`, etc. | ✅︎ | ✅︎ |
@@ -762,10 +762,8 @@ Some models are supported only via the [Transformers modeling backend](#transfor
762
|--------------|--------|--------|-------------------|-----------------------------|-----------------------------------------|
763
| `Emu3ForConditionalGeneration` | Emu3 | T + I | `BAAI/Emu3-Chat-hf` | ✅︎ | ✅︎ |
764
765
-<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.
766
- • For example, to use DeepSeek-VL2 series models:
767
- `--hf-overrides '{"architectures": ["DeepseekVLV2ForCausalLM"]}'`
768
-<sup>E</sup> Pre-computed embeddings can be inputted for this modality.
765
+<sup>^</sup> You need to set the architecture name via `--hf-overrides` to match the one in vLLM.</br>
766
+<sup>E</sup> Pre-computed embeddings can be inputted for this modality.</br>
767
<sup>+</sup> Multiple items can be inputted per text prompt for this modality.
768
769
!!! note
tests/models/registry.py
-1
@@ -715,7 +715,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
715
extras={"fork": "Isotr0py/deepseek-vl2-tiny"},
716
max_transformers_version="4.48",
717
transformers_version_reason={"hf": "HF model is not compatible."},
718
- hf_overrides={"architectures": ["DeepseekVLV2ForCausalLM"]},
718
),
719
"DeepseekOCRForCausalLM": _HfExamplesInfo(
720
"deepseek-ai/DeepSeek-OCR",
tests/v1/kv_connector/nixl_integration/run_accuracy_test.sh
-23
@@ -95,18 +95,6 @@ cleanup_instances() {
95
sleep 2
96
}
97
98
-# Handle to get model-specific arguments for deepseek
99
-get_model_args() {
100
- local model_name=$1
101
- local extra_args=""
102
-
103
- if [[ "$model_name" == "deepseek-ai/deepseek-vl2-tiny" ]]; then
104
- extra_args="--hf_overrides '{\"architectures\": [\"DeepseekVLV2ForCausalLM\"]}' --trust-remote-code"
105
- fi
106
-
107
- echo "$extra_args"
108
-}
109
-
98
get_num_gpus() {
99
if [[ "$SMI_BIN" == *"nvidia"* ]]; then
100
$SMI_BIN --query-gpu=name --format=csv,noheader | wc -l
@@ -127,9 +115,6 @@ run_tests_for_model() {
115
echo "Testing model: $model_name"
116
echo "================================"
117
130
- # Get model-specific arguments
131
- local model_args=$(get_model_args "$model_name")
132
-
118
# Arrays to store all hosts and ports
119
PREFILL_HOSTS=()
120
PREFILL_PORTS=()
@@ -172,11 +157,7 @@ run_tests_for_model() {
157
BASE_CMD="${BASE_CMD} --attention-backend=$ATTENTION_BACKEND"
158
fi
159
175
- if [ -n "$model_args" ]; then
176
- FULL_CMD="$BASE_CMD $model_args"
177
- else
160
FULL_CMD="$BASE_CMD"
179
- fi
161
162
eval "$FULL_CMD &"
163
@@ -227,11 +208,7 @@ run_tests_for_model() {
208
--tensor-parallel-size 1 --enable-expert-parallel"
209
fi
210
230
- if [ -n "$model_args" ]; then
231
- FULL_CMD="$BASE_CMD $model_args"
232
- else
211
FULL_CMD="$BASE_CMD"
234
- fi
212
213
eval "$FULL_CMD &"
214
tests/v1/kv_connector/nixl_integration/run_edge_case_test.sh
-24
@@ -55,19 +55,6 @@ cleanup_instances() {
55
sleep 2
56
}
57
58
-# Handle to get model-specific arguments for deepseek
59
-get_model_args() {
60
- local model_name=$1
61
- local extra_args=""
62
-
63
- if [[ "$model_name" == "deepseek-ai/deepseek-vl2-tiny" ]]; then
64
- extra_args="--hf_overrides '{\"architectures\": [\"DeepseekVLV2ForCausalLM\"]}' --trust-remote-code"
65
- fi
66
-
67
- echo "$extra_args"
68
-}
69
-
70
-
58
# Function to run tests for a specific model
59
run_tests_for_model() {
60
local model_name=$1
@@ -75,9 +62,6 @@ run_tests_for_model() {
62
echo "Testing model: $model_name"
63
echo "================================"
64
78
- # Get model-specific arguments
79
- local model_args=$(get_model_args "$model_name")
80
-
65
# Start prefill instance
66
PREFILL_PORT=8001
67
@@ -87,11 +71,7 @@ run_tests_for_model() {
71
--gpu-memory-utilization 0.2 \
72
--kv-transfer-config '$KV_CONFIG'"
73
90
- if [ -n "$model_args" ]; then
91
- FULL_CMD="$BASE_CMD $model_args"
92
- else
74
FULL_CMD="$BASE_CMD"
94
- fi
75
76
eval "$FULL_CMD &"
77
@@ -105,11 +85,7 @@ run_tests_for_model() {
85
--gpu-memory-utilization 0.2 \
86
--kv-transfer-config '$KV_CONFIG'"
87
108
- if [ -n "$model_args" ]; then
109
- FULL_CMD="$BASE_CMD $model_args"
110
- else
88
FULL_CMD="$BASE_CMD"
112
- fi
89
90
eval "$FULL_CMD &"
91
vllm/transformers_utils/configs/deepseek_vl2.py
+6
-3
@@ -89,6 +89,7 @@ class MlpProjectorConfig(PretrainedConfig):
89
90
class DeepseekVLV2Config(PretrainedConfig):
91
model_type = "deepseek_vl_v2"
92
+ architectures: list[str] | None = None
93
vision_config: VisionEncoderConfig
94
projector_config: MlpProjectorConfig
95
@@ -105,6 +106,9 @@ class DeepseekVLV2Config(PretrainedConfig):
106
):
107
super().__init__(**kwargs)
108
109
+ if self.architectures is None:
110
+ self.architectures = ["DeepseekVLV2ForCausalLM"]
111
+
112
vision_config = kwargs.get("vision_config", {})
113
self.vision_config = VisionEncoderConfig(**vision_config)
114
@@ -120,8 +124,7 @@ class DeepseekVLV2Config(PretrainedConfig):
124
self.vocab_size = self.text_config.vocab_size
125
126
# update model_type for OCR models
123
- architectures = self.architectures or kwargs.get("architectures", [])
124
- if "DeepseekOCRForCausalLM" in architectures:
127
+ if "DeepseekOCRForCausalLM" in self.architectures:
128
self.model_type = "deepseek_ocr"
126
- elif "DeepseekOCR2ForCausalLM" in architectures:
129
+ elif "DeepseekOCR2ForCausalLM" in self.architectures:
130
self.model_type = "deepseek_ocr2"