A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 19:00 UTC
[Core] Parse vLLM engine required fields from hf_config to model_arch_config (#28454)
Signed-off-by: Xingyu Liu <charlotteliu12x@gmail.com> Signed-off-by: Xingyu Liu <38244988+charlotte12l@users.noreply.github.com>
Xingyu Liu committed
Jan 2, 2026 at 16:13 UTC
0eee877f672934c69eeaeb2ffc84faa3e7511f71
11 files changed
+1121
-287
tests/config/base_model_arch_groundtruth.json
new
+359
@@ -0,0 +1,359 @@
1
+{
2
+ "state-spaces/mamba-130m-hf": {
3
+ "architectures": [
4
+ "MambaForCausalLM"
5
+ ],
6
+ "model_type": "mamba",
7
+ "text_model_type": "mamba",
8
+ "hidden_size": 768,
9
+ "total_num_hidden_layers": 24,
10
+ "total_num_attention_heads": 0,
11
+ "head_size": 0,
12
+ "vocab_size": 50280,
13
+ "total_num_kv_heads": 0,
14
+ "num_experts": 0,
15
+ "is_deepseek_mla": false,
16
+ "is_multimodal_model": false,
17
+ "dtype": "torch.float32"
18
+ },
19
+ "mistralai/Mamba-Codestral-7B-v0.1": {
20
+ "architectures": [
21
+ "Mamba2ForCausalLM"
22
+ ],
23
+ "model_type": "mamba",
24
+ "text_model_type": "mamba",
25
+ "hidden_size": 4096,
26
+ "total_num_hidden_layers": 64,
27
+ "total_num_attention_heads": 0,
28
+ "head_size": 0,
29
+ "vocab_size": 32768,
30
+ "total_num_kv_heads": 0,
31
+ "num_experts": 0,
32
+ "is_deepseek_mla": false,
33
+ "is_multimodal_model": false,
34
+ "dtype": "torch.bfloat16"
35
+ },
36
+ "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11": {
37
+ "architectures": [
38
+ "Terratorch"
39
+ ],
40
+ "model_type": "timm_wrapper",
41
+ "text_model_type": "timm_wrapper",
42
+ "hidden_size": 0,
43
+ "total_num_hidden_layers": 0,
44
+ "total_num_attention_heads": 0,
45
+ "head_size": 0,
46
+ "vocab_size": 0,
47
+ "total_num_kv_heads": 0,
48
+ "num_experts": 0,
49
+ "is_deepseek_mla": false,
50
+ "is_multimodal_model": true,
51
+ "dtype": "torch.float32"
52
+ },
53
+ "tiiuae/falcon-mamba-7b-instruct": {
54
+ "architectures": [
55
+ "FalconMambaForCausalLM"
56
+ ],
57
+ "model_type": "falcon_mamba",
58
+ "text_model_type": "falcon_mamba",
59
+ "hidden_size": 4096,
60
+ "total_num_hidden_layers": 64,
61
+ "total_num_attention_heads": 0,
62
+ "head_size": 0,
63
+ "vocab_size": 65024,
64
+ "total_num_kv_heads": 0,
65
+ "num_experts": 0,
66
+ "is_deepseek_mla": false,
67
+ "is_multimodal_model": false,
68
+ "dtype": "torch.bfloat16"
69
+ },
70
+ "Zyphra/Zamba2-7B-instruct": {
71
+ "architectures": [
72
+ "Zamba2ForCausalLM"
73
+ ],
74
+ "model_type": "zamba2",
75
+ "text_model_type": "zamba2",
76
+ "hidden_size": 3584,
77
+ "total_num_hidden_layers": 81,
78
+ "total_num_attention_heads": 32,
79
+ "head_size": 224,
80
+ "vocab_size": 32000,
81
+ "total_num_kv_heads": 32,
82
+ "num_experts": 0,
83
+ "is_deepseek_mla": false,
84
+ "is_multimodal_model": false,
85
+ "dtype": "torch.bfloat16"
86
+ },
87
+ "mosaicml/mpt-7b": {
88
+ "architectures": [
89
+ "MPTForCausalLM"
90
+ ],
91
+ "model_type": "mpt",
92
+ "text_model_type": "mpt",
93
+ "hidden_size": 4096,
94
+ "total_num_hidden_layers": 32,
95
+ "total_num_attention_heads": 32,
96
+ "head_size": 128,
97
+ "vocab_size": 50432,
98
+ "total_num_kv_heads": 32,
99
+ "num_experts": 0,
100
+ "is_deepseek_mla": false,
101
+ "is_multimodal_model": false,
102
+ "dtype": "torch.bfloat16"
103
+ },
104
+ "databricks/dbrx-instruct": {
105
+ "architectures": [
106
+ "DbrxForCausalLM"
107
+ ],
108
+ "model_type": "dbrx",
109
+ "text_model_type": "dbrx",
110
+ "hidden_size": 6144,
111
+ "total_num_hidden_layers": 40,
112
+ "total_num_attention_heads": 48,
113
+ "head_size": 128,
114
+ "vocab_size": 100352,
115
+ "total_num_kv_heads": 8,
116
+ "num_experts": 0,
117
+ "is_deepseek_mla": false,
118
+ "is_multimodal_model": false,
119
+ "dtype": "torch.bfloat16"
120
+ },
121
+ "tiiuae/falcon-7b": {
122
+ "architectures": [
123
+ "FalconForCausalLM"
124
+ ],
125
+ "model_type": "falcon",
126
+ "text_model_type": "falcon",
127
+ "hidden_size": 4544,
128
+ "total_num_hidden_layers": 32,
129
+ "total_num_attention_heads": 71,
130
+ "head_size": 64,
131
+ "vocab_size": 65024,
132
+ "total_num_kv_heads": 1,
133
+ "num_experts": 0,
134
+ "is_deepseek_mla": false,
135
+ "is_multimodal_model": false,
136
+ "dtype": "torch.bfloat16"
137
+ },
138
+ "tiiuae/falcon-40b": {
139
+ "architectures": [
140
+ "FalconForCausalLM"
141
+ ],
142
+ "model_type": "falcon",
143
+ "text_model_type": "falcon",
144
+ "hidden_size": 8192,
145
+ "total_num_hidden_layers": 60,
146
+ "total_num_attention_heads": 128,
147
+ "head_size": 64,
148
+ "vocab_size": 65024,
149
+ "total_num_kv_heads": 8,
150
+ "num_experts": 0,
151
+ "is_deepseek_mla": false,
152
+ "is_multimodal_model": false,
153
+ "dtype": "torch.bfloat16"
154
+ },
155
+ "luccafong/deepseek_mtp_main_random": {
156
+ "architectures": [
157
+ "DeepseekV3ForCausalLM"
158
+ ],
159
+ "model_type": "deepseek_v3",
160
+ "text_model_type": "deepseek_v3",
161
+ "hidden_size": 2560,
162
+ "total_num_hidden_layers": 5,
163
+ "total_num_attention_heads": 32,
164
+ "head_size": 576,
165
+ "vocab_size": 129280,
166
+ "total_num_kv_heads": 32,
167
+ "num_experts": 72,
168
+ "is_deepseek_mla": true,
169
+ "is_multimodal_model": false,
170
+ "dtype": "torch.bfloat16"
171
+ },
172
+ "luccafong/deepseek_mtp_draft_random": {
173
+ "architectures": [
174
+ "DeepseekV3ForCausalLM"
175
+ ],
176
+ "model_type": "deepseek_v3",
177
+ "text_model_type": "deepseek_v3",
178
+ "hidden_size": 2560,
179
+ "total_num_hidden_layers": 10,
180
+ "total_num_attention_heads": 32,
181
+ "head_size": 576,
182
+ "vocab_size": 129280,
183
+ "total_num_kv_heads": 32,
184
+ "num_experts": 72,
185
+ "is_deepseek_mla": true,
186
+ "is_multimodal_model": false,
187
+ "dtype": "torch.bfloat16"
188
+ },
189
+ "Qwen/Qwen3-Next-80B-A3B-Instruct": {
190
+ "architectures": [
191
+ "Qwen3NextForCausalLM"
192
+ ],
193
+ "model_type": "qwen3_next",
194
+ "text_model_type": "qwen3_next",
195
+ "hidden_size": 2048,
196
+ "total_num_hidden_layers": 48,
197
+ "total_num_attention_heads": 16,
198
+ "head_size": 256,
199
+ "vocab_size": 151936,
200
+ "total_num_kv_heads": 2,
201
+ "num_experts": 512,
202
+ "is_deepseek_mla": false,
203
+ "is_multimodal_model": false,
204
+ "dtype": "torch.bfloat16"
205
+ },
206
+ "tiny-random/qwen3-next-moe": {
207
+ "architectures": [
208
+ "Qwen3NextForCausalLM"
209
+ ],
210
+ "model_type": "qwen3_next",
211
+ "text_model_type": "qwen3_next",
212
+ "hidden_size": 8,
213
+ "total_num_hidden_layers": 4,
214
+ "total_num_attention_heads": 16,
215
+ "head_size": 32,
216
+ "vocab_size": 151936,
217
+ "total_num_kv_heads": 8,
218
+ "num_experts": 32,
219
+ "is_deepseek_mla": false,
220
+ "is_multimodal_model": false,
221
+ "dtype": "torch.bfloat16"
222
+ },
223
+ "zai-org/GLM-4.5": {
224
+ "architectures": [
225
+ "Glm4MoeForCausalLM"
226
+ ],
227
+ "model_type": "glm4_moe",
228
+ "text_model_type": "glm4_moe",
229
+ "hidden_size": 5120,
230
+ "total_num_hidden_layers": 92,
231
+ "total_num_attention_heads": 96,
232
+ "head_size": 128,
233
+ "vocab_size": 151552,
234
+ "total_num_kv_heads": 8,
235
+ "num_experts": 160,
236
+ "is_deepseek_mla": false,
237
+ "is_multimodal_model": false,
238
+ "dtype": "torch.bfloat16"
239
+ },
240
+ "baidu/ERNIE-4.5-21B-A3B-PT": {
241
+ "architectures": [
242
+ "Ernie4_5_MoeForCausalLM"
243
+ ],
244
+ "model_type": "ernie4_5_moe",
245
+ "text_model_type": "ernie4_5_moe",
246
+ "hidden_size": 2560,
247
+ "total_num_hidden_layers": 28,
248
+ "total_num_attention_heads": 20,
249
+ "head_size": 128,
250
+ "vocab_size": 103424,
251
+ "total_num_kv_heads": 4,
252
+ "num_experts": 64,
253
+ "is_deepseek_mla": false,
254
+ "is_multimodal_model": false,
255
+ "dtype": "torch.bfloat16"
256
+ },
257
+ "lmsys/gpt-oss-20b-bf16": {
258
+ "architectures": [
259
+ "GptOssForCausalLM"
260
+ ],
261
+ "model_type": "gpt_oss",
262
+ "text_model_type": "gpt_oss",
263
+ "hidden_size": 2880,
264
+ "total_num_hidden_layers": 24,
265
+ "total_num_attention_heads": 64,
266
+ "head_size": 64,
267
+ "vocab_size": 201088,
268
+ "total_num_kv_heads": 8,
269
+ "num_experts": 32,
270
+ "is_deepseek_mla": false,
271
+ "is_multimodal_model": false,
272
+ "dtype": "torch.bfloat16"
273
+ },
274
+ "deepseek-ai/DeepSeek-V3.2-Exp": {
275
+ "architectures": [
276
+ "DeepseekV32ForCausalLM"
277
+ ],
278
+ "model_type": "deepseek_v32",
279
+ "text_model_type": "deepseek_v32",
280
+ "hidden_size": 7168,
281
+ "total_num_hidden_layers": 61,
282
+ "total_num_attention_heads": 128,
283
+ "head_size": 576,
284
+ "vocab_size": 129280,
285
+ "total_num_kv_heads": 128,
286
+ "num_experts": 256,
287
+ "is_deepseek_mla": true,
288
+ "is_multimodal_model": false,
289
+ "dtype": "torch.bfloat16"
290
+ },
291
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct": {
292
+ "architectures": [
293
+ "Llama4ForConditionalGeneration"
294
+ ],
295
+ "model_type": "llama4",
296
+ "text_model_type": "llama4_text",
297
+ "hidden_size": 5120,
298
+ "total_num_hidden_layers": 48,
299
+ "total_num_attention_heads": 40,
300
+ "head_size": 128,
301
+ "vocab_size": 202048,
302
+ "total_num_kv_heads": 8,
303
+ "num_experts": 16,
304
+ "is_deepseek_mla": false,
305
+ "is_multimodal_model": true,
306
+ "dtype": "torch.bfloat16"
307
+ },
308
+ "nvidia/Llama-3_3-Nemotron-Super-49B-v1": {
309
+ "architectures": [
310
+ "DeciLMForCausalLM"
311
+ ],
312
+ "model_type": "nemotron-nas",
313
+ "text_model_type": "nemotron-nas",
314
+ "hidden_size": 8192,
315
+ "total_num_hidden_layers": 80,
316
+ "total_num_attention_heads": 64,
317
+ "head_size": 128,
318
+ "vocab_size": 128256,
319
+ "total_num_kv_heads": 8,
320
+ "num_experts": 0,
321
+ "is_deepseek_mla": false,
322
+ "is_multimodal_model": false,
323
+ "dtype": "torch.bfloat16"
324
+ },
325
+ "XiaomiMiMo/MiMo-7B-RL": {
326
+ "architectures": [
327
+ "MiMoForCausalLM"
328
+ ],
329
+ "model_type": "mimo",
330
+ "text_model_type": "mimo",
331
+ "hidden_size": 4096,
332
+ "total_num_hidden_layers": 36,
333
+ "total_num_attention_heads": 32,
334
+ "head_size": 128,
335
+ "vocab_size": 151680,
336
+ "total_num_kv_heads": 8,
337
+ "num_experts": 0,
338
+ "is_deepseek_mla": false,
339
+ "is_multimodal_model": false,
340
+ "dtype": "torch.bfloat16"
341
+ },
342
+ "meituan-longcat/LongCat-Flash-Chat": {
343
+ "architectures": [
344
+ "LongcatFlashForCausalLM"
345
+ ],
346
+ "model_type": "longcat_flash",
347
+ "text_model_type": "longcat_flash",
348
+ "hidden_size": 6144,
349
+ "total_num_hidden_layers": 28,
350
+ "total_num_attention_heads": 64,
351
+ "head_size": 576,
352
+ "vocab_size": 131072,
353
+ "total_num_kv_heads": 64,
354
+ "num_experts": 512,
355
+ "is_deepseek_mla": true,
356
+ "is_multimodal_model": false,
357
+ "dtype": "torch.float32"
358
+ }
359
+}
tests/config/draft_model_arch_groundtruth.json
new
+87
@@ -0,0 +1,87 @@
1
+{
2
+ "abhigoyal/vllm-medusa-llama-68m-random": {
3
+ "architectures": [
4
+ "MedusaModel"
5
+ ],
6
+ "model_type": "medusa",
7
+ "text_model_type": "medusa",
8
+ "hidden_size": 768,
9
+ "total_num_hidden_layers": 1,
10
+ "total_num_attention_heads": 0,
11
+ "head_size": "Error: integer division or modulo by zero",
12
+ "vocab_size": 32000,
13
+ "total_num_kv_heads": 0,
14
+ "num_experts": 0,
15
+ "is_deepseek_mla": false,
16
+ "is_multimodal_model": false,
17
+ "dtype": "torch.float32"
18
+ },
19
+ "luccafong/deepseek_mtp_draft_random": {
20
+ "architectures": [
21
+ "DeepSeekMTPModel"
22
+ ],
23
+ "model_type": "deepseek_mtp",
24
+ "text_model_type": "deepseek_mtp",
25
+ "hidden_size": 2560,
26
+ "total_num_hidden_layers": 1,
27
+ "total_num_attention_heads": 32,
28
+ "head_size": 576,
29
+ "vocab_size": 129280,
30
+ "total_num_kv_heads": 32,
31
+ "num_experts": 72,
32
+ "is_deepseek_mla": true,
33
+ "is_multimodal_model": false,
34
+ "dtype": "torch.bfloat16"
35
+ },
36
+ "eagle618/eagle-deepseek-v3-random": {
37
+ "architectures": [
38
+ "EagleDeepSeekMTPModel"
39
+ ],
40
+ "model_type": "eagle",
41
+ "text_model_type": "deepseek_mtp",
42
+ "hidden_size": 2560,
43
+ "total_num_hidden_layers": 1,
44
+ "total_num_attention_heads": 32,
45
+ "head_size": 576,
46
+ "vocab_size": 129280,
47
+ "total_num_kv_heads": 32,
48
+ "num_experts": 72,
49
+ "is_deepseek_mla": true,
50
+ "is_multimodal_model": false,
51
+ "dtype": "bfloat16"
52
+ },
53
+ "yuhuili/EAGLE-LLaMA3-Instruct-8B": {
54
+ "architectures": [
55
+ "EagleLlamaForCausalLM"
56
+ ],
57
+ "model_type": "eagle",
58
+ "text_model_type": "llama",
59
+ "hidden_size": 4096,
60
+ "total_num_hidden_layers": 1,
61
+ "total_num_attention_heads": 32,
62
+ "head_size": 128,
63
+ "vocab_size": 128256,
64
+ "total_num_kv_heads": 8,
65
+ "num_experts": 0,
66
+ "is_deepseek_mla": false,
67
+ "is_multimodal_model": false,
68
+ "dtype": "float16"
69
+ },
70
+ "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B": {
71
+ "architectures": [
72
+ "Eagle3LlamaForCausalLM"
73
+ ],
74
+ "model_type": "eagle",
75
+ "text_model_type": "llama",
76
+ "hidden_size": 4096,
77
+ "total_num_hidden_layers": 1,
78
+ "total_num_attention_heads": 32,
79
+ "head_size": 128,
80
+ "vocab_size": 128256,
81
+ "total_num_kv_heads": 8,
82
+ "num_experts": 0,
83
+ "is_deepseek_mla": false,
84
+ "is_multimodal_model": false,
85
+ "dtype": "float16"
86
+ }
87
+}
tests/config/test_model_arch_config.py
new
+152
@@ -0,0 +1,152 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Tests for ModelArchitectureConfig and its integration with ModelConfig."""
4
+
5
+import json
6
+from pathlib import Path
7
+
8
+import pytest
9
+
10
+from vllm.config import ModelConfig, ParallelConfig, SpeculativeConfig
11
+from vllm.transformers_utils.model_arch_config_convertor import (
12
+ ModelArchConfigConvertorBase,
13
+)
14
+
15
+BASE_TRUST_REMOTE_CODE_MODELS = {
16
+ "nvidia/Llama-3_3-Nemotron-Super-49B-v1",
17
+ "XiaomiMiMo/MiMo-7B-RL",
18
+ # Excluded: Not available online right now
19
+ # "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",
20
+ "meituan-longcat/LongCat-Flash-Chat",
21
+}
22
+
23
+BASE_MODELS_TO_TEST = [
24
+ "state-spaces/mamba-130m-hf",
25
+ "mistralai/Mamba-Codestral-7B-v0.1",
26
+ # Excluded: terratorch/torchgeo version mismatch in CPU CI environment
27
+ # (NonGeoDataset import error). Tested in model initialization tests.
28
+ # "ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11",
29
+ "Zyphra/Zamba2-7B-instruct",
30
+ # FIXME: mosaicml/mpt-7b has been deleted
31
+ # "mosaicml/mpt-7b",
32
+ # FIXME: databricks/dbrx-instruct has been deleted
33
+ # "databricks/dbrx-instruct",
34
+ "tiiuae/falcon-7b",
35
+ "tiiuae/falcon-40b",
36
+ "luccafong/deepseek_mtp_main_random",
37
+ "Qwen/Qwen3-Next-80B-A3B-Instruct",
38
+ "tiny-random/qwen3-next-moe",
39
+ "zai-org/GLM-4.5",
40
+ "baidu/ERNIE-4.5-21B-A3B-PT",
41
+ # Models using base convertor
42
+ "lmsys/gpt-oss-20b-bf16",
43
+ "deepseek-ai/DeepSeek-V3.2-Exp",
44
+ "meta-llama/Llama-4-Scout-17B-16E-Instruct",
45
+] + list(BASE_TRUST_REMOTE_CODE_MODELS)
46
+
47
+# (target_model, draft_model, trust_remote_code)
48
+SPECULATIVE_MODELS = [
49
+ ("JackFram/llama-68m", "abhigoyal/vllm-medusa-llama-68m-random", False),
50
+ ("luccafong/deepseek_mtp_main_random", "luccafong/deepseek_mtp_draft_random", True),
51
+ ("eagle618/deepseek-v3-random", "eagle618/eagle-deepseek-v3-random", True),
52
+ ("meta-llama/Meta-Llama-3-8B-Instruct", "yuhuili/EAGLE-LLaMA3-Instruct-8B", True),
53
+ ("meta-llama/Llama-3.1-8B-Instruct", "yuhuili/EAGLE3-LLaMA3.1-Instruct-8B", True),
54
+]
55
+
56
+
57
+def _load_groundtruth(filename: str) -> dict:
58
+ """Load groundtruth JSON from the test directory."""
59
+ groundtruth_path = Path(__file__).parent / filename
60
+ with open(groundtruth_path) as f:
61
+ return json.load(f)
62
+
63
+
64
+def _assert_model_arch_config(
65
+ model_config, expected: dict, check_head_size: bool = True
66
+):
67
+ """Assert model_arch_config matches expected values."""
68
+ model_arch_config = model_config.model_arch_config
69
+ assert model_arch_config.architectures == expected["architectures"]
70
+ assert model_arch_config.model_type == expected["model_type"]
71
+ assert model_arch_config.text_model_type == expected["text_model_type"]
72
+ assert model_arch_config.hidden_size == expected["hidden_size"]
73
+ assert (
74
+ model_arch_config.total_num_hidden_layers == expected["total_num_hidden_layers"]
75
+ )
76
+ assert (
77
+ model_arch_config.total_num_attention_heads
78
+ == expected["total_num_attention_heads"]
79
+ )
80
+ assert model_arch_config.vocab_size == expected["vocab_size"]
81
+ assert model_arch_config.total_num_kv_heads == expected["total_num_kv_heads"]
82
+ assert model_arch_config.num_experts == expected["num_experts"]
83
+ assert model_arch_config.is_deepseek_mla == expected["is_deepseek_mla"]
84
+
85
+ torch_dtype = ModelArchConfigConvertorBase.get_torch_dtype(
86
+ model_config.hf_config, model_config.model, revision=model_config.revision
87
+ )
88
+ assert str(torch_dtype) == expected["dtype"]
89
+
90
+ if check_head_size:
91
+ assert model_arch_config.head_size == expected["head_size"]
92
+
93
+
94
+def _assert_model_config_methods(
95
+ model_config, expected: dict, check_head_size: bool = True
96
+):
97
+ """Assert model_config methods return expected values."""
98
+ assert model_config.architectures == expected["architectures"]
99
+ assert model_config.get_vocab_size() == expected["vocab_size"]
100
+ assert model_config.get_hidden_size() == expected["hidden_size"]
101
+ assert model_config.get_total_num_kv_heads() == expected["total_num_kv_heads"]
102
+ assert model_config.get_num_experts() == expected["num_experts"]
103
+ assert (
104
+ model_config.get_total_num_hidden_layers()
105
+ == expected["total_num_hidden_layers"]
106
+ )
107
+
108
+ if check_head_size:
109
+ assert model_config.get_head_size() == expected["head_size"]
110
+
111
+
112
+@pytest.mark.parametrize("model", BASE_MODELS_TO_TEST)
113
+def test_base_model_arch_config(model: str):
114
+ """Test model architecture config for base models."""
115
+ groundtruth = _load_groundtruth("base_model_arch_groundtruth.json")
116
+ expected = groundtruth[model]
117
+
118
+ model_config = ModelConfig(
119
+ model, trust_remote_code=model in BASE_TRUST_REMOTE_CODE_MODELS
120
+ )
121
+
122
+ _assert_model_arch_config(model_config, expected)
123
+ _assert_model_config_methods(model_config, expected)
124
+
125
+
126
+@pytest.mark.parametrize(
127
+ "target_model,draft_model,trust_remote_code", SPECULATIVE_MODELS
128
+)
129
+def test_draft_model_arch_config(
130
+ target_model: str, draft_model: str, trust_remote_code: bool
131
+):
132
+ """Test model architecture config for draft/speculative models."""
133
+ groundtruth = _load_groundtruth("draft_model_arch_groundtruth.json")
134
+ expected = groundtruth[draft_model]
135
+
136
+ target_model_config = ModelConfig(target_model, trust_remote_code=trust_remote_code)
137
+ speculative_config = SpeculativeConfig(
138
+ model=draft_model,
139
+ num_speculative_tokens=1,
140
+ target_model_config=target_model_config,
141
+ target_parallel_config=ParallelConfig(),
142
+ )
143
+ model_config = speculative_config.draft_model_config
144
+
145
+ # For medusa models, head_size may cause division by zero before
146
+ # model_arch_config was introduced, so we conditionally check it
147
+ check_head_size = isinstance(expected["head_size"], int)
148
+
149
+ _assert_model_arch_config(model_config, expected, check_head_size=check_head_size)
150
+ _assert_model_config_methods(
151
+ model_config, expected, check_head_size=check_head_size
152
+ )
tests/models/utils.py
+5
-1
@@ -471,12 +471,16 @@ def dummy_hf_overrides(
471
"num_kv_shared_layers": 1,
472
}
473
474
+ _hf_config = hf_config
475
+
476
class DummyConfig:
477
+ hf_config = _hf_config
478
hf_text_config = text_config
479
480
+ model_arch_config = ModelConfig.get_model_arch_config(DummyConfig)
481
# Only set MoE related config when the model has MoE layers.
482
# Otherwise all models detected as MoE by _get_transformers_backend_cls.
479
- if ModelConfig.get_num_experts(DummyConfig) > 0:
483
+ if model_arch_config.num_experts > 0:
484
update_dict.update(
485
{
486
"num_experts": num_experts,
tests/test_config.py
+1
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
import logging
5
import os
6
from dataclasses import MISSING, Field, asdict, dataclass, field
tests/v1/metrics/test_perf_metrics.py
+10
@@ -16,6 +16,10 @@ from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
16
from transformers.models.qwen3_moe.configuration_qwen3_moe import Qwen3MoeConfig
17
18
from vllm.config.model import ModelConfig, get_hf_text_config
19
+from vllm.transformers_utils.model_arch_config_convertor import (
20
+ MODEL_ARCH_CONFIG_CONVERTORS,
21
+ ModelArchConfigConvertorBase,
22
+)
23
from vllm.v1.metrics.perf import (
24
AttentionMetrics,
25
BaseConfigParser,
@@ -33,6 +37,12 @@ class MockModelConfig:
37
def __init__(self, hf_config, dtype):
38
self.hf_config = hf_config
39
self.hf_text_config = get_hf_text_config(hf_config)
40
+ convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get(
41
+ self.hf_config.model_type, ModelArchConfigConvertorBase
42
+ )
43
+ self.model_arch_config = convertor_cls(
44
+ self.hf_config, self.hf_text_config
45
+ ).convert()
46
self.dtype = dtype
47
self.is_attention_free = False
48
vllm/config/model.py
+44
-286
@@ -10,10 +10,12 @@ from typing import TYPE_CHECKING, Any, Literal, cast, get_args
10
import torch
11
from pydantic import ConfigDict, Field, field_validator, model_validator
12
from pydantic.dataclasses import dataclass
13
-from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
13
14
import vllm.envs as envs
15
from vllm.attention.backends.registry import AttentionBackendEnum
16
+from vllm.config.model_arch import (
17
+ ModelArchitectureConfig,
18
+)
19
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MultiModalConfig
20
from vllm.config.pooler import PoolerConfig
21
from vllm.config.scheduler import RunnerType
@@ -31,7 +33,6 @@ from vllm.transformers_utils.config import (
33
is_rope_parameters_nested,
34
try_get_dense_modules,
35
try_get_generation_config,
34
- try_get_safetensors_metadata,
36
try_get_tokenizer_config,
37
uses_mrope,
38
uses_xdrope_dim,
@@ -42,10 +43,13 @@ from vllm.transformers_utils.gguf_utils import (
43
maybe_patch_hf_config_from_gguf,
44
split_remote_gguf,
45
)
46
+from vllm.transformers_utils.model_arch_config_convertor import (
47
+ MODEL_ARCH_CONFIG_CONVERTORS,
48
+ ModelArchConfigConvertorBase,
49
+)
50
from vllm.transformers_utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
51
from vllm.transformers_utils.utils import maybe_model_redirect
52
from vllm.utils.import_utils import LazyLoader
48
-from vllm.utils.torch_utils import common_broadcastable_dtype
53
54
if TYPE_CHECKING:
55
from transformers import PretrainedConfig
@@ -483,6 +487,7 @@ class ModelConfig:
487
self.hf_image_processor_config = get_hf_image_processor_config(
488
self.model, hf_token=self.hf_token, revision=self.revision
489
)
490
+ self.model_arch_config = self.get_model_arch_config()
491
492
architectures = self.architectures
493
registry = self.registry
@@ -600,6 +605,15 @@ class ModelConfig:
605
self._verify_cuda_graph()
606
self._verify_bnb_config()
607
608
+ def get_model_arch_config(
609
+ self,
610
+ ) -> ModelArchitectureConfig:
611
+ convertor_cls = MODEL_ARCH_CONFIG_CONVERTORS.get(
612
+ self.hf_config.model_type, ModelArchConfigConvertorBase
613
+ )
614
+ convertor = convertor_cls(self.hf_config, self.hf_text_config)
615
+ return convertor.convert()
616
+
617
@field_validator("tokenizer", "max_model_len", mode="wrap")
618
@classmethod
619
def _skip_none_validation(cls, value: Any, handler: Callable) -> Any:
@@ -675,7 +689,7 @@ class ModelConfig:
689
690
@property
691
def architectures(self) -> list[str]:
678
- return getattr(self.hf_config, "architectures", [])
692
+ return self.model_arch_config.architectures
693
694
@property
695
def architecture(self) -> str:
@@ -835,56 +849,16 @@ class ModelConfig:
849
850
return convert_type
851
838
- def _parse_quant_hf_config(self, hf_config: PretrainedConfig):
839
- quant_cfg = getattr(hf_config, "quantization_config", None)
840
- if quant_cfg is None:
841
- # compressed-tensors uses a "compression_config" key
842
- quant_cfg = getattr(hf_config, "compression_config", None)
843
-
844
- else:
845
- # Set quant_method for ModelOpt models.
846
- producer_name = quant_cfg.get("producer", {}).get("name")
847
- if producer_name == "modelopt":
848
- quant_algo = quant_cfg.get("quantization", {}).get("quant_algo")
849
- if quant_algo is not None:
850
- quant_algo_upper = str(quant_algo).upper()
851
- if quant_algo_upper in {
852
- "FP8",
853
- "FP8_PER_CHANNEL_PER_TOKEN",
854
- "FP8_PB_WO",
855
- }:
856
- quant_cfg["quant_method"] = "modelopt"
857
- elif quant_algo_upper == "NVFP4":
858
- quant_cfg["quant_method"] = "modelopt_fp4"
859
- else:
860
- raise ValueError(f"Unknown ModelOpt quant algo: {quant_algo}")
861
-
862
- return quant_cfg
863
-
852
def _verify_quantization(self) -> None:
853
supported_quantization = me_quant.QUANTIZATION_METHODS
854
if self.quantization is not None:
855
self.quantization = cast(me_quant.QuantizationMethods, self.quantization)
856
857
# Parse quantization method from the HF model config, if available.
870
- quant_cfg = self._parse_quant_hf_config(self.hf_config)
871
- if quant_cfg is None and (
872
- text_config := getattr(self.hf_config, "text_config", None)
873
- ):
874
- # Check the text config as well for multi-modal models.
875
- quant_cfg = self._parse_quant_hf_config(text_config)
858
+ quant_cfg = self.model_arch_config.quantization_config
859
860
if quant_cfg is not None:
878
- # Use the community standard 'quant_method'
879
- quant_method = quant_cfg.get("quant_method", "").lower()
880
-
881
- # Normalize library names
882
- quant_method = quant_method.replace(
883
- "compressed_tensors", "compressed-tensors"
884
- )
885
-
886
- quant_cfg["quant_method"] = quant_method
887
-
861
+ quant_method = quant_cfg["quant_method"]
862
# Quantization methods which are overrides (i.e. they have a
863
# `override_quantization_method` method) must be checked in order
864
# of preference (this is particularly important for GPTQ).
@@ -966,7 +940,7 @@ class ModelConfig:
940
logger.warning(
941
"CUDA graph is not supported for %s on ROCm yet, fallback "
942
"to eager mode.",
969
- self.hf_config.model_type,
943
+ self.model_arch_config.model_type,
944
)
945
self.enforce_eager = True
946
@@ -977,11 +951,9 @@ class ModelConfig:
951
# TODO Remove this when bitsandbytes supports.
952
"""
953
is_bitsandbytes = self.quantization == "bitsandbytes"
980
- has_quantization_config = (
981
- getattr(self.hf_config, "quantization_config", None) is not None
982
- )
954
+ has_quantization_config = self.model_arch_config.quantization_config is not None
955
is_8bit = (
984
- self.hf_config.quantization_config.get("load_in_8bit", False)
956
+ self.model_arch_config.quantization_config.get("load_in_8bit", False)
957
if has_quantization_config
958
else False
959
)
@@ -1051,9 +1023,7 @@ class ModelConfig:
1023
self,
1024
parallel_config: ParallelConfig,
1025
) -> None:
1054
- total_num_attention_heads = getattr(
1055
- self.hf_text_config, "num_attention_heads", 0
1056
- )
1026
+ total_num_attention_heads = self.model_arch_config.total_num_attention_heads
1027
tensor_parallel_size = parallel_config.tensor_parallel_size
1028
if total_num_attention_heads % tensor_parallel_size != 0:
1029
raise ValueError(
@@ -1104,10 +1074,10 @@ class ModelConfig:
1074
return getattr(self.hf_text_config, "sliding_window", None)
1075
1076
def get_vocab_size(self) -> int:
1107
- return getattr(self.hf_text_config, "vocab_size", 0)
1077
+ return self.model_arch_config.vocab_size
1078
1079
def get_hidden_size(self) -> int:
1110
- return getattr(self.hf_text_config, "hidden_size", 0)
1080
+ return self.model_arch_config.hidden_size
1081
1082
def get_inputs_embeds_size(self) -> int:
1083
# The size of inputs_embeds is usually identical to the size
@@ -1120,29 +1090,7 @@ class ModelConfig:
1090
1091
@property
1092
def is_deepseek_mla(self) -> bool:
1123
- if not hasattr(self.hf_text_config, "model_type"):
1124
- return False
1125
- elif self.hf_text_config.model_type in (
1126
- "deepseek_v2",
1127
- "deepseek_v3",
1128
- "deepseek_v32",
1129
- "deepseek_mtp",
1130
- "kimi_k2",
1131
- "kimi_linear",
1132
- "longcat_flash",
1133
- "pangu_ultra_moe",
1134
- "pangu_ultra_moe_mtp",
1135
- ):
1136
- return self.hf_text_config.kv_lora_rank is not None
1137
- elif self.hf_text_config.model_type == "eagle":
1138
- # if the model is an EAGLE module, check for the
1139
- # underlying architecture
1140
- return (
1141
- self.hf_text_config.model.model_type
1142
- in ("deepseek_v2", "deepseek_v3", "deepseek_v32")
1143
- and self.hf_text_config.kv_lora_rank is not None
1144
- )
1145
- return False
1093
+ return self.model_arch_config.is_deepseek_mla
1094
1095
@cached_property
1096
def is_mm_prefix_lm(self) -> bool:
@@ -1158,103 +1106,11 @@ class ModelConfig:
1106
return self.hf_config.model_type in MM_PREFIX_LM_MODELS
1107
1108
def get_head_size(self) -> int:
1161
- # TODO remove hard code
1162
- if self.is_deepseek_mla:
1163
- qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim", 0)
1164
- if self.use_mla:
1165
- return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
1166
- else:
1167
- qk_nope_head_dim = getattr(self.hf_text_config, "qk_nope_head_dim", 0)
1168
- if qk_rope_head_dim and qk_nope_head_dim:
1169
- return qk_rope_head_dim + qk_nope_head_dim
1170
-
1171
- if hasattr(self.hf_text_config, "model_type") and (
1172
- self.hf_text_config.model_type == "zamba2"
1173
- ):
1174
- return self.hf_text_config.attention_head_dim
1175
-
1176
- if self.is_attention_free:
1177
- return 0
1178
-
1179
- # NOTE: Some configs may set head_dim=None in the config
1180
- if getattr(self.hf_text_config, "head_dim", None) is not None:
1181
- return self.hf_text_config.head_dim
1182
-
1183
- # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head`
1184
- if getattr(self.hf_text_config, "hidden_size_per_head", None) is not None:
1185
- return self.hf_text_config.hidden_size_per_head
1186
-
1187
- # FIXME(woosuk): This may not be true for all models.
1188
- return (
1189
- self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads
1190
- )
1109
+ return self.model_arch_config.head_size
1110
1111
def get_total_num_kv_heads(self) -> int:
1112
"""Returns the total number of KV heads."""
1194
- # For GPTBigCode & Falcon:
1195
- # NOTE: for falcon, when new_decoder_architecture is True, the
1196
- # multi_query flag is ignored and we use n_head_kv for the number of
1197
- # KV heads.
1198
- falcon_model_types = ["falcon", "RefinedWeb", "RefinedWebModel"]
1199
- new_decoder_arch_falcon = (
1200
- self.hf_config.model_type in falcon_model_types
1201
- and getattr(self.hf_config, "new_decoder_architecture", False)
1202
- )
1203
- if not new_decoder_arch_falcon and getattr(
1204
- self.hf_text_config, "multi_query", False
1205
- ):
1206
- # Multi-query attention, only one KV head.
1207
- # Currently, tensor parallelism is not supported in this case.
1208
- return 1
1209
-
1210
- # For DBRX and MPT
1211
- if self.hf_config.model_type == "mpt":
1212
- if "kv_n_heads" in self.hf_config.attn_config:
1213
- return self.hf_config.attn_config["kv_n_heads"]
1214
- return self.hf_config.num_attention_heads
1215
- if self.hf_config.model_type == "dbrx":
1216
- return getattr(
1217
- self.hf_config.attn_config,
1218
- "kv_n_heads",
1219
- self.hf_config.num_attention_heads,
1220
- )
1221
-
1222
- if self.hf_config.model_type == "nemotron-nas":
1223
- for block in self.hf_config.block_configs:
1224
- if not block.attention.no_op:
1225
- return (
1226
- self.hf_config.num_attention_heads
1227
- // block.attention.n_heads_in_group
1228
- )
1229
-
1230
- raise RuntimeError(
1231
- "Could not determine the number of key-value attention heads "
1232
- "from model configuration. "
1233
- f"Model: {self.model}, Architecture: {self.architectures}. "
1234
- "This usually indicates an unsupported model architecture or "
1235
- "missing configuration. "
1236
- "Please check if your model is supported at: "
1237
- "https://docs.vllm.ai/en/latest/models/supported_models.html"
1238
- )
1239
-
1240
- if self.is_attention_free:
1241
- return 0
1242
-
1243
- attributes = [
1244
- # For Falcon:
1245
- "n_head_kv",
1246
- "num_kv_heads",
1247
- # For LLaMA-2:
1248
- "num_key_value_heads",
1249
- # For ChatGLM:
1250
- "multi_query_group_num",
1251
- ]
1252
- # For non-grouped-query attention models, the number of KV heads is
1253
- # equal to the number of attention heads.
1254
- default_factory = lambda: self.hf_text_config.num_attention_heads
1255
- return getattr_iter(
1256
- self.hf_text_config, attributes, default_factory=default_factory
1257
- )
1113
+ return self.model_arch_config.total_num_kv_heads
1114
1115
def get_num_kv_heads(self, parallel_config: ParallelConfig) -> int:
1116
"""Returns the number of KV heads per GPU."""
@@ -1270,46 +1126,14 @@ class ModelConfig:
1126
return max(1, total_num_kv_heads // parallel_config.tensor_parallel_size)
1127
1128
def get_num_attention_heads(self, parallel_config: ParallelConfig) -> int:
1273
- num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
1129
+ num_heads = self.model_arch_config.total_num_attention_heads
1130
return num_heads // parallel_config.tensor_parallel_size
1131
1132
def get_num_experts(self) -> int:
1277
- """Returns the number of experts in the model."""
1278
- num_expert_names = [
1279
- "num_experts", # Jamba
1280
- "moe_num_experts", # Dbrx
1281
- "n_routed_experts", # DeepSeek
1282
- "num_local_experts", # Mixtral
1283
- ]
1284
- num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
1285
- if isinstance(num_experts, list):
1286
- # Ernie VL's remote code uses list[int]...
1287
- # The values are always the same so we just take the first one.
1288
- return num_experts[0]
1289
- # Coerce to 0 if explicitly set to None
1290
- return num_experts or 0
1133
+ return self.model_arch_config.num_experts
1134
1135
def get_total_num_hidden_layers(self) -> int:
1293
- if (
1294
- self.hf_text_config.model_type == "deepseek_mtp"
1295
- or self.hf_config.model_type == "mimo_mtp"
1296
- or self.hf_config.model_type == "glm4_moe_mtp"
1297
- or self.hf_config.model_type == "ernie_mtp"
1298
- or self.hf_config.model_type == "qwen3_next_mtp"
1299
- or self.hf_config.model_type == "pangu_ultra_moe_mtp"
1300
- ):
1301
- total_num_hidden_layers = getattr(
1302
- self.hf_text_config, "num_nextn_predict_layers", 0
1303
- )
1304
- elif self.hf_config.model_type == "longcat_flash_mtp":
1305
- total_num_hidden_layers = getattr(
1306
- self.hf_text_config, "num_nextn_predict_layers", 1
1307
- )
1308
- else:
1309
- total_num_hidden_layers = getattr(
1310
- self.hf_text_config, "num_hidden_layers", 0
1311
- )
1312
- return total_num_hidden_layers
1136
+ return self.model_arch_config.total_num_hidden_layers
1137
1138
def get_layers_start_end_indices(
1139
self, parallel_config: ParallelConfig
@@ -1360,9 +1184,7 @@ class ModelConfig:
1184
self.hf_text_config, "layers_block_type", None
1185
)
1186
if layers_block_type_value is not None:
1363
- if hasattr(self.hf_text_config, "model_type") and (
1364
- self.hf_text_config.model_type == "zamba2"
1365
- ):
1187
+ if self.model_arch_config.text_model_type == "zamba2":
1188
if attn_block_type:
1189
return sum(
1190
t == "hybrid" for t in layers_block_type_value[start:end]
@@ -1677,6 +1499,7 @@ class ModelConfig:
1499
)
1500
max_model_len = _get_and_verify_max_len(
1501
hf_config=self.hf_text_config,
1502
+ model_arch_config=self.model_arch_config,
1503
tokenizer_config=tokenizer_config,
1504
max_model_len=max_model_len,
1505
disable_sliding_window=self.disable_sliding_window,
@@ -1907,46 +1730,6 @@ def _check_valid_dtype(model_type: str, dtype: torch.dtype):
1730
return True
1731
1732
1910
-def _find_dtype(
1911
- model_id: str,
1912
- config: PretrainedConfig,
1913
- *,
1914
- revision: str | None,
1915
-):
1916
- # NOTE: getattr(config, "dtype", torch.float32) is not correct
1917
- # because config.dtype can be None.
1918
- config_dtype = getattr(config, "dtype", None)
1919
-
1920
- # Fallbacks for multi-modal models if the root config
1921
- # does not define dtype
1922
- if config_dtype is None:
1923
- config_dtype = getattr(config.get_text_config(), "dtype", None)
1924
- if config_dtype is None and hasattr(config, "vision_config"):
1925
- config_dtype = getattr(config.vision_config, "dtype", None)
1926
- if config_dtype is None and hasattr(config, "encoder_config"):
1927
- config_dtype = getattr(config.encoder_config, "dtype", None)
1928
-
1929
- # Try to read the dtype of the weights if they are in safetensors format
1930
- if config_dtype is None:
1931
- repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
1932
-
1933
- if repo_mt and (files_mt := repo_mt.files_metadata):
1934
- param_dtypes: set[torch.dtype] = {
1935
- _SAFETENSORS_TO_TORCH_DTYPE[dtype_str]
1936
- for file_mt in files_mt.values()
1937
- for dtype_str in file_mt.parameter_count
1938
- if dtype_str in _SAFETENSORS_TO_TORCH_DTYPE
1939
- }
1940
-
1941
- if param_dtypes:
1942
- return common_broadcastable_dtype(param_dtypes)
1943
-
1944
- if config_dtype is None:
1945
- config_dtype = torch.float32
1946
-
1947
- return config_dtype
1948
-
1949
-
1733
def _resolve_auto_dtype(
1734
model_type: str,
1735
config_dtype: torch.dtype,
@@ -2001,7 +1784,9 @@ def _get_and_verify_dtype(
1784
is_pooling_model: bool,
1785
revision: str | None = None,
1786
) -> torch.dtype:
2004
- config_dtype = _find_dtype(model_id, config, revision=revision)
1787
+ config_dtype = ModelArchConfigConvertorBase.get_torch_dtype(
1788
+ config, model_id, revision=revision
1789
+ )
1790
model_type = config.model_type
1791
1792
if isinstance(dtype, str):
@@ -2064,6 +1849,7 @@ def _get_head_dtype(
1849
1850
def _get_and_verify_max_len(
1851
hf_config: PretrainedConfig,
1852
+ model_arch_config: ModelArchitectureConfig,
1853
tokenizer_config: dict | None,
1854
max_model_len: int | None,
1855
disable_sliding_window: bool,
@@ -2072,36 +1858,9 @@ def _get_and_verify_max_len(
1858
encoder_config: Any | None = None,
1859
) -> int:
1860
"""Get and verify the model's maximum length."""
2075
- derived_max_model_len = float("inf")
2076
- possible_keys = [
2077
- # OPT
2078
- "max_position_embeddings",
2079
- # GPT-2
2080
- "n_positions",
2081
- # MPT
2082
- "max_seq_len",
2083
- # ChatGLM2
2084
- "seq_length",
2085
- # Command-R
2086
- "model_max_length",
2087
- # Whisper
2088
- "max_target_positions",
2089
- # Others
2090
- "max_sequence_length",
2091
- "max_seq_length",
2092
- "seq_len",
2093
- ]
2094
- # Choose the smallest "max_length" from the possible keys
2095
- max_len_key = None
2096
- for key in possible_keys:
2097
- max_len = getattr(hf_config, key, None)
2098
- if max_len is not None:
2099
- max_len_key = key if max_len < derived_max_model_len else max_len_key
2100
- derived_max_model_len = min(derived_max_model_len, max_len)
2101
- # For Command-R / Cohere, Cohere2 / Aya Vision models
2102
- if tmp_max_len := getattr(hf_config, "model_max_length", None):
2103
- max_len_key = "model_max_length"
2104
- derived_max_model_len = tmp_max_len
1861
+ (derived_max_model_len, max_len_key) = (
1862
+ model_arch_config.derived_max_model_len_and_key
1863
+ )
1864
1865
# If sliding window is manually disabled, max_length should be less
1866
# than the sliding window length in the model config.
@@ -2134,10 +1893,9 @@ def _get_and_verify_max_len(
1893
1894
default_max_len = 2048
1895
logger.warning(
2137
- "The model's config.json does not contain any of the following "
2138
- "keys to determine the original maximum length of the model: "
2139
- "%s. Assuming the model's maximum length is %d.",
2140
- possible_keys,
1896
+ "The model's config.json does not contain any of the keys "
1897
+ "to determine the original maximum length of the model. "
1898
+ "Assuming the model's maximum length is %d.",
1899
default_max_len,
1900
)
1901
derived_max_model_len = default_max_len
vllm/config/model_arch.py
new
+57
@@ -0,0 +1,57 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+from typing import Any
4
+
5
+from pydantic import ConfigDict
6
+from pydantic.dataclasses import dataclass
7
+
8
+from vllm.logger import init_logger
9
+
10
+logger = init_logger(__name__)
11
+
12
+
13
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
14
+class ModelArchitectureConfig:
15
+ """
16
+ Configuration for model architecture that required by vLLM runtime
17
+ """
18
+
19
+ architectures: list[str] | None
20
+ """List of model architecture class names (e.g., ['LlamaForCausalLM']).
21
+ It can be None upon calling `vllm_config.with_hf_config(config.text_config)`"""
22
+
23
+ model_type: str
24
+ """Model type identifier (e.g., 'llama', 'gpt_oss')."""
25
+
26
+ text_model_type: str | None
27
+ """Text model type identifier (e.g., 'llama4_text')."""
28
+
29
+ hidden_size: int
30
+ """Hidden size of the model."""
31
+
32
+ total_num_hidden_layers: int
33
+ """Number of hidden layers in the model."""
34
+
35
+ total_num_attention_heads: int
36
+ """Number of attention heads in the model."""
37
+
38
+ head_size: int
39
+ """Head dimension of the model."""
40
+
41
+ vocab_size: int
42
+ """Vocabulary size of the model."""
43
+
44
+ total_num_kv_heads: int
45
+ """Number of key value heads in the model."""
46
+
47
+ num_experts: int
48
+ """Number of experts in the model."""
49
+
50
+ quantization_config: dict[str, Any] | None
51
+ """Quantization configuration dictionary containing quantization parameters."""
52
+
53
+ is_deepseek_mla: bool
54
+ """Whether the model is a DeepSeek MLA model."""
55
+
56
+ derived_max_model_len_and_key: tuple[float, str | None]
57
+ """Derived maximum model length and key from the hf config."""
vllm/config/speculative.py
+3
@@ -401,6 +401,9 @@ class SpeculativeConfig:
401
model_type="eagle",
402
)
403
self.draft_model_config.hf_config = eagle_config
404
+ self.draft_model_config.model_arch_config = (
405
+ self.draft_model_config.get_model_arch_config()
406
+ )
407
408
if self.num_speculative_tokens is not None and hasattr(
409
self.draft_model_config.hf_config, "num_lookahead_tokens"
vllm/config/vllm.py
+1
@@ -444,6 +444,7 @@ class VllmConfig:
444
445
model_config = copy.deepcopy(self.model_config)
446
model_config.hf_config = hf_config
447
+ model_config.model_arch_config = model_config.get_model_arch_config()
448
449
return replace(self, model_config=model_config)
450
vllm/transformers_utils/model_arch_config_convertor.py
new
+402
@@ -0,0 +1,402 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from typing import final
5
+
6
+import torch
7
+from safetensors.torch import _TYPES as _SAFETENSORS_TO_TORCH_DTYPE
8
+from transformers import PretrainedConfig
9
+
10
+from vllm import envs
11
+from vllm.config.model_arch import (
12
+ ModelArchitectureConfig,
13
+)
14
+from vllm.config.utils import getattr_iter
15
+from vllm.logger import init_logger
16
+from vllm.transformers_utils.config import (
17
+ try_get_safetensors_metadata,
18
+)
19
+from vllm.utils.torch_utils import common_broadcastable_dtype
20
+
21
+logger = init_logger(__name__)
22
+
23
+
24
+class ModelArchConfigConvertorBase:
25
+ def __init__(self, hf_config: PretrainedConfig, hf_text_config: PretrainedConfig):
26
+ self.hf_config = hf_config
27
+ self.hf_text_config = hf_text_config
28
+
29
+ def get_architectures(self) -> list[str]:
30
+ return getattr(self.hf_config, "architectures", [])
31
+
32
+ def get_num_hidden_layers(self) -> int:
33
+ return getattr(self.hf_text_config, "num_hidden_layers", 0)
34
+
35
+ def get_total_num_attention_heads(self) -> int:
36
+ return getattr(self.hf_text_config, "num_attention_heads", 0)
37
+
38
+ def get_vocab_size(self) -> int:
39
+ return getattr(self.hf_text_config, "vocab_size", 0)
40
+
41
+ def get_hidden_size(self) -> int:
42
+ return getattr(self.hf_text_config, "hidden_size", 0)
43
+
44
+ def get_head_size(self) -> int:
45
+ if self.is_deepseek_mla():
46
+ qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim", 0)
47
+ if not envs.VLLM_MLA_DISABLE:
48
+ return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
49
+ else:
50
+ qk_nope_head_dim = getattr(self.hf_text_config, "qk_nope_head_dim", 0)
51
+ if qk_rope_head_dim and qk_nope_head_dim:
52
+ return qk_rope_head_dim + qk_nope_head_dim
53
+
54
+ # NOTE: Some configs may set head_dim=None in the config
55
+ if getattr(self.hf_text_config, "head_dim", None) is not None:
56
+ return self.hf_text_config.head_dim
57
+
58
+ # NOTE: Some models (such as PLaMo2.1) use `hidden_size_per_head`
59
+ if getattr(self.hf_text_config, "hidden_size_per_head", None) is not None:
60
+ return self.hf_text_config.hidden_size_per_head
61
+
62
+ # FIXME(woosuk): This may not be true for all models.
63
+ return (
64
+ self.hf_text_config.hidden_size // self.hf_text_config.num_attention_heads
65
+ )
66
+
67
+ def get_total_num_kv_heads(self) -> int:
68
+ attributes = [
69
+ # For Falcon:
70
+ "n_head_kv",
71
+ "num_kv_heads",
72
+ # For LLaMA-2:
73
+ "num_key_value_heads",
74
+ # For ChatGLM:
75
+ "multi_query_group_num",
76
+ ]
77
+ # For non-grouped-query attention models, the number of KV heads is
78
+ # equal to the number of attention heads.
79
+ default_factory = lambda: self.hf_text_config.num_attention_heads
80
+ return getattr_iter(
81
+ self.hf_text_config, attributes, default_factory=default_factory
82
+ )
83
+
84
+ def get_num_experts(self) -> int:
85
+ """Returns the number of experts in the model."""
86
+ num_expert_names = [
87
+ "num_experts", # Jamba
88
+ "moe_num_experts", # Dbrx
89
+ "n_routed_experts", # DeepSeek
90
+ "num_local_experts", # Mixtral
91
+ ]
92
+ num_experts = getattr_iter(self.hf_text_config, num_expert_names, 0)
93
+ if isinstance(num_experts, list):
94
+ # Ernie VL's remote code uses list[int]...
95
+ # The values are always the same so we just take the first one.
96
+ return num_experts[0]
97
+ # Coerce to 0 if explicitly set to None
98
+ return num_experts or 0
99
+
100
+ @final
101
+ @classmethod
102
+ def get_torch_dtype(
103
+ cls, hf_config: PretrainedConfig, model_id: str, revision: str | None
104
+ ):
105
+ # NOTE: getattr(config, "dtype", torch.float32) is not correct
106
+ # because config.dtype can be None.
107
+ config_dtype = getattr(hf_config, "dtype", None)
108
+
109
+ # Fallbacks for multi-modal models if the root config
110
+ # does not define dtype
111
+ if config_dtype is None:
112
+ config_dtype = getattr(hf_config.get_text_config(), "dtype", None)
113
+ if config_dtype is None and hasattr(hf_config, "vision_config"):
114
+ config_dtype = getattr(hf_config.vision_config, "dtype", None)
115
+ if config_dtype is None and hasattr(hf_config, "encoder_config"):
116
+ config_dtype = getattr(hf_config.encoder_config, "dtype", None)
117
+
118
+ # Try to read the dtype of the weights if they are in safetensors format
119
+ if config_dtype is None:
120
+ repo_mt = try_get_safetensors_metadata(model_id, revision=revision)
121
+
122
+ if repo_mt and (files_mt := repo_mt.files_metadata):
123
+ param_dtypes: set[torch.dtype] = {
124
+ _SAFETENSORS_TO_TORCH_DTYPE[dtype_str]
125
+ for file_mt in files_mt.values()
126
+ for dtype_str in file_mt.parameter_count
127
+ if dtype_str in _SAFETENSORS_TO_TORCH_DTYPE
128
+ }
129
+
130
+ if param_dtypes:
131
+ return common_broadcastable_dtype(param_dtypes)
132
+
133
+ if config_dtype is None:
134
+ config_dtype = torch.float32
135
+
136
+ return config_dtype
137
+
138
+ def _normalize_quantization_config(self, config: PretrainedConfig):
139
+ quant_cfg = getattr(config, "quantization_config", None)
140
+ if quant_cfg is None:
141
+ # compressed-tensors uses a "compression_config" key
142
+ quant_cfg = getattr(config, "compression_config", None)
143
+
144
+ else:
145
+ # Set quant_method for ModelOpt models.
146
+ producer_name = quant_cfg.get("producer", {}).get("name")
147
+ if producer_name == "modelopt":
148
+ quant_algo = quant_cfg.get("quantization", {}).get("quant_algo")
149
+ if quant_algo is not None:
150
+ quant_algo_upper = str(quant_algo).upper()
151
+ if quant_algo_upper in {
152
+ "FP8",
153
+ "FP8_PER_CHANNEL_PER_TOKEN",
154
+ "FP8_PB_WO",
155
+ }:
156
+ quant_cfg["quant_method"] = "modelopt"
157
+ elif quant_algo_upper == "NVFP4":
158
+ quant_cfg["quant_method"] = "modelopt_fp4"
159
+ else:
160
+ raise ValueError(f"Unknown ModelOpt quant algo: {quant_algo}")
161
+
162
+ if quant_cfg is not None:
163
+ # Use the community standard 'quant_method'
164
+ quant_method = quant_cfg.get("quant_method", "").lower()
165
+
166
+ # Normalize library names
167
+ quant_method = quant_method.replace(
168
+ "compressed_tensors", "compressed-tensors"
169
+ )
170
+
171
+ quant_cfg["quant_method"] = quant_method
172
+
173
+ return quant_cfg
174
+
175
+ def get_quantization_config(self):
176
+ quant_cfg = self._normalize_quantization_config(self.hf_config)
177
+ if quant_cfg is None and (
178
+ text_config := getattr(self.hf_config, "text_config", None)
179
+ ):
180
+ # Check the text config as well for multi-modal models.
181
+ quant_cfg = self._normalize_quantization_config(text_config)
182
+ return quant_cfg
183
+
184
+ def is_deepseek_mla(self) -> bool:
185
+ if not hasattr(self.hf_text_config, "model_type"):
186
+ return False
187
+ elif self.hf_text_config.model_type in (
188
+ "deepseek_v2",
189
+ "deepseek_v3",
190
+ "deepseek_v32",
191
+ "deepseek_mtp",
192
+ "kimi_k2",
193
+ "kimi_linear",
194
+ "longcat_flash",
195
+ "pangu_ultra_moe",
196
+ "pangu_ultra_moe_mtp",
197
+ ):
198
+ return self.hf_text_config.kv_lora_rank is not None
199
+ elif self.hf_text_config.model_type == "eagle":
200
+ # if the model is an EAGLE module, check for the
201
+ # underlying architecture
202
+ return (
203
+ self.hf_text_config.model.model_type
204
+ in ("deepseek_v2", "deepseek_v3", "deepseek_v32")
205
+ and self.hf_text_config.kv_lora_rank is not None
206
+ )
207
+ return False
208
+
209
+ def derive_max_model_len_and_key(self) -> tuple[float, str | None]:
210
+ derived_max_model_len = float("inf")
211
+ possible_keys = [
212
+ # OPT
213
+ "max_position_embeddings",
214
+ # GPT-2
215
+ "n_positions",
216
+ # MPT
217
+ "max_seq_len",
218
+ # ChatGLM2
219
+ "seq_length",
220
+ # Command-R
221
+ "model_max_length",
222
+ # Whisper
223
+ "max_target_positions",
224
+ # Others
225
+ "max_sequence_length",
226
+ "max_seq_length",
227
+ "seq_len",
228
+ ]
229
+ # Choose the smallest "max_length" from the possible keys
230
+ max_len_key = None
231
+ for key in possible_keys:
232
+ max_len = getattr(self.hf_text_config, key, None)
233
+ if max_len is not None:
234
+ if max_len < derived_max_model_len:
235
+ max_len_key = key
236
+ derived_max_model_len = min(derived_max_model_len, max_len)
237
+
238
+ # For Command-R / Cohere, Cohere2 / Aya Vision models
239
+ if tmp_max_len := getattr(self.hf_text_config, "model_max_length", None):
240
+ max_len_key = "model_max_length"
241
+ derived_max_model_len = tmp_max_len
242
+ return derived_max_model_len, max_len_key
243
+
244
+ def convert(self) -> ModelArchitectureConfig:
245
+ model_arch_config = ModelArchitectureConfig(
246
+ architectures=self.get_architectures(),
247
+ model_type=self.hf_config.model_type,
248
+ text_model_type=getattr(self.hf_text_config, "model_type", None),
249
+ hidden_size=self.get_hidden_size(),
250
+ total_num_hidden_layers=self.get_num_hidden_layers(),
251
+ total_num_attention_heads=self.get_total_num_attention_heads(),
252
+ head_size=self.get_head_size(),
253
+ vocab_size=self.get_vocab_size(),
254
+ total_num_kv_heads=self.get_total_num_kv_heads(),
255
+ num_experts=self.get_num_experts(),
256
+ quantization_config=self.get_quantization_config(),
257
+ is_deepseek_mla=self.is_deepseek_mla(),
258
+ derived_max_model_len_and_key=self.derive_max_model_len_and_key(),
259
+ )
260
+
261
+ return model_arch_config
262
+
263
+
264
+class MambaModelArchConfigConvertor(ModelArchConfigConvertorBase):
265
+ def get_head_size(self) -> int:
266
+ return 0
267
+
268
+ def get_total_num_kv_heads(self) -> int:
269
+ return 0
270
+
271
+
272
+class TerratorchModelArchConfigConvertor(ModelArchConfigConvertorBase):
273
+ def get_head_size(self) -> int:
274
+ return 0
275
+
276
+ def get_total_num_kv_heads(self) -> int:
277
+ return 0
278
+
279
+
280
+class MedusaModelArchConfigConvertor(ModelArchConfigConvertorBase):
281
+ def get_head_size(self) -> int:
282
+ return 0
283
+
284
+ def get_total_num_kv_heads(self) -> int:
285
+ return 0
286
+
287
+
288
+class Zamba2ModelArchConfigConvertor(ModelArchConfigConvertorBase):
289
+ def get_head_size(self) -> int:
290
+ return getattr(self.hf_text_config, "attention_head_dim", 0)
291
+
292
+
293
+class FalconModelArchConfigConvertor(ModelArchConfigConvertorBase):
294
+ def get_total_num_kv_heads(self) -> int:
295
+ # NOTE: for falcon, when new_decoder_architecture is True, the
296
+ # multi_query flag is ignored and we use n_head_kv for the number of
297
+ # KV heads.
298
+ new_decoder_arch_falcon = getattr(
299
+ self.hf_text_config, "new_decoder_architecture", False
300
+ )
301
+
302
+ if not new_decoder_arch_falcon and getattr(
303
+ self.hf_text_config, "multi_query", False
304
+ ):
305
+ # Multi-query attention, only one KV head.
306
+ return 1
307
+
308
+ # Use the base implementation which checks n_head_kv, num_kv_heads, etc.
309
+ return super().get_total_num_kv_heads()
310
+
311
+
312
+class MPTModelArchConfigConvertor(ModelArchConfigConvertorBase):
313
+ def get_total_num_kv_heads(self) -> int:
314
+ if "kv_n_heads" in self.hf_text_config.attn_config:
315
+ return self.hf_text_config.attn_config["kv_n_heads"]
316
+ return self.hf_text_config.num_attention_heads
317
+
318
+
319
+class DbrxModelArchConfigConvertor(ModelArchConfigConvertorBase):
320
+ def get_total_num_kv_heads(self) -> int:
321
+ return getattr(
322
+ self.hf_text_config.attn_config,
323
+ "kv_n_heads",
324
+ self.hf_text_config.num_attention_heads,
325
+ )
326
+
327
+
328
+class NemotronNasModelArchConfigConvertor(ModelArchConfigConvertorBase):
329
+ def get_total_num_kv_heads(self) -> int:
330
+ for block in self.hf_text_config.block_configs:
331
+ if not block.attention.no_op:
332
+ return (
333
+ self.hf_text_config.num_attention_heads
334
+ // block.attention.n_heads_in_group
335
+ )
336
+ raise RuntimeError(
337
+ "Could not determine the number of key-value attention heads "
338
+ "from model configuration. "
339
+ f"Architecture: {self.get_architectures()}. "
340
+ "This usually indicates an unsupported model architecture or "
341
+ "missing configuration. "
342
+ "Please check if your model is supported at: "
343
+ "https://docs.vllm.ai/en/latest/models/supported_models.html"
344
+ )
345
+
346
+
347
+class DeepSeekMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
348
+ def get_num_hidden_layers(self) -> int:
349
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
350
+
351
+
352
+class MimoMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
353
+ def get_num_hidden_layers(self) -> int:
354
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
355
+
356
+
357
+class GLM4MoeMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
358
+ def get_num_hidden_layers(self) -> int:
359
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
360
+
361
+
362
+class ErnieMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
363
+ def get_num_hidden_layers(self) -> int:
364
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
365
+
366
+
367
+class Qwen3NextMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
368
+ def get_num_hidden_layers(self) -> int:
369
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
370
+
371
+
372
+class PanguUltraMoeMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
373
+ def get_num_hidden_layers(self) -> int:
374
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
375
+
376
+
377
+class LongCatFlashMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
378
+ def get_num_hidden_layers(self) -> int:
379
+ return getattr(self.hf_text_config, "num_nextn_predict_layers", 1)
380
+
381
+
382
+# hf_config.model_type -> convertor class
383
+MODEL_ARCH_CONFIG_CONVERTORS = {
384
+ "mamba": MambaModelArchConfigConvertor,
385
+ "falcon_mamba": MambaModelArchConfigConvertor,
386
+ "timm_wrapper": TerratorchModelArchConfigConvertor,
387
+ "medusa": MedusaModelArchConfigConvertor,
388
+ "zamba2": Zamba2ModelArchConfigConvertor,
389
+ "mpt": MPTModelArchConfigConvertor,
390
+ "dbrx": DbrxModelArchConfigConvertor,
391
+ "falcon": FalconModelArchConfigConvertor,
392
+ "RefinedWeb": FalconModelArchConfigConvertor,
393
+ "RefinedWebModel": FalconModelArchConfigConvertor,
394
+ "nemotron-nas": NemotronNasModelArchConfigConvertor,
395
+ "deepseek_mtp": DeepSeekMTPModelArchConfigConvertor,
396
+ "qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
397
+ "mimo_mtp": MimoMTPModelArchConfigConvertor,
398
+ "glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor,
399
+ "ernie_mtp": ErnieMTPModelArchConfigConvertor,
400
+ "pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
401
+ "longcat_flash_mtp": LongCatFlashMTPModelArchConfigConvertor,
402
+}