A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 07:30 UTC
[Misc] Port Qwen3.5 Configs (#34512)
Signed-off-by: Roger Wang <hey@rogerw.io>
Roger Wang committed
Feb 13, 2026 at 05:24 UTC
5885e330efea5212733375f3573990de791d5042
6 files changed
+410
-12
vllm/model_executor/models/qwen3_5.py
+8
-8
@@ -31,14 +31,6 @@ import torch
31
from einops import rearrange
32
from torch import nn
33
from transformers.activations import ACT2FN
34
-from transformers.models.qwen3_5.configuration_qwen3_5 import (
35
- Qwen3_5Config,
36
- Qwen3_5TextConfig,
37
-)
38
-from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
39
- Qwen3_5MoeConfig,
40
- Qwen3_5MoeTextConfig,
41
-)
34
35
from vllm.compilation.decorators import support_torch_compile
36
from vllm.config import (
@@ -87,6 +79,14 @@ from vllm.model_executor.utils import set_weight_attrs
79
from vllm.multimodal import MULTIMODAL_REGISTRY
80
from vllm.platforms import current_platform
81
from vllm.sequence import IntermediateTensors
82
+from vllm.transformers_utils.configs.qwen3_5 import (
83
+ Qwen3_5Config,
84
+ Qwen3_5TextConfig,
85
+)
86
+from vllm.transformers_utils.configs.qwen3_5_moe import (
87
+ Qwen3_5MoeConfig,
88
+ Qwen3_5MoeTextConfig,
89
+)
90
91
from .interfaces import (
92
HasInnerState,
vllm/model_executor/models/qwen3_5_mtp.py
+2
-4
@@ -7,10 +7,6 @@ from collections.abc import Callable, Iterable
7
8
import torch
9
from torch import nn
10
-from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
11
-from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
12
- Qwen3_5MoeTextConfig,
13
-)
10
11
from vllm.compilation.decorators import support_torch_compile
12
from vllm.config import VllmConfig
@@ -27,6 +23,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
23
from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
24
from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
25
from vllm.sequence import IntermediateTensors
26
+from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5TextConfig
27
+from vllm.transformers_utils.configs.qwen3_5_moe import Qwen3_5MoeTextConfig
28
29
from .interfaces import (
30
MultiModalEmbeddings,
vllm/transformers_utils/config.py
+2
@@ -100,6 +100,8 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
100
step3p5="Step3p5Config",
101
qwen3_asr="Qwen3ASRConfig",
102
qwen3_next="Qwen3NextConfig",
103
+ qwen3_5="Qwen3_5Config",
104
+ qwen3_5_moe="Qwen3_5MoeConfig",
105
lfm2_moe="Lfm2MoeConfig",
106
tarsier2="Tarsier2Config",
107
)
vllm/transformers_utils/configs/__init__.py
+8
@@ -55,6 +55,10 @@ _CLASS_TO_MODULE: dict[str, str] = {
55
"Step3p5Config": "vllm.transformers_utils.configs.step3p5",
56
"Qwen3ASRConfig": "vllm.transformers_utils.configs.qwen3_asr",
57
"Qwen3NextConfig": "vllm.transformers_utils.configs.qwen3_next",
58
+ "Qwen3_5Config": "vllm.transformers_utils.configs.qwen3_5",
59
+ "Qwen3_5TextConfig": "vllm.transformers_utils.configs.qwen3_5",
60
+ "Qwen3_5MoeConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
61
+ "Qwen3_5MoeTextConfig": "vllm.transformers_utils.configs.qwen3_5_moe",
62
"Tarsier2Config": "vllm.transformers_utils.configs.tarsier2",
63
# Special case: DeepseekV3Config is from HuggingFace Transformers
64
"DeepseekV3Config": "transformers",
@@ -99,6 +103,10 @@ __all__ = [
103
"Step3p5Config",
104
"Qwen3ASRConfig",
105
"Qwen3NextConfig",
106
+ "Qwen3_5Config",
107
+ "Qwen3_5TextConfig",
108
+ "Qwen3_5MoeConfig",
109
+ "Qwen3_5MoeTextConfig",
110
"Tarsier2Config",
111
]
112
vllm/transformers_utils/configs/qwen3_5.py
new
+189
@@ -0,0 +1,189 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
4
+# All rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Qwen3.5 model configuration"""
18
+
19
+from transformers.configuration_utils import PretrainedConfig, layer_type_validation
20
+
21
+
22
+class Qwen3_5TextConfig(PretrainedConfig):
23
+ model_type = "qwen3_5_text"
24
+ keys_to_ignore_at_inference = ["past_key_values"]
25
+
26
+ base_model_tp_plan = {
27
+ "layers.*.self_attn.q_proj": "colwise",
28
+ "layers.*.self_attn.k_proj": "colwise",
29
+ "layers.*.self_attn.v_proj": "colwise",
30
+ "layers.*.self_attn.o_proj": "rowwise",
31
+ "layers.*.mlp.gate_proj": "colwise",
32
+ "layers.*.mlp.up_proj": "colwise",
33
+ "layers.*.mlp.down_proj": "rowwise",
34
+ }
35
+ base_model_pp_plan = {
36
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
37
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
38
+ "norm": (["hidden_states"], ["hidden_states"]),
39
+ }
40
+ base_config_key = "text_config"
41
+
42
+ def __init__(
43
+ self,
44
+ vocab_size=248320,
45
+ hidden_size=4096,
46
+ intermediate_size=12288,
47
+ num_hidden_layers=32,
48
+ num_attention_heads=16,
49
+ num_key_value_heads=4,
50
+ hidden_act="silu",
51
+ max_position_embeddings=32768,
52
+ initializer_range=0.02,
53
+ rms_norm_eps=1e-6,
54
+ use_cache=True,
55
+ tie_word_embeddings=False,
56
+ rope_parameters=None,
57
+ attention_bias=False,
58
+ attention_dropout=0.0,
59
+ head_dim=256,
60
+ linear_conv_kernel_dim=4,
61
+ linear_key_head_dim=128,
62
+ linear_value_head_dim=128,
63
+ linear_num_key_heads=16,
64
+ linear_num_value_heads=32,
65
+ layer_types=None,
66
+ pad_token_id=None,
67
+ bos_token_id=None,
68
+ eos_token_id=None,
69
+ **kwargs,
70
+ ):
71
+ kwargs["ignore_keys_at_rope_validation"] = [
72
+ "mrope_section",
73
+ "mrope_interleaved",
74
+ ]
75
+ self.pad_token_id = pad_token_id
76
+ self.bos_token_id = bos_token_id
77
+ self.eos_token_id = eos_token_id
78
+ self.tie_word_embeddings = tie_word_embeddings
79
+ self.vocab_size = vocab_size
80
+ self.max_position_embeddings = max_position_embeddings
81
+ self.hidden_size = hidden_size
82
+ self.intermediate_size = intermediate_size
83
+ self.num_hidden_layers = num_hidden_layers
84
+ self.num_attention_heads = num_attention_heads
85
+ self.num_key_value_heads = num_key_value_heads
86
+ self.hidden_act = hidden_act
87
+ self.initializer_range = initializer_range
88
+ self.rms_norm_eps = rms_norm_eps
89
+ self.use_cache = use_cache
90
+ self.attention_bias = attention_bias
91
+ self.attention_dropout = attention_dropout
92
+ self.head_dim = head_dim
93
+ self.rope_parameters = rope_parameters
94
+ kwargs.setdefault("partial_rotary_factor", 0.25)
95
+
96
+ self.layer_types = layer_types
97
+ if self.layer_types is None:
98
+ interval_pattern = kwargs.get("full_attention_interval", 4)
99
+ self.layer_types = [
100
+ "linear_attention"
101
+ if bool((i + 1) % interval_pattern)
102
+ else "full_attention"
103
+ for i in range(self.num_hidden_layers)
104
+ ]
105
+ layer_type_validation(self.layer_types, self.num_hidden_layers)
106
+
107
+ # linear attention part
108
+ self.linear_conv_kernel_dim = linear_conv_kernel_dim
109
+ self.linear_key_head_dim = linear_key_head_dim
110
+ self.linear_value_head_dim = linear_value_head_dim
111
+ self.linear_num_key_heads = linear_num_key_heads
112
+ self.linear_num_value_heads = linear_num_value_heads
113
+ super().__init__(**kwargs)
114
+
115
+
116
+class Qwen3_5VisionConfig(PretrainedConfig):
117
+ model_type = "qwen3_5"
118
+ base_config_key = "vision_config"
119
+
120
+ def __init__(
121
+ self,
122
+ depth=27,
123
+ hidden_size=1152,
124
+ hidden_act="gelu_pytorch_tanh",
125
+ intermediate_size=4304,
126
+ num_heads=16,
127
+ in_channels=3,
128
+ patch_size=16,
129
+ spatial_merge_size=2,
130
+ temporal_patch_size=2,
131
+ out_hidden_size=3584,
132
+ num_position_embeddings=2304,
133
+ initializer_range=0.02,
134
+ **kwargs,
135
+ ):
136
+ super().__init__(**kwargs)
137
+
138
+ self.depth = depth
139
+ self.hidden_size = hidden_size
140
+ self.hidden_act = hidden_act
141
+ self.intermediate_size = intermediate_size
142
+ self.num_heads = num_heads
143
+ self.in_channels = in_channels
144
+ self.patch_size = patch_size
145
+ self.spatial_merge_size = spatial_merge_size
146
+ self.temporal_patch_size = temporal_patch_size
147
+ self.out_hidden_size = out_hidden_size
148
+ self.num_position_embeddings = num_position_embeddings
149
+ self.initializer_range = initializer_range
150
+
151
+
152
+class Qwen3_5Config(PretrainedConfig):
153
+ model_type = "qwen3_5"
154
+ sub_configs = {
155
+ "vision_config": Qwen3_5VisionConfig,
156
+ "text_config": Qwen3_5TextConfig,
157
+ }
158
+ keys_to_ignore_at_inference = ["past_key_values"]
159
+
160
+ def __init__(
161
+ self,
162
+ text_config=None,
163
+ vision_config=None,
164
+ image_token_id=248056,
165
+ video_token_id=248057,
166
+ vision_start_token_id=248053,
167
+ vision_end_token_id=248054,
168
+ tie_word_embeddings=False,
169
+ **kwargs,
170
+ ):
171
+ if isinstance(vision_config, dict):
172
+ self.vision_config = self.sub_configs["vision_config"](**vision_config)
173
+ elif vision_config is None:
174
+ self.vision_config = self.sub_configs["vision_config"]()
175
+
176
+ if isinstance(text_config, dict):
177
+ self.text_config = self.sub_configs["text_config"](**text_config)
178
+ elif text_config is None:
179
+ self.text_config = self.sub_configs["text_config"]()
180
+
181
+ self.image_token_id = image_token_id
182
+ self.video_token_id = video_token_id
183
+ self.vision_start_token_id = vision_start_token_id
184
+ self.vision_end_token_id = vision_end_token_id
185
+ self.tie_word_embeddings = tie_word_embeddings
186
+ super().__init__(**kwargs)
187
+
188
+
189
+__all__ = ["Qwen3_5Config", "Qwen3_5TextConfig"]
vllm/transformers_utils/configs/qwen3_5_moe.py
new
+201
@@ -0,0 +1,201 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+# Copyright 2025 The Qwen Team and The HuggingFace Inc. team.
4
+# All rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+# http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+"""Qwen3.5-MoE model configuration"""
18
+
19
+from transformers.configuration_utils import PretrainedConfig, layer_type_validation
20
+
21
+
22
+class Qwen3_5MoeTextConfig(PretrainedConfig):
23
+ model_type = "qwen3_5_moe_text"
24
+ keys_to_ignore_at_inference = ["past_key_values"]
25
+
26
+ base_model_tp_plan = {
27
+ "layers.*.self_attn.q_proj": "colwise",
28
+ "layers.*.self_attn.k_proj": "colwise",
29
+ "layers.*.self_attn.v_proj": "colwise",
30
+ "layers.*.self_attn.o_proj": "rowwise",
31
+ "layers.*.mlp.experts.gate_up_proj": "packed_colwise",
32
+ "layers.*.mlp.experts.down_proj": "rowwise",
33
+ "layers.*.mlp.shared_expert.gate_proj": "colwise",
34
+ "layers.*.mlp.shared_expert.up_proj": "colwise",
35
+ "layers.*.mlp.shared_expert.down_proj": "rowwise",
36
+ }
37
+ base_model_pp_plan = {
38
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
39
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
40
+ "norm": (["hidden_states"], ["hidden_states"]),
41
+ }
42
+ base_config_key = "text_config"
43
+
44
+ def __init__(
45
+ self,
46
+ vocab_size=248320,
47
+ hidden_size=2048,
48
+ num_hidden_layers=40,
49
+ num_attention_heads=16,
50
+ num_key_value_heads=2,
51
+ hidden_act="silu",
52
+ max_position_embeddings=32768,
53
+ initializer_range=0.02,
54
+ rms_norm_eps=1e-6,
55
+ use_cache=True,
56
+ tie_word_embeddings=False,
57
+ rope_parameters=None,
58
+ attention_bias=False,
59
+ attention_dropout=0.0,
60
+ head_dim=256,
61
+ linear_conv_kernel_dim=4,
62
+ linear_key_head_dim=128,
63
+ linear_value_head_dim=128,
64
+ linear_num_key_heads=16,
65
+ linear_num_value_heads=32,
66
+ moe_intermediate_size=512,
67
+ shared_expert_intermediate_size=512,
68
+ num_experts_per_tok=8,
69
+ num_experts=256,
70
+ output_router_logits=False,
71
+ router_aux_loss_coef=0.001,
72
+ layer_types=None,
73
+ pad_token_id=None,
74
+ bos_token_id=None,
75
+ eos_token_id=None,
76
+ **kwargs,
77
+ ):
78
+ kwargs["ignore_keys_at_rope_validation"] = [
79
+ "mrope_section",
80
+ "mrope_interleaved",
81
+ ]
82
+ self.pad_token_id = pad_token_id
83
+ self.bos_token_id = bos_token_id
84
+ self.eos_token_id = eos_token_id
85
+ self.tie_word_embeddings = tie_word_embeddings
86
+ self.vocab_size = vocab_size
87
+ self.max_position_embeddings = max_position_embeddings
88
+ self.hidden_size = hidden_size
89
+ self.num_hidden_layers = num_hidden_layers
90
+ self.num_attention_heads = num_attention_heads
91
+ self.num_key_value_heads = num_key_value_heads
92
+ self.hidden_act = hidden_act
93
+ self.initializer_range = initializer_range
94
+ self.rms_norm_eps = rms_norm_eps
95
+ self.use_cache = use_cache
96
+ self.attention_bias = attention_bias
97
+ self.attention_dropout = attention_dropout
98
+ self.head_dim = head_dim
99
+ self.rope_parameters = rope_parameters
100
+ kwargs.setdefault("partial_rotary_factor", 0.25)
101
+
102
+ self.layer_types = layer_types
103
+ if self.layer_types is None:
104
+ interval_pattern = kwargs.get("full_attention_interval", 4)
105
+ self.layer_types = [
106
+ "linear_attention"
107
+ if bool((i + 1) % interval_pattern)
108
+ else "full_attention"
109
+ for i in range(self.num_hidden_layers)
110
+ ]
111
+ layer_type_validation(self.layer_types, self.num_hidden_layers)
112
+
113
+ # linear attention part
114
+ self.linear_conv_kernel_dim = linear_conv_kernel_dim
115
+ self.linear_key_head_dim = linear_key_head_dim
116
+ self.linear_value_head_dim = linear_value_head_dim
117
+ self.linear_num_key_heads = linear_num_key_heads
118
+ self.linear_num_value_heads = linear_num_value_heads
119
+ self.moe_intermediate_size = moe_intermediate_size
120
+ self.shared_expert_intermediate_size = shared_expert_intermediate_size
121
+ self.num_experts_per_tok = num_experts_per_tok
122
+ self.num_experts = num_experts
123
+ self.output_router_logits = output_router_logits
124
+ self.router_aux_loss_coef = router_aux_loss_coef
125
+ super().__init__(**kwargs)
126
+
127
+
128
+class Qwen3_5MoeVisionConfig(PretrainedConfig):
129
+ model_type = "qwen3_5_moe"
130
+ base_config_key = "vision_config"
131
+
132
+ def __init__(
133
+ self,
134
+ depth=27,
135
+ hidden_size=1152,
136
+ hidden_act="gelu_pytorch_tanh",
137
+ intermediate_size=4304,
138
+ num_heads=16,
139
+ in_channels=3,
140
+ patch_size=16,
141
+ spatial_merge_size=2,
142
+ temporal_patch_size=2,
143
+ out_hidden_size=3584,
144
+ num_position_embeddings=2304,
145
+ initializer_range=0.02,
146
+ **kwargs,
147
+ ):
148
+ super().__init__(**kwargs)
149
+
150
+ self.depth = depth
151
+ self.hidden_size = hidden_size
152
+ self.hidden_act = hidden_act
153
+ self.intermediate_size = intermediate_size
154
+ self.num_heads = num_heads
155
+ self.in_channels = in_channels
156
+ self.patch_size = patch_size
157
+ self.spatial_merge_size = spatial_merge_size
158
+ self.temporal_patch_size = temporal_patch_size
159
+ self.out_hidden_size = out_hidden_size
160
+ self.num_position_embeddings = num_position_embeddings
161
+ self.initializer_range = initializer_range
162
+
163
+
164
+class Qwen3_5MoeConfig(PretrainedConfig):
165
+ model_type = "qwen3_5_moe"
166
+ sub_configs = {
167
+ "vision_config": Qwen3_5MoeVisionConfig,
168
+ "text_config": Qwen3_5MoeTextConfig,
169
+ }
170
+ keys_to_ignore_at_inference = ["past_key_values"]
171
+
172
+ def __init__(
173
+ self,
174
+ text_config=None,
175
+ vision_config=None,
176
+ image_token_id=248056,
177
+ video_token_id=248057,
178
+ vision_start_token_id=248053,
179
+ vision_end_token_id=248054,
180
+ tie_word_embeddings=False,
181
+ **kwargs,
182
+ ):
183
+ if isinstance(vision_config, dict):
184
+ self.vision_config = self.sub_configs["vision_config"](**vision_config)
185
+ elif vision_config is None:
186
+ self.vision_config = self.sub_configs["vision_config"]()
187
+
188
+ if isinstance(text_config, dict):
189
+ self.text_config = self.sub_configs["text_config"](**text_config)
190
+ elif text_config is None:
191
+ self.text_config = self.sub_configs["text_config"]()
192
+
193
+ self.image_token_id = image_token_id
194
+ self.video_token_id = video_token_id
195
+ self.vision_start_token_id = vision_start_token_id
196
+ self.vision_end_token_id = vision_end_token_id
197
+ self.tie_word_embeddings = tie_word_embeddings
198
+ super().__init__(**kwargs)
199
+
200
+
201
+__all__ = ["Qwen3_5MoeConfig", "Qwen3_5MoeTextConfig"]