@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9562912cea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 8, 16:30 UTC

[MODEL] Adding Support for Qwen3.5 Models (#34110)

Signed-off-by: JJJYmmm <1650675829@qq.com> Signed-off-by: JJJYmmm <92386084+JJJYmmm@users.noreply.github.com> Signed-off-by: Roger Wang <hey@rogerw.io> Co-authored-by: wulipc <wulipc@users.noreply.github.com> Co-authored-by: ywang96 <ywang96@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Roger Wang <hey@rogerw.io>

JJJYmmm committed Feb 9, 2026 at 21:12 UTC 9562912cead1f11e8540fb91306c5cbda66f0007
11 files changed +1501 -9
docs/models/supported_models.md
+2
@@ -738,6 +738,8 @@ These models primarily accept the [`LLM.generate`](./generative_models.md#llmgen
738 | `Qwen2VLForConditionalGeneration` | QVQ, Qwen2-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/QVQ-72B-Preview`, `Qwen/Qwen2-VL-7B-Instruct`, `Qwen/Qwen2-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
739 | `Qwen2_5_VLForConditionalGeneration` | Qwen2.5-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc. | ✅︎ | ✅︎ |
740 | `Qwen2_5OmniThinkerForConditionalGeneration` | Qwen2.5-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen2.5-Omni-3B`, `Qwen/Qwen2.5-Omni-7B` | ✅︎ | ✅︎ |
741 +| `Qwen3_5ForConditionalGeneration` | Qwen3.5 | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3.5-9B-Instruct`, etc. | ✅︎ | ✅︎ |
742 +| `Qwen3_5MoeForConditionalGeneration` | Qwen3.5-MOE | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3.5-35B-A3B-Instruct`, etc. | ✅︎ | ✅︎ |
743 | `Qwen3VLForConditionalGeneration` | Qwen3-VL | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-4B-Instruct`, etc. | ✅︎ | ✅︎ |
744 | `Qwen3VLMoeForConditionalGeneration` | Qwen3-VL-MOE | T + I<sup>E+</sup> + V<sup>E+</sup> | `Qwen/Qwen3-VL-30B-A3B-Instruct`, etc. | ✅︎ | ✅︎ |
745 | `Qwen3OmniMoeThinkerForConditionalGeneration` | Qwen3-Omni | T + I<sup>E+</sup> + V<sup>E+</sup> + A<sup>+</sup> | `Qwen/Qwen3-Omni-30B-A3B-Instruct`, `Qwen/Qwen3-Omni-30B-A3B-Thinking` | ✅︎ | ✅︎ |
tests/models/registry.py
+20
@@ -970,6 +970,26 @@ _MULTIMODAL_EXAMPLE_MODELS = {
970 max_model_len=4096,
971 min_transformers_version="4.57",
972 ),
973 + "Qwen3_5ForConditionalGeneration": _HfExamplesInfo(
974 + "Qwen/Qwen3.5-9B-Instruct",
975 + max_model_len=4096,
976 + min_transformers_version="5.1.0",
977 + ),
978 + "Qwen3_5MoeForConditionalGeneration": _HfExamplesInfo(
979 + "Qwen/Qwen3.5-35B-A3B-Instruct",
980 + max_model_len=4096,
981 + min_transformers_version="5.1.0",
982 + ),
983 + "Qwen3_5MTP": _HfExamplesInfo(
984 + "Qwen/Qwen3.5-9B-Instruct",
985 + speculative_model="Qwen/Qwen3.5-9B-Instruct",
986 + min_transformers_version="5.1.0",
987 + ),
988 + "Qwen3_5MoeMTP": _HfExamplesInfo(
989 + "Qwen/Qwen3.5-35B-A3B-Instruct",
990 + speculative_model="Qwen/Qwen3.5-35B-A3B-Instruct",
991 + min_transformers_version="5.1.0",
992 + ),
993 "Qwen3OmniMoeForConditionalGeneration": _HfExamplesInfo(
994 "Qwen/Qwen3-Omni-30B-A3B-Instruct",
995 max_model_len=4096,
vllm/config/model.py
+2 -2
@@ -1218,8 +1218,8 @@ class ModelConfig:
1218 if attn_type_list:
1219 return sum(t == 1 for t in attn_type_list[start:end])
1220
1221 - # Hybrid model Qwen3Next
1222 - layer_types_value = getattr(self.hf_config, "layer_types", None)
1221 + # Hybrid model Qwen3Next Qwen3.5 Series
1222 + layer_types_value = getattr(self.hf_text_config, "layer_types", None)
1223 if layer_types_value is not None:
1224 if block_type == "attention":
1225 return sum(
vllm/config/speculative.py
+11
@@ -37,6 +37,7 @@ MTPModelTypes = Literal[
37 "ernie_mtp",
38 "exaone_moe_mtp",
39 "qwen3_next_mtp",
40 + "qwen3_5_mtp",
41 "longcat_flash_mtp",
42 "mtp",
43 "pangu_ultra_moe_mtp",
@@ -263,6 +264,16 @@ class SpeculativeConfig:
264 {"n_predict": n_predict, "architectures": ["ExaoneMoeMTP"]}
265 )
266
267 + if hf_config.model_type in ("qwen3_5", "qwen3_5_moe"):
268 + is_moe = hf_config.model_type == "qwen3_5_moe"
269 + hf_config.model_type = "qwen3_5_mtp"
270 + n_predict = getattr(hf_config, "mtp_num_hidden_layers", None)
271 + hf_config.update(
272 + {
273 + "n_predict": n_predict,
274 + "architectures": ["Qwen3_5MoeMTP" if is_moe else "Qwen3_5MTP"],
275 + }
276 + )
277 if hf_config.model_type == "longcat_flash":
278 hf_config.model_type = "longcat_flash_mtp"
279 n_predict = getattr(hf_config, "num_nextn_predict_layers", 1)
vllm/model_executor/layers/mamba/abstract.py
+2 -1
@@ -43,7 +43,8 @@ class MambaBase(AttentionLayerBase):
43 def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
44 if (
45 vllm_config.speculative_config is not None
46 - and vllm_config.model_config.hf_config.model_type not in ["qwen3_next"]
46 + and vllm_config.model_config.hf_config.model_type
47 + not in ["qwen3_next", "qwen3_5", "qwen3_5_moe"]
48 ):
49 raise NotImplementedError(
50 "Mamba with speculative decoding is not supported yet."
vllm/model_executor/models/qwen3_5.py new
+993
@@ -0,0 +1,993 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +# Copyright 2025 The vLLM team.
5 +# Copyright 2025 The Qwen Team.
6 +# Copyright 2025 The HuggingFace Inc. team.
7 +# All rights reserved.
8 +#
9 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
10 +# and OPT implementations in this library. It has been modified from its
11 +# original forms to accommodate minor architectural differences compared
12 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
13 +#
14 +# Licensed under the Apache License, Version 2.0 (the "License");
15 +# you may not use this file except in compliance with the License.
16 +# You may obtain a copy of the License at
17 +#
18 +# http://www.apache.org/licenses/LICENSE-2.0
19 +#
20 +# Unless required by applicable law or agreed to in writing, software
21 +# distributed under the License is distributed on an "AS IS" BASIS,
22 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
23 +# See the License for the specific language governing permissions and
24 +# limitations under the License.
25 +"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
26 +
27 +import typing
28 +from collections.abc import Callable, Iterable
29 +
30 +import torch
31 +from einops import rearrange
32 +from torch import nn
33 +from transformers.activations import ACT2FN
34 +from transformers.models.qwen3_5.configuration_qwen3_5 import (
35 + Qwen3_5Config,
36 + Qwen3_5TextConfig,
37 +)
38 +from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
39 + Qwen3_5MoeConfig,
40 + Qwen3_5MoeTextConfig,
41 +)
42 +
43 +from vllm.compilation.decorators import support_torch_compile
44 +from vllm.config import (
45 + CacheConfig,
46 + ModelConfig,
47 + SpeculativeConfig,
48 + VllmConfig,
49 + get_current_vllm_config,
50 +)
51 +from vllm.distributed import (
52 + divide,
53 + get_pp_group,
54 + get_tensor_model_parallel_rank,
55 + get_tensor_model_parallel_world_size,
56 +)
57 +from vllm.logger import init_logger
58 +from vllm.model_executor.layers.layernorm import (
59 + GemmaRMSNorm as Qwen3_5RMSNorm,
60 +)
61 +from vllm.model_executor.layers.layernorm import RMSNormGated
62 +from vllm.model_executor.layers.linear import (
63 + ColumnParallelLinear,
64 + MergedColumnParallelLinear,
65 + RowParallelLinear,
66 +)
67 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
68 +from vllm.model_executor.layers.mamba.mamba_mixer2 import (
69 + mamba_v2_sharded_weight_loader,
70 +)
71 +from vllm.model_executor.layers.mamba.mamba_utils import (
72 + MambaStateCopyFunc,
73 + MambaStateCopyFuncCalculator,
74 + MambaStateDtypeCalculator,
75 + MambaStateShapeCalculator,
76 +)
77 +from vllm.model_executor.layers.quantization import QuantizationConfig
78 +from vllm.model_executor.layers.vocab_parallel_embedding import (
79 + ParallelLMHead,
80 + VocabParallelEmbedding,
81 +)
82 +from vllm.model_executor.model_loader.weight_utils import (
83 + default_weight_loader,
84 + sharded_weight_loader,
85 +)
86 +from vllm.model_executor.utils import set_weight_attrs
87 +from vllm.multimodal import MULTIMODAL_REGISTRY
88 +from vllm.platforms import current_platform
89 +from vllm.sequence import IntermediateTensors
90 +
91 +from .interfaces import (
92 + HasInnerState,
93 + IsHybrid,
94 + MixtureOfExperts,
95 + MultiModalEmbeddings,
96 + SupportsLoRA,
97 + SupportsPP,
98 + _require_is_multimodal,
99 +)
100 +from .qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
101 +from .qwen3_next import (
102 + Qwen3NextAttention,
103 + Qwen3NextDecoderLayer,
104 + Qwen3NextGatedDeltaNet,
105 + Qwen3NextModel,
106 + Qwen3NextSparseMoeBlock,
107 + QwenNextMixtureOfExperts,
108 +)
109 +from .qwen3_vl import (
110 + Qwen3_VisionTransformer,
111 + Qwen3VLDummyInputsBuilder,
112 + Qwen3VLForConditionalGeneration,
113 + Qwen3VLMultiModalProcessor,
114 + Qwen3VLProcessingInfo,
115 +)
116 +from .utils import (
117 + AutoWeightsLoader,
118 + PPMissingLayer,
119 + _merge_multimodal_embeddings,
120 + extract_layer_index,
121 + is_pp_missing_parameter,
122 + make_empty_intermediate_tensors_factory,
123 + make_layers,
124 + maybe_prefix,
125 +)
126 +
127 +logger = init_logger(__name__)
128 +
129 +
130 +class Qwen3_5ProcessingInfo(Qwen3VLProcessingInfo):
131 + def get_hf_config(self):
132 + return self.ctx.get_hf_config(Qwen3_5Config)
133 +
134 +
135 +class Qwen3_5MoeProcessingInfo(Qwen3VLProcessingInfo):
136 + def get_hf_config(self):
137 + return self.ctx.get_hf_config(Qwen3_5MoeConfig)
138 +
139 +
140 +class Qwen3_5GatedDeltaNet(Qwen3NextGatedDeltaNet):
141 + def __init__(
142 + self,
143 + config: Qwen3_5TextConfig | Qwen3_5MoeTextConfig,
144 + model_config: ModelConfig | None = None,
145 + cache_config: CacheConfig | None = None,
146 + quant_config: QuantizationConfig | None = None,
147 + speculative_config: SpeculativeConfig | None = None,
148 + prefix: str = "",
149 + ) -> None:
150 + super(Qwen3NextGatedDeltaNet, self).__init__()
151 + self.tp_size = get_tensor_model_parallel_world_size()
152 + self.tp_rank = get_tensor_model_parallel_rank()
153 + self.hidden_size = config.hidden_size
154 + self.num_v_heads = config.linear_num_value_heads
155 + self.num_k_heads = config.linear_num_key_heads
156 + self.head_k_dim = config.linear_key_head_dim
157 + self.head_v_dim = config.linear_value_head_dim
158 + self.key_dim = self.head_k_dim * self.num_k_heads
159 + self.value_dim = self.head_v_dim * self.num_v_heads
160 +
161 + self.conv_kernel_size = config.linear_conv_kernel_dim
162 + self.layer_idx = extract_layer_index(prefix)
163 + self.activation = config.hidden_act
164 + self.act = ACT2FN[config.hidden_act]
165 + self.layer_norm_epsilon = config.rms_norm_eps
166 + self.prefix = prefix
167 +
168 + self.config = config
169 + self.model_config = model_config
170 + self.cache_config = cache_config
171 + self.quant_config = quant_config
172 + self.speculative_config = speculative_config
173 + self.num_spec = (
174 + self.speculative_config.num_speculative_tokens
175 + if self.speculative_config
176 + else 0
177 + )
178 +
179 + # QKV
180 + self.conv_dim = self.key_dim * 2 + self.value_dim
181 + self.conv1d = ColumnParallelLinear(
182 + input_size=self.conv_kernel_size,
183 + output_size=self.conv_dim,
184 + bias=False,
185 + prefix=f"{prefix}.conv1d",
186 + )
187 + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
188 +
189 + self.in_proj_qkv = MergedColumnParallelLinear(
190 + input_size=self.hidden_size,
191 + output_sizes=[self.key_dim, self.key_dim, self.value_dim],
192 + bias=False,
193 + quant_config=quant_config,
194 + prefix=f"{prefix}.in_proj_qkv",
195 + )
196 + self.in_proj_z = ColumnParallelLinear(
197 + input_size=self.hidden_size,
198 + output_size=self.value_dim,
199 + bias=False,
200 + quant_config=quant_config,
201 + prefix=f"{prefix}.in_proj_z",
202 + )
203 + self.in_proj_b = ColumnParallelLinear(
204 + input_size=self.hidden_size,
205 + output_size=self.num_v_heads,
206 + bias=False,
207 + quant_config=quant_config,
208 + prefix=f"{prefix}.in_proj_ba",
209 + )
210 + self.in_proj_a = ColumnParallelLinear(
211 + input_size=self.hidden_size,
212 + output_size=self.num_v_heads,
213 + bias=False,
214 + quant_config=quant_config,
215 + prefix=f"{prefix}.in_proj_a",
216 + )
217 +
218 + query_key_settings = (self.key_dim, 0, False)
219 + value_settings = (self.value_dim, 0, False)
220 +
221 + delattr(self.conv1d.weight, "weight_loader")
222 + set_weight_attrs(
223 + self.conv1d.weight,
224 + {
225 + "weight_loader": mamba_v2_sharded_weight_loader(
226 + [
227 + query_key_settings,
228 + query_key_settings,
229 + value_settings,
230 + ],
231 + self.tp_size,
232 + self.tp_rank,
233 + )
234 + },
235 + )
236 +
237 + # selective projection used to make dt, B and C input dependant
238 +
239 + # time step projection (discretization)
240 + # instantiate once and copy inv_dt in init_weights of PretrainedModel
241 + self.dt_bias = nn.Parameter(
242 + torch.ones(self.num_v_heads // self.tp_size),
243 + )
244 + self.A_log = nn.Parameter(
245 + torch.empty(
246 + divide(self.num_v_heads, self.tp_size),
247 + )
248 + )
249 +
250 + set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
251 + set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
252 +
253 + self.norm = RMSNormGated(
254 + self.head_v_dim,
255 + eps=self.layer_norm_epsilon,
256 + group_size=None,
257 + norm_before_gate=True,
258 + device=current_platform.current_device(),
259 + dtype=config.dtype,
260 + )
261 +
262 + self.out_proj = RowParallelLinear(
263 + self.value_dim,
264 + self.hidden_size,
265 + bias=False,
266 + input_is_parallel=True,
267 + quant_config=quant_config,
268 + prefix=f"{prefix}.out_proj",
269 + )
270 +
271 + compilation_config = get_current_vllm_config().compilation_config
272 + if prefix in compilation_config.static_forward_context:
273 + raise ValueError(f"Duplicate layer name: {prefix}")
274 + compilation_config.static_forward_context[prefix] = self
275 +
276 + def fix_query_key_value_ordering(
277 + self,
278 + mixed_qkv,
279 + z,
280 + b,
281 + a,
282 + ):
283 + raise NotImplementedError(
284 + "Qwen3.5 Series dont need to fix query key value ordering"
285 + )
286 +
287 + def forward(
288 + self,
289 + hidden_states: torch.Tensor,
290 + output: torch.Tensor,
291 + ):
292 + """
293 + Forward pass with three parts:
294 + 1. Input projection
295 + 2. Core attention (custom op)
296 + 3. Output projection
297 + """
298 + num_tokens = hidden_states.size(0)
299 +
300 + # ============================================================
301 + # Part 1: Input Projection
302 + # ============================================================
303 + mixed_qkv, _ = self.in_proj_qkv(hidden_states)
304 + z, _ = self.in_proj_z(hidden_states)
305 + z = z.reshape(z.size(0), -1, self.head_v_dim)
306 + b, _ = self.in_proj_b(hidden_states)
307 + a, _ = self.in_proj_a(hidden_states)
308 +
309 + b = b.contiguous()
310 + a = a.contiguous()
311 +
312 + # ============================================================
313 + # Part 2: Core Attention (Custom Op)
314 + # ============================================================
315 + # Note: we should not use torch.empty here like other attention backends,
316 + # see discussions in https://github.com/vllm-project/vllm/pull/28182
317 + core_attn_out = torch.zeros(
318 + (num_tokens, self.num_v_heads // self.tp_size, self.head_v_dim),
319 + dtype=hidden_states.dtype,
320 + device=hidden_states.device,
321 + )
322 +
323 + torch.ops.vllm.gdn_attention_core(
324 + mixed_qkv,
325 + b,
326 + a,
327 + core_attn_out,
328 + self.prefix,
329 + )
330 +
331 + # ============================================================
332 + # Part 3: Output Projection
333 + # ============================================================
334 + z_shape_og = z.shape
335 + # Reshape input data into 2D tensor
336 + core_attn_out = core_attn_out.reshape(-1, core_attn_out.shape[-1])
337 + z = z.reshape(-1, z.shape[-1])
338 + core_attn_out = self.norm(core_attn_out, z)
339 + core_attn_out = core_attn_out.reshape(z_shape_og)
340 + core_attn_out = rearrange(core_attn_out, "... h d -> ... (h d)")
341 + output[:num_tokens], _ = self.out_proj(core_attn_out)
342 +
343 +
344 +class Qwen3_5DecoderLayer(Qwen3NextDecoderLayer):
345 + def __init__(
346 + self,
347 + vllm_config: VllmConfig,
348 + layer_type: str,
349 + prefix: str = "",
350 + ) -> None:
351 + super(Qwen3NextDecoderLayer, self).__init__()
352 +
353 + config = vllm_config.model_config.hf_text_config
354 + model_config = vllm_config.model_config
355 + cache_config = vllm_config.cache_config
356 + quant_config = vllm_config.quant_config
357 + speculative_config = vllm_config.speculative_config
358 +
359 + self.layer_type = layer_type
360 + self.layer_idx = extract_layer_index(prefix)
361 +
362 + if self.layer_type == "linear_attention":
363 + self.linear_attn = Qwen3_5GatedDeltaNet(
364 + config,
365 + model_config=model_config,
366 + cache_config=cache_config,
367 + quant_config=quant_config,
368 + speculative_config=speculative_config,
369 + prefix=f"{prefix}.linear_attn",
370 + )
371 + elif self.layer_type == "full_attention":
372 + self.self_attn = Qwen3NextAttention(
373 + config,
374 + model_config=model_config,
375 + cache_config=cache_config,
376 + quant_config=quant_config,
377 + prefix=f"{prefix}.self_attn",
378 + )
379 + else:
380 + raise ValueError(f"Invalid layer_type {self.layer_type}")
381 +
382 + # NOTE: Determine the MLP type based on the model type
383 + # Qwen3.5 use all layers for MLP / Qwen3.5-MoE use sparse MoE blocks
384 + if config.model_type == "qwen3_5_moe_text":
385 + self.mlp = Qwen3NextSparseMoeBlock(
386 + vllm_config=vllm_config,
387 + prefix=f"{prefix}.mlp",
388 + )
389 + elif config.model_type == "qwen3_5_text":
390 + self.mlp = Qwen3NextMLP(
391 + hidden_size=config.hidden_size,
392 + intermediate_size=config.intermediate_size,
393 + hidden_act=config.hidden_act,
394 + quant_config=quant_config,
395 + prefix=f"{prefix}.mlp",
396 + )
397 + else:
398 + raise ValueError(f"Invalid model_type {config.model_type}")
399 +
400 + self.input_layernorm = Qwen3_5RMSNorm(
401 + config.hidden_size, eps=config.rms_norm_eps
402 + )
403 + self.post_attention_layernorm = Qwen3_5RMSNorm(
404 + config.hidden_size, eps=config.rms_norm_eps
405 + )
406 +
407 + self.layer_scale = getattr(config, "layer_scale", False)
408 + if self.layer_scale:
409 + self.attn_layer_scale = torch.nn.Parameter(
410 + torch.zeros(
411 + 1,
412 + 1,
413 + config.hidden_size,
414 + dtype=config.dtype,
415 + ),
416 + )
417 + self.ffn_layer_scale = torch.nn.Parameter(
418 + torch.zeros(
419 + 1,
420 + 1,
421 + config.hidden_size,
422 + dtype=config.dtype,
423 + ),
424 + )
425 +
426 +
427 +@support_torch_compile(
428 + dynamic_arg_dims={
429 + "input_ids": 0,
430 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
431 + # otherwise (seq_len, ).
432 + "positions": -1,
433 + "intermediate_tensors": 0,
434 + "inputs_embeds": 0,
435 + }
436 +)
437 +class Qwen3_5Model(Qwen3NextModel):
438 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
439 + super(Qwen3NextModel, self).__init__()
440 +
441 + config: Qwen3_5TextConfig | Qwen3_5MoeTextConfig = (
442 + vllm_config.model_config.hf_text_config
443 + )
444 + parallel_config = vllm_config.parallel_config
445 +
446 + eplb_config = parallel_config.eplb_config
447 + self.num_redundant_experts = eplb_config.num_redundant_experts
448 +
449 + self.config = config
450 +
451 + self.vocab_size = config.vocab_size
452 +
453 + self.embed_tokens = VocabParallelEmbedding(
454 + self.vocab_size,
455 + config.hidden_size,
456 + )
457 +
458 + def get_layer(prefix: str):
459 + return Qwen3_5DecoderLayer(
460 + vllm_config,
461 + layer_type=config.layer_types[extract_layer_index(prefix)],
462 + prefix=prefix,
463 + )
464 +
465 + self.start_layer, self.end_layer, self.layers = make_layers(
466 + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers"
467 + )
468 + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
469 + ["hidden_states", "residual"], config.hidden_size
470 + )
471 +
472 + if get_pp_group().is_last_rank:
473 + self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
474 + else:
475 + self.norm = PPMissingLayer()
476 +
477 + def load_fused_expert_weights(
478 + self,
479 + name: str,
480 + params_dict: dict,
481 + loaded_weight: torch.Tensor,
482 + shard_id: str,
483 + num_experts: int,
484 + ) -> bool:
485 + param = params_dict[name]
486 + weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
487 + loaded_local_expert = False
488 + for expert_id in range(num_experts):
489 + curr_expert_weight = loaded_weight[expert_id]
490 + success = weight_loader(
491 + param,
492 + curr_expert_weight,
493 + name,
494 + shard_id,
495 + expert_id,
496 + return_success=True,
497 + )
498 + if success:
499 + loaded_local_expert = True
500 +
501 + return loaded_local_expert
502 +
503 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
504 + stacked_params_mapping = [
505 + # (param_name, shard_name, shard_id)
506 + ("qkv_proj", "q_proj", "q"),
507 + ("qkv_proj", "k_proj", "k"),
508 + ("qkv_proj", "v_proj", "v"),
509 + ("gate_up_proj", "gate_proj", 0),
510 + ("gate_up_proj", "up_proj", 1),
511 + ]
512 +
513 + params_dict = dict(self.named_parameters())
514 + loaded_params: set[str] = set()
515 + expert_params_mapping = self.get_expert_mapping()
516 + is_fused_expert = False
517 + fused_expert_params_mapping = [
518 + ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
519 + ("experts.w2_weight", "experts.down_proj", 0, "w2"),
520 + ]
521 + num_experts = (
522 + self.config.num_experts if hasattr(self.config, "num_experts") else 0
523 + )
524 + for name, loaded_weight in weights:
525 + if "rotary_emb.inv_freq" in name:
526 + continue
527 +
528 + if name.startswith("mtp."):
529 + continue
530 +
531 + for param_name, weight_name, shard_id in stacked_params_mapping:
532 + if "experts.gate_up_proj" in name or "experts.down_proj" in name:
533 + is_fused_expert = True
534 + expert_params_mapping = fused_expert_params_mapping
535 +
536 + if weight_name not in name:
537 + continue
538 +
539 + if "mlp.experts" in name:
540 + continue
541 +
542 + name = name.replace(weight_name, param_name)
543 + # Skip loading extra bias for GPTQ models.
544 + if name.endswith(".bias") and name not in params_dict:
545 + continue
546 + # Skip layers on other devices.
547 + if is_pp_missing_parameter(name, self):
548 + continue
549 + # name = apply_attn_prefix(name, params_dict)
550 + if name not in params_dict:
551 + continue
552 + param = params_dict[name]
553 + weight_loader = param.weight_loader
554 + weight_loader(param, loaded_weight, shard_id)
555 + break
556 + else:
557 + is_expert_weight = False
558 + for mapping in expert_params_mapping:
559 + param_name, weight_name, expert_id, shard_id = mapping
560 + if weight_name not in name:
561 + continue
562 + is_expert_weight = True
563 + name_mapped = name.replace(weight_name, param_name)
564 + # Skip layers on other devices.
565 + if is_pp_missing_parameter(name_mapped, self):
566 + continue
567 + if is_fused_expert:
568 + # qwen3.5 no need to transpose
569 + # loaded_weight = loaded_weight.transpose(-1, -2)
570 + if "experts.gate_up_proj" in name:
571 + loaded_weight = loaded_weight.chunk(2, dim=-2)
572 + success_w1 = self.load_fused_expert_weights(
573 + name_mapped,
574 + params_dict,
575 + loaded_weight[0],
576 + "w1",
577 + num_experts,
578 + )
579 + success_w3 = self.load_fused_expert_weights(
580 + name_mapped,
581 + params_dict,
582 + loaded_weight[1],
583 + "w3",
584 + num_experts,
585 + )
586 + success = success_w1 and success_w3
587 + else:
588 + # down_proj
589 + success = self.load_fused_expert_weights(
590 + name_mapped,
591 + params_dict,
592 + loaded_weight,
593 + shard_id,
594 + num_experts,
595 + )
596 + if success:
597 + name = name_mapped
598 + break
599 + else:
600 + # Skip loading extra bias for GPTQ models.
601 + if (
602 + name_mapped.endswith(".bias")
603 + or name_mapped.endswith("_bias")
604 + ) and name_mapped not in params_dict:
605 + continue
606 + param = params_dict[name_mapped]
607 + weight_loader = param.weight_loader
608 + success = weight_loader(
609 + param,
610 + loaded_weight,
611 + name_mapped,
612 + shard_id=shard_id,
613 + expert_id=expert_id,
614 + return_success=True,
615 + )
616 + if success:
617 + name = name_mapped
618 + break
619 + else:
620 + if is_expert_weight:
621 + # We've checked that this is an expert weight
622 + # However it's not mapped locally to this rank
623 + # So we simply skip it
624 + continue
625 + # Skip loading extra bias for GPTQ models.
626 + if name.endswith(".bias") and name not in params_dict:
627 + continue
628 + if is_pp_missing_parameter(name, self):
629 + continue
630 + if name not in params_dict:
631 + logger.warning_once(
632 + f"Parameter {name} not found in params_dict, skip loading"
633 + )
634 + continue
635 + param = params_dict[name]
636 + weight_loader = getattr(
637 + param, "weight_loader", default_weight_loader
638 + )
639 + weight_loader(param, loaded_weight)
640 + loaded_params.add(name)
641 + return loaded_params
642 +
643 +
644 +class Qwen3_5ForCausalLMBase(
645 + nn.Module,
646 + HasInnerState,
647 + SupportsLoRA,
648 + SupportsPP,
649 +):
650 + packed_modules_mapping = {
651 + "qkv_proj": [
652 + "q_proj",
653 + "k_proj",
654 + "v_proj",
655 + ],
656 + "gate_up_proj": ["gate_proj", "up_proj"],
657 + }
658 +
659 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
660 + config = vllm_config.model_config.hf_text_config
661 + self.vllm_config = vllm_config
662 + self.model_config = vllm_config.model_config
663 + cache_config = vllm_config.cache_config
664 +
665 + scheduler_config = vllm_config.scheduler_config
666 + if cache_config.mamba_cache_mode == "all":
667 + raise NotImplementedError(
668 + "Qwen3.5 currently does not support 'all' prefix caching, "
669 + "please use '--mamba-cache-mode=align' instead"
670 + )
671 + self.quant_config = vllm_config.quant_config
672 +
673 + super().__init__()
674 + self.config = config
675 + self.scheduler_config = scheduler_config
676 + self.model = Qwen3_5Model(
677 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
678 + )
679 +
680 + if get_pp_group().is_last_rank:
681 + if config.tie_word_embeddings:
682 + self.lm_head = self.model.embed_tokens
683 + else:
684 + self.lm_head = ParallelLMHead(
685 + config.vocab_size,
686 + config.hidden_size,
687 + prefix=maybe_prefix(prefix, "lm_head"),
688 + )
689 + else:
690 + self.lm_head = PPMissingLayer()
691 +
692 + self.logits_processor = LogitsProcessor(config.vocab_size)
693 + self.make_empty_intermediate_tensors = (
694 + self.model.make_empty_intermediate_tensors
695 + )
696 +
697 + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
698 + return self.model.embed_input_ids(input_ids)
699 +
700 + def forward(
701 + self,
702 + input_ids: torch.Tensor,
703 + positions: torch.Tensor,
704 + intermediate_tensors: IntermediateTensors | None = None,
705 + inputs_embeds: torch.Tensor | None = None,
706 + **kwargs: object,
707 + ):
708 + hidden_states = self.model(
709 + input_ids, positions, intermediate_tensors, inputs_embeds
710 + )
711 +
712 + return hidden_states
713 +
714 + def compute_logits(
715 + self,
716 + hidden_states: torch.Tensor,
717 + ) -> torch.Tensor | None:
718 + return self.logits_processor(self.lm_head, hidden_states)
719 +
720 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
721 + loader = AutoWeightsLoader(
722 + self,
723 + skip_prefixes=["mtp."],
724 + )
725 + return loader.load_weights(weights)
726 +
727 +
728 +class Qwen3_5ForCausalLM(Qwen3_5ForCausalLMBase):
729 + pass
730 +
731 +
732 +class Qwen3_5MoeForCausalLM(Qwen3_5ForCausalLMBase, QwenNextMixtureOfExperts):
733 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
734 + super().__init__(vllm_config=vllm_config, prefix=prefix)
735 +
736 + # set MoE hyperparameters
737 + self.set_moe_parameters()
738 +
739 + def get_expert_mapping(self) -> list[tuple[str, str, int, str]]:
740 + return self.model.get_expert_mapping()
741 +
742 +
743 +########################################################
744 +# Qwen3_5-Dense
745 +########################################################
746 +
747 +
748 +@MULTIMODAL_REGISTRY.register_processor(
749 + Qwen3VLMultiModalProcessor,
750 + info=Qwen3_5ProcessingInfo,
751 + dummy_inputs=Qwen3VLDummyInputsBuilder,
752 +)
753 +class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid):
754 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
755 + # protocols have not __init__ method, so we need to use nn.Module.__init__
756 + nn.Module.__init__(self)
757 + config: Qwen3_5Config = vllm_config.model_config.hf_config
758 + quant_config = vllm_config.quant_config
759 + multimodal_config = vllm_config.model_config.multimodal_config
760 +
761 + self.config = config
762 + self.multimodal_config = multimodal_config
763 + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
764 + self.video_pruning_rate = multimodal_config.video_pruning_rate
765 + self.is_multimodal_pruning_enabled = (
766 + multimodal_config.is_multimodal_pruning_enabled()
767 + )
768 +
769 + with self._mark_tower_model(vllm_config, {"image", "video"}):
770 + self.visual = Qwen3_VisionTransformer(
771 + config.vision_config,
772 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
773 + quant_config=quant_config,
774 + prefix=maybe_prefix(prefix, "visual"),
775 + )
776 +
777 + with self._mark_language_model(vllm_config):
778 + self.language_model = Qwen3_5ForCausalLM(
779 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
780 + )
781 +
782 + self.make_empty_intermediate_tensors = (
783 + self.language_model.make_empty_intermediate_tensors
784 + )
785 +
786 + def embed_input_ids(
787 + self,
788 + input_ids: torch.Tensor,
789 + multimodal_embeddings: MultiModalEmbeddings | None = None,
790 + *,
791 + is_multimodal: torch.Tensor | None = None,
792 + handle_oov_mm_token: bool = False,
793 + ) -> torch.Tensor:
794 + inputs_embeds = self._embed_text_input_ids(
795 + input_ids,
796 + self.language_model.embed_input_ids,
797 + is_multimodal=is_multimodal,
798 + handle_oov_mm_token=handle_oov_mm_token,
799 + )
800 +
801 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
802 + return inputs_embeds
803 +
804 + is_multimodal = _require_is_multimodal(is_multimodal)
805 +
806 + inputs_embeds = _merge_multimodal_embeddings(
807 + inputs_embeds=inputs_embeds,
808 + multimodal_embeddings=multimodal_embeddings,
809 + is_multimodal=is_multimodal,
810 + )
811 +
812 + return inputs_embeds
813 +
814 + def forward(
815 + self,
816 + input_ids: torch.Tensor,
817 + positions: torch.Tensor,
818 + intermediate_tensors: IntermediateTensors | None = None,
819 + inputs_embeds: torch.Tensor | None = None,
820 + **kwargs: object,
821 + ) -> torch.Tensor | IntermediateTensors:
822 + """Run forward pass for Qwen3.5.
823 +
824 + Args:
825 + input_ids: Flattened (concatenated) input_ids corresponding to a
826 + batch.
827 + positions: Flattened (concatenated) position ids corresponding to a
828 + batch.
829 + **NOTE**: If mrope is enabled (default setting for Qwen3VL
830 + opensource models), the shape will be `(3, seq_len)`,
831 + otherwise it will be `(seq_len,).
832 + intermediate_tensors: Intermediate tensors from previous pipeline
833 + stages.
834 + inputs_embeds: Pre-computed input embeddings.
835 + **kwargs: Additional keyword arguments including:
836 + - pixel_values: Pixel values to be fed to a model.
837 + `None` if no images are passed.
838 + - image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in
839 + LLM. `None` if no images are passed.
840 + - pixel_values_videos: Pixel values of videos to be fed to a
841 + model. `None` if no videos are passed.
842 + - video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in
843 + LLM. `None` if no videos are passed.
844 + """
845 +
846 + if intermediate_tensors is not None:
847 + inputs_embeds = None
848 +
849 + hidden_states = self.language_model.model(
850 + input_ids=input_ids,
851 + positions=positions,
852 + intermediate_tensors=intermediate_tensors,
853 + inputs_embeds=inputs_embeds,
854 + )
855 +
856 + return hidden_states
857 +
858 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
859 + loader = AutoWeightsLoader(
860 + self,
861 + skip_prefixes=["mtp."],
862 + )
863 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
864 +
865 + @classmethod
866 + def get_mamba_state_dtype_from_config(
867 + cls,
868 + vllm_config: "VllmConfig",
869 + ) -> tuple[torch.dtype, torch.dtype]:
870 + return MambaStateDtypeCalculator.gated_delta_net_state_dtype(
871 + vllm_config.model_config.dtype, vllm_config.cache_config.mamba_cache_dtype
872 + )
873 +
874 + @classmethod
875 + def get_mamba_state_shape_from_config(
876 + cls, vllm_config: "VllmConfig"
877 + ) -> tuple[tuple[int, int], tuple[int, int]]:
878 + parallel_config = vllm_config.parallel_config
879 + hf_config = vllm_config.model_config.hf_text_config
880 + tp_size = parallel_config.tensor_parallel_size
881 + num_spec = (
882 + vllm_config.speculative_config.num_speculative_tokens
883 + if vllm_config.speculative_config
884 + else 0
885 + )
886 + return MambaStateShapeCalculator.gated_delta_net_state_shape(
887 + tp_size,
888 + hf_config.linear_num_key_heads,
889 + hf_config.linear_num_value_heads,
890 + hf_config.linear_key_head_dim,
891 + hf_config.linear_value_head_dim,
892 + hf_config.linear_conv_kernel_dim,
893 + num_spec,
894 + )
895 +
896 + @classmethod
897 + def get_mamba_state_copy_func(cls) -> tuple[MambaStateCopyFunc, MambaStateCopyFunc]:
898 + return MambaStateCopyFuncCalculator.gated_delta_net_state_copy_func()
899 +
900 +
901 +########################################################
902 +# Qwen3_5-MoE
903 +########################################################
904 +
905 +
906 +class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
907 + def update_physical_experts_metadata(
908 + self,
909 + num_physical_experts: int,
910 + num_local_physical_experts: int,
911 + ) -> None:
912 + assert self.num_local_physical_experts == num_local_physical_experts
913 + self.num_physical_experts = num_physical_experts
914 + self.num_local_physical_experts = num_local_physical_experts
915 + self.num_redundant_experts = num_physical_experts - self.num_logical_experts
916 + for layer in self.language_model.model.layers:
917 + if isinstance(layer.mlp, Qwen3NextSparseMoeBlock):
918 + moe = layer.mlp
919 + moe.n_local_physical_experts = num_local_physical_experts
920 + moe.n_physical_experts = num_physical_experts
921 + moe.n_redundant_experts = self.num_redundant_experts
922 + moe.experts.update_expert_map()
923 +
924 + def set_moe_parameters(self):
925 + self.expert_weights = []
926 +
927 + self.moe_layers = []
928 + example_moe = None
929 + for layer in self.language_model.model.layers:
930 + if isinstance(layer, Qwen3_5DecoderLayer) and isinstance(
931 + layer.mlp, Qwen3NextSparseMoeBlock
932 + ):
933 + example_moe = layer.mlp
934 + self.moe_layers.append(layer.mlp.experts)
935 +
936 + if example_moe is None:
937 + raise RuntimeError(
938 + "No Qwen3_5 layer found in the language_model.model.layers."
939 + )
940 +
941 + # Set MoE hyperparameters
942 + self.num_moe_layers = len(self.moe_layers)
943 + self.num_expert_groups = 1
944 + self.num_shared_experts = 0
945 + self.num_logical_experts = example_moe.n_logical_experts
946 + self.num_physical_experts = example_moe.n_physical_experts
947 + self.num_local_physical_experts = example_moe.n_local_physical_experts
948 + self.num_routed_experts = example_moe.n_routed_experts
949 + self.num_redundant_experts = example_moe.n_redundant_experts
950 +
951 +
952 +@MULTIMODAL_REGISTRY.register_processor(
953 + Qwen3VLMultiModalProcessor,
954 + info=Qwen3_5MoeProcessingInfo,
955 + dummy_inputs=Qwen3VLDummyInputsBuilder,
956 +)
957 +class Qwen3_5MoeForConditionalGeneration(
958 + Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
959 +):
960 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
961 + # protocols have not __init__ method, so we need to use nn.Module.__init__
962 + nn.Module.__init__(self)
963 + config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config
964 + quant_config = vllm_config.quant_config
965 + multimodal_config = vllm_config.model_config.multimodal_config
966 +
967 + self.config = config
968 + self.multimodal_config = multimodal_config
969 + self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
970 + self.video_pruning_rate = multimodal_config.video_pruning_rate
971 + self.is_multimodal_pruning_enabled = (
972 + multimodal_config.is_multimodal_pruning_enabled()
973 + )
974 +
975 + with self._mark_tower_model(vllm_config, {"image", "video"}):
976 + self.visual = Qwen3_VisionTransformer(
977 + config.vision_config,
978 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
979 + quant_config=quant_config,
980 + prefix=maybe_prefix(prefix, "visual"),
981 + )
982 +
983 + with self._mark_language_model(vllm_config):
984 + self.language_model = Qwen3_5MoeForCausalLM(
985 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "language_model")
986 + )
987 +
988 + self.make_empty_intermediate_tensors = (
989 + self.language_model.make_empty_intermediate_tensors
990 + )
991 +
992 + # set MoE hyperparameters
993 + self.set_moe_parameters()
vllm/model_executor/models/qwen3_5_mtp.py new
+447
@@ -0,0 +1,447 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Inference-only Qwen3_5 MTP model."""
4 +
5 +import typing
6 +from collections.abc import Callable, Iterable
7 +
8 +import torch
9 +from torch import nn
10 +from transformers.models.qwen3_5.configuration_qwen3_5 import Qwen3_5TextConfig
11 +from transformers.models.qwen3_5_moe.configuration_qwen3_5_moe import (
12 + Qwen3_5MoeTextConfig,
13 +)
14 +
15 +from vllm.compilation.decorators import support_torch_compile
16 +from vllm.config import VllmConfig
17 +from vllm.distributed.parallel_state import get_pp_group
18 +from vllm.logger import init_logger
19 +from vllm.model_executor.layers.fused_moe import FusedMoE
20 +from vllm.model_executor.layers.linear import ColumnParallelLinear
21 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
22 +from vllm.model_executor.layers.vocab_parallel_embedding import (
23 + ParallelLMHead,
24 + VocabParallelEmbedding,
25 +)
26 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 +from vllm.model_executor.models.qwen3_5 import Qwen3_5DecoderLayer, Qwen3_5RMSNorm
28 +from vllm.model_executor.models.qwen3_next import QwenNextMixtureOfExperts
29 +from vllm.sequence import IntermediateTensors
30 +
31 +from .interfaces import (
32 + MultiModalEmbeddings,
33 + SupportsMultiModal,
34 + _require_is_multimodal,
35 +)
36 +from .utils import (
37 + AutoWeightsLoader,
38 + PPMissingLayer,
39 + _merge_multimodal_embeddings,
40 + is_pp_missing_parameter,
41 + make_empty_intermediate_tensors_factory,
42 + maybe_prefix,
43 +)
44 +
45 +logger = init_logger(__name__)
46 +
47 +
48 +@support_torch_compile(
49 + dynamic_arg_dims={
50 + "input_ids": 0,
51 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
52 + # otherwise (seq_len, ).
53 + "positions": -1,
54 + "intermediate_tensors": 0,
55 + "inputs_embeds": 0,
56 + "hidden_states": 0,
57 + }
58 +)
59 +class Qwen3_5MultiTokenPredictor(nn.Module):
60 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
61 + super().__init__()
62 +
63 + model_config = vllm_config.model_config
64 + quant_config = vllm_config.quant_config
65 +
66 + config: Qwen3_5TextConfig | Qwen3_5MoeTextConfig = model_config.hf_text_config
67 +
68 + self.config = config
69 +
70 + self.vocab_size = config.vocab_size
71 +
72 + self.mtp_start_layer_idx = config.num_hidden_layers
73 + self.num_mtp_layers = getattr(config, "mtp_num_hidden_layers", 1)
74 +
75 + self.embed_tokens = VocabParallelEmbedding(
76 + self.vocab_size,
77 + config.hidden_size,
78 + )
79 +
80 + self.fc = ColumnParallelLinear(
81 + self.config.hidden_size * 2,
82 + self.config.hidden_size,
83 + gather_output=True,
84 + bias=False,
85 + return_bias=False,
86 + quant_config=quant_config,
87 + prefix=f"{prefix}.fc",
88 + )
89 +
90 + self.layers = torch.nn.ModuleList(
91 + Qwen3_5DecoderLayer(
92 + vllm_config,
93 + layer_type="full_attention",
94 + prefix=f"{prefix}.layers.{idx}",
95 + )
96 + for idx in range(self.num_mtp_layers)
97 + )
98 +
99 + self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
100 + ["hidden_states", "residual"], config.hidden_size
101 + )
102 +
103 + self.norm = Qwen3_5RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
104 + self.pre_fc_norm_hidden = Qwen3_5RMSNorm(
105 + config.hidden_size, eps=config.rms_norm_eps
106 + )
107 + self.pre_fc_norm_embedding = Qwen3_5RMSNorm(
108 + config.hidden_size, eps=config.rms_norm_eps
109 + )
110 +
111 + def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
112 + return self.embed_tokens(input_ids)
113 +
114 + def forward(
115 + self,
116 + input_ids: torch.Tensor,
117 + positions: torch.Tensor,
118 + hidden_states: torch.Tensor,
119 + intermediate_tensors: IntermediateTensors | None = None,
120 + inputs_embeds: torch.Tensor | None = None,
121 + spec_step_idx: int = 0,
122 + ) -> torch.Tensor:
123 + if get_pp_group().is_first_rank:
124 + if inputs_embeds is None:
125 + inputs_embeds = self.embed_input_ids(input_ids)
126 + assert hidden_states.shape[-1] == inputs_embeds.shape[-1]
127 + inputs_embeds = self.pre_fc_norm_embedding(inputs_embeds)
128 + hidden_states = self.pre_fc_norm_hidden(hidden_states)
129 + hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1)
130 + hidden_states = self.fc(hidden_states)
131 + residual = None
132 + else:
133 + assert intermediate_tensors is not None
134 + hidden_states = intermediate_tensors["hidden_states"]
135 + residual = intermediate_tensors["residual"]
136 +
137 + current_step_idx = spec_step_idx % self.num_mtp_layers
138 + hidden_states, residual = self.layers[current_step_idx](
139 + positions=positions,
140 + hidden_states=hidden_states,
141 + residual=residual,
142 + )
143 +
144 + if not get_pp_group().is_last_rank:
145 + return IntermediateTensors(
146 + {"hidden_states": hidden_states, "residual": residual}
147 + )
148 +
149 + hidden_states, _ = self.norm(hidden_states, residual)
150 + return hidden_states
151 +
152 + def load_fused_expert_weights(
153 + self,
154 + name: str,
155 + params_dict: dict,
156 + loaded_weight: torch.Tensor,
157 + shard_id: str,
158 + num_experts: int,
159 + ) -> bool:
160 + param = params_dict[name]
161 + weight_loader = typing.cast(Callable[..., bool], param.weight_loader)
162 + loaded_local_expert = False
163 + for expert_id in range(num_experts):
164 + curr_expert_weight = loaded_weight[expert_id]
165 + success = weight_loader(
166 + param,
167 + curr_expert_weight,
168 + name,
169 + shard_id,
170 + expert_id,
171 + return_success=True,
172 + )
173 + if success:
174 + loaded_local_expert = True
175 +
176 + return loaded_local_expert
177 +
178 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
179 + stacked_params_mapping = [
180 + # (param_name, shard_name, shard_id)
181 + ("qkv_proj", "q_proj", "q"),
182 + ("qkv_proj", "k_proj", "k"),
183 + ("qkv_proj", "v_proj", "v"),
184 + ("gate_up_proj", "gate_proj", 0),
185 + ("gate_up_proj", "up_proj", 1),
186 + ]
187 +
188 + # Params for weights, fp8 weight scales, fp8 activation scales
189 + # (param_name, weight_name, expert_id, shard_id)
190 + expert_params_mapping = FusedMoE.make_expert_params_mapping(
191 + self,
192 + ckpt_gate_proj_name="gate_proj",
193 + ckpt_down_proj_name="down_proj",
194 + ckpt_up_proj_name="up_proj",
195 + num_experts=self.config.num_experts
196 + if hasattr(self.config, "num_experts")
197 + else 0,
198 + )
199 +
200 + params_dict = dict(self.named_parameters())
201 + loaded_params: set[str] = set()
202 + is_fused_expert = False
203 + fused_expert_params_mapping = [
204 + ("experts.w13_weight", "experts.gate_up_proj", 0, "w1"),
205 + ("experts.w2_weight", "experts.down_proj", 0, "w2"),
206 + ]
207 + num_experts = (
208 + self.config.num_experts if hasattr(self.config, "num_experts") else 0
209 + )
210 + for name, loaded_weight in weights:
211 + if "rotary_emb.inv_freq" in name:
212 + continue
213 +
214 + for param_name, weight_name, shard_id in stacked_params_mapping:
215 + if "experts.gate_up_proj" in name or "experts.down_proj" in name:
216 + is_fused_expert = True
217 + expert_params_mapping = fused_expert_params_mapping
218 +
219 + if weight_name not in name:
220 + continue
221 +
222 + if "mlp.experts" in name:
223 + continue
224 +
225 + name = name.replace(weight_name, param_name)
226 + # Skip loading extra bias for GPTQ models.
227 + if name.endswith(".bias") and name not in params_dict:
228 + continue
229 + # Skip layers on other devices.
230 + if is_pp_missing_parameter(name, self):
231 + continue
232 + if name not in params_dict:
233 + continue
234 + param = params_dict[name]
235 + weight_loader = param.weight_loader
236 + weight_loader(param, loaded_weight, shard_id)
237 + break
238 + else:
239 + is_expert_weight = False
240 + for mapping in expert_params_mapping:
241 + param_name, weight_name, expert_id, shard_id = mapping
242 + if weight_name not in name:
243 + continue
244 + is_expert_weight = True
245 + name_mapped = name.replace(weight_name, param_name)
246 + # Skip layers on other devices.
247 + if is_pp_missing_parameter(name_mapped, self):
248 + continue
249 + if is_fused_expert:
250 + # qwen3.5 no need to transpose
251 + # loaded_weight = loaded_weight.transpose(-1, -2)
252 + if "experts.gate_up_proj" in name:
253 + loaded_weight = loaded_weight.chunk(2, dim=-2)
254 + success_w1 = self.load_fused_expert_weights(
255 + name_mapped,
256 + params_dict,
257 + loaded_weight[0],
258 + "w1",
259 + num_experts,
260 + )
261 + success_w3 = self.load_fused_expert_weights(
262 + name_mapped,
263 + params_dict,
264 + loaded_weight[1],
265 + "w3",
266 + num_experts,
267 + )
268 + success = success_w1 and success_w3
269 + else:
270 + # down_proj
271 + success = self.load_fused_expert_weights(
272 + name_mapped,
273 + params_dict,
274 + loaded_weight,
275 + shard_id,
276 + num_experts,
277 + )
278 + if success:
279 + name = name_mapped
280 + break
281 + else:
282 + # Skip loading extra bias for GPTQ models.
283 + if (
284 + name_mapped.endswith(".bias")
285 + or name_mapped.endswith("_bias")
286 + ) and name_mapped not in params_dict:
287 + continue
288 + param = params_dict[name_mapped]
289 + weight_loader = param.weight_loader
290 + success = weight_loader(
291 + param,
292 + loaded_weight,
293 + name_mapped,
294 + shard_id=shard_id,
295 + expert_id=expert_id,
296 + return_success=True,
297 + )
298 + if success:
299 + name = name_mapped
300 + break
301 + else:
302 + if is_expert_weight:
303 + # We've checked that this is an expert weight
304 + # However it's not mapped locally to this rank
305 + # So we simply skip it
306 + continue
307 + # Skip loading extra bias for GPTQ models.
308 + if name.endswith(".bias") and name not in params_dict:
309 + continue
310 + if is_pp_missing_parameter(name, self):
311 + continue
312 + if name not in params_dict:
313 + logger.warning_once(
314 + f"Parameter {name} not found in params_dict, skip loading"
315 + )
316 + continue
317 + param = params_dict[name]
318 + weight_loader = getattr(
319 + param, "weight_loader", default_weight_loader
320 + )
321 + weight_loader(param, loaded_weight)
322 + loaded_params.add(name)
323 + return loaded_params
324 +
325 +
326 +@support_torch_compile(
327 + dynamic_arg_dims={
328 + "input_ids": 0,
329 + # positions is of shape (3, seq_len) if mrope is enabled for qwen2-vl,
330 + # otherwise (seq_len, ).
331 + "positions": -1,
332 + "intermediate_tensors": 0,
333 + "inputs_embeds": 0,
334 + "hidden_states": 0,
335 + }
336 +)
337 +class Qwen3_5MTP(nn.Module, SupportsMultiModal):
338 + packed_modules_mapping = {
339 + "qkv_proj": [
340 + "q_proj",
341 + "k_proj",
342 + "v_proj",
343 + ],
344 + "gate_up_proj": ["up_proj", "down_proj"],
345 + }
346 +
347 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
348 + config = vllm_config.model_config.hf_text_config
349 + self.vllm_config = vllm_config
350 + cache_config = vllm_config.cache_config
351 + if cache_config.mamba_cache_mode == "all":
352 + raise NotImplementedError(
353 + "Qwen3_5MTP currently does not support 'all' prefix caching, "
354 + "please use '--mamba-cache-mode=align' instead"
355 + )
356 +
357 + self.quant_config = vllm_config.quant_config
358 +
359 + super().__init__()
360 + self.config = config
361 + self.model = Qwen3_5MultiTokenPredictor(
362 + vllm_config=vllm_config, prefix=maybe_prefix(prefix, "mtp")
363 + )
364 +
365 + if get_pp_group().is_last_rank:
366 + if config.tie_word_embeddings:
367 + self.lm_head = self.model.embed_tokens
368 + else:
369 + self.lm_head = ParallelLMHead(
370 + config.vocab_size,
371 + config.hidden_size,
372 + prefix=maybe_prefix(prefix, "lm_head"),
373 + )
374 + else:
375 + self.lm_head = PPMissingLayer()
376 +
377 + self.logits_processor = LogitsProcessor(config.vocab_size)
378 +
379 + def embed_input_ids(
380 + self,
381 + input_ids: torch.Tensor,
382 + multimodal_embeddings: MultiModalEmbeddings | None = None,
383 + *,
384 + is_multimodal: torch.Tensor | None = None,
385 + handle_oov_mm_token: bool = False,
386 + ) -> torch.Tensor:
387 + inputs_embeds = self._embed_text_input_ids(
388 + input_ids,
389 + self.model.embed_input_ids,
390 + is_multimodal=is_multimodal,
391 + handle_oov_mm_token=handle_oov_mm_token,
392 + )
393 +
394 + if multimodal_embeddings is None or len(multimodal_embeddings) == 0:
395 + return inputs_embeds
396 +
397 + is_multimodal = _require_is_multimodal(is_multimodal)
398 +
399 + inputs_embeds = _merge_multimodal_embeddings(
400 + inputs_embeds=inputs_embeds,
401 + multimodal_embeddings=multimodal_embeddings,
402 + is_multimodal=is_multimodal,
403 + )
404 +
405 + return inputs_embeds
406 +
407 + def forward(
408 + self,
409 + input_ids: torch.Tensor,
410 + positions: torch.Tensor,
411 + hidden_states: torch.Tensor,
412 + intermediate_tensors: IntermediateTensors | None = None,
413 + inputs_embeds: torch.Tensor | None = None,
414 + **kwargs: object,
415 + ):
416 + hidden_states = self.model(
417 + input_ids, positions, hidden_states, intermediate_tensors, inputs_embeds
418 + )
419 + return hidden_states
420 +
421 + def compute_logits(
422 + self,
423 + hidden_states: torch.Tensor,
424 + spec_step_idx: int = 0,
425 + ) -> torch.Tensor | None:
426 + return self.logits_processor(self.lm_head, hidden_states)
427 +
428 + def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
429 + def remap_weight_names(weights):
430 + for name, weight in weights:
431 + if name.startswith("mtp."):
432 + name = name.replace("mtp.", "model.")
433 + elif any(key in name for key in ["embed_tokens", "lm_head"]):
434 + if "embed_tokens" in name:
435 + name = name.replace("language_model.", "")
436 + else:
437 + continue
438 + yield name, weight
439 +
440 + loader = AutoWeightsLoader(self)
441 + return loader.load_weights(remap_weight_names(weights))
442 +
443 +
444 +class Qwen3_5MoeMTP(Qwen3_5MTP, QwenNextMixtureOfExperts):
445 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
446 + super().__init__(vllm_config=vllm_config, prefix=prefix)
447 + self.set_moe_parameters()
vllm/model_executor/models/qwen3_next.py
+6 -6
@@ -105,7 +105,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
105 def __init__(self, vllm_config: VllmConfig, prefix: str = ""):
106 super().__init__()
107
108 - config = vllm_config.model_config.hf_config
108 + config = vllm_config.model_config.hf_text_config
109 parallel_config = vllm_config.parallel_config
110 quant_config = vllm_config.quant_config
111
@@ -176,7 +176,7 @@ class Qwen3NextSparseMoeBlock(nn.Module):
176 hidden_size=config.hidden_size,
177 intermediate_size=config.moe_intermediate_size,
178 reduce_results=False,
179 - renormalize=config.norm_topk_prob,
179 + renormalize=getattr(config, "norm_topk_prob", True),
180 quant_config=quant_config,
181 prefix=f"{prefix}.experts",
182 enable_eplb=self.enable_eplb,
@@ -965,7 +965,7 @@ class Qwen3NextModel(nn.Module):
965 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
966 super().__init__()
967
968 - config: Qwen3NextConfig = vllm_config.model_config.hf_config
968 + config: Qwen3NextConfig = vllm_config.model_config.hf_text_config
969 parallel_config = vllm_config.parallel_config
970
971 eplb_config = parallel_config.eplb_config
@@ -1042,7 +1042,7 @@ class Qwen3NextModel(nn.Module):
1042 ckpt_gate_proj_name="gate_proj",
1043 ckpt_down_proj_name="down_proj",
1044 ckpt_up_proj_name="up_proj",
1045 - num_experts=self.config.num_experts,
1045 + num_experts=getattr(self.config, "num_experts", 0),
1046 num_redundant_experts=self.num_redundant_experts,
1047 )
1048
@@ -1201,7 +1201,7 @@ class Qwen3NextForCausalLM(
1201 }
1202
1203 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1204 - config = vllm_config.model_config.hf_config
1204 + config = vllm_config.model_config.hf_text_config
1205 self.vllm_config = vllm_config
1206 self.model_config = vllm_config.model_config
1207 cache_config = vllm_config.cache_config
@@ -1265,7 +1265,7 @@ class Qwen3NextForCausalLM(
1265 cls, vllm_config: "VllmConfig"
1266 ) -> tuple[tuple[int, int], tuple[int, int]]:
1267 parallel_config = vllm_config.parallel_config
1268 - hf_config = vllm_config.model_config.hf_config
1268 + hf_config = vllm_config.model_config.hf_text_config
1269 tp_size = parallel_config.tensor_parallel_size
1270 num_spec = (
1271 vllm_config.speculative_config.num_speculative_tokens
vllm/model_executor/models/registry.py
+10
@@ -466,6 +466,14 @@ _MULTIMODAL_MODELS = {
466 "qwen3_vl_moe",
467 "Qwen3VLMoeForConditionalGeneration",
468 ),
469 + "Qwen3_5ForConditionalGeneration": (
470 + "qwen3_5",
471 + "Qwen3_5ForConditionalGeneration",
472 + ),
473 + "Qwen3_5MoeForConditionalGeneration": (
474 + "qwen3_5",
475 + "Qwen3_5MoeForConditionalGeneration",
476 + ),
477 "SkyworkR1VChatModel": ("skyworkr1v", "SkyworkR1VChatModel"),
478 "Step3VLForConditionalGeneration": ("step3_vl", "Step3VLForConditionalGeneration"), # noqa: E501
479 "TarsierForConditionalGeneration": ("tarsier", "TarsierForConditionalGeneration"), # noqa: E501
@@ -509,6 +517,8 @@ _SPECULATIVE_DECODING_MODELS = {
517 "OpenPanguMTPModel": ("openpangu_mtp", "OpenPanguMTP"),
518 "Qwen3NextMTP": ("qwen3_next_mtp", "Qwen3NextMTP"),
519 "Step3p5MTP": ("step3p5_mtp", "Step3p5MTP"),
520 + "Qwen3_5MTP": ("qwen3_5_mtp", "Qwen3_5MTP"),
521 + "Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
522 # Temporarily disabled.
523 # # TODO(woosuk): Re-enable this once the MLP Speculator is supported in V1.
524 # "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
vllm/transformers_utils/model_arch_config_convertor.py
+6
@@ -420,6 +420,11 @@ class Qwen3NextMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
420 return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
421
422
423 +class Qwen3_5MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
424 + def get_num_hidden_layers(self) -> int:
425 + return getattr(self.hf_text_config, "mtp_num_hidden_layers", 0)
426 +
427 +
428 class PanguUltraMoeMTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
429 def get_num_hidden_layers(self) -> int:
430 return getattr(self.hf_text_config, "num_nextn_predict_layers", 0)
@@ -445,6 +450,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
450 "nemotron-nas": NemotronNasModelArchConfigConvertor,
451 "deepseek_mtp": DeepSeekMTPModelArchConfigConvertor,
452 "qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
453 + "qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor,
454 "mimo_mtp": MimoMTPModelArchConfigConvertor,
455 "glm4_moe_mtp": GLM4MoeMTPModelArchConfigConvertor,
456 "glm_ocr_mtp": GLM4MoeMTPModelArchConfigConvertor,
vllm/v1/spec_decode/eagle.py
+2
@@ -1356,6 +1356,8 @@ class SpecDecodeBaseProposer:
1356 "Qwen3VLMoeForConditionalGeneration",
1357 "HunYuanVLForConditionalGeneration",
1358 "GlmOcrForConditionalGeneration",
1359 + "Qwen3_5ForConditionalGeneration",
1360 + "Qwen3_5MoeForConditionalGeneration",
1361 ]:
1362 self.model.config.image_token_index = target_model.config.image_token_id
1363 elif self.get_model_name(target_model) == "PixtralForConditionalGeneration":