A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
[Models] Fuse Qwen3.5 GDN's qkvz_proj and ba_proj (#34492)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
Isotr0py committed
Feb 16, 2026 at 23:32 UTC
3bb4e4311c6da31257e6c8e5b1027ef516e025c8
3 files changed
+87
-182
vllm/model_executor/layers/linear.py
+28
-6
@@ -685,8 +685,13 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
685
self,
686
param: Parameter,
687
loaded_weight: torch.Tensor,
688
- loaded_shard_id: int | None = None,
688
+ loaded_shard_id: tuple[int, ...] | int | None = None,
689
):
690
+ if isinstance(loaded_shard_id, tuple):
691
+ raise NotImplementedError(
692
+ "Shard id with multiple indices is not supported in weight_loader, "
693
+ "please use weight_loader_v2 instead."
694
+ )
695
# Special case for GGUF
696
# initialize GGUF param after we know the quantize type
697
is_gguf_weight = getattr(param, "is_gguf_weight", False)
@@ -825,7 +830,10 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
830
param_data.copy_(loaded_weight)
831
832
def _load_fused_module_from_checkpoint(
828
- self, param: BasevLLMParameter, loaded_weight: torch.Tensor
833
+ self,
834
+ param: BasevLLMParameter,
835
+ loaded_weight: torch.Tensor,
836
+ output_sizes: list[int] | None = None,
837
):
838
"""
839
Handle special case for models where MLP layers are already
@@ -839,7 +847,8 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
847
848
current_shard_offset = 0
849
shard_offsets: list[tuple[int, int, int]] = []
842
- for i, output_size in enumerate(self.output_sizes):
850
+ output_sizes = output_sizes or self.output_sizes
851
+ for i, output_size in enumerate(output_sizes):
852
shard_offsets.append((i, current_shard_offset, output_size))
853
current_shard_offset += output_size
854
@@ -864,17 +873,30 @@ class MergedColumnParallelLinear(ColumnParallelLinear):
873
self,
874
param: BasevLLMParameter,
875
loaded_weight: torch.Tensor,
867
- loaded_shard_id: int | None = None,
876
+ loaded_shard_id: tuple[int, ...] | int | None = None,
877
):
869
- if loaded_shard_id is None:
878
+ if loaded_shard_id is None or isinstance(loaded_shard_id, tuple):
879
if isinstance(param, PerTensorScaleParameter):
880
param.load_merged_column_weight(loaded_weight=loaded_weight, shard_id=0)
881
return
882
elif type(param) in (RowvLLMParameter, BasevLLMParameter):
883
param.load_merged_column_weight(loaded_weight=loaded_weight)
884
return
885
+ output_sizes = (
886
+ [self.output_sizes[idx] for idx in loaded_shard_id]
887
+ if loaded_shard_id
888
+ else None
889
+ )
890
+ if isinstance(param, BlockQuantScaleParameter):
891
+ weight_block_size = getattr(self, "weight_block_size", None)
892
+ output_sizes = [
893
+ adjust_block_scale_shard(weight_block_size, size, 0)[0]
894
+ for size in (output_sizes or self.output_sizes)
895
+ ]
896
# TODO: @dsikka - move to parameter.py
877
- self._load_fused_module_from_checkpoint(param, loaded_weight)
897
+ self._load_fused_module_from_checkpoint(
898
+ param, loaded_weight, output_sizes=output_sizes
899
+ )
900
return
901
902
assert loaded_shard_id < len(self.output_sizes)
vllm/model_executor/models/qwen3_5.py
+32
-166
@@ -30,36 +30,20 @@ from collections.abc import Callable, Iterable
30
import torch
31
from einops import rearrange
32
from torch import nn
33
-from transformers.activations import ACT2FN
33
34
from vllm.compilation.decorators import support_torch_compile
35
from vllm.config import (
37
- CacheConfig,
38
- ModelConfig,
39
- SpeculativeConfig,
36
VllmConfig,
41
- get_current_vllm_config,
37
)
38
from vllm.distributed import (
44
- divide,
39
get_pp_group,
46
- get_tensor_model_parallel_rank,
47
- get_tensor_model_parallel_world_size,
40
)
41
from vllm.logger import init_logger
42
from vllm.model_executor.layers.layernorm import (
43
GemmaRMSNorm as Qwen3_5RMSNorm,
44
)
53
-from vllm.model_executor.layers.layernorm import RMSNormGated
54
-from vllm.model_executor.layers.linear import (
55
- ColumnParallelLinear,
56
- MergedColumnParallelLinear,
57
- RowParallelLinear,
58
-)
45
+from vllm.model_executor.layers.linear import MergedColumnParallelLinear
46
from vllm.model_executor.layers.logits_processor import LogitsProcessor
60
-from vllm.model_executor.layers.mamba.mamba_mixer2 import (
61
- mamba_v2_sharded_weight_loader,
62
-)
47
from vllm.model_executor.layers.mamba.mamba_utils import (
48
MambaStateCopyFunc,
49
MambaStateCopyFuncCalculator,
@@ -73,11 +57,8 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
57
)
58
from vllm.model_executor.model_loader.weight_utils import (
59
default_weight_loader,
76
- sharded_weight_loader,
60
)
78
-from vllm.model_executor.utils import set_weight_attrs
61
from vllm.multimodal import MULTIMODAL_REGISTRY
80
-from vllm.platforms import current_platform
62
from vllm.sequence import IntermediateTensors
63
from vllm.transformers_utils.configs.qwen3_5 import (
64
Qwen3_5Config,
@@ -99,7 +80,6 @@ from .interfaces import (
80
)
81
from .qwen2_moe import Qwen2MoeMLP as Qwen3NextMLP
82
from .qwen3_next import (
102
- ChunkGatedDeltaRule,
83
Qwen3NextAttention,
84
Qwen3NextDecoderLayer,
85
Qwen3NextGatedDeltaNet,
@@ -139,154 +119,31 @@ class Qwen3_5MoeProcessingInfo(Qwen3VLProcessingInfo):
119
120
121
class Qwen3_5GatedDeltaNet(Qwen3NextGatedDeltaNet):
142
- def __init__(
143
- self,
144
- config: Qwen3_5TextConfig | Qwen3_5MoeTextConfig,
145
- model_config: ModelConfig | None = None,
146
- cache_config: CacheConfig | None = None,
147
- quant_config: QuantizationConfig | None = None,
148
- speculative_config: SpeculativeConfig | None = None,
149
- prefix: str = "",
150
- ) -> None:
151
- super(Qwen3NextGatedDeltaNet, self).__init__()
152
- self.tp_size = get_tensor_model_parallel_world_size()
153
- self.tp_rank = get_tensor_model_parallel_rank()
154
- self.hidden_size = config.hidden_size
155
- self.num_v_heads = config.linear_num_value_heads
156
- self.num_k_heads = config.linear_num_key_heads
157
- self.head_k_dim = config.linear_key_head_dim
158
- self.head_v_dim = config.linear_value_head_dim
159
- self.key_dim = self.head_k_dim * self.num_k_heads
160
- self.value_dim = self.head_v_dim * self.num_v_heads
161
-
162
- self.conv_kernel_size = config.linear_conv_kernel_dim
163
- self.layer_idx = extract_layer_index(prefix)
164
- self.activation = config.hidden_act
165
- self.act = ACT2FN[config.hidden_act]
166
- self.layer_norm_epsilon = config.rms_norm_eps
167
- self.prefix = prefix
168
-
169
- self.config = config
170
- self.model_config = model_config
171
- self.cache_config = cache_config
172
- self.quant_config = quant_config
173
- self.speculative_config = speculative_config
174
- self.num_spec = (
175
- self.speculative_config.num_speculative_tokens
176
- if self.speculative_config
177
- else 0
178
- )
179
-
180
- # QKV
181
- self.conv_dim = self.key_dim * 2 + self.value_dim
182
- self.conv1d = ColumnParallelLinear(
183
- input_size=self.conv_kernel_size,
184
- output_size=self.conv_dim,
185
- bias=False,
186
- prefix=f"{prefix}.conv1d",
187
- )
188
- self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
189
-
190
- self.in_proj_qkv = MergedColumnParallelLinear(
191
- input_size=self.hidden_size,
192
- output_sizes=[self.key_dim, self.key_dim, self.value_dim],
193
- bias=False,
194
- quant_config=quant_config,
195
- prefix=f"{prefix}.in_proj_qkv",
196
- )
197
- self.in_proj_z = ColumnParallelLinear(
198
- input_size=self.hidden_size,
199
- output_size=self.value_dim,
200
- bias=False,
201
- quant_config=quant_config,
202
- prefix=f"{prefix}.in_proj_z",
203
- )
204
- self.in_proj_b = ColumnParallelLinear(
205
- input_size=self.hidden_size,
206
- output_size=self.num_v_heads,
207
- bias=False,
208
- quant_config=quant_config,
209
- prefix=f"{prefix}.in_proj_b",
210
- )
211
- self.in_proj_a = ColumnParallelLinear(
212
- input_size=self.hidden_size,
213
- output_size=self.num_v_heads,
214
- bias=False,
215
- quant_config=quant_config,
216
- prefix=f"{prefix}.in_proj_a",
217
- )
218
-
219
- query_key_settings = (self.key_dim, 0, False)
220
- value_settings = (self.value_dim, 0, False)
221
-
222
- delattr(self.conv1d.weight, "weight_loader")
223
- set_weight_attrs(
224
- self.conv1d.weight,
225
- {
226
- "weight_loader": mamba_v2_sharded_weight_loader(
227
- [
228
- query_key_settings,
229
- query_key_settings,
230
- value_settings,
231
- ],
232
- self.tp_size,
233
- self.tp_rank,
234
- )
235
- },
236
- )
237
-
238
- # selective projection used to make dt, B and C input dependant
239
-
240
- # time step projection (discretization)
241
- # instantiate once and copy inv_dt in init_weights of PretrainedModel
242
- self.dt_bias = nn.Parameter(
243
- torch.ones(self.num_v_heads // self.tp_size),
244
- )
245
- self.A_log = nn.Parameter(
246
- torch.empty(
247
- divide(self.num_v_heads, self.tp_size),
248
- )
249
- )
250
-
251
- set_weight_attrs(self.A_log, {"weight_loader": sharded_weight_loader(0)})
252
- set_weight_attrs(self.dt_bias, {"weight_loader": sharded_weight_loader(0)})
253
-
254
- self.norm = RMSNormGated(
255
- self.head_v_dim,
256
- eps=self.layer_norm_epsilon,
257
- group_size=None,
258
- norm_before_gate=True,
259
- device=current_platform.current_device(),
260
- dtype=config.dtype,
261
- )
262
-
263
- self.out_proj = RowParallelLinear(
264
- self.value_dim,
265
- self.hidden_size,
266
- bias=False,
267
- input_is_parallel=True,
268
- quant_config=quant_config,
269
- prefix=f"{prefix}.out_proj",
270
- )
271
-
272
- self.chunk_gated_delta_rule = ChunkGatedDeltaRule()
273
-
274
- compilation_config = get_current_vllm_config().compilation_config
275
- if prefix in compilation_config.static_forward_context:
276
- raise ValueError(f"Duplicate layer name: {prefix}")
277
- compilation_config.static_forward_context[prefix] = self
278
-
122
def fix_query_key_value_ordering(
123
self,
281
- mixed_qkv,
282
- z,
283
- b,
284
- a,
124
+ mixed_qkvz: torch.Tensor,
125
+ mixed_ba: torch.Tensor,
126
):
127
raise NotImplementedError(
128
"Qwen3.5 Series dont need to fix query key value ordering"
129
)
130
131
+ def create_qkvz_proj(
132
+ self,
133
+ hidden_size: int,
134
+ key_dim: int,
135
+ value_dim: int,
136
+ quant_config: QuantizationConfig | None,
137
+ prefix: str,
138
+ ) -> MergedColumnParallelLinear:
139
+ return MergedColumnParallelLinear(
140
+ input_size=hidden_size,
141
+ output_sizes=[key_dim, key_dim, value_dim, value_dim],
142
+ bias=False,
143
+ quant_config=quant_config,
144
+ prefix=prefix,
145
+ )
146
+
147
def forward(
148
self,
149
hidden_states: torch.Tensor,
@@ -303,11 +160,13 @@ class Qwen3_5GatedDeltaNet(Qwen3NextGatedDeltaNet):
160
# ============================================================
161
# Part 1: Input Projection
162
# ============================================================
306
- mixed_qkv, _ = self.in_proj_qkv(hidden_states)
307
- z, _ = self.in_proj_z(hidden_states)
163
+ mixed_qkvz, _ = self.in_proj_qkvz(hidden_states)
164
+ qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
165
+ z_size = self.value_dim // self.tp_size
166
+ mixed_qkv, z = mixed_qkvz.split([qkv_size, z_size], dim=-1)
167
z = z.reshape(z.size(0), -1, self.head_v_dim)
309
- b, _ = self.in_proj_b(hidden_states)
310
- a, _ = self.in_proj_a(hidden_states)
168
+ ba, _ = self.in_proj_ba(hidden_states)
169
+ b, a = ba.chunk(2, dim=-1)
170
171
b = b.contiguous()
172
a = a.contiguous()
@@ -506,11 +365,18 @@ class Qwen3_5Model(Qwen3NextModel):
365
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
366
stacked_params_mapping = [
367
# (param_name, shard_name, shard_id)
368
+ # self attention
369
("qkv_proj", "q_proj", "q"),
370
("qkv_proj", "k_proj", "k"),
371
("qkv_proj", "v_proj", "v"),
372
+ # mlp
373
("gate_up_proj", "gate_proj", 0),
374
("gate_up_proj", "up_proj", 1),
375
+ # GDN
376
+ ("in_proj_qkvz", "in_proj_qkv", (0, 1, 2)),
377
+ ("in_proj_qkvz", "in_proj_z", 3),
378
+ ("in_proj_ba", "in_proj_b", 0),
379
+ ("in_proj_ba", "in_proj_a", 1),
380
]
381
382
params_dict = dict(self.named_parameters())
vllm/model_executor/models/qwen3_next.py
+27
-10
@@ -44,6 +44,7 @@ from vllm.model_executor.layers.layernorm import (
44
from vllm.model_executor.layers.layernorm import RMSNormGated
45
from vllm.model_executor.layers.linear import (
46
ColumnParallelLinear,
47
+ MergedColumnParallelLinear,
48
QKVParallelLinear,
49
ReplicatedLinear,
50
RowParallelLinear,
@@ -406,19 +407,19 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
407
self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
408
409
# projection of the input hidden states
409
- self.projection_size_qkvz = self.key_dim * 2 + self.value_dim * 2
410
- self.projection_size_ba = self.num_v_heads * 2
411
- self.in_proj_qkvz = ColumnParallelLinear(
412
- input_size=self.hidden_size,
413
- output_size=self.projection_size_qkvz,
414
- bias=False,
410
+ # Qwen3-Next and Qwen3.5 has a different qkv_proj layout,
411
+ # we need to create qkvz_proj adaptively here.
412
+ self.in_proj_qkvz = self.create_qkvz_proj(
413
+ hidden_size=self.hidden_size,
414
+ key_dim=self.key_dim,
415
+ value_dim=self.value_dim,
416
quant_config=quant_config,
417
prefix=f"{prefix}.in_proj_qkvz",
418
)
419
# ba_proj doesn't support blockwise fp8 quantization.
419
- self.in_proj_ba = ColumnParallelLinear(
420
+ self.in_proj_ba = MergedColumnParallelLinear(
421
input_size=self.hidden_size,
421
- output_size=self.projection_size_ba,
422
+ output_sizes=[self.num_v_heads] * 2,
423
bias=False,
424
quant_config=quant_config,
425
prefix=f"{prefix}.in_proj_ba",
@@ -484,10 +485,26 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
485
raise ValueError(f"Duplicate layer name: {prefix}")
486
compilation_config.static_forward_context[prefix] = self
487
488
+ def create_qkvz_proj(
489
+ self,
490
+ hidden_size: int,
491
+ key_dim: int,
492
+ value_dim: int,
493
+ quant_config: QuantizationConfig | None,
494
+ prefix: str,
495
+ ) -> MergedColumnParallelLinear:
496
+ return MergedColumnParallelLinear(
497
+ input_size=hidden_size,
498
+ output_sizes=[sum((key_dim, key_dim, value_dim)), value_dim],
499
+ bias=False,
500
+ quant_config=quant_config,
501
+ prefix=f"{prefix}.in_proj_qkvz",
502
+ )
503
+
504
def fix_query_key_value_ordering(
505
self,
489
- mixed_qkvz,
490
- mixed_ba,
506
+ mixed_qkvz: torch.Tensor,
507
+ mixed_ba: torch.Tensor,
508
):
509
"""
510
Derives `query`, `key` and `value` tensors from `mixed_qkvzba`.