A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 09:30 UTC
Fix AttributeError in Qwen3.5 GDN layers with quantized models (#37448)
Signed-off-by: Jim Smith <jim@joshua8.ai> Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: Xin Yang <105740670+xyang16@users.noreply.github.com>
Jim Smith committed
Mar 19, 2026 at 19:21 UTC
4120a05ff1d03797dbcd506110f0a997520a6395
2 files changed
+4
-4
vllm/model_executor/models/qwen3_5.py
+2
-2
@@ -182,8 +182,8 @@ class Qwen3_5GatedDeltaNet(Qwen3NextGatedDeltaNet):
182
# ============================================================
183
mixed_qkvz, ba = torch.ops.vllm.gdn_in_proj(
184
hidden_states,
185
- self.in_proj_qkvz.weight.shape[0],
186
- self.in_proj_ba.weight.shape[0],
185
+ sum(self.in_proj_qkvz.output_sizes) // self.tp_size,
186
+ sum(self.in_proj_ba.output_sizes) // self.tp_size,
187
self.prefix,
188
)
189
qkv_size = (self.key_dim * 2 + self.value_dim) // self.tp_size
vllm/model_executor/models/qwen3_next.py
+2
-2
@@ -660,8 +660,8 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):
660
# ============================================================
661
projected_states_qkvz, projected_states_ba = torch.ops.vllm.gdn_in_proj(
662
hidden_states,
663
- self.in_proj_qkvz.weight.shape[0],
664
- self.in_proj_ba.weight.shape[0],
663
+ sum(self.in_proj_qkvz.output_sizes) // self.tp_size,
664
+ sum(self.in_proj_ba.output_sizes) // self.tp_size,
665
self.prefix,
666
)
667
query, key, value, z, b, a = self.fix_query_key_value_ordering(