[Bugfix] Preserve FP8 indexer WK pairs across incremental load_weights (#46168)
Signed-off-by: lcheng <lcheng321@gatech.edu> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
lcheng committed
Jul 7, 2026 at 00:16 UTC
24dd2aec81d780b9d3085fbc83fa81af970a3ba8
1 file changed
+4
-1
vllm/model_executor/models/deepseek_v2.py
+4
-1
@@ -1511,7 +1511,10 @@ class DeepseekV2Model(nn.Module):
1511
("qkv_proj", "v_proj", "v"),
1512
]
1513
# Fused indexer wk + weights_proj (shard 0 = wk, shard 1 = weights_proj)
1514
- _pending_wk_fp8: dict = {} # When WK is in FP8, we dequant to BF16 for fusion
1514
+ _pending_wk_fp8 = getattr(self, "_pending_indexer_wk_fp8", None)
1515
+ if _pending_wk_fp8 is None:
1516
+ self._pending_indexer_wk_fp8 = _pending_wk_fp8 = {}
1517
+
1518
indexer_fused_mapping = [
1519
("wk_weights_proj", "wk", 0),
1520
("wk_weights_proj", "weights_proj", 1),