A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
[Bugfix][ROCm] Fix load issue on deepseek quark quantization when shared expert enabled (#31261)
Signed-off-by: ganyi <ygan@amd.com>
Pleaplusone committed
Dec 24, 2025 at 16:47 UTC
b41aeb3468fecb33b7934fc7ca5406bdba84dace
1 file changed
+11
-8
vllm/model_executor/models/deepseek_v2.py
+11
-8
@@ -1598,7 +1598,11 @@ class DeepseekV2ForCausalLM(
1598
# Determine split axis based on op type
1599
# gate/up: ColumnParallel → split along dim 0
1600
# down: RowParallel → split along dim 1
1601
- split_dim = 1 if "down_proj.weight" in name else 0
1601
+ split_dim = (
1602
+ 1
1603
+ if ("down_proj.weight" in name and loaded_weight.ndim > 1)
1604
+ else 0
1605
+ )
1606
total = loaded_weight.shape[split_dim]
1607
assert total % num_chunks == 0, (
1608
f"Shared expert weight dim {total} "
@@ -1611,14 +1615,13 @@ class DeepseekV2ForCausalLM(
1615
weight_to_load = loaded_weight
1616
1617
if is_fusion_moe_shared_experts_layer:
1614
- if split_dim == 0:
1615
- weight_to_load = loaded_weight[
1616
- j * chunk_size : (j + 1) * chunk_size, :
1617
- ]
1618
+ chunk_slice = slice(j * chunk_size, (j + 1) * chunk_size)
1619
+ if loaded_weight.ndim == 1:
1620
+ weight_to_load = loaded_weight[chunk_slice]
1621
+ elif split_dim == 0:
1622
+ weight_to_load = loaded_weight[chunk_slice, :]
1623
else:
1619
- weight_to_load = loaded_weight[
1620
- :, j * chunk_size : (j + 1) * chunk_size
1621
- ]
1624
+ weight_to_load = loaded_weight[:, chunk_slice]
1625
# Synthesize an expert-style name so expert mapping
1626
# can route it
1627
chunk_name = name.replace(