A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 12:30 UTC
[Misc][Quantization] Clarify the intent of GGUF `FusedMoE` weight materialization (#30310)
Signed-off-by: Tsukasa OI <floss_llm@irq.a4lg.com>
Tsukasa OI committed
Dec 13, 2025 at 14:55 UTC
fdc135d768267b3a0ae8ed6fc3eca6a68d75f7a6
1 file changed
+6
-2
vllm/model_executor/layers/fused_moe/layer.py
+6
-2
@@ -1200,10 +1200,14 @@ class FusedMoE(CustomOp):
1200
if full_load:
1201
shard_dim += 1
1202
1203
- # Materialize GGUF UninitializedParameter
1203
+ # Materialize GGUF UninitializedParameter accounting merged weights
1204
if is_gguf_weight and isinstance(param, UninitializedParameter):
1205
+ # To materialize a tensor, we must have full shape including
1206
+ # number of experts, making this portion to require `full_load`.
1207
+ assert full_load
1208
final_shape = list(loaded_weight.shape)
1206
- if shard_id in ["w1", "w3"]:
1209
+ # w1 and w3 are merged per expert.
1210
+ if shard_id in {"w1", "w3"}:
1211
final_shape[1] *= 2
1212
final_shape[shard_dim] = final_shape[shard_dim] // self.tp_size
1213
param.materialize(final_shape, dtype=loaded_weight.dtype)