A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[feat] move WEIGHT_SCALE_SUPPORTED into raise block to accelerate RLHF weight loading (#21164)
Signed-off-by: huangweixiao <huangweixiao@msh.team>
Weixiao Huang committed
Aug 4, 2025 at 15:43 UTC
c1b4eb048a286ea5e7bcca730ae5676625f06541
1 file changed
+3
-3
vllm/model_executor/layers/fused_moe/layer.py
+3
-3
@@ -1079,9 +1079,6 @@ class FusedMoE(torch.nn.Module):
1079
raise ValueError(f"shard_id must be ['w1','w2','w3'] but "
1080
f"got {shard_id}.")
1081
1082
- WEIGHT_SCALE_SUPPORTED = [
1083
- e.value for e in FusedMoeWeightScaleSupported
1084
- ]
1082
# Fetch the dim to shard the parameter/loaded weight
1083
# based on the shard id. This will be whatever
1084
# dimension intermediate_size_per_partition is used.
@@ -1230,6 +1227,9 @@ class FusedMoE(torch.nn.Module):
1227
loaded_weight=loaded_weight,
1228
expert_id=expert_id)
1229
else:
1230
+ WEIGHT_SCALE_SUPPORTED = [
1231
+ e.value for e in FusedMoeWeightScaleSupported
1232
+ ]
1233
raise ValueError(
1234
f"quant method must be one of {WEIGHT_SCALE_SUPPORTED}")
1235
return True if return_success else None