A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Fix benchmark_moe.py for blockwise fp8. (#23823)
Signed-off-by: crischeng <420985011@qq.com> Co-authored-by: cris <grace@guisenbindeMacBook-Pro.local>
YUQI.CHENG committed
Aug 28, 2025 at 21:44 UTC
66548f66031006ca873ac799d2dc8497fec33339
1 file changed
+4
-1
benchmarks/kernels/benchmark_moe.py
+4
-1
@@ -419,8 +419,10 @@ class BenchmarkWorker:
419
)
420
# NOTE(woosuk): The current naming convention uses w2.shape[2], which
421
# is the intermediate size after silu_and_mul.
422
+ block_n = block_quant_shape[0] if block_quant_shape else None
423
+ block_k = block_quant_shape[1] if block_quant_shape else None
424
op_config = get_moe_configs(
423
- num_experts, shard_intermediate_size // 2, dtype_str
425
+ num_experts, shard_intermediate_size // 2, dtype_str, block_n, block_k
426
)
427
if op_config is None:
428
config = get_default_config(
@@ -430,6 +432,7 @@ class BenchmarkWorker:
432
hidden_size,
433
topk,
434
dtype_str,
435
+ block_quant_shape,
436
)
437
else:
438
config = op_config[min(op_config.keys(), key=lambda x: abs(x - num_tokens))]