@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 27208be66e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Kernel] Add back batch size 1536 and 3072 to MoE tuning (#5242)

Woosuk Kwon committed Jun 4, 2024 at 09:58 UTC 27208be66e6529d016358d15fe87e95810698227
1 file changed +3 -1
benchmarks/kernels/benchmark_moe.py
+3 -1
@@ -254,7 +254,9 @@ def main(args: argparse.Namespace):
254 use_fp8 = args.dtype == "fp8"
255
256 if args.batch_size is None:
257 - batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
257 + batch_sizes = [
258 + 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096
259 + ]
260 else:
261 batch_sizes = [args.batch_size]
262