A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Kernel] Add back batch size 1536 and 3072 to MoE tuning (#5242)
Woosuk Kwon committed
Jun 4, 2024 at 09:58 UTC
27208be66e6529d016358d15fe87e95810698227
1 file changed
+3
-1
benchmarks/kernels/benchmark_moe.py
+3
-1
@@ -254,7 +254,9 @@ def main(args: argparse.Namespace):
254
use_fp8 = args.dtype == "fp8"
255
256
if args.batch_size is None:
257
- batch_sizes = [1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096]
257
+ batch_sizes = [
258
+ 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 1536, 2048, 3072, 4096
259
+ ]
260
else:
261
batch_sizes = [args.batch_size]
262