A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Bugfix/CI] Fixup benchmark_moe.py (#12562)
Fixes `is_marlin` not being passed into `get_default_config` Also allow `--tensor-parallel-size` in addition to `-tp` and `--tp-size` Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Feb 1, 2025 at 00:41 UTC
cfa134d2475096eae47a58d14a6dec4c3fba9294
1 file changed
+12
-4
benchmarks/kernels/benchmark_moe.py
+12
-4
@@ -343,9 +343,13 @@ class BenchmarkWorker:
343
op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
344
dtype_str)
345
if op_config is None:
346
- config = get_default_config(num_tokens, num_experts,
347
- shard_intermediate_size, hidden_size,
348
- topk, dtype_str)
346
+ config = get_default_config(num_tokens,
347
+ num_experts,
348
+ shard_intermediate_size,
349
+ hidden_size,
350
+ topk,
351
+ dtype_str,
352
+ is_marlin=False)
353
else:
354
config = op_config[min(op_config.keys(),
355
key=lambda x: abs(x - num_tokens))]
@@ -536,7 +540,11 @@ if __name__ == "__main__":
540
parser.add_argument("--model",
541
type=str,
542
default="mistralai/Mixtral-8x7B-Instruct-v0.1")
539
- parser.add_argument("--tp-size", "-tp", type=int, default=2)
543
+ parser.add_argument("--tp-size",
544
+ "-tp",
545
+ "--tensor-parallel-size",
546
+ type=int,
547
+ default=2)
548
parser.add_argument("--dtype",
549
type=str,
550
choices=["auto", "fp8_w8a8", "int8_w8a16"],