@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfa134d247
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Bugfix/CI] Fixup benchmark_moe.py (#12562)

Fixes `is_marlin` not being passed into `get_default_config` Also allow `--tensor-parallel-size` in addition to `-tp` and `--tp-size` Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Feb 1, 2025 at 00:41 UTC cfa134d2475096eae47a58d14a6dec4c3fba9294
1 file changed +12 -4
benchmarks/kernels/benchmark_moe.py
+12 -4
@@ -343,9 +343,13 @@ class BenchmarkWorker:
343 op_config = get_moe_configs(num_experts, shard_intermediate_size // 2,
344 dtype_str)
345 if op_config is None:
346 - config = get_default_config(num_tokens, num_experts,
347 - shard_intermediate_size, hidden_size,
348 - topk, dtype_str)
346 + config = get_default_config(num_tokens,
347 + num_experts,
348 + shard_intermediate_size,
349 + hidden_size,
350 + topk,
351 + dtype_str,
352 + is_marlin=False)
353 else:
354 config = op_config[min(op_config.keys(),
355 key=lambda x: abs(x - num_tokens))]
@@ -536,7 +540,11 @@ if __name__ == "__main__":
540 parser.add_argument("--model",
541 type=str,
542 default="mistralai/Mixtral-8x7B-Instruct-v0.1")
539 - parser.add_argument("--tp-size", "-tp", type=int, default=2)
543 + parser.add_argument("--tp-size",
544 + "-tp",
545 + "--tensor-parallel-size",
546 + type=int,
547 + default=2)
548 parser.add_argument("--dtype",
549 type=str,
550 choices=["auto", "fp8_w8a8", "int8_w8a16"],