@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1c1bb0bbf2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc][MoE] add Deepseek-V3 moe tuning support (#12558)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>

Divakar Verma committed Jan 29, 2025 at 18:47 UTC 1c1bb0bbf20955d346f66bb25d349c1bd9fe6ea2
1 file changed +8 -1
benchmarks/kernels/benchmark_moe.py
+8 -1
@@ -450,7 +450,8 @@ def save_configs(configs: Dict[int, BenchmarkConfig], num_experts: int,
450 def main(args: argparse.Namespace):
451 print(args)
452
453 - config = AutoConfig.from_pretrained(args.model)
453 + config = AutoConfig.from_pretrained(
454 + args.model, trust_remote_code=args.trust_remote_code)
455 if config.architectures[0] == "DbrxForCausalLM":
456 E = config.ffn_config.moe_num_experts
457 topk = config.ffn_config.moe_top_k
@@ -461,6 +462,11 @@ def main(args: argparse.Namespace):
462 topk = config.num_experts_per_tok
463 intermediate_size = config.intermediate_size
464 shard_intermediate_size = 2 * intermediate_size // args.tp_size
465 + elif config.architectures[0] == "DeepseekV3ForCausalLM":
466 + E = config.n_routed_experts
467 + topk = config.num_experts_per_tok
468 + intermediate_size = config.moe_intermediate_size
469 + shard_intermediate_size = 2 * intermediate_size // args.tp_size
470 else:
471 # Default: Mixtral.
472 E = config.num_local_experts
@@ -538,6 +544,7 @@ if __name__ == "__main__":
544 parser.add_argument("--seed", type=int, default=0)
545 parser.add_argument("--batch-size", type=int, required=False)
546 parser.add_argument("--tune", action="store_true")
547 + parser.add_argument("--trust-remote-code", action="store_true")
548 args = parser.parse_args()
549
550 main(args)