A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc][MoE] add Deepseek-V3 moe tuning support (#12558)
Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Divakar Verma committed
Jan 29, 2025 at 18:47 UTC
1c1bb0bbf20955d346f66bb25d349c1bd9fe6ea2
1 file changed
+8
-1
benchmarks/kernels/benchmark_moe.py
+8
-1
@@ -450,7 +450,8 @@ def save_configs(configs: Dict[int, BenchmarkConfig], num_experts: int,
450
def main(args: argparse.Namespace):
451
print(args)
452
453
- config = AutoConfig.from_pretrained(args.model)
453
+ config = AutoConfig.from_pretrained(
454
+ args.model, trust_remote_code=args.trust_remote_code)
455
if config.architectures[0] == "DbrxForCausalLM":
456
E = config.ffn_config.moe_num_experts
457
topk = config.ffn_config.moe_top_k
@@ -461,6 +462,11 @@ def main(args: argparse.Namespace):
462
topk = config.num_experts_per_tok
463
intermediate_size = config.intermediate_size
464
shard_intermediate_size = 2 * intermediate_size // args.tp_size
465
+ elif config.architectures[0] == "DeepseekV3ForCausalLM":
466
+ E = config.n_routed_experts
467
+ topk = config.num_experts_per_tok
468
+ intermediate_size = config.moe_intermediate_size
469
+ shard_intermediate_size = 2 * intermediate_size // args.tp_size
470
else:
471
# Default: Mixtral.
472
E = config.num_local_experts
@@ -538,6 +544,7 @@ if __name__ == "__main__":
544
parser.add_argument("--seed", type=int, default=0)
545
parser.add_argument("--batch-size", type=int, required=False)
546
parser.add_argument("--tune", action="store_true")
547
+ parser.add_argument("--trust-remote-code", action="store_true")
548
args = parser.parse_args()
549
550
main(args)