A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Benchmarks] Add support for Qwen 3 VL MoE tuning (#26419)
Signed-off-by: Lukas Geiger <lukas.geiger94@gmail.com>
Lukas Geiger committed
Oct 8, 2025 at 15:01 UTC
338b1bf04f9aa0d2f85aef1f3a82b12ba1a7a0c4
1 file changed
+12
-1
benchmarks/kernels/benchmark_moe.py
+12
-1
@@ -579,10 +579,12 @@ def main(args: argparse.Namespace):
579
E = config.ffn_config.moe_num_experts
580
topk = config.ffn_config.moe_top_k
581
intermediate_size = config.ffn_config.ffn_hidden_size
582
+ hidden_size = config.hidden_size
583
elif config.architectures[0] == "JambaForCausalLM":
584
E = config.num_experts
585
topk = config.num_experts_per_tok
586
intermediate_size = config.intermediate_size
587
+ hidden_size = config.hidden_size
588
elif config.architectures[0] in (
589
"DeepseekV2ForCausalLM",
590
"DeepseekV3ForCausalLM",
@@ -592,6 +594,7 @@ def main(args: argparse.Namespace):
594
E = config.n_routed_experts
595
topk = config.num_experts_per_tok
596
intermediate_size = config.moe_intermediate_size
597
+ hidden_size = config.hidden_size
598
elif config.architectures[0] in (
599
"Qwen2MoeForCausalLM",
600
"Qwen3MoeForCausalLM",
@@ -600,10 +603,18 @@ def main(args: argparse.Namespace):
603
E = config.num_experts
604
topk = config.num_experts_per_tok
605
intermediate_size = config.moe_intermediate_size
606
+ hidden_size = config.hidden_size
607
+ elif config.architectures[0] == "Qwen3VLMoeForConditionalGeneration":
608
+ text_config = config.get_text_config()
609
+ E = text_config.num_experts
610
+ topk = text_config.num_experts_per_tok
611
+ intermediate_size = text_config.moe_intermediate_size
612
+ hidden_size = text_config.hidden_size
613
elif config.architectures[0] in ("HunYuanMoEV1ForCausalLM"):
614
E = config.num_experts
615
topk = config.moe_topk[0]
616
intermediate_size = config.moe_intermediate_size[0]
617
+ hidden_size = config.hidden_size
618
else:
619
# Support for llama4
620
config = config.get_text_config()
@@ -611,6 +622,7 @@ def main(args: argparse.Namespace):
622
E = config.num_local_experts
623
topk = config.num_experts_per_tok
624
intermediate_size = config.intermediate_size
625
+ hidden_size = config.hidden_size
626
enable_ep = bool(args.enable_expert_parallel)
627
if enable_ep:
628
ensure_divisibility(E, args.tp_size, "Number of experts")
@@ -619,7 +631,6 @@ def main(args: argparse.Namespace):
631
else:
632
ensure_divisibility(intermediate_size, args.tp_size, "intermediate_size")
633
shard_intermediate_size = 2 * intermediate_size // args.tp_size
622
- hidden_size = config.hidden_size
634
dtype = torch.float16 if current_platform.is_rocm() else config.torch_dtype
635
use_fp8_w8a8 = args.dtype == "fp8_w8a8"
636
use_int8_w8a16 = args.dtype == "int8_w8a16"