de527e1cec · tduncan/Nvidia-Optimization-Stack-VLLM

tests/evals/gsm8k/configs/Qwen3-Next-80B-A3B-NVFP4-EP2.yaml

+1 -2

@@ -8,5 +8,4 @@ server_args: >-

8 --tensor-parallel-size 2

9 --enable-expert-parallel

10 --speculative-config '{"method":"qwen3_next_mtp","num_speculative_tokens":1}'

11 -env:

12 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

11 + --moe-backend=flashinfer_trtllm

tests/evals/gsm8k/configs/Qwen3-Next-FP8-EP2.yaml

+1 -2

@@ -7,5 +7,4 @@ server_args: >-

7 --tensor-parallel-size 2

8 --enable-expert-parallel

9 --async-scheduling

10 -env:

11 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

10 + --moe-backend=flashinfer_trtllm

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Llama-4-Scout-Fp8-ModelOpt-triton.yaml

+1 -2

@@ -2,7 +2,6 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --moe-backend=triton"

6 env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "0"

7 VLLM_USE_DEEP_GEMM: "0"

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "masked_gemm"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl"

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "masked_gemm"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency --moe-backend=flashinfer_cutedsl"

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-BF16-fi-cutlass.yaml

+1 -5

@@ -2,8 +2,4 @@ model_name: "meta-llama/Llama-4-Scout-17B-16E-Instruct"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP16: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

9 -

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-triton.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "0"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=triton"

tests/evals/gsm8k/configs/moe-refactor/Mixtral-8x7B-BF16-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "mistralai/Mixtral-8x7B-v0.1"

2 accuracy_threshold: 0.58

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP16: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Mixtral-8x7B-Fp8-AutoFp8-fi-cutlass.yaml

+1 -4

@@ -3,7 +3,4 @@

3 # accuracy_threshold: 0.62

4 # num_questions: 1319

5 # num_fewshot: 5

6 -# server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

7 -# env:

8 -# VLLM_USE_FLASHINFER_MOE_FP8: "1"

9 -# VLLM_FLASHINFER_MOE_BACKEND: "throughput"

6 +# server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Nemotron-Nano-30B-Fp8-ModelOpt-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"

2 accuracy_threshold: 0.29

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Nemotron-Nano-30B-NvFp4-ModelOpt-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"

2 accuracy_threshold: 0.29

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-BF16-fi-cutlass.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-30B-A3B"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP16: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --enable-expert-parallel --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml

+1 -2

@@ -2,7 +2,6 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=triton"

6 env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "0"

7 VLLM_USE_DEEP_GEMM: "0"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"

2 accuracy_threshold: 0.85

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-triton.yaml

+1 -2

@@ -2,7 +2,6 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"

2 accuracy_threshold: 0.85

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=triton"

6 env:

7 - VLLM_USE_FLASHINFER_MOE_FP8: "0"

7 VLLM_USE_DEEP_GEMM: "0"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-vllm-cutlass.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "0"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_cutlass"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-trtllm.yaml

+1 -4

@@ -2,7 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "1"

8 - VLLM_FLASHINFER_MOE_BACKEND: "latency"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=flashinfer_trtllm"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-vllm-cutlass.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_USE_FLASHINFER_MOE_FP4: "0"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --moe-backend=cutlass"

tests/quantization/test_blackwell_moe.py

+42 -28

@@ -85,34 +85,34 @@ def can_initialize(

85 )

86 )

87 def test_llama4_fp8_tensor_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):

88 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")

89 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")

88 can_initialize(

91 - "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", hf_overrides=HF_OVERRIDE_MM

89 + "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",

90 + hf_overrides=HF_OVERRIDE_MM,

91 + extra_args=["--moe-backend=flashinfer_cutlass"],

92 )

93

94

95 def test_llama4_fp8_tensor_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

96 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")

97 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")

96 can_initialize(

99 - "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8", hf_overrides=HF_OVERRIDE_MM

97 + "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",

98 + hf_overrides=HF_OVERRIDE_MM,

99 + extra_args=["--moe-backend=flashinfer_trtllm"],

100 )

101

102

103 def test_llama4_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):

104 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")

105 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")

104 can_initialize(

107 - "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", hf_overrides=HF_OVERRIDE_MM

105 + "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",

106 + hf_overrides=HF_OVERRIDE_MM,

107 + extra_args=["--moe-backend=flashinfer_cutlass"],

108 )

109

110

111 def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

112 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")

113 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")

112 can_initialize(

115 - "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4", hf_overrides=HF_OVERRIDE_MM

113 + "nvidia/Llama-4-Scout-17B-16E-Instruct-FP4",

114 + hf_overrides=HF_OVERRIDE_MM,

115 + extra_args=["--moe-backend=flashinfer_trtllm"],

116 )

117

118

@@ -120,8 +120,11 @@ def test_llama4_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

120

121

122 def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):

123 - monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "1")

124 - can_initialize("deepseek-ai/DeepSeek-V3.1", hf_overrides=HF_OVERRIDE_TEXT)

123 + can_initialize(

124 + "deepseek-ai/DeepSeek-V3.1",

125 + hf_overrides=HF_OVERRIDE_TEXT,

126 + extra_args=["--moe-backend=deep_gemm"],

127 + )

128

129

130 @pytest.mark.skip(

@@ -131,27 +134,35 @@ def test_deepseek_fp8_block_moe_deep_gemm(monkeypatch: pytest.MonkeyPatch):

134 )

135 )

136 def test_deepseek_fp8_block_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):

134 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")

135 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")

136 - can_initialize("deepseek-ai/DeepSeek-V3.1", hf_overrides=HF_OVERRIDE_TEXT)

137 + can_initialize(

138 + "deepseek-ai/DeepSeek-V3.1",

139 + hf_overrides=HF_OVERRIDE_TEXT,

140 + extra_args=["--moe-backend=flashinfer_cutlass"],

141 + )

142

143

144 def test_deepseek_fp8_block_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

140 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP8", "1")

141 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")

142 - can_initialize("deepseek-ai/DeepSeek-V3.1", hf_overrides=HF_OVERRIDE_TEXT)

145 + can_initialize(

146 + "deepseek-ai/DeepSeek-V3.1",

147 + hf_overrides=HF_OVERRIDE_TEXT,

148 + extra_args=["--moe-backend=flashinfer_trtllm"],

149 + )

150

151

152 def test_deepseek_nvfp4_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):

146 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")

147 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")

148 - can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2", hf_overrides=HF_OVERRIDE_TEXT)

153 + can_initialize(

154 + "nvidia/DeepSeek-R1-0528-FP4-v2",

155 + hf_overrides=HF_OVERRIDE_TEXT,

156 + extra_args=["--moe-backend=flashinfer_cutlass"],

157 + )

158

159

160 def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

152 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")

153 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "latency")

154 - can_initialize("nvidia/DeepSeek-R1-0528-FP4-v2", hf_overrides=HF_OVERRIDE_TEXT)

161 + can_initialize(

162 + "nvidia/DeepSeek-R1-0528-FP4-v2",

163 + hf_overrides=HF_OVERRIDE_TEXT,

164 + extra_args=["--moe-backend=flashinfer_trtllm"],

165 + )

166

167

168 ## GPT-OSS ##

@@ -184,5 +195,8 @@ def test_gptoss_eager(monkeypatch: pytest.MonkeyPatch):

195

196

197 def test_qwen3_next_bf16_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):

187 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")

188 - can_initialize("Qwen/Qwen3-Next-80B-A3B-Instruct", hf_overrides=HF_OVERRIDE_TEXT)

198 + can_initialize(

199 + "Qwen/Qwen3-Next-80B-A3B-Instruct",

200 + hf_overrides=HF_OVERRIDE_TEXT,

201 + extra_args=["--moe-backend=flashinfer_trtllm"],

202 + )

vllm/config/kernel.py

+33 -1

@@ -2,13 +2,25 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3

4 from collections.abc import Callable

5 -from typing import Any

5 +from typing import Any, Literal

6

7 from pydantic import Field, field_validator

8

9 from vllm.config.utils import config

10 from vllm.utils.hashing import safe_hash

11

12 +MoEBackend = Literal[

13 + "auto",

14 + "triton",

15 + "deep_gemm",

16 + "cutlass",

17 + "flashinfer_trtllm",

18 + "flashinfer_cutlass",

19 + "flashinfer_cutedsl",

20 + "marlin",

21 + "aiter",

22 +]

23 +

24

25 @config

26 class KernelConfig:

@@ -17,6 +29,26 @@ class KernelConfig:

29 enable_flashinfer_autotune: bool = Field(default=None)

30 """If True, run FlashInfer autotuning during kernel warmup."""

31

32 + moe_backend: MoEBackend = "auto"

33 + """Backend for MoE expert computation kernels. Available options:

34 +

35 + - "auto": Automatically select the best backend based on model and hardware\n

36 + - "triton": Use Triton-based fused MoE kernels\n

37 + - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)\n

38 + - "cutlass": Use vLLM CUTLASS kernels\n

39 + - "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels\n

40 + - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels\n

41 + - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)\n

42 + - "marlin": Use Marlin kernels (weight-only quantization)\n

43 + - "aiter": Use AMD AITer kernels (ROCm only)"""

44 +

45 + @field_validator("moe_backend", mode="before")

46 + @classmethod

47 + def _normalize_moe_backend(cls, value: Any) -> Any:

48 + if isinstance(value, str):

49 + return value.lower().replace("-", "_")

50 + return value

51 +

52 def compute_hash(self) -> str:

53 """

54 WARNING: Whenever a new field is added to this config,

vllm/engine/arg_utils.py

+7

@@ -70,6 +70,7 @@ from vllm.config.cache import (

70 PrefixCachingHashAlgo,

71 )

72 from vllm.config.device import Device

73 +from vllm.config.kernel import MoEBackend

74 from vllm.config.lora import MaxLoRARanks

75 from vllm.config.model import (

76 ConvertOption,

@@ -416,6 +417,7 @@ class EngineArgs:

417 data_parallel_external_lb: bool = False

418 data_parallel_backend: DataParallelBackend = ParallelConfig.data_parallel_backend

419 enable_expert_parallel: bool = ParallelConfig.enable_expert_parallel

420 + moe_backend: MoEBackend = KernelConfig.moe_backend

421 all2all_backend: All2AllBackend = ParallelConfig.all2all_backend

422 enable_dbo: bool = ParallelConfig.enable_dbo

423 ubatch_size: int = ParallelConfig.ubatch_size

@@ -1227,6 +1229,9 @@ class EngineArgs:

1229 "--enable-flashinfer-autotune",

1230 **kernel_kwargs["enable_flashinfer_autotune"],

1231 )

1232 + moe_backend_kwargs = kernel_kwargs["moe_backend"]

1233 + moe_backend_kwargs["type"] = lambda s: s.lower().replace("-", "_")

1234 + kernel_group.add_argument("--moe-backend", **moe_backend_kwargs)

1235

1236 # vLLM arguments

1237 vllm_kwargs = get_kwargs(VllmConfig)

@@ -1817,6 +1822,8 @@ class EngineArgs:

1822 "are mutually exclusive"

1823 )

1824 kernel_config.enable_flashinfer_autotune = self.enable_flashinfer_autotune

1825 + if self.moe_backend != "auto":

1826 + kernel_config.moe_backend = self.moe_backend

1827

1828 load_config = self.create_load_config()

1829

vllm/model_executor/layers/fused_moe/config.py

+1 -1

@@ -1066,7 +1066,6 @@ class FusedMoEParallelConfig:

1066 - Comment: There are 2 engine instances and the experts are split

1067 between the 4 devices.

1068 """

1069 -

1069 use_ep = (

1070 dp_size_ * pcp_size_ * tp_size_ > 1

1071 and vllm_parallel_config.enable_expert_parallel

@@ -1155,6 +1154,7 @@ class FusedMoEConfig:

1154 # Defaults to in_dtype if not specified.

1155 router_logits_dtype: torch.dtype | None = None

1156

1157 + moe_backend: str = "auto"

1158 max_num_tokens: int = envs.VLLM_MOE_DP_CHUNK_SIZE

1159 has_bias: bool = False

1160 is_act_and_mul: bool = True

vllm/model_executor/layers/fused_moe/deepep_ll_prepare_finalize.py

-1

@@ -198,7 +198,6 @@ class DeepEPLLPrepareAndFinalize(mk.FusedMoEPrepareAndFinalize):

198 x = x[0].permute(2, 0, 1)

199 num_experts, max_tokens, hidden_dim_by_2 = x.shape

200 hidden_dim = hidden_dim_by_2 * 2

201 - assert envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm"

201 logger.info_once(

202 "Quantization is fused with DeepEP nvfp4 dispatch for "

203 "FlashInfer CUTEDSL as VLLM_DEEPEPLL_NVFP4_DISPATCH==1"

vllm/model_executor/layers/fused_moe/layer.py

+1

@@ -550,6 +550,7 @@ class FusedMoE(CustomOp):

550 num_logical_experts=self.logical_num_experts,

551 moe_parallel_config=self.moe_parallel_config,

552 in_dtype=moe_in_dtype,

553 + moe_backend=vllm_config.kernel_config.moe_backend,

554 router_logits_dtype=router_logits_dtype,

555 max_num_tokens=envs.VLLM_MOE_DP_CHUNK_SIZE,

556 has_bias=has_bias,

vllm/model_executor/layers/fused_moe/oracle/fp8.py

+59

@@ -7,6 +7,7 @@ import torch

7 import vllm.model_executor.layers.fused_moe.modular_kernel as mk

8 from vllm import envs

9 from vllm._aiter_ops import rocm_aiter_ops

10 +from vllm.config.kernel import MoEBackend

11 from vllm.logger import init_logger

12 from vllm.model_executor.layers.fused_moe.all2all_utils import (

13 maybe_make_prepare_finalize,

@@ -180,6 +181,25 @@ def backend_to_kernel_cls(

181 raise ValueError(f"Unknown FP8 MoE backend: {backend.value}")

182

183

184 +def map_fp8_backend(runner_backend: MoEBackend) -> Fp8MoeBackend:

185 + """Map user's MoEBackend to Fp8MoeBackend."""

186 + mapping = {

187 + "triton": Fp8MoeBackend.TRITON,

188 + "deep_gemm": Fp8MoeBackend.DEEPGEMM,

189 + "cutlass": Fp8MoeBackend.VLLM_CUTLASS,

190 + "flashinfer_trtllm": Fp8MoeBackend.FLASHINFER_TRTLLM,

191 + "flashinfer_cutlass": Fp8MoeBackend.FLASHINFER_CUTLASS,

192 + "marlin": Fp8MoeBackend.MARLIN,

193 + "aiter": Fp8MoeBackend.AITER,

194 + }

195 + if backend := mapping.get(runner_backend):

196 + return backend

197 + raise ValueError(

198 + f"moe_backend='{runner_backend}' is not supported for FP8 MoE. "

199 + f"Expected one of {list(mapping.keys())}."

200 + )

201 +

202 +

203 def select_fp8_moe_backend(

204 config: FusedMoEConfig,

205 weight_key: QuantKey | None,

@@ -242,6 +262,45 @@ def select_fp8_moe_backend(

262 return backend, k_cls

263 raise ValueError(_make_log_unsupported(backend, reason))

264

265 + # Handle explicit moe_backend from user.

266 + runner_backend = config.moe_backend

267 + if runner_backend != "auto":

268 + requested_backend = map_fp8_backend(runner_backend)

269 + # For batched activation format, use batched variants if available.

270 + if activation_format == mk.FusedMoEActivationFormat.BatchedExperts:

271 + if requested_backend == Fp8MoeBackend.DEEPGEMM:

272 + requested_backend = Fp8MoeBackend.BATCHED_DEEPGEMM

273 + elif requested_backend == Fp8MoeBackend.TRITON:

274 + requested_backend = Fp8MoeBackend.BATCHED_TRITON

275 + elif requested_backend == Fp8MoeBackend.VLLM_CUTLASS:

276 + requested_backend = Fp8MoeBackend.BATCHED_VLLM_CUTLASS

277 +

278 + if (

279 + requested_backend

280 + in [

281 + Fp8MoeBackend.VLLM_CUTLASS,

282 + Fp8MoeBackend.BATCHED_VLLM_CUTLASS,

283 + ]

284 + and not allow_vllm_cutlass

285 + ):

286 + raise ValueError(

287 + "vLLM CUTLASS FP8 MoE backend is disabled for this configuration."

288 + )

289 +

290 + # Handle FLASHINFER_TRTLLM specially (no kernel class).

291 + if requested_backend == Fp8MoeBackend.FLASHINFER_TRTLLM:

292 + supported, reason = is_supported_config_trtllm_fp8(

293 + config, weight_key, activation_key, activation_format

294 + )

295 + if supported:

296 + logger.info_once(_make_log_backend(requested_backend))

297 + return requested_backend, None

298 + raise ValueError(_make_log_unsupported(requested_backend, reason))

299 +

300 + return _return_or_raise(

301 + requested_backend, config, weight_key, activation_key, activation_format

302 + )

303 +

304 # Handle explicit FlashInfer FP8 configuration.

305 if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):

306 if not envs.VLLM_USE_FLASHINFER_MOE_FP8:

vllm/model_executor/layers/fused_moe/oracle/nvfp4.py

+35

@@ -6,6 +6,7 @@ import torch

6

7 import vllm.envs as envs

8 import vllm.model_executor.layers.fused_moe.modular_kernel as mk

9 +from vllm.config.kernel import MoEBackend

10 from vllm.logger import init_logger

11 from vllm.model_executor.layers.fused_moe.all2all_utils import (

12 maybe_make_prepare_finalize,

@@ -103,6 +104,23 @@ def backend_to_kernel_cls(

104 raise ValueError(f"Unknown NvFP4 MoE backend: {backend.value}")

105

106

107 +def map_nvfp4_backend(runner_backend: MoEBackend) -> NvFp4MoeBackend:

108 + """Map user's MoEBackend to NvFp4MoeBackend."""

109 + mapping = {

110 + "cutlass": NvFp4MoeBackend.VLLM_CUTLASS,

111 + "flashinfer_trtllm": NvFp4MoeBackend.FLASHINFER_TRTLLM,

112 + "flashinfer_cutlass": NvFp4MoeBackend.FLASHINFER_CUTLASS,

113 + "flashinfer_cutedsl": NvFp4MoeBackend.FLASHINFER_CUTEDSL,

114 + "marlin": NvFp4MoeBackend.MARLIN,

115 + }

116 + if backend := mapping.get(runner_backend):

117 + return backend

118 + raise ValueError(

119 + f"moe_backend='{runner_backend}' is not supported for NvFP4 MoE. "

120 + f"Expected one of {list(mapping.keys())}."

121 + )

122 +

123 +

124 def select_nvfp4_moe_backend(

125 config: FusedMoEConfig,

126 weight_key: QuantKey | None,

@@ -170,6 +188,23 @@ def select_nvfp4_moe_backend(

188 return backend, k_cls

189 raise ValueError(_make_log_unsupported(backend, reason))

190

191 + # Handle explicit moe_backend from user.

192 + runner_backend = config.moe_backend

193 + if runner_backend != "auto":

194 + requested_backend = map_nvfp4_backend(runner_backend)

195 + if requested_backend == NvFp4MoeBackend.FLASHINFER_TRTLLM:

196 + supported, reason = is_supported_config_trtllm(

197 + config, weight_key, activation_key, activation_format

198 + )

199 + if supported:

200 + logger.info_once(_make_log_backend(requested_backend))

201 + return requested_backend, None

202 + raise ValueError(_make_log_unsupported(requested_backend, reason))

203 +

204 + return _return_or_raise(

205 + requested_backend, config, weight_key, activation_key, activation_format

206 + )

207 +

208 if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):

209 if not envs.VLLM_USE_FLASHINFER_MOE_FP4:

210 # If the user rejects FlashInfer remove those backends.

vllm/model_executor/layers/fused_moe/oracle/unquantized.py

+54 -10

@@ -9,6 +9,7 @@ from torch.nn import Module

9 import vllm.envs as envs

10 import vllm.model_executor.layers.fused_moe.modular_kernel as mk

11 from vllm._aiter_ops import rocm_aiter_ops

12 +from vllm.config.kernel import MoEBackend

13 from vllm.logger import init_logger

14 from vllm.model_executor.layers.fused_moe.config import (

15 FusedMoEConfig,

@@ -51,6 +52,22 @@ UNSUPPORTED_BACKEND = [

52 ]

53

54

55 +def map_unquantized_backend(runner_backend: MoEBackend) -> UnquantizedMoeBackend:

56 + """Map user's MoEBackend to UnquantizedMoeBackend."""

57 + mapping = {

58 + "triton": UnquantizedMoeBackend.TRITON,

59 + "flashinfer_trtllm": UnquantizedMoeBackend.FLASHINFER_TRTLLM,

60 + "flashinfer_cutlass": UnquantizedMoeBackend.FLASHINFER_CUTLASS,

61 + "aiter": UnquantizedMoeBackend.AITER,

62 + }

63 + if backend := mapping.get(runner_backend):

64 + return backend

65 + raise ValueError(

66 + f"moe_backend='{runner_backend}' is not supported for unquantized MoE. "

67 + f"Expected one of {list(mapping.keys())}."

68 + )

69 +

70 +

71 def select_unquantized_moe_backend(

72 moe_config: FusedMoEConfig,

73 use_ep: bool,

@@ -64,8 +81,6 @@ def select_unquantized_moe_backend(

81 def _make_log_backend(backend: UnquantizedMoeBackend):

82 return f"Using {backend.value} backend for Unquantized MoE"

83

67 - rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()

68 -

84 activation_format = (

85 mk.FusedMoEActivationFormat.BatchedExperts

86 if moe_config.moe_parallel_config.use_batched_activation_format

@@ -77,20 +92,49 @@ def select_unquantized_moe_backend(

92 moe_config=moe_config,

93 activation_format=activation_format,

94 )

80 - flashinfer_trtllm_moe_enabled = (

81 - has_flashinfer()

82 - and envs.VLLM_USE_FLASHINFER_MOE_FP16

83 - and trtllm_supported

84 - and envs.VLLM_FLASHINFER_MOE_BACKEND == "latency"

85 - )

95 + flashinfer_trtllm_available = has_flashinfer() and trtllm_supported

96 # FlashInfer CUTLASS MoE is only supported on Hopper and later GPUS

87 - flashinfer_cutlass_moe_enabled = (

97 + flashinfer_cutlass_available = (

98 has_flashinfer_cutlass_fused_moe()

89 - and envs.VLLM_USE_FLASHINFER_MOE_FP16

99 and use_ep

100 and (not use_dp)

101 and current_platform.has_device_capability(90)

102 )

103 + flashinfer_trtllm_moe_enabled = (

104 + flashinfer_trtllm_available

105 + and envs.VLLM_USE_FLASHINFER_MOE_FP16

106 + and envs.VLLM_FLASHINFER_MOE_BACKEND == "latency"

107 + )

108 + flashinfer_cutlass_moe_enabled = (

109 + flashinfer_cutlass_available and envs.VLLM_USE_FLASHINFER_MOE_FP16

110 + )

111 + rocm_aiter_moe_enabled = rocm_aiter_ops.is_fused_moe_enabled()

112 +

113 + # Handle explicit moe_backend from user.

114 + runner_backend = moe_config.moe_backend

115 + if runner_backend != "auto":

116 + requested_backend = map_unquantized_backend(runner_backend)

117 + if requested_backend == UnquantizedMoeBackend.FLASHINFER_TRTLLM:

118 + if not flashinfer_trtllm_available:

119 + raise ValueError(

120 + "FlashInfer TRTLLM MoE backend is not available for this "

121 + "configuration."

122 + )

123 + elif requested_backend == UnquantizedMoeBackend.FLASHINFER_CUTLASS:

124 + if not flashinfer_cutlass_available:

125 + raise ValueError(

126 + "FlashInfer CUTLASS MoE backend is not available for this "

127 + "configuration."

128 + )

129 + elif requested_backend == UnquantizedMoeBackend.AITER and not (

130 + current_platform.is_rocm() and rocm_aiter_moe_enabled

131 + ):

132 + raise ValueError(

133 + "ROCm AITer MoE backend is not available for this configuration."

134 + )

135 + logger.info_once(_make_log_backend(requested_backend), scope="local")

136 + return requested_backend

137 +

138 if current_platform.is_rocm():

139 if rocm_aiter_moe_enabled:

140 backend = UnquantizedMoeBackend.AITER

[UX] Add `--moe-backend` arg for explicit kernel selection (#33807)