A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[MoE Refactor] Add Temporary Integration Tests - H100/B200 (#31759)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Jan 6, 2026 at 10:34 UTC
d3e477c01342f800790b170b8fa799b467621358
30 files changed
+247
.buildkite/test-pipeline.yaml
+16
@@ -1406,3 +1406,19 @@ steps:
1406
working_dir: "/vllm-workspace"
1407
commands:
1408
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020 2 1
1409
+
1410
+##### MoE Refactor (Temporary) Tests #####
1411
+
1412
+- label: MoE Refactor Integration Test (H100 - TEMPORARY) # optional
1413
+ gpu: h100
1414
+ optional: true
1415
+ num_gpus: 2
1416
+ commands:
1417
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-h100.txt
1418
+
1419
+- label: MoE Refactor Integration Test (B200 - TEMPORARY) # optional
1420
+ gpu: b200
1421
+ optional: true
1422
+ num_gpus: 2
1423
+ commands:
1424
+ - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor/config-b200.txt
tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-fi-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-fi-trtllm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "latency"
tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml
new
+7
@@ -0,0 +1,7 @@
1
+model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-triton.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"
2
+accuracy_threshold: 0.92
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
tests/evals/gsm8k/configs/moe-refactor/Mixtral-8x7B-Fp8-AutoFp8-fi-cutlass.yaml
new
+9
@@ -0,0 +1,9 @@
1
+# TODO(rob): enable
2
+# model_name: "amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV"
3
+# accuracy_threshold: 0.62
4
+# num_questions: 1319
5
+# num_fewshot: 5
6
+# server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
7
+# env:
8
+# VLLM_USE_FLASHINFER_MOE_FP8: "1"
9
+# VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Mixtral-8x7B-Fp8-AutoFp8-triton.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "amd/Mixtral-8x7B-Instruct-v0.1-FP8-KV"
2
+accuracy_threshold: 0.62
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "1"
8
+ VLLM_USE_DEEP_GEMM_MOE: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-fi-cutlass.yaml
new
+10
@@ -0,0 +1,10 @@
1
+model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
9
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
10
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-fi-trtllm.yaml
new
+10
@@ -0,0 +1,10 @@
1
+model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
9
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
10
+ VLLM_FLASHINFER_MOE_BACKEND: "latency"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml
new
+9
@@ -0,0 +1,9 @@
1
+model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
9
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "1"
8
+ VLLM_USE_DEEP_GEMM_MOE: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-fi-cutlass.yaml
new
+10
@@ -0,0 +1,10 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
9
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
10
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml
new
+9
@@ -0,0 +1,9 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
9
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-vllm-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_DEEP_GEMM: "0"
8
+ VLLM_USE_DEEP_GEMM_MOE: "0"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml
new
+7
@@ -0,0 +1,7 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-vllm-cutlass.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"
2
+accuracy_threshold: 0.85
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-fi-cutlass-dp-ep.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-fi-trtllm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "latency"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml
new
+7
@@ -0,0 +1,7 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-vllm-cutlass.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass-dp-ep.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-trtllm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "latency"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml
new
+7
@@ -0,0 +1,7 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_TEST_FORCE_FP8_MARLIN: "1"
tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-vllm-cutlass.yaml
new
+5
@@ -0,0 +1,5 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
tests/evals/gsm8k/configs/moe-refactor/config-b200.txt
new
+12
@@ -0,0 +1,12 @@
1
+Llama-4-Scout-Fp8-ModelOpt-fi-trtllm.yaml
2
+Qwen3-30B-A3B-Fp8-AutoFp8-fi-trtllm.yaml
3
+Qwen3-30B-A3B-NvFp4-CT-vllm-cutlass.yaml
4
+Qwen3-30B-A3B-NvFp4-CT-marlin.yaml
5
+Qwen3-30B-A3B-NvFp4-CT-fi-trtllm.yaml
6
+Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
7
+Qwen3-30B-A3B-NvFp4-CT-fi-cutlass-dp-ep.yaml
8
+Qwen3-30B-A3B-NvFp4-ModelOpt-vllm-cutlass.yaml
9
+Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml
10
+Qwen3-30B-A3B-NvFp4-ModelOpt-fi-trtllm.yaml
11
+Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml
12
+Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass-dp-ep.yaml
tests/evals/gsm8k/configs/moe-refactor/config-h100.txt
new
+13
@@ -0,0 +1,13 @@
1
+Mixtral-8x7B-Fp8-AutoFp8-triton.yaml
2
+Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm.yaml
3
+Qwen3-30B-A3B-Fp8-AutoFp8-fi-cutlass.yaml
4
+Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml
5
+Qwen3-30B-A3B-Fp8-AutoFp8-triton.yaml
6
+Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
7
+Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml
8
+Qwen3-30B-A3B-Fp8-CT-Block-vllm-cutlass.yaml
9
+Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml
10
+Qwen3-30B-A3B-Fp8-CT-Channel-vllm-cutlass.yaml
11
+Llama-4-Scout-Fp8-ModelOpt-fi-cutlass.yaml
12
+Llama-4-Scout-Fp8-ModelOpt-marlin.yaml
13
+Llama-4-Scout-Fp8-ModelOpt-triton.yaml