A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 11:30 UTC
[Bugfix] Fix Typo from NVFP4 Refactor (#31977)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Jan 8, 2026 at 19:18 UTC
0fa8dd24d281f71c0b0f6fb5b6033ca30d97f932
6 files changed
+30
-3
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "masked_gemm"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel --all2all-backend deepep_low_latency"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "masked_gemm"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/Qwen3-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.88
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --data-parallel-size 2 --enable-expert-parallel"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
+4
-2
@@ -1,8 +1,10 @@
1
+Qwen3-30B-A3B-NvFp4-CT-fi-cutedsl-deepep-ll.yaml
2
+Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutedsl-deepep-ll.yaml
3
+Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
4
+Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass.yaml
5
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ht.yaml
6
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm-deepep-ll.yaml
7
Qwen3-30B-A3B-Fp8-AutoFp8-deepgemm.yaml
8
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ht.yaml
9
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm-deepep-ll.yaml
10
Qwen3-30B-A3B-Fp8-CT-Block-deepgemm.yaml
7
-Qwen3-30B-A3B-NvFp4-CT-fi-cutlass.yaml
8
-Qwen3-30B-A3B-NvFp4-ModelOpt-fi-cutlass-fi-a2av.yaml
tests/evals/gsm8k/configs/moe-refactor/config-test.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen3-30B-A3B-NvFp4-CT-marlin.yaml
\ No newline at end of file
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
+1
-1
@@ -456,7 +456,7 @@ def prepare_nvfp4_moe_layer_for_fi_or_cutlass(
456
NvFp4MoeBackend.VLLM_CUTLASS,
457
NvFp4MoeBackend.FLASHINFER_CUTLASS,
458
NvFp4MoeBackend.FLASHINFER_TRTLLM,
459
- NvFp4MoeBackend.FLASHINFER_TRTLLM,
459
+ NvFp4MoeBackend.FLASHINFER_CUTEDSL,
460
]
461
462
# Reorder [w1, w3] to [w3, w1] for FI NVFP4 MoE kernels.