A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 02:30 UTC
[Bugfix] Fix NVFP4 TRTLLM MoE non-gated support; add gsm8k for Nemotron-3-Nano FP8+NVFP4 (#34725)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Feb 18, 2026 at 12:39 UTC
caeb887bf6332b74b229e90804fc6ab4f1361eda
4 files changed
+20
tests/evals/gsm8k/configs/moe-refactor/Nemotron-Nano-30B-Fp8-ModelOpt-fi-trtllm.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
2
+accuracy_threshold: 0.29
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "latency"
tests/evals/gsm8k/configs/moe-refactor/Nemotron-Nano-30B-NvFp4-ModelOpt-fi-cutlass.yaml
new
+8
@@ -0,0 +1,8 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4"
2
+accuracy_threshold: 0.29
3
+num_questions: 1319
4
+num_fewshot: 5
5
+server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"
6
+env:
7
+ VLLM_USE_FLASHINFER_MOE_FP4: "1"
8
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
tests/evals/gsm8k/configs/moe-refactor/config-b200.txt
+2
@@ -13,3 +13,5 @@ Llama-4-Scout-BF16-fi-cutlass.yaml
13
Llama-4-Scout-BF16-triton.yaml
14
Mixtral-8x7B-BF16-fi-cutlass.yaml
15
Mixtral-8x7B-BF16-triton.yaml
16
+Nemotron-Nano-30B-Fp8-ModelOpt-fi-trtllm.yaml
17
+Nemotron-Nano-30B-NvFp4-ModelOpt-fi-cutlass.yaml
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
+2
@@ -122,6 +122,8 @@ def is_supported_config_trtllm(
122
return False, _make_reason("routing method")
123
elif activation_format != mk.FusedMoEActivationFormat.Standard:
124
return False, _make_reason("activation format")
125
+ elif moe_config.hidden_dim % 512 != 0:
126
+ return False, _make_reason("hidden_dim must be divisible by 512")
127
128
return True, None
129