A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 03:15 UTC
[Test] E2E Nemotron-3-Super tests (#36803)
Signed-off-by: Roi Koren <roik@nvidia.com>
roikoren755 committed
Mar 24, 2026 at 02:49 UTC
56777b5c898dcb92dbe6d64fecfccb670f919a0d
6 files changed
+37
.buildkite/test_areas/lm_eval.yaml
+1
@@ -90,6 +90,7 @@ steps:
90
commands:
91
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/moe-refactor-dp-ep/config-b200.txt
92
93
+
94
- label: GPQA Eval (GPT-OSS) (H100)
95
timeout_in_minutes: 120
96
device: h100
tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-BF16.yaml
new
+11
@@ -0,0 +1,11 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16"
2
+accuracy_threshold: 0.93
3
+num_questions: 1319
4
+num_fewshot: 5
5
+startup_max_wait_seconds: 1200
6
+server_args: >-
7
+ --enforce-eager
8
+ --max-model-len 4096
9
+ --tensor-parallel-size 8
10
+ --enable-expert-parallel
11
+ --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-FP8.yaml
new
+11
@@ -0,0 +1,11 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8"
2
+accuracy_threshold: 0.93
3
+num_questions: 1319
4
+num_fewshot: 5
5
+startup_max_wait_seconds: 1200
6
+server_args: >-
7
+ --enforce-eager
8
+ --max-model-len 4096
9
+ --tensor-parallel-size 8
10
+ --enable-expert-parallel
11
+ --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
tests/evals/gsm8k/configs/Nemotron-3-Super-120B-A12B-NVFP4.yaml
new
+11
@@ -0,0 +1,11 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4"
2
+accuracy_threshold: 0.93
3
+num_questions: 1319
4
+num_fewshot: 5
5
+startup_max_wait_seconds: 1200
6
+server_args: >-
7
+ --enforce-eager
8
+ --max-model-len 4096
9
+ --tensor-parallel-size 2
10
+ --enable-expert-parallel
11
+ --speculative-config '{"method":"mtp","num_speculative_tokens":5}'
tests/evals/gsm8k/configs/models-blackwell.txt
+1
@@ -5,3 +5,4 @@ DeepSeek-V2-Lite-Instruct-FP8.yaml
5
Qwen3-30B-A3B-NVFP4.yaml
6
Qwen3-Next-80B-A3B-NVFP4-EP2.yaml
7
Qwen3-Next-FP8-EP2.yaml
8
+Nemotron-3-Super-120B-A12B-NVFP4.yaml
tests/evals/gsm8k/configs/models-h200.txt
+2
@@ -2,3 +2,5 @@ DeepSeek-R1-TP.yaml
2
DeepSeek-R1-DP.yaml
3
DeepSeek-V3.2-TP.yaml
4
DeepSeek-V3.2-DP.yaml
5
+Nemotron-3-Super-120B-A12B-BF16.yaml
6
+Nemotron-3-Super-120B-A12B-FP8.yaml