A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 00:00 UTC
Feat/add nemotron nano v3 tests (#33345)
shaharmor98 committed
Feb 3, 2026 at 15:52 UTC
4bc913aeeca39a304e4ace51febf55f142c8c86e
6 files changed
+54
.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml
new
+15
@@ -0,0 +1,15 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16"
2
+tasks:
3
+- name: "gsm8k"
4
+ metrics:
5
+ - name: "exact_match,strict-match"
6
+ value: 0.695
7
+ - name: "exact_match,flexible-extract"
8
+ value: 0.447
9
+limit: 1319
10
+num_fewshot: 5
11
+max_model_len: 262144
12
+enforce_eager: false
13
+apply_chat_template: true
14
+fewshot_as_multiturn: true
15
+trust_remote_code: true
\ No newline at end of file
.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml
new
+19
@@ -0,0 +1,19 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
2
+tasks:
3
+- name: "gsm8k"
4
+ metrics:
5
+ - name: "exact_match,strict-match"
6
+ value: 0.7142
7
+ - name: "exact_match,flexible-extract"
8
+ value: 0.4579
9
+env_vars:
10
+ VLLM_USE_FLASHINFER_MOE_FP8: "1"
11
+ VLLM_FLASHINFER_MOE_BACKEND: "throughput"
12
+limit: 1319
13
+num_fewshot: 5
14
+max_model_len: 262144
15
+kv_cache_dtype: fp8
16
+enforce_eager: false
17
+apply_chat_template: true
18
+fewshot_as_multiturn: true
19
+trust_remote_code: true
.buildkite/lm-eval-harness/configs/models-large-hopper.txt
+1
@@ -1 +1,2 @@
1
Qwen3-235B-A22B-Instruct-2507-FP8.yaml
2
+NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -3,3 +3,4 @@ Meta-Llama-3-70B-Instruct.yaml
3
Mixtral-8x7B-Instruct-v0.1.yaml
4
Qwen2-57B-A14-Instruct.yaml
5
DeepSeek-V2-Lite-Chat.yaml
6
+NVIDIA-Nemotron-3-Nano-30B-A3B-BF16.yaml
tests/config/base_model_arch_groundtruth.json
+17
@@ -355,5 +355,22 @@
355
"is_deepseek_mla": true,
356
"is_multimodal_model": false,
357
"dtype": "torch.float32"
358
+ },
359
+ "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": {
360
+ "architectures": [
361
+ "NemotronHForCausalLM"
362
+ ],
363
+ "model_type": "nemotron_h",
364
+ "text_model_type": "nemotron_h",
365
+ "hidden_size": 2688,
366
+ "total_num_hidden_layers": 52,
367
+ "total_num_attention_heads": 32,
368
+ "head_size": 128,
369
+ "vocab_size": 131072,
370
+ "total_num_kv_heads": 2,
371
+ "num_experts": 128,
372
+ "is_deepseek_mla": false,
373
+ "is_multimodal_model": false,
374
+ "dtype": "torch.bfloat16"
375
}
376
}
tests/config/test_model_arch_config.py
+1
@@ -14,6 +14,7 @@ from vllm.transformers_utils.model_arch_config_convertor import (
14
15
BASE_TRUST_REMOTE_CODE_MODELS = {
16
"nvidia/Llama-3_3-Nemotron-Super-49B-v1",
17
+ "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16",
18
"XiaomiMiMo/MiMo-7B-RL",
19
# Excluded: Not available online right now
20
# "FreedomIntelligence/openPangu-Ultra-MoE-718B-V1.1",