[XPU] Fix Nemotron FP8 LM-eval config: drop CUDA-only moe_backend and wire to new Buildkite job (#49685)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Chaojun Zhang committed
Sep 22, 2026 at 10:05 UTC
1fee69838eeb72622b441a593888687831c1e434
3 files changed
+41
.buildkite/intel_jobs/lm_eval_intel.yaml
+24
@@ -26,3 +26,27 @@ steps:
26
'uv pip install "gpt-oss[eval]==0.0.5" &&
27
cd tests &&
28
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
29
+- label: LM Eval Large Models(2 GPUs)
30
+ key: lm-eval-large-models-2-gpus
31
+ timeout_in_minutes: 45
32
+ device: intel_gpu
33
+ agent_tags:
34
+ label: production
35
+ gpu: 2+
36
+ mem: 24+
37
+ no_plugin: true
38
+ optional: true
39
+ working_dir: "."
40
+ env:
41
+ REGISTRY: "public.ecr.aws/q9t5s3a7"
42
+ REPO: "vllm-ci-test-repo"
43
+ VLLM_TEST_DEVICE: "xpu"
44
+ source_file_dependencies:
45
+ - vllm/
46
+ - .buildkite/lm-eval-harness/
47
+ commands:
48
+ - >-
49
+ bash .buildkite/scripts/hardware_ci/run-intel-test.sh
50
+ 'cd .buildkite/lm-eval-harness &&
51
+ export VLLM_USE_DEEP_GEMM=0 &&
52
+ pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-xpu-fp8.txt --tp-size=2'
.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml
new
+16
@@ -0,0 +1,16 @@
1
+model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8"
2
+tasks:
3
+- name: "gsm8k"
4
+ metrics:
5
+ - name: "exact_match,strict-match"
6
+ value: 0.7142
7
+ - name: "exact_match,flexible-extract"
8
+ value: 0.4579
9
+limit: 256
10
+num_fewshot: 5
11
+max_model_len: 262144
12
+kv_cache_dtype: fp8
13
+enforce_eager: true
14
+apply_chat_template: true
15
+fewshot_as_multiturn: true
16
+trust_remote_code: true
.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt
new
+1
@@ -0,0 +1 @@
1
+NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml
\ No newline at end of file