A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] GPT-OSS GPQA eval test for Blackwell (#24920)
Signed-off-by: mgoin <mgoin64@gmail.com> Signed-off-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Michael Goin committed
Sep 16, 2025 at 21:13 UTC
493b10f8bf38495654baa601e8ed8dc4ce1565b7
4 files changed
+136
.buildkite/test-pipeline.yaml
+14
@@ -821,6 +821,20 @@ steps:
821
- pytest -v -s tests/kernels/moe/test_flashinfer.py
822
- pytest -v -s tests/compile/test_silu_mul_quant_fusion.py
823
824
+- label: GPT-OSS Eval (Blackwell)
825
+ timeout_in_minutes: 60
826
+ working_dir: "/vllm-workspace/"
827
+ gpu: b200
828
+ # optional: true
829
+ source_file_dependencies:
830
+ - tests/evals/gpt_oss
831
+ - vllm/model_executor/models/gpt_oss.py
832
+ - vllm/model_executor/layers/quantization/mxfp4.py
833
+ - vllm/v1/attention/backends/flashinfer.py
834
+ commands:
835
+ - uv pip install --system 'gpt-oss[eval]==0.0.5'
836
+ - pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py --model openai/gpt-oss-20b --metric 0.58 --server-args '--tensor-parallel-size 2'
837
+
838
##### 1 GPU test #####
839
##### multi gpus test #####
840
tests/evals/gpt_oss/__init__.py
new
+2
@@ -0,0 +1,2 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
\ No newline at end of file
tests/evals/gpt_oss/conftest.py
new
+18
@@ -0,0 +1,18 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+Pytest configuration for GPT-OSS evaluation tests.
5
+"""
6
+
7
+
8
+def pytest_addoption(parser):
9
+ """Add command line options for pytest."""
10
+ parser.addoption("--model", action="store", help="Model name to evaluate")
11
+ parser.addoption("--metric",
12
+ action="store",
13
+ type=float,
14
+ help="Expected metric threshold")
15
+ parser.addoption("--server-args",
16
+ action="store",
17
+ default="",
18
+ help="Additional server arguments")
tests/evals/gpt_oss/test_gpqa_correctness.py
new
+102
@@ -0,0 +1,102 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""
4
+GPQA evaluation using vLLM server and GPT-OSS evaluation package.
5
+
6
+Usage:
7
+pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \
8
+ --model openai/gpt-oss-20b \
9
+ --metric 0.58 \
10
+ --server-args "--tensor-parallel-size 2"
11
+"""
12
+
13
+import subprocess
14
+import sys
15
+
16
+import regex as re
17
+
18
+from tests.utils import RemoteOpenAIServer
19
+
20
+TOL = 0.05 # Absolute tolerance for accuracy comparison
21
+
22
+
23
+def run_gpqa_eval(model_name: str, base_url: str) -> float:
24
+ """Run GPQA evaluation using the gpt-oss evaluation package."""
25
+
26
+ # Build the command to run the evaluation
27
+ cmd = [
28
+ sys.executable, "-m", "gpt_oss.evals", "--eval", "gpqa", "--model",
29
+ model_name, "--reasoning-effort", "low", "--base-url", base_url
30
+ ]
31
+
32
+ try:
33
+ # Run the evaluation
34
+ result = subprocess.run(
35
+ cmd,
36
+ text=True,
37
+ capture_output=True,
38
+ timeout=1800, # 30 minute timeout
39
+ env={"OPENAI_API_KEY": "dummy"})
40
+
41
+ print("Evaluation process output:\n", result.stdout)
42
+
43
+ # Parse the output to extract the score
44
+ match = re.search(r"'metric':\s*([\d.]+)", result.stdout)
45
+ if match:
46
+ return float(match.group(1))
47
+
48
+ # If we still can't find it, raise an error
49
+ raise ValueError(
50
+ f"Could not parse score from evaluation output:\n{result.stdout}")
51
+
52
+ except subprocess.TimeoutExpired as e:
53
+ raise RuntimeError("Evaluation timed out") from e
54
+ except subprocess.CalledProcessError as e:
55
+ raise RuntimeError(
56
+ f"Evaluation failed with exit code {e.returncode}:\n"
57
+ f"stdout: {e.stdout}\nstderr: {e.stderr}") from e
58
+
59
+
60
+def test_gpqa_correctness(request):
61
+ """Test GPQA correctness for GPT-OSS model."""
62
+
63
+ # Get command line arguments
64
+ model_name = request.config.getoption("--model")
65
+ expected_metric = request.config.getoption("--metric")
66
+ server_args_str = request.config.getoption("--server-args")
67
+
68
+ # Parse server arguments
69
+ server_args = []
70
+ if server_args_str:
71
+ server_args = server_args_str.split()
72
+
73
+ # Add standard server arguments
74
+ server_args.extend([
75
+ "--max-model-len",
76
+ "32768",
77
+ "--trust-remote-code",
78
+ ])
79
+
80
+ print(f"Starting GPQA evaluation for model: {model_name}")
81
+ print(f"Expected metric threshold: {expected_metric}")
82
+ print(f"Server args: {' '.join(server_args)}")
83
+
84
+ # Launch server and run evaluation
85
+ with RemoteOpenAIServer(model_name, server_args,
86
+ max_wait_seconds=1800) as remote_server:
87
+ base_url = remote_server.url_for("v1")
88
+ print(f"Server started at: {base_url}")
89
+
90
+ measured_metric = run_gpqa_eval(model_name, base_url)
91
+
92
+ print(f"GPQA Results for {model_name}:")
93
+ print(f" Measured metric: {measured_metric:.4f}")
94
+ print(f" Expected metric: {expected_metric:.4f}")
95
+ print(f" Tolerance: {TOL:.4f}")
96
+
97
+ # Verify metric is within tolerance
98
+ assert measured_metric >= expected_metric - TOL, (
99
+ f"GPQA metric too low: {measured_metric:.4f} < "
100
+ f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}")
101
+
102
+ print(f"✅ GPQA test passed for {model_name}")