@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
py 175 lines 5.95 KB
Raw
1 # SPDX-License-Identifier: Apache-2.0
2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 """GPQA evaluation using vLLM server and GPT-OSS evaluation package.
4
5 Usage:
6 pytest -s -v tests/evals/gpt_oss/test_gpqa_correctness.py \
7 --config-list-file=configs/models-h200.txt
8 """
9
10 import os
11 import shlex
12 import subprocess
13 import sys
14 import urllib.request
15 from pathlib import Path
16
17 import regex as re
18 import yaml
19
20 from tests.utils import RemoteOpenAIServer
21
22 TOL = 0.05 # Absolute tolerance for accuracy comparison
23
24 # gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
25 # budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
26 # This test originally ran with a 32768-token model context, so retain that
27 # proven generation budget while using the current evaluator package.
28 MAX_OUTPUT_TOKENS = 32_768
29
30 # Path to tiktoken encoding files
31 TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
32
33 # Tiktoken encoding files to download
34 TIKTOKEN_FILES = {
35 "cl100k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken",
36 "o200k_base.tiktoken": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
37 }
38
39
40 def ensure_tiktoken_files():
41 """Download tiktoken encoding files if they don't exist."""
42 TIKTOKEN_DATA_DIR.mkdir(parents=True, exist_ok=True)
43
44 for filename, url in TIKTOKEN_FILES.items():
45 filepath = TIKTOKEN_DATA_DIR / filename
46 if not filepath.exists():
47 print(f"Downloading {filename} from {url}...")
48 urllib.request.urlretrieve(url, filepath)
49 print(f" Downloaded to {filepath}")
50 else:
51 print(f" {filename} already exists.")
52
53
54 def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> float:
55 """Run GPQA evaluation using the gpt-oss evaluation package."""
56 # Build the command to run the evaluation
57 cmd = [
58 sys.executable,
59 str(Path(__file__).with_name("run_gpqa_eval.py")),
60 "--model",
61 model_name,
62 "--reasoning-effort",
63 reasoning_effort,
64 "--base-url",
65 base_url,
66 "--n-threads",
67 "200",
68 "--max-output-tokens",
69 str(MAX_OUTPUT_TOKENS),
70 ]
71
72 try:
73 # Set up environment for the evaluation subprocess
74 # Inherit current environment and add required variables
75 eval_env = os.environ.copy()
76 eval_env["OPENAI_API_KEY"] = "dummy"
77
78 # Run the evaluation
79 result = subprocess.run(
80 cmd,
81 text=True,
82 capture_output=True,
83 timeout=1800, # 30 minute timeout
84 env=eval_env,
85 )
86
87 print("Evaluation process stdout:\n", result.stdout)
88 print("Evaluation process stderr:\n", result.stderr)
89 print(f"Evaluation process return code: {result.returncode}")
90
91 if result.returncode != 0:
92 raise RuntimeError(
93 f"Evaluation failed with exit code {result.returncode}:\n"
94 f"stdout: {result.stdout}\nstderr: {result.stderr}"
95 )
96
97 # Parse the output to extract the score
98 match = re.search(r"'metric':\s*([\d.]+)", result.stdout)
99 if match:
100 return float(match.group(1))
101
102 # If we still can't find it, raise an error
103 raise ValueError(
104 f"Could not parse score from evaluation output:\n{result.stdout}"
105 )
106
107 except subprocess.TimeoutExpired as e:
108 raise RuntimeError("Evaluation timed out") from e
109
110
111 def test_gpqa_correctness(config_filename):
112 """Test GPQA correctness for a given model configuration."""
113 # Ensure tiktoken files are downloaded
114 ensure_tiktoken_files()
115
116 # Verify tiktoken files exist
117 for filename in TIKTOKEN_FILES:
118 filepath = TIKTOKEN_DATA_DIR / filename
119 assert filepath.exists(), f"Tiktoken file not found: {filepath}"
120
121 eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
122
123 # Parse server arguments from config (use shlex to handle quoted strings)
124 server_args_str = eval_config.get("server_args", "")
125 server_args = shlex.split(server_args_str) if server_args_str else []
126
127 # Add standard server arguments
128 server_args.extend(
129 [
130 "--trust-remote-code",
131 "--enforce-eager",
132 "--disable-uvicorn-access-log",
133 ]
134 )
135
136 # Build server environment with tiktoken path and any config-specified vars
137 server_env = {"TIKTOKEN_ENCODINGS_BASE": str(TIKTOKEN_DATA_DIR)}
138 if eval_config.get("env"):
139 server_env.update(eval_config["env"])
140
141 reasoning_effort = eval_config.get("reasoning_effort", "low")
142
143 print(f"Starting GPQA evaluation for model: {eval_config['model_name']}")
144 print(f"Expected metric threshold: {eval_config['metric_threshold']}")
145 print(f"Reasoning effort: {reasoning_effort}")
146 print(f"Server args: {' '.join(server_args)}")
147 print(f"Server environment variables: {server_env}")
148
149 # Launch server and run evaluation
150 with RemoteOpenAIServer(
151 eval_config["model_name"],
152 server_args,
153 env_dict=server_env,
154 max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800),
155 ) as remote_server:
156 base_url = remote_server.url_for("v1")
157 print(f"Server started at: {base_url}")
158
159 measured_metric = run_gpqa_eval(
160 eval_config["model_name"], base_url, reasoning_effort
161 )
162 expected_metric = eval_config["metric_threshold"]
163
164 print(f"GPQA Results for {eval_config['model_name']}:")
165 print(f" Measured metric: {measured_metric:.4f}")
166 print(f" Expected metric: {expected_metric:.4f}")
167 print(f" Tolerance: {TOL:.4f}")
168
169 # Verify metric is within tolerance
170 assert measured_metric >= expected_metric - TOL, (
171 f"GPQA metric too low: {measured_metric:.4f} < "
172 f"{expected_metric:.4f} - {TOL:.4f} = {expected_metric - TOL:.4f}"
173 )
174
175 print(f"GPQA test passed for {eval_config['model_name']}")