@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0f4f0191d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[CI/Build] Replace lm-eval gsm8k tests with faster implementation (#23002)

Signed-off-by: mgoin <mgoin64@gmail.com>

Michael Goin committed Aug 19, 2025 at 18:07 UTC 0f4f0191d842831854c49dfbe589f7b31e638dc7
12 files changed +476 -3
.buildkite/test-pipeline.yaml
+1 -3
@@ -451,13 +451,11 @@ steps:
451
452 - label: LM Eval Small Models # 53min
453 mirror_hardwares: [amdexperimental]
454 - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
454 source_file_dependencies:
455 - csrc/
456 - vllm/model_executor/layers/quantization
457 commands:
459 - - export VLLM_WORKER_MULTIPROC_METHOD=spawn
460 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
458 + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
459
460 - label: OpenAI API correctness
461 mirror_hardwares: [amdexperimental]
tests/evals/gsm8k/README.md new
+35
@@ -0,0 +1,35 @@
1 +# GSM8K Accuracy Evaluation
2 +
3 +This directory contains a replacement for the lm-eval-harness GSM8K evaluation, using an isolated GSM8K script and vLLM server for better performance and control.
4 +
5 +## Usage
6 +
7 +### Run tests with pytest (like buildkite)
8 +
9 +```bash
10 +pytest -s -v tests/gsm8k/test_gsm8k_correctness.py \
11 + --config-list-file=configs/models-small.txt \
12 + --tp-size=1
13 +```
14 +
15 +### Run standalone evaluation script
16 +
17 +```bash
18 +# Start vLLM server first
19 +vllm serve Qwen/Qwen2.5-1.5B-Instruct --port 8000
20 +
21 +# Run evaluation
22 +python tests/gsm8k/gsm8k_eval.py --port 8000
23 +```
24 +
25 +## Configuration Format
26 +
27 +Model configs in `configs/` directory use this YAML format:
28 +
29 +```yaml
30 +model_name: "Qwen/Qwen2.5-1.5B-Instruct"
31 +accuracy_threshold: 0.54 # Minimum expected accuracy
32 +num_questions: 1319 # Number of questions (default: full test set)
33 +num_fewshot: 5 # Few-shot examples from train set
34 +max_model_len: 4096 # Model context length
35 +```
tests/evals/gsm8k/__init__.py new
+2
@@ -0,0 +1,2 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
\ No newline at end of file
tests/evals/gsm8k/configs/Llama-3-8B-Instruct-nonuniform-CT.yaml new
+5
@@ -0,0 +1,5 @@
1 +model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test"
2 +accuracy_threshold: 0.74
3 +num_questions: 1319
4 +num_fewshot: 5
5 +max_model_len: 4096
\ No newline at end of file
tests/evals/gsm8k/configs/Llama-3.2-1B-Instruct-INT8-CT.yaml new
+5
@@ -0,0 +1,5 @@
1 +model_name: "RedHatAI/Llama-3.2-1B-Instruct-quantized.w8a8"
2 +accuracy_threshold: 0.31
3 +num_questions: 1319
4 +num_fewshot: 5
5 +max_model_len: 4096
\ No newline at end of file
tests/evals/gsm8k/configs/Qwen1.5-MoE-W4A16-CT.yaml new
+5
@@ -0,0 +1,5 @@
1 +model_name: "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16"
2 +accuracy_threshold: 0.45
3 +num_questions: 1319
4 +num_fewshot: 5
5 +max_model_len: 4096
\ No newline at end of file
tests/evals/gsm8k/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml new
+5
@@ -0,0 +1,5 @@
1 +model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
2 +accuracy_threshold: 0.60
3 +num_questions: 1319
4 +num_fewshot: 5
5 +max_model_len: 4096
\ No newline at end of file
tests/evals/gsm8k/configs/Qwen3-0.6B-FP8.yaml new
+5
@@ -0,0 +1,5 @@
1 +model_name: "Qwen/Qwen3-0.6B-FP8"
2 +accuracy_threshold: 0.375
3 +num_questions: 1319
4 +num_fewshot: 5
5 +max_model_len: 4096
\ No newline at end of file
tests/evals/gsm8k/configs/models-small.txt new
+5
@@ -0,0 +1,5 @@
1 +Qwen3-0.6B-FP8.yaml
2 +Llama-3.2-1B-Instruct-INT8-CT.yaml
3 +Llama-3-8B-Instruct-nonuniform-CT.yaml
4 +Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
5 +Qwen1.5-MoE-W4A16-CT.yaml
tests/evals/gsm8k/conftest.py new
+66
@@ -0,0 +1,66 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +from pathlib import Path
5 +
6 +
7 +def pytest_addoption(parser):
8 + """Add custom command line options."""
9 + parser.addoption("--config-list-file",
10 + default="configs/models-small.txt",
11 + help="File containing list of config files to test")
12 + parser.addoption("--tp-size",
13 + default=1,
14 + type=int,
15 + help="Tensor parallel size")
16 +
17 +
18 +def pytest_generate_tests(metafunc):
19 + """Generate test parameters from config files."""
20 + if "config_filename" in metafunc.fixturenames:
21 + config_list_file = metafunc.config.getoption("--config-list-file")
22 + tp_size = metafunc.config.getoption("--tp-size")
23 +
24 + # Handle both relative and absolute paths
25 + config_list_path = Path(config_list_file)
26 + if not config_list_path.is_absolute():
27 + # If relative, try relative to test directory first
28 + test_dir_path = Path(__file__).parent / config_list_file
29 + if test_dir_path.exists():
30 + config_list_path = test_dir_path
31 + else:
32 + # Try relative to current working directory
33 + config_list_path = Path.cwd() / config_list_file
34 +
35 + print(f"Looking for config list at: {config_list_path}")
36 +
37 + config_files = []
38 + if config_list_path.exists():
39 + # Determine config directory (same directory as the list file)
40 + config_dir = config_list_path.parent
41 +
42 + with open(config_list_path) as f:
43 + for line in f:
44 + line = line.strip()
45 + if line and not line.startswith("#"):
46 + config_path = config_dir / line
47 + print(f"Checking config file: {config_path}")
48 + if config_path.exists():
49 + config_files.append(config_path)
50 + print(f" ✓ Found: {config_path}")
51 + else:
52 + print(f" ✗ Missing: {config_path}")
53 + else:
54 + print(f"Config list file not found: {config_list_path}")
55 +
56 + # Generate test parameters
57 + if config_files:
58 + metafunc.parametrize(["config_filename", "tp_size"],
59 + [(config_file, int(tp_size))
60 + for config_file in config_files],
61 + ids=[
62 + f"{config_file.stem}-tp{tp_size}"
63 + for config_file in config_files
64 + ])
65 + else:
66 + print("No config files found, test will be skipped")
tests/evals/gsm8k/gsm8k_eval.py new
+252
@@ -0,0 +1,252 @@
1 +#!/usr/bin/env python3
2 +# SPDX-License-Identifier: Apache-2.0
3 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
4 +"""
5 +Isolated GSM8K evaluation script for vLLM serve endpoint.
6 +"""
7 +
8 +import argparse
9 +import ast
10 +import asyncio
11 +import json
12 +import os
13 +import time
14 +from collections.abc import Generator
15 +from typing import Optional, Union
16 +
17 +import aiohttp
18 +import numpy as np
19 +import regex as re
20 +import requests
21 +from tqdm.asyncio import tqdm
22 +
23 +INVALID = -9999999
24 +
25 +
26 +def download_and_cache_file(url: str, filename: Optional[str] = None) -> str:
27 + """Download and cache a file from a URL."""
28 + if filename is None:
29 + filename = os.path.join("/tmp", url.split("/")[-1])
30 +
31 + if os.path.exists(filename):
32 + return filename
33 +
34 + print(f"Downloading from {url} to {filename}")
35 + response = requests.get(url, stream=True)
36 + response.raise_for_status()
37 +
38 + with open(filename, "wb") as f:
39 + for chunk in response.iter_content(chunk_size=1024):
40 + f.write(chunk)
41 +
42 + return filename
43 +
44 +
45 +def load_gsm8k_data() -> tuple[list[dict], list[dict]]:
46 + """Load GSM8K train and test data"""
47 + train_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/train.jsonl"
48 + test_url = "https://raw.githubusercontent.com/openai/grade-school-math/master/grade_school_math/data/test.jsonl"
49 +
50 + train_file = download_and_cache_file(train_url)
51 + test_file = download_and_cache_file(test_url)
52 +
53 + train_data = list(read_jsonl(train_file))
54 + test_data = list(read_jsonl(test_file))
55 +
56 + return train_data, test_data
57 +
58 +
59 +def read_jsonl(filename: str) -> Generator[dict, None, None]:
60 + """Read a JSONL file."""
61 + with open(filename) as fin:
62 + for line in fin:
63 + if not line.startswith("#"):
64 + yield json.loads(line)
65 +
66 +
67 +def get_answer_value(answer_str: str) -> int:
68 + """Extract the numerical answer from the response."""
69 + answer_str = answer_str.replace(",", "")
70 + numbers = re.findall(r"\d+", answer_str)
71 + if len(numbers) < 1:
72 + return INVALID
73 + try:
74 + return ast.literal_eval(numbers[-1])
75 + except SyntaxError:
76 + return INVALID
77 +
78 +
79 +async def call_vllm_api(session: aiohttp.ClientSession,
80 + prompt: str,
81 + temperature: float,
82 + max_tokens: int,
83 + stop: Optional[list[str]] = None,
84 + url: Optional[str] = None,
85 + seed: Optional[int] = None) -> str:
86 + """Call vLLM's OpenAI-compatible completions endpoint."""
87 + data = {
88 + "prompt": prompt,
89 + "temperature": temperature,
90 + "max_tokens": max_tokens,
91 + "stop": stop,
92 + }
93 + if seed is not None:
94 + data["seed"] = seed
95 +
96 + try:
97 + async with session.post(f"{url}/v1/completions",
98 + json=data) as response:
99 + response.raise_for_status()
100 + result = await response.json()
101 + return result["choices"][0]["text"]
102 + except Exception as e:
103 + print(f"Error calling vLLM API: {e}")
104 + return ""
105 +
106 +
107 +def evaluate_gsm8k(num_questions: int = 1319,
108 + num_shots: int = 5,
109 + max_tokens: int = 256,
110 + host: str = "http://127.0.0.1",
111 + port: int = 8000,
112 + temperature: float = 0.0,
113 + seed: Optional[int] = 42) -> dict[str, Union[float, int]]:
114 + """
115 + Evaluate GSM8K accuracy using vLLM serve endpoint.
116 +
117 + Returns dict with accuracy, invalid_rate, latency, etc.
118 + """
119 + base_url = f"{host}:{port}"
120 +
121 + # Load GSM8K train and test data
122 + train_data, test_data = load_gsm8k_data()
123 +
124 + # Limit to available test questions
125 + num_questions = min(num_questions, len(test_data))
126 +
127 + # Build few-shot examples from train split (like lm-eval does)
128 + few_shot_examples = ""
129 + for i in range(num_shots):
130 + few_shot_examples += (f"Question: {train_data[i]['question']}\n"
131 + f"Answer: {train_data[i]['answer']}\n\n")
132 +
133 + # Prepare test questions and labels from test split
134 + questions = []
135 + labels = []
136 + for i in range(num_questions):
137 + questions.append(f"Question: {test_data[i]['question']}\nAnswer:")
138 + labels.append(get_answer_value(test_data[i]["answer"]))
139 +
140 + assert all(label != INVALID for label in labels), "Some labels are invalid"
141 +
142 + # Run evaluation
143 + async def run_async_evaluation():
144 + states: list[str] = [""] * num_questions
145 +
146 + async def get_answer(session: aiohttp.ClientSession, i: int) -> str:
147 + prompt = few_shot_examples + questions[i]
148 + answer = await call_vllm_api(
149 + session=session,
150 + prompt=prompt,
151 + temperature=temperature,
152 + max_tokens=max_tokens,
153 + stop=["Question", "Assistant:", "<|separator|>"],
154 + url=base_url,
155 + seed=seed,
156 + )
157 + states[i] = answer
158 + return answer
159 +
160 + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(
161 + total=600)) as session:
162 + tasks = [get_answer(session, i) for i in range(num_questions)]
163 + await tqdm.gather(*tasks, desc="Evaluating")
164 +
165 + return states
166 +
167 + print(f"Running GSM8K evaluation: {num_questions} questions, "
168 + f"{num_shots}-shot")
169 +
170 + tic = time.perf_counter()
171 + states = asyncio.run(run_async_evaluation())
172 + latency = time.perf_counter() - tic
173 +
174 + # Compute metrics
175 + preds = [get_answer_value(state) for state in states]
176 + accuracy = np.mean(np.array(preds) == np.array(labels))
177 + invalid_rate = np.mean(np.array(preds) == INVALID)
178 +
179 + result = {
180 + "accuracy": accuracy,
181 + "invalid_rate": invalid_rate,
182 + "latency": latency,
183 + "questions_per_second": num_questions / latency,
184 + "num_questions": num_questions,
185 + "num_shots": num_shots,
186 + "max_tokens": max_tokens,
187 + "timestamp": time.time(),
188 + }
189 +
190 + return result
191 +
192 +
193 +def main() -> None:
194 + parser = argparse.ArgumentParser(
195 + description="GSM8K evaluation for vLLM serve")
196 + parser.add_argument("--num-shots",
197 + type=int,
198 + default=5,
199 + help="Number of few-shot examples")
200 + parser.add_argument("--num-questions",
201 + type=int,
202 + default=1319,
203 + help="Number of questions to evaluate")
204 + parser.add_argument("--max-tokens",
205 + type=int,
206 + default=256,
207 + help="Max tokens for generation")
208 + parser.add_argument("--host",
209 + type=str,
210 + default="http://127.0.0.1",
211 + help="Host URL")
212 + parser.add_argument("--port", type=int, default=8000, help="Port number")
213 + parser.add_argument("--temperature",
214 + type=float,
215 + default=0.0,
216 + help="Temperature for generation")
217 + parser.add_argument("--seed",
218 + type=int,
219 + default=42,
220 + help="Random seed for reproducibility")
221 + parser.add_argument("--save-results",
222 + type=str,
223 + help="Save results to JSON file")
224 +
225 + args = parser.parse_args()
226 +
227 + result = evaluate_gsm8k(
228 + num_questions=args.num_questions,
229 + num_shots=args.num_shots,
230 + max_tokens=args.max_tokens,
231 + host=args.host,
232 + port=args.port,
233 + temperature=args.temperature,
234 + seed=args.seed,
235 + )
236 +
237 + # Print results to terminal
238 + print("\nResults:")
239 + print(f"Accuracy: {result['accuracy']:.3f}")
240 + print(f"Invalid responses: {result['invalid_rate']:.3f}")
241 + print(f"Total latency: {result['latency']:.3f} s")
242 + print(f"Questions per second: {result['questions_per_second']:.3f}")
243 +
244 + # Optional file saving
245 + if args.save_results:
246 + with open(args.save_results, "w") as f:
247 + json.dump(result, f, indent=2)
248 + print(f"Results saved to {args.save_results}")
249 +
250 +
251 +if __name__ == "__main__":
252 + main()
tests/evals/gsm8k/test_gsm8k_correctness.py new
+90
@@ -0,0 +1,90 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""
4 +GSM8K evaluation using vLLM server and isolated GSM8K script.
5 +Replacement for lm-eval-harness with better performance and control.
6 +
7 +Usage:
8 +pytest -s -v test_gsm8k_correctness.py \
9 + --config-list-file=configs/models-small.txt \
10 + --tp-size=1
11 +"""
12 +
13 +import yaml
14 +
15 +from tests.utils import RemoteOpenAIServer
16 +
17 +from .gsm8k_eval import evaluate_gsm8k
18 +
19 +RTOL = 0.08 # Relative tolerance for accuracy comparison
20 +
21 +
22 +def launch_gsm8k_eval(eval_config, server_url, tp_size):
23 + """Launch GSM8K evaluation using our isolated script."""
24 + # Extract host and port from server URL
25 + if "://" in server_url:
26 + server_url = server_url.split("://")[1]
27 +
28 + host_port = server_url.split("/")[0] # Remove path if present
29 + if ":" in host_port:
30 + host, port = host_port.split(":")
31 + port = int(port)
32 + else:
33 + host = host_port
34 + port = 8000
35 +
36 + # Add http:// prefix if not present
37 + if not host.startswith("http"):
38 + host = f"http://{host}"
39 +
40 + # Run GSM8K evaluation
41 + results = evaluate_gsm8k(
42 + num_questions=eval_config["num_questions"],
43 + num_shots=eval_config["num_fewshot"],
44 + host=host,
45 + port=port,
46 + )
47 +
48 + return results
49 +
50 +
51 +def test_gsm8k_correctness_param(config_filename, tp_size):
52 + """Test GSM8K correctness for a given model configuration."""
53 + eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
54 +
55 + # Server arguments
56 + server_args = [
57 + "--max-model-len",
58 + str(eval_config.get("max_model_len", 4096)),
59 + "--enforce-eager",
60 + "--trust-remote-code",
61 + "--tensor-parallel-size",
62 + str(tp_size),
63 + ]
64 +
65 + # Launch server and run evaluation
66 + with RemoteOpenAIServer(eval_config["model_name"],
67 + server_args,
68 + max_wait_seconds=480) as remote_server:
69 + server_url = remote_server.url_for("v1")
70 +
71 + results = launch_gsm8k_eval(eval_config, server_url, tp_size)
72 +
73 + # Check accuracy against threshold
74 + measured_accuracy = results["accuracy"]
75 + expected_accuracy = eval_config["accuracy_threshold"]
76 +
77 + print(f"GSM8K Results for {eval_config['model_name']}:")
78 + print(f" Accuracy: {measured_accuracy:.3f}")
79 + print(f" Expected: {expected_accuracy:.3f}")
80 + print(f" Questions: {results['num_questions']}")
81 + print(f" Invalid rate: {results['invalid_rate']:.3f}")
82 + print(f" Latency: {results['latency']:.1f}s")
83 + print(f" QPS: {results['questions_per_second']:.1f}")
84 +
85 + # Verify accuracy is within tolerance
86 + assert measured_accuracy >= expected_accuracy - RTOL, (
87 + f"Accuracy too low: {measured_accuracy:.3f} < "
88 + f"{expected_accuracy:.3f} - {RTOL:.3f}")
89 +
90 + print(f"✅ GSM8K test passed for {eval_config['model_name']}")