| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """GSM8K evaluation using vLLM server and isolated GSM8K script. |
| 4 | Replacement for lm-eval-harness with better performance and control. |
| 5 | |
| 6 | Usage: |
| 7 | pytest -s -v tests/evals/gsm8k/test_gsm8k_correctness.py \ |
| 8 | --config-list-file=configs/models-small.txt |
| 9 | """ |
| 10 | |
| 11 | import shlex |
| 12 | |
| 13 | import pytest |
| 14 | import requests |
| 15 | import yaml |
| 16 | |
| 17 | from tests.utils import RemoteOpenAIServer |
| 18 | from vllm.platforms import current_platform |
| 19 | |
| 20 | from .gsm8k_eval import evaluate_gsm8k |
| 21 | |
| 22 | DEFAULT_STARTUP_MAX_WAIT_SECONDS = 1200 |
| 23 | |
| 24 | |
| 25 | def run_gsm8k_eval(eval_config: dict, server_url: str) -> dict: |
| 26 | """Run GSM8K evaluation using our isolated script.""" |
| 27 | # Extract host and port from server URL |
| 28 | if "://" in server_url: |
| 29 | server_url = server_url.split("://")[1] |
| 30 | |
| 31 | host_port = server_url.split("/")[0] # Remove path if present |
| 32 | if ":" in host_port: |
| 33 | host, p = host_port.split(":") |
| 34 | port = int(p) |
| 35 | else: |
| 36 | host = host_port |
| 37 | port = 8000 |
| 38 | |
| 39 | # Add http:// prefix if not present |
| 40 | if not host.startswith("http"): |
| 41 | host = f"http://{host}" |
| 42 | |
| 43 | # Run GSM8K evaluation |
| 44 | request_timeout_seconds = eval_config.get("request_timeout_seconds", 600) |
| 45 | if current_platform.is_rocm(): |
| 46 | request_timeout_seconds = eval_config.get( |
| 47 | "rocm_request_timeout_seconds", request_timeout_seconds |
| 48 | ) |
| 49 | |
| 50 | results = evaluate_gsm8k( |
| 51 | num_questions=eval_config["num_questions"], |
| 52 | num_shots=eval_config["num_fewshot"], |
| 53 | max_tokens=eval_config.get("max_tokens", 256), |
| 54 | model=eval_config["model_name"], |
| 55 | use_chat_completions=eval_config.get("use_chat_completions", False), |
| 56 | host=host, |
| 57 | port=port, |
| 58 | temperature=eval_config.get("temperature", 0.0), |
| 59 | seed=eval_config.get("seed", 42), |
| 60 | request_timeout_seconds=request_timeout_seconds, |
| 61 | gen_prefix=eval_config.get("gen_prefix", ""), |
| 62 | max_concurrency=eval_config.get("max_concurrency"), |
| 63 | ) |
| 64 | |
| 65 | return results |
| 66 | |
| 67 | |
| 68 | def get_acceptance_length(server_url: str) -> float: |
| 69 | """Mean tokens emitted per verification step, from the server's counters. |
| 70 | |
| 71 | 1.0 means every draft was rejected (speculation bought nothing); the |
| 72 | theoretical maximum is 1 + num_speculative_tokens. |
| 73 | """ |
| 74 | response = requests.get(f"{server_url.rstrip('/').removesuffix('/v1')}/metrics") |
| 75 | response.raise_for_status() |
| 76 | counters: dict[str, float] = {} |
| 77 | for line in response.text.splitlines(): |
| 78 | if line.startswith("vllm:spec_decode_num_"): |
| 79 | name, _, value = line.partition(" ") |
| 80 | counters[name.split("{")[0]] = float(value) |
| 81 | |
| 82 | num_drafts = counters.get("vllm:spec_decode_num_drafts_total", 0.0) |
| 83 | num_accepted = counters.get("vllm:spec_decode_num_accepted_tokens_total", 0.0) |
| 84 | assert num_drafts > 0, ( |
| 85 | "no drafts recorded; speculative decoding did not run for this config" |
| 86 | ) |
| 87 | return 1.0 + num_accepted / num_drafts |
| 88 | |
| 89 | |
| 90 | def test_gsm8k_correctness(config_filename): |
| 91 | """Test GSM8K correctness for a given model configuration.""" |
| 92 | eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) |
| 93 | |
| 94 | if ( |
| 95 | not current_platform.is_cuda() |
| 96 | and "Qwen3-30B-A3B-MXFP4A16" in eval_config["model_name"] |
| 97 | ): |
| 98 | pytest.skip( |
| 99 | "Skipping Qwen3-30B-A3B-MXFP4A16 on non-CUDA platforms. " |
| 100 | "Marlin kernels are not supported." |
| 101 | ) |
| 102 | |
| 103 | if ( |
| 104 | not current_platform.is_cuda() |
| 105 | and "gemma-4-E4B-it-qat-mobile-ct" in eval_config["model_name"] |
| 106 | ): |
| 107 | pytest.skip( |
| 108 | "Skipping gemma-4-E4B-it-qat-mobile-ct on non-CUDA platforms. " |
| 109 | "Its W2A16 (uint2b2) scheme has no kernel outside CUDA." |
| 110 | ) |
| 111 | |
| 112 | if current_platform.is_rocm() and ( |
| 113 | "deepseek-ai/DeepSeek-V3.2" in eval_config["model_name"] |
| 114 | or "deepseek-ai/DeepSeek-R1" in eval_config["model_name"] |
| 115 | ): |
| 116 | from vllm.platforms.rocm import get_cdna_version |
| 117 | |
| 118 | if get_cdna_version() < 4: |
| 119 | pytest.skip( |
| 120 | "Skipping DeepSeek-V3.2 and DeepSeek-R1 on ROCm platforms below " |
| 121 | "CDNA 4 due to agent pool disk space issues and pod evictions." |
| 122 | ) |
| 123 | if current_platform.is_rocm() and ( |
| 124 | "Qwen3.5-35B-A3B-MXFP4-AITER-TP2" in config_filename.name |
| 125 | ): |
| 126 | from vllm.platforms.rocm import on_gfx950 |
| 127 | |
| 128 | if not on_gfx950(): |
| 129 | pytest.skip( |
| 130 | "Skipping Qwen3.5-35B-A3B-MXFP4-AITER-TP2 on non-GFX950 platforms. " |
| 131 | "The quantization scheme is not supported on non-GFX950 platforms." |
| 132 | ) |
| 133 | if "--all2all-backend=moonep" in eval_config.get("server_args", ""): |
| 134 | from vllm.utils.import_utils import has_moonep |
| 135 | |
| 136 | if not has_moonep(): |
| 137 | pytest.skip("Skipping MoonEP config: the moonep package is not installed.") |
| 138 | |
| 139 | # Parse server arguments from config (use shlex to handle quoted strings) |
| 140 | server_args_str = eval_config.get("server_args", "") |
| 141 | server_args = shlex.split(server_args_str) if server_args_str else [] |
| 142 | |
| 143 | # Add standard server arguments |
| 144 | server_args.extend( |
| 145 | [ |
| 146 | "--trust-remote-code", |
| 147 | "--disable-uvicorn-access-log", |
| 148 | ] |
| 149 | ) |
| 150 | |
| 151 | startup_max_wait_seconds = eval_config.get( |
| 152 | "startup_max_wait_seconds", DEFAULT_STARTUP_MAX_WAIT_SECONDS |
| 153 | ) |
| 154 | env_dict = dict(eval_config.get("env") or {}) |
| 155 | env_dict["VLLM_ENGINE_READY_TIMEOUT_S"] = str(int(startup_max_wait_seconds)) |
| 156 | |
| 157 | print(f"Starting GSM8K evaluation for model: {eval_config['model_name']}") |
| 158 | print(f"Expected metric threshold: {eval_config['accuracy_threshold']}") |
| 159 | print(f"Number of questions: {eval_config['num_questions']}") |
| 160 | print(f"Number of few-shot examples: {eval_config['num_fewshot']}") |
| 161 | request_timeout_seconds = eval_config.get("request_timeout_seconds", 600) |
| 162 | if current_platform.is_rocm(): |
| 163 | request_timeout_seconds = eval_config.get( |
| 164 | "rocm_request_timeout_seconds", request_timeout_seconds |
| 165 | ) |
| 166 | print(f"Request timeout: {request_timeout_seconds}s") |
| 167 | print(f"Startup max wait: {startup_max_wait_seconds}s") |
| 168 | print(f"Server args: {' '.join(server_args)}") |
| 169 | print(f"Environment variables: {env_dict}") |
| 170 | |
| 171 | # Launch server and run evaluation |
| 172 | with RemoteOpenAIServer( |
| 173 | eval_config["model_name"], |
| 174 | server_args, |
| 175 | env_dict=env_dict, |
| 176 | max_wait_seconds=startup_max_wait_seconds, |
| 177 | ) as remote_server: |
| 178 | server_url = remote_server.url_for("v1") |
| 179 | print(f"Server started at: {server_url}") |
| 180 | |
| 181 | results = run_gsm8k_eval(eval_config, server_url) |
| 182 | |
| 183 | measured_metric = results["accuracy"] |
| 184 | expected_metric = eval_config["accuracy_threshold"] |
| 185 | tol = eval_config.get("tolerance", 0.08) |
| 186 | |
| 187 | print(f"GSM8K Results for {eval_config['model_name']}:") |
| 188 | print(f" Measured metric: {measured_metric:.4f}") |
| 189 | print(f" Expected metric: {expected_metric:.4f}") |
| 190 | print(f" Tolerance: {tol:.4f}") |
| 191 | print(f" Questions: {results['num_questions']}") |
| 192 | print(f" Invalid rate: {results['invalid_rate']:.3f}") |
| 193 | print(f" Latency: {results['latency']:.1f}s") |
| 194 | print(f" QPS: {results['questions_per_second']:.1f}") |
| 195 | |
| 196 | assert measured_metric >= expected_metric - tol, ( |
| 197 | f"GSM8K metric too low: {measured_metric:.4f} < " |
| 198 | f"{expected_metric:.4f} - {tol:.4f} = {expected_metric - tol:.4f}" |
| 199 | ) |
| 200 | |
| 201 | # Speculative configs additionally assert that drafts are actually |
| 202 | # landing: accuracy alone passes even when every draft is rejected. |
| 203 | min_acceptance_length = eval_config.get("min_acceptance_length") |
| 204 | if min_acceptance_length is not None: |
| 205 | acceptance_length = get_acceptance_length(server_url) |
| 206 | print(f" Mean acceptance length: {acceptance_length:.3f}") |
| 207 | print(f" Minimum acceptance length: {min_acceptance_length:.3f}") |
| 208 | assert acceptance_length >= min_acceptance_length, ( |
| 209 | f"Acceptance length too low: {acceptance_length:.3f} < " |
| 210 | f"{min_acceptance_length:.3f}" |
| 211 | ) |
| 212 | |
| 213 | print(f"✅ GSM8K test passed for {eval_config['model_name']}") |