| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """GSM8K and AIME25 accuracy evaluation for Qwen3.8-Flash-Next-FP8.""" |
| 4 | |
| 5 | import shlex |
| 6 | from pathlib import Path |
| 7 | from typing import Any |
| 8 | |
| 9 | import yaml |
| 10 | |
| 11 | from tests.utils import RemoteOpenAIServer |
| 12 | |
| 13 | |
| 14 | def run_evalscope( |
| 15 | eval_config: dict[str, Any], base_url: str, work_dir: Path |
| 16 | ) -> dict[str, float]: |
| 17 | from evalscope.run import run_task |
| 18 | |
| 19 | reports = run_task( |
| 20 | { |
| 21 | "model": eval_config["model_name"], |
| 22 | "api_url": base_url, |
| 23 | "api_key": "EMPTY_TOKEN", |
| 24 | "datasets": list(eval_config["datasets"]), |
| 25 | "eval_batch_size": eval_config.get("eval_batch_size", 32), |
| 26 | "generation_config": eval_config["generation_config"], |
| 27 | "work_dir": str(work_dir), |
| 28 | "no_timestamp": True, |
| 29 | } |
| 30 | ) |
| 31 | if not isinstance(reports, dict): |
| 32 | raise TypeError(f"Expected EvalScope reports to be a dict, got {type(reports)}") |
| 33 | |
| 34 | return {dataset: float(report.score) for dataset, report in reports.items()} |
| 35 | |
| 36 | |
| 37 | def test_qwen4_exp_accuracy(config_filename: Path, tmp_path: Path): |
| 38 | eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8")) |
| 39 | server_args = shlex.split(eval_config.get("server_args", "")) |
| 40 | server_args.extend(["--trust-remote-code", "--disable-uvicorn-access-log"]) |
| 41 | |
| 42 | model_name = eval_config["model_name"] |
| 43 | print(f"Starting Qwen4Exp evaluation for model: {model_name}") |
| 44 | print(f"Datasets: {', '.join(eval_config['datasets'])}") |
| 45 | print(f"Server args: {' '.join(server_args)}") |
| 46 | |
| 47 | with RemoteOpenAIServer( |
| 48 | model_name, |
| 49 | server_args, |
| 50 | env_dict=eval_config.get("env"), |
| 51 | max_wait_seconds=eval_config.get("startup_max_wait_seconds", 1800), |
| 52 | ) as remote_server: |
| 53 | scores = run_evalscope(eval_config, remote_server.url_for("v1"), tmp_path) |
| 54 | |
| 55 | for dataset, metric_config in eval_config["datasets"].items(): |
| 56 | score = scores[dataset] |
| 57 | threshold = metric_config["metric_threshold"] |
| 58 | tolerance = metric_config.get("tolerance", 0.0) |
| 59 | minimum_score = threshold - tolerance |
| 60 | |
| 61 | print( |
| 62 | f"{dataset}: measured={score:.4f}, expected={threshold:.4f}, " |
| 63 | f"tolerance={tolerance:.4f}" |
| 64 | ) |
| 65 | assert score >= minimum_score, ( |
| 66 | f"{dataset} score too low: {score:.4f} < {threshold:.4f} - " |
| 67 | f"{tolerance:.4f} = {minimum_score:.4f}" |
| 68 | ) |