A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
Replace lm-eval bash script with pytest and use enforce_eager for faster CI (#17717)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 6, 2025 at 21:00 UTC
950b71186f9696a60f1a53cd1a033f7fef952500
4 files changed
+52
-91
.buildkite/lm-eval-harness/conftest.py
new
+39
@@ -0,0 +1,39 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from pathlib import Path
3
+
4
+import pytest
5
+
6
+
7
+def pytest_addoption(parser):
8
+ parser.addoption(
9
+ "--config-list-file",
10
+ action="store",
11
+ help="Path to the file listing model config YAMLs (one per line)")
12
+ parser.addoption("--tp-size",
13
+ action="store",
14
+ default="1",
15
+ help="Tensor parallel size to use for evaluation")
16
+
17
+
18
+@pytest.fixture(scope="session")
19
+def config_list_file(pytestconfig, config_dir):
20
+ rel_path = pytestconfig.getoption("--config-list-file")
21
+ return config_dir / rel_path
22
+
23
+
24
+@pytest.fixture(scope="session")
25
+def tp_size(pytestconfig):
26
+ return pytestconfig.getoption("--tp-size")
27
+
28
+
29
+def pytest_generate_tests(metafunc):
30
+ if "config_filename" in metafunc.fixturenames:
31
+ rel_path = metafunc.config.getoption("--config-list-file")
32
+ config_list_file = Path(rel_path).resolve()
33
+ config_dir = config_list_file.parent
34
+ with open(config_list_file, encoding="utf-8") as f:
35
+ configs = [
36
+ config_dir / line.strip() for line in f
37
+ if line.strip() and not line.startswith("#")
38
+ ]
39
+ metafunc.parametrize("config_filename", configs)
.buildkite/lm-eval-harness/run-tests.sh
deleted
-59
@@ -1,59 +0,0 @@
1
-#!/bin/bash
2
-
3
-usage() {
4
- echo``
5
- echo "Runs lm eval harness on GSM8k using vllm and compares to "
6
- echo "precomputed baseline (measured by HF transformers.)"
7
- echo
8
- echo "usage: ${0} <options>"
9
- echo
10
- echo " -c - path to the test data config (e.g. configs/small-models.txt)"
11
- echo " -t - tensor parallel size"
12
- echo
13
-}
14
-
15
-SUCCESS=0
16
-
17
-while getopts "c:t:" OPT; do
18
- case ${OPT} in
19
- c )
20
- CONFIG="$OPTARG"
21
- ;;
22
- t )
23
- TP_SIZE="$OPTARG"
24
- ;;
25
- \? )
26
- usage
27
- exit 1
28
- ;;
29
- esac
30
-done
31
-
32
-# Parse list of configs.
33
-IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < "$CONFIG"
34
-
35
-for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
36
-do
37
- LOCAL_SUCCESS=0
38
-
39
- echo "=== RUNNING MODEL: $MODEL_CONFIG WITH TP SIZE: $TP_SIZE==="
40
-
41
- export LM_EVAL_TEST_DATA_FILE=$PWD/configs/${MODEL_CONFIG}
42
- export LM_EVAL_TP_SIZE=$TP_SIZE
43
- pytest -s test_lm_eval_correctness.py || LOCAL_SUCCESS=$?
44
-
45
- if [[ $LOCAL_SUCCESS == 0 ]]; then
46
- echo "=== PASSED MODEL: ${MODEL_CONFIG} ==="
47
- else
48
- echo "=== FAILED MODEL: ${MODEL_CONFIG} ==="
49
- fi
50
-
51
- SUCCESS=$((SUCCESS + LOCAL_SUCCESS))
52
-
53
-done
54
-
55
-if [ "${SUCCESS}" -eq "0" ]; then
56
- exit 0
57
-else
58
- exit 1
59
-fi
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+11
-30
@@ -3,35 +3,25 @@
3
LM eval harness on model to compare vs HF baseline computed offline.
4
Configs are found in configs/$MODEL.yaml
5
6
-* export LM_EVAL_TEST_DATA_FILE=configs/Meta-Llama-3-70B-Instruct.yaml
7
-* export LM_EVAL_TP_SIZE=4
8
-* pytest -s test_lm_eval_correctness.py
6
+pytest -s -v test_lm_eval_correctness.py \
7
+ --config-list-file=configs/models-small.txt \
8
+ --tp-size=1
9
"""
10
11
-import os
12
-from pathlib import Path
13
-
11
import lm_eval
15
-import numpy
16
-import pytest
12
+import numpy as np
13
import yaml
14
15
RTOL = 0.08
20
-TEST_DATA_FILE = os.environ.get(
21
- "LM_EVAL_TEST_DATA_FILE",
22
- ".buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml")
23
-
24
-TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
16
17
27
-def launch_lm_eval(eval_config):
18
+def launch_lm_eval(eval_config, tp_size):
19
trust_remote_code = eval_config.get('trust_remote_code', False)
29
-
20
model_args = f"pretrained={eval_config['model_name']}," \
31
- f"tensor_parallel_size={TP_SIZE}," \
21
+ f"tensor_parallel_size={tp_size}," \
22
+ f"enforce_eager=true," \
23
f"add_bos_token=true," \
24
f"trust_remote_code={trust_remote_code}"
34
-
25
results = lm_eval.simple_evaluate(
26
model="vllm",
27
model_args=model_args,
@@ -39,22 +29,14 @@ def launch_lm_eval(eval_config):
29
num_fewshot=eval_config["num_fewshot"],
30
limit=eval_config["limit"],
31
batch_size="auto")
42
-
32
return results
33
34
46
-def test_lm_eval_correctness():
47
- eval_config = yaml.safe_load(
48
- Path(TEST_DATA_FILE).read_text(encoding="utf-8"))
49
-
50
- if eval_config[
51
- "model_name"] == "nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform": #noqa: E501
52
- pytest.skip("FBGEMM is currently failing on main.")
35
+def test_lm_eval_correctness_param(config_filename, tp_size):
36
+ eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
37
54
- # Launch eval requests.
55
- results = launch_lm_eval(eval_config)
38
+ results = launch_lm_eval(eval_config, tp_size)
39
57
- # Confirm scores match ground truth.
40
success = True
41
for task in eval_config["tasks"]:
42
for metric in task["metrics"]:
@@ -62,8 +44,7 @@ def test_lm_eval_correctness():
44
measured_value = results["results"][task["name"]][metric["name"]]
45
print(f'{task["name"]} | {metric["name"]}: '
46
f'ground_truth={ground_truth} | measured={measured_value}')
65
- success = success and numpy.isclose(
47
+ success = success and np.isclose(
48
ground_truth, measured_value, rtol=RTOL)
49
68
- # Assert at the end, print all scores even on failure for debugging.
50
assert success
.buildkite/test-pipeline.yaml
+2
-2
@@ -408,7 +408,7 @@ steps:
408
- vllm/model_executor/layers/quantization
409
commands:
410
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
411
- - bash ./run-tests.sh -c configs/models-small.txt -t 1
411
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-small.txt --tp-size=1
412
413
- label: OpenAI API correctness
414
source_file_dependencies:
@@ -713,4 +713,4 @@ steps:
713
- vllm/model_executor/layers/quantization
714
commands:
715
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
716
- - bash ./run-tests.sh -c configs/models-large.txt -t 4
716
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4