@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 75aa1442db
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[ CI/Build ] LM Eval Harness Based CI Testing (#5838)

Co-authored-by: Robert Shaw <rshaw@neuralmagic>

Robert Shaw committed Jun 29, 2024 at 13:04 UTC 75aa1442dbc76c43804e8dd528eff1aae3b45d1e
11 files changed +274
.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-70B-Instruct -b 32 -l 250 -f 5
2 +model_name: "meta-llama/Meta-Llama-3-70B-Instruct"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.892
8 + - name: "exact_match,flexible-extract"
9 + value: 0.892
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
2 +model_name: "neuralmagic/Meta-Llama-3-8B-Instruct-FP8"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.756
8 + - name: "exact_match,flexible-extract"
9 + value: 0.752
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-8B-Instruct -b 32 -l 250 -f 5 -t 1
2 +model_name: "meta-llama/Meta-Llama-3-8B-Instruct"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.756
8 + - name: "exact_match,flexible-extract"
9 + value: 0.752
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1.yaml new
+11
@@ -0,0 +1,11 @@
1 +# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1 -b 32 -l 250 -f 5 -t 4
2 +model_name: "mistralai/Mixtral-8x7B-Instruct-v0.1"
3 +tasks:
4 +- name: "gsm8k"
5 + metrics:
6 + - name: "exact_match,strict-match"
7 + value: 0.616
8 + - name: "exact_match,flexible-extract"
9 + value: 0.632
10 +limit: 250
11 +num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt new
+2
@@ -0,0 +1,2 @@
1 +Meta-Llama-3-70B-Instruct.yaml
2 +Mixtral-8x7B-Instruct-v0.1.yaml
.buildkite/lm-eval-harness/configs/models-small.txt new
+2
@@ -0,0 +1,2 @@
1 +Meta-Llama-3-8B-Instruct.yaml
2 +Meta-Llama-3-8B-Instruct-FP8.yaml
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh new
+46
@@ -0,0 +1,46 @@
1 +#!/bin/bash
2 +# We can use this script to compute baseline accuracy on GSM for transformers.
3 +#
4 +# Make sure you have lm-eval-harness installed:
5 +# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@9516087b81a61d0e220b22cc1b75be76de23bc10
6 +
7 +usage() {
8 + echo``
9 + echo "Runs lm eval harness on GSM8k using huggingface transformers."
10 + echo "This pathway is intended to be used to create baselines for "
11 + echo "our automated nm-test-accuracy workflow"
12 + echo
13 + echo "usage: ${0} <options>"
14 + echo
15 + echo " -m - huggingface stub or local directory of the model"
16 + echo " -b - batch size to run the evaluation at"
17 + echo " -l - limit number of samples to run"
18 + echo " -f - number of fewshot samples to use"
19 + echo
20 +}
21 +
22 +while getopts "m:b:l:f:" OPT; do
23 + case ${OPT} in
24 + m )
25 + MODEL="$OPTARG"
26 + ;;
27 + b )
28 + BATCH_SIZE="$OPTARG"
29 + ;;
30 + l )
31 + LIMIT="$OPTARG"
32 + ;;
33 + f )
34 + FEWSHOT="$OPTARG"
35 + ;;
36 + \? )
37 + usage
38 + exit 1
39 + ;;
40 + esac
41 +done
42 +
43 +lm_eval --model hf \
44 + --model_args pretrained=$MODEL,parallelize=True \
45 + --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
46 + --batch_size $BATCH_SIZE
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh new
+51
@@ -0,0 +1,51 @@
1 +#!/bin/bash
2 +# We can use this script to compute baseline accuracy on GSM for vllm.
3 +# We use this for fp8, which HF does not support.
4 +#
5 +# Make sure you have lm-eval-harness installed:
6 +# pip install lm-eval==0.4.2
7 +
8 +usage() {
9 + echo``
10 + echo "Runs lm eval harness on GSM8k using huggingface transformers."
11 + echo "This pathway is intended to be used to create baselines for "
12 + echo "our automated nm-test-accuracy workflow"
13 + echo
14 + echo "usage: ${0} <options>"
15 + echo
16 + echo " -m - huggingface stub or local directory of the model"
17 + echo " -b - batch size to run the evaluation at"
18 + echo " -l - limit number of samples to run"
19 + echo " -f - number of fewshot samples to use"
20 + echo " -t - tensor parallel size to run at"
21 + echo
22 +}
23 +
24 +while getopts "m:b:l:f:t:" OPT; do
25 + case ${OPT} in
26 + m )
27 + MODEL="$OPTARG"
28 + ;;
29 + b )
30 + BATCH_SIZE="$OPTARG"
31 + ;;
32 + l )
33 + LIMIT="$OPTARG"
34 + ;;
35 + f )
36 + FEWSHOT="$OPTARG"
37 + ;;
38 + t )
39 + TP_SIZE="$OPTARG"
40 + ;;
41 + \? )
42 + usage
43 + exit 1
44 + ;;
45 + esac
46 +done
47 +
48 +lm_eval --model vllm \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE \
50 + --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 + --batch_size $BATCH_SIZE
.buildkite/lm-eval-harness/run-tests.sh new
+59
@@ -0,0 +1,59 @@
1 +#!/bin/bash
2 +
3 +usage() {
4 + echo``
5 + echo "Runs lm eval harness on GSM8k using vllm and compares to "
6 + echo "precomputed baseline (measured by HF transformers.)"
7 + echo
8 + echo "usage: ${0} <options>"
9 + echo
10 + echo " -c - path to the test data config (e.g. configs/small-models.txt)"
11 + echo " -t - tensor parallel size"
12 + echo
13 +}
14 +
15 +SUCCESS=0
16 +
17 +while getopts "c:t:" OPT; do
18 + case ${OPT} in
19 + c )
20 + CONFIG="$OPTARG"
21 + ;;
22 + t )
23 + TP_SIZE="$OPTARG"
24 + ;;
25 + \? )
26 + usage
27 + exit 1
28 + ;;
29 + esac
30 +done
31 +
32 +# Parse list of configs.
33 +IFS=$'\n' read -d '' -r -a MODEL_CONFIGS < $CONFIG
34 +
35 +for MODEL_CONFIG in "${MODEL_CONFIGS[@]}"
36 +do
37 + LOCAL_SUCCESS=0
38 +
39 + echo "=== RUNNING MODEL: $MODEL_CONFIG WITH TP SIZE: $TP_SIZE==="
40 +
41 + export LM_EVAL_TEST_DATA_FILE=$PWD/configs/${MODEL_CONFIG}
42 + export LM_EVAL_TP_SIZE=$TP_SIZE
43 + pytest -s test_lm_eval_correctness.py || LOCAL_SUCCESS=$?
44 +
45 + if [[ $LOCAL_SUCCESS == 0 ]]; then
46 + echo "=== PASSED MODEL: ${MODEL_CONFIG} ==="
47 + else
48 + echo "=== FAILED MODEL: ${MODEL_CONFIG} ==="
49 + fi
50 +
51 + SUCCESS=$((SUCCESS + LOCAL_SUCCESS))
52 +
53 +done
54 +
55 +if [ "${SUCCESS}" -eq "0" ]; then
56 + exit 0
57 +else
58 + exit 1
59 +fi
.buildkite/lm-eval-harness/test_lm_eval_correctness.py new
+54
@@ -0,0 +1,54 @@
1 +"""
2 +LM eval harness on model to compare vs HF baseline computed offline.
3 +Configs are found in configs/$MODEL.yaml
4 +
5 +* export LM_EVAL_TEST_DATA_FILE=configs/Meta-Llama-3-70B-Instruct.yaml
6 +* export LM_EVAL_TP_SIZE=4
7 +* pytest -s test_lm_eval_correctness.py
8 +"""
9 +
10 +import os
11 +from pathlib import Path
12 +
13 +import lm_eval
14 +import numpy
15 +import yaml
16 +
17 +RTOL = 0.02
18 +TEST_DATA_FILE = os.environ.get(
19 + "LM_EVAL_TEST_DATA_FILE",
20 + ".buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml")
21 +
22 +TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
23 +
24 +
25 +def launch_lm_eval(eval_config):
26 + model_args = f"pretrained={eval_config['model_name']}," \
27 + f"tensor_parallel_size={TP_SIZE}"
28 +
29 + results = lm_eval.simple_evaluate(
30 + model="vllm",
31 + model_args=model_args,
32 + tasks=[task["name"] for task in eval_config["tasks"]],
33 + num_fewshot=eval_config["num_fewshot"],
34 + limit=eval_config["limit"],
35 + batch_size="auto")
36 +
37 + return results
38 +
39 +
40 +def test_lm_eval_correctness():
41 + eval_config = yaml.safe_load(
42 + Path(TEST_DATA_FILE).read_text(encoding="utf-8"))
43 +
44 + # Launch eval requests.
45 + results = launch_lm_eval(eval_config)
46 +
47 + # Confirm scores match ground truth.
48 + for task in eval_config["tasks"]:
49 + for metric in task["metrics"]:
50 + ground_truth = metric["value"]
51 + measured_value = results["results"][task["name"]][metric["name"]]
52 + print(f'{task["name"]} | {metric["name"]}: '
53 + f'ground_truth={ground_truth} | measured={measured_value}')
54 + assert numpy.isclose(ground_truth, measured_value, rtol=RTOL)
.buildkite/test-pipeline.yaml
+16
@@ -197,6 +197,22 @@ steps:
197 - pip install aiohttp
198 - bash run-benchmarks.sh
199
200 +- label: LM Eval Small Models
201 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
202 + commands:
203 + - pip install lm-eval
204 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
205 + - bash ./run-tests.sh -c configs/models-small.txt -t 1
206 +
207 +- label: LM Eval Large Models
208 + gpu: a100
209 + num_gpus: 4
210 + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
211 + commands:
212 + - pip install lm-eval
213 + - export VLLM_WORKER_MULTIPROC_METHOD=spawn
214 + - bash ./run-tests.sh -c configs/models-large.txt -t 4
215 +
216 - label: Documentation Build
217 working_dir: "/vllm-workspace/test_docs/docs"
218 no_gpu: True