| 1 | #!/bin/bash |
| 2 | # We can use this script to compute baseline accuracy on GSM for vllm. |
| 3 | # We use this for fp8, which HF does not support. |
| 4 | # |
| 5 | # Make sure you have lm-eval-harness installed: |
| 6 | # pip install "lm-eval[api]>=0.4.12" |
| 7 | |
| 8 | usage() { |
| 9 | echo`` |
| 10 | echo "Runs lm eval harness on GSM8k using huggingface transformers." |
| 11 | echo "This pathway is intended to be used to create baselines for " |
| 12 | echo "our automated nm-test-accuracy workflow" |
| 13 | echo |
| 14 | echo "usage: ${0} <options>" |
| 15 | echo |
| 16 | echo " -m - huggingface stub or local directory of the model" |
| 17 | echo " -b - batch size to run the evaluation at" |
| 18 | echo " -l - limit number of samples to run" |
| 19 | echo " -f - number of fewshot samples to use" |
| 20 | echo " -t - tensor parallel size to run at" |
| 21 | echo |
| 22 | } |
| 23 | |
| 24 | while getopts "m:b:l:f:t:" OPT; do |
| 25 | case ${OPT} in |
| 26 | m ) |
| 27 | MODEL="$OPTARG" |
| 28 | ;; |
| 29 | b ) |
| 30 | BATCH_SIZE="$OPTARG" |
| 31 | ;; |
| 32 | l ) |
| 33 | LIMIT="$OPTARG" |
| 34 | ;; |
| 35 | f ) |
| 36 | FEWSHOT="$OPTARG" |
| 37 | ;; |
| 38 | t ) |
| 39 | TP_SIZE="$OPTARG" |
| 40 | ;; |
| 41 | \? ) |
| 42 | usage |
| 43 | exit 1 |
| 44 | ;; |
| 45 | esac |
| 46 | done |
| 47 | |
| 48 | lm_eval --model vllm \ |
| 49 | --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \ |
| 50 | --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ |
| 51 | --batch_size "$BATCH_SIZE" |