| 1 | #!/bin/bash |
| 2 | # We can use this script to compute baseline accuracy on GSM for transformers. |
| 3 | # |
| 4 | # Make sure you have lm-eval-harness installed: |
| 5 | # pip install "lm-eval[api]>=0.4.12" |
| 6 | |
| 7 | usage() { |
| 8 | echo`` |
| 9 | echo "Runs lm eval harness on GSM8k using huggingface transformers." |
| 10 | echo "This pathway is intended to be used to create baselines for " |
| 11 | echo "our automated nm-test-accuracy workflow" |
| 12 | echo |
| 13 | echo "usage: ${0} <options>" |
| 14 | echo |
| 15 | echo " -m - huggingface stub or local directory of the model" |
| 16 | echo " -b - batch size to run the evaluation at" |
| 17 | echo " -l - limit number of samples to run" |
| 18 | echo " -f - number of fewshot samples to use" |
| 19 | echo |
| 20 | } |
| 21 | |
| 22 | while getopts "m:b:l:f:" OPT; do |
| 23 | case ${OPT} in |
| 24 | m ) |
| 25 | MODEL="$OPTARG" |
| 26 | ;; |
| 27 | b ) |
| 28 | BATCH_SIZE="$OPTARG" |
| 29 | ;; |
| 30 | l ) |
| 31 | LIMIT="$OPTARG" |
| 32 | ;; |
| 33 | f ) |
| 34 | FEWSHOT="$OPTARG" |
| 35 | ;; |
| 36 | \? ) |
| 37 | usage |
| 38 | exit 1 |
| 39 | ;; |
| 40 | esac |
| 41 | done |
| 42 | |
| 43 | lm_eval --model hf \ |
| 44 | --model_args "pretrained=$MODEL,parallelize=True" \ |
| 45 | --tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ |
| 46 | --batch_size "$BATCH_SIZE" |