| 1 | #!/bin/bash |
| 2 | # We can use this script to compute baseline accuracy on MMLUPRO for vllm. |
| 3 | # We use this for fp8, which HF does not support. |
| 4 | # |
| 5 | # Make sure you have lm-eval-harness installed: |
| 6 | # pip install "lm-eval[api]>=0.4.12" |
| 7 | |
| 8 | usage() { |
| 9 | echo`` |
| 10 | echo "Runs lm eval harness on MMLU Pro using huggingface transformers." |
| 11 | echo "This pathway is intended to be used to create baselines for " |
| 12 | echo "our automated nm-test-accuracy workflow" |
| 13 | echo |
| 14 | echo "usage: ${0} <options>" |
| 15 | echo |
| 16 | echo " -m - huggingface stub or local directory of the model" |
| 17 | echo " -l - limit number of samples to run" |
| 18 | echo " -f - number of fewshot samples to use" |
| 19 | echo " -t - tensor parallel size to run at" |
| 20 | echo |
| 21 | } |
| 22 | |
| 23 | while getopts "m:l:f:t:" OPT; do |
| 24 | case ${OPT} in |
| 25 | m ) |
| 26 | MODEL="$OPTARG" |
| 27 | ;; |
| 28 | l ) |
| 29 | LIMIT="$OPTARG" |
| 30 | ;; |
| 31 | f ) |
| 32 | FEWSHOT="$OPTARG" |
| 33 | ;; |
| 34 | t ) |
| 35 | TP_SIZE="$OPTARG" |
| 36 | ;; |
| 37 | \? ) |
| 38 | usage |
| 39 | exit 1 |
| 40 | ;; |
| 41 | esac |
| 42 | done |
| 43 | |
| 44 | lm_eval --model vllm \ |
| 45 | --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \ |
| 46 | --tasks mmlu_pro --num_fewshot "$FEWSHOT" --limit "$LIMIT" \ |
| 47 | --batch_size auto |