| 1 | #!/bin/bash |
| 2 | # We can use this script to compute baseline accuracy on chartqa for vllm. |
| 3 | # |
| 4 | # Make sure you have lm-eval-harness installed: |
| 5 | # pip install "lm-eval[api]>=0.4.12" |
| 6 | |
| 7 | usage() { |
| 8 | echo`` |
| 9 | echo "Runs lm eval harness on ChartQA using multimodal vllm." |
| 10 | echo "This pathway is intended to be used to create baselines for " |
| 11 | echo "our correctness tests in vllm's CI." |
| 12 | echo |
| 13 | echo "usage: ${0} <options>" |
| 14 | echo |
| 15 | echo " -m - huggingface stub or local directory of the model" |
| 16 | echo " -l - limit number of samples to run" |
| 17 | echo " -t - tensor parallel size to run at" |
| 18 | echo |
| 19 | } |
| 20 | |
| 21 | while getopts "m:l:t:" OPT; do |
| 22 | case ${OPT} in |
| 23 | m ) |
| 24 | MODEL="$OPTARG" |
| 25 | ;; |
| 26 | l ) |
| 27 | LIMIT="$OPTARG" |
| 28 | ;; |
| 29 | t ) |
| 30 | TP_SIZE="$OPTARG" |
| 31 | ;; |
| 32 | \? ) |
| 33 | usage |
| 34 | exit 1 |
| 35 | ;; |
| 36 | esac |
| 37 | done |
| 38 | |
| 39 | lm_eval --model vllm-vlm \ |
| 40 | --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE" \ |
| 41 | --tasks chartqa \ |
| 42 | --batch_size auto \ |
| 43 | --apply_chat_template \ |
| 44 | --limit "$LIMIT" |