A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI/Build] Add Qwen2.5-VL-7B-Instruct ChartQA Accuracy Tests in CI (#21810)
Signed-off-by: Ye (Charlotte) Qi <yeq@meta.com> Signed-off-by: zhewenli <zhewenli@meta.com> Co-authored-by: Ye (Charlotte) Qi <yeq@meta.com> Co-authored-by: Ye (Charlotte) Qi <ye.charlotte.qi@gmail.com>
Zhewen Li committed
Oct 15, 2025 at 01:09 UTC
f3c378ffa7f95317497a2cf64ac52b09a8708bc9
13 files changed
+164
-4
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
new
+12
@@ -0,0 +1,12 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m HandH1998/QQQ-Llama-3-8b-g128 -b 32 -l 1000 -f 5 -t 1
3
+model_name: "HandH1998/QQQ-Llama-3-8b-g128"
4
+tasks:
5
+- name: "gsm8k"
6
+ metrics:
7
+ - name: "exact_match,strict-match"
8
+ value: 0.419
9
+ - name: "exact_match,flexible-extract"
10
+ value: 0.416
11
+limit: 1000
12
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# For hf script, without -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -b 32 -l 100 -t 8
3
+model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
4
+backend: "vllm-vlm"
5
+tasks:
6
+- name: "chartqa"
7
+ metrics:
8
+ - name: "relaxed_accuracy,none"
9
+ value: 0.90
10
+limit: 100
11
+num_fewshot: 0
.buildkite/lm-eval-harness/configs/Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# For hf script, without -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8 -b 32 -l 250 -t 8 -f 5
3
+model_name: "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8"
4
+backend: "vllm-vlm"
5
+tasks:
6
+- name: "mmlu_pro"
7
+ metrics:
8
+ - name: "exact_match,custom-extract"
9
+ value: 0.80
10
+limit: 250 # will run on 250 * 14 subjects = 3500 samples
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
+2
-1
@@ -1,4 +1,5 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -b auto -l 1319 -f 5 -t 1
1
+# For vllm script, with -t option (tensor parallel size)
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -l 1319 -t 1
3
model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
4
tasks:
5
- name: "gsm8k"
.buildkite/lm-eval-harness/configs/Qwen2.5-VL-7B-Instruct.yaml
new
+12
@@ -0,0 +1,12 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh -m Qwen/Qwen2.5-VL-7B-Instruct -l 2500 -t 1
3
+
4
+model_name: "Qwen/Qwen2.5-VL-7B-Instruct"
5
+backend: "vllm-vlm"
6
+tasks:
7
+- name: "chartqa"
8
+ metrics:
9
+ - name: "relaxed_accuracy,none"
10
+ value: 0.855
11
+limit: 2500
12
+num_fewshot: 0
.buildkite/lm-eval-harness/configs/models-large-h100.txt
new
+1
@@ -0,0 +1 @@
1
+Meta-Llama-4-Maverick-17B-128E-Instruct-FP8.yaml
.buildkite/lm-eval-harness/configs/models-mm-large-h100.txt
new
+1
@@ -0,0 +1 @@
1
+Meta-Llama-4-Maverick-17B-128E-Instruct-FP8-MM.yaml
.buildkite/lm-eval-harness/configs/models-mm-small.txt
new
+1
@@ -0,0 +1 @@
1
+Qwen2.5-VL-7B-Instruct.yaml
\ No newline at end of file
.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh
new
+44
@@ -0,0 +1,44 @@
1
+#!/bin/bash
2
+# We can use this script to compute baseline accuracy on chartqa for vllm.
3
+#
4
+# Make sure you have lm-eval-harness installed:
5
+# pip install lm-eval==0.4.9
6
+
7
+usage() {
8
+ echo``
9
+ echo "Runs lm eval harness on ChartQA using multimodal vllm."
10
+ echo "This pathway is intended to be used to create baselines for "
11
+ echo "our correctness tests in vllm's CI."
12
+ echo
13
+ echo "usage: ${0} <options>"
14
+ echo
15
+ echo " -m - huggingface stub or local directory of the model"
16
+ echo " -l - limit number of samples to run"
17
+ echo " -t - tensor parallel size to run at"
18
+ echo
19
+}
20
+
21
+while getopts "m:l:t:" OPT; do
22
+ case ${OPT} in
23
+ m )
24
+ MODEL="$OPTARG"
25
+ ;;
26
+ l )
27
+ LIMIT="$OPTARG"
28
+ ;;
29
+ t )
30
+ TP_SIZE="$OPTARG"
31
+ ;;
32
+ \? )
33
+ usage
34
+ exit 1
35
+ ;;
36
+ esac
37
+done
38
+
39
+lm_eval --model vllm-vlm \
40
+ --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE" \
41
+ --tasks chartqa \
42
+ --batch_size auto \
43
+ --apply_chat_template \
44
+ --limit $LIMIT
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh
new
+50
@@ -0,0 +1,50 @@
1
+#!/bin/bash
2
+# We can use this script to compute baseline accuracy on MMLUPRO for vllm.
3
+# We use this for fp8, which HF does not support.
4
+#
5
+# Make sure you have lm-eval-harness installed:
6
+# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
7
+
8
+usage() {
9
+ echo``
10
+ echo "Runs lm eval harness on MMLU Pro using huggingface transformers."
11
+ echo "This pathway is intended to be used to create baselines for "
12
+ echo "our automated nm-test-accuracy workflow"
13
+ echo
14
+ echo "usage: ${0} <options>"
15
+ echo
16
+ echo " -m - huggingface stub or local directory of the model"
17
+ echo " -l - limit number of samples to run"
18
+ echo " -f - number of fewshot samples to use"
19
+ echo " -t - tensor parallel size to run at"
20
+ echo
21
+}
22
+
23
+while getopts "m:b:l:f:t:" OPT; do
24
+ case ${OPT} in
25
+ m )
26
+ MODEL="$OPTARG"
27
+ ;;
28
+ b )
29
+ BATCH_SIZE="$OPTARG"
30
+ ;;
31
+ l )
32
+ LIMIT="$OPTARG"
33
+ ;;
34
+ f )
35
+ FEWSHOT="$OPTARG"
36
+ ;;
37
+ t )
38
+ TP_SIZE="$OPTARG"
39
+ ;;
40
+ \? )
41
+ usage
42
+ exit 1
43
+ ;;
44
+ esac
45
+done
46
+
47
+lm_eval --model vllm \
48
+ --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \
49
+ --tasks mmlu_pro --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
50
+ --batch_size auto
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+9
-3
@@ -19,21 +19,27 @@ RTOL = 0.08
19
def launch_lm_eval(eval_config, tp_size):
20
trust_remote_code = eval_config.get("trust_remote_code", False)
21
max_model_len = eval_config.get("max_model_len", 4096)
22
+ batch_size = eval_config.get("batch_size", "auto")
23
+ backend = eval_config.get("backend", "vllm")
24
model_args = (
25
f"pretrained={eval_config['model_name']},"
26
f"tensor_parallel_size={tp_size},"
27
f"enforce_eager=true,"
28
f"add_bos_token=true,"
29
f"trust_remote_code={trust_remote_code},"
28
- f"max_model_len={max_model_len}"
30
+ f"max_model_len={max_model_len},"
31
)
32
results = lm_eval.simple_evaluate(
31
- model="vllm",
33
+ model=backend,
34
model_args=model_args,
35
tasks=[task["name"] for task in eval_config["tasks"]],
36
num_fewshot=eval_config["num_fewshot"],
37
limit=eval_config["limit"],
36
- batch_size="auto",
38
+ # TODO(yeq): using chat template w/ fewshot_as_multiturn is supposed help
39
+ # text models. however, this is regressing measured strict-match for
40
+ # existing text models in CI, so only apply it for mm.
41
+ apply_chat_template=backend == "vllm-vlm",
42
+ batch_size=batch_size,
43
)
44
return results
45
.buildkite/test-pipeline.yaml
+10
@@ -734,6 +734,16 @@ steps:
734
- pytest -v -s models/multimodal -m core_model --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/processing
735
- cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work
736
737
+- label: Multi-Modal Accuracy Eval (Small Models) # 50min
738
+ timeout_in_minutes: 70
739
+ working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
740
+ source_file_dependencies:
741
+ - vllm/multimodal/
742
+ - vllm/inputs/
743
+ - vllm/v1/core/
744
+ commands:
745
+ - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-mm-small.txt --tp-size=1
746
+
747
- label: Multi-Modal Models Test (Extended) 1
748
mirror_hardwares: [amdexperimental]
749
optional: true