A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[CI] Prune down lm-eval small tests (#17012)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
May 8, 2025 at 15:00 UTC
8342e3abd118a933bf0da28686d9ef9cfcbe9f81
5 files changed
+36
-6
.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-FP8-compressed-tensors.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Llama-3.2-1B-Instruct-FP8 -b "auto" -l 1319 -f 5 -t 1
2
+model_name: "RedHatAI/Llama-3.2-1B-Instruct-FP8"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.335
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.323
10
+limit: 1319
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2.5-1.5B-Instruct.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2.5-1.5B-Instruct -b auto -l 1319 -f 5 -t 1
2
+model_name: "Qwen/Qwen2.5-1.5B-Instruct"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.54
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.59
10
+limit: 1319
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
new
+11
@@ -0,0 +1,11 @@
1
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic -b auto -l 1319 -f 5 -t 1
2
+model_name: "RedHatAI/Qwen2.5-VL-3B-Instruct-FP8-Dynamic"
3
+tasks:
4
+- name: "gsm8k"
5
+ metrics:
6
+ - name: "exact_match,strict-match"
7
+ value: 0.47
8
+ - name: "exact_match,flexible-extract"
9
+ value: 0.64
10
+limit: 1319
11
+num_fewshot: 5
.buildkite/lm-eval-harness/configs/models-large.txt
+1
@@ -3,3 +3,4 @@ Meta-Llama-3-70B-Instruct.yaml
3
Mixtral-8x7B-Instruct-v0.1.yaml
4
Qwen2-57B-A14-Instruct.yaml
5
DeepSeek-V2-Lite-Chat.yaml
6
+Meta-Llama-3-8B-QQQ.yaml
.buildkite/lm-eval-harness/configs/models-small.txt
+2
-6
@@ -1,10 +1,6 @@
1
-Meta-Llama-3-8B-Instruct.yaml
2
-Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
1
+Qwen2.5-1.5B-Instruct.yaml
2
Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
3
Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
4
Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
6
-Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
5
+Qwen2.5-VL-3B-Instruct-FP8-dynamic.yaml
6
Qwen1.5-MoE-W4A16-compressed-tensors.yaml
8
-Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
9
-Qwen2-1.5B-Instruct-FP8W8.yaml
10
-Meta-Llama-3-8B-QQQ.yaml