A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[CI/Build] remove -t for run-lm-eval-gsm-hf-baseline.sh (#16271)
Signed-off-by: reidliu41 <reid201711@gmail.com> Co-authored-by: reidliu41 <reid201711@gmail.com>
Reid committed
Apr 27, 2025 at 02:25 UTC
3642c59aa8a52d5565cdb070b06b8ed3a3f62303
23 files changed
+25
-2
.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash ./run-lm-eval-gsm-vllm-baseline.sh -m deepseek-ai/DeepSeek-V2-Lite-Chat -b "auto" -l 1000 -f 5 -t 2
3
model_name: "deepseek-ai/DeepSeek-V2-Lite-Chat"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform.yaml
+1
@@ -1,3 +1,4 @@
1
+# For hf script, without -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5
3
model_name: "nm-testing/Meta-Llama-3-70B-Instruct-FBGEMM-nonuniform"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-70B-Instruct.yaml
+1
@@ -1,3 +1,4 @@
1
+# For hf script, without -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-70B-Instruct -b 32 -l 250 -f 5
3
model_name: "meta-llama/Meta-Llama-3-70B-Instruct"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-Channelwise-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors -b auto -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8A8-FP8-Channelwise-compressed-tensors"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform -b auto -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-Instruct-FBGEMM-nonuniform"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test -b 32 -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-FP8.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Meta-Llama-3-8B-Instruct-FP8 -b 32 -l 250 -f 5 -t 1
3
model_name: "neuralmagic/Meta-Llama-3-8B-Instruct-FP8"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors-asym.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test -b "auto" -l 250 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Asym-Per-Token-Test"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-INT8-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test -b "auto" -l 250 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-Instruct-W8-Channel-A8-Dynamic-Per-Token-Test"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct-nonuniform-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test -b auto -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Meta-Llama-3-8B-Instruct-nonuniform-test"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-Instruct.yaml
+2
-1
@@ -1,4 +1,5 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-8B-Instruct -b 32 -l 250 -f 5 -t 1
1
+# For hf script, without -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m meta-llama/Meta-Llama-3-8B-Instruct -b 32 -l 250 -f 5
3
model_name: "meta-llama/Meta-Llama-3-8B-Instruct"
4
tasks:
5
- name: "gsm8k"
.buildkite/lm-eval-harness/configs/Meta-Llama-3-8B-QQQ.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m HandH1998/QQQ-Llama-3-8b-g128 -b 32 -l 1000 -f 5 -t 1
3
model_name: "HandH1998/QQQ-Llama-3-8b-g128"
4
tasks:
.buildkite/lm-eval-harness/configs/Meta-Llama-3.2-1B-Instruct-INT8-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
3
model_name: "neuralmagic/Llama-3.2-1B-Instruct-quantized.w8a8"
4
tasks:
.buildkite/lm-eval-harness/configs/Minitron-4B-Base-FP8.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m mgoin/Minitron-4B-Base-FP8 -b auto -l 1000 -f 5 -t 1
3
model_name: "mgoin/Minitron-4B-Base-FP8"
4
tasks:
.buildkite/lm-eval-harness/configs/Mixtral-8x22B-Instruct-v0.1-FP8-Dynamic.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic -b "auto" -l 250 -f 5 -t 8
3
model_name: "neuralmagic/Mixtral-8x22B-Instruct-v0.1-FP8-dynamic"
4
tasks:
.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1-FP8.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash ./run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8 -b "auto" -l 250 -f 5 -t 4
3
model_name: "neuralmagic/Mixtral-8x7B-Instruct-v0.1-FP8"
4
tasks:
.buildkite/lm-eval-harness/configs/Mixtral-8x7B-Instruct-v0.1.yaml
+2
-1
@@ -1,4 +1,5 @@
1
-# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1 -b 32 -l 250 -f 5 -t 4
1
+# For hf script, without -t option (tensor parallel size).
2
+# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh -m neuralmagic/Mixtral-8x7B-Instruct-v0.1 -b 32 -l 250 -f 5
3
model_name: "mistralai/Mixtral-8x7B-Instruct-v0.1"
4
tasks:
5
- name: "gsm8k"
.buildkite/lm-eval-harness/configs/Qwen1.5-MoE-W4A16-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16 -b auto -l 1319 -f 5 -t 1
3
model_name: "nm-testing/Qwen1.5-MoE-A2.7B-Chat-quantized.w4a16"
4
tasks:
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-FP8W8.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-FP8W8 -b auto -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Qwen2-1.5B-Instruct-FP8W8"
4
tasks:
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-INT8-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8 -b "auto" -l 1000 -f 5 -t 1
3
model_name: "neuralmagic/Qwen2-1.5B-Instruct-quantized.w8a8"
4
tasks:
.buildkite/lm-eval-harness/configs/Qwen2-1.5B-Instruct-W8A16-compressed-tensors.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash .buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise -b "auto" -l 1000 -f 5 -t 1
3
model_name: "nm-testing/Qwen2-1.5B-Instruct-W8A16-Channelwise"
4
tasks:
.buildkite/lm-eval-harness/configs/Qwen2-57B-A14-Instruct.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash ./run-lm-eval-gsm-vllm-baseline.sh -m Qwen/Qwen2-57B-A14B-Instruct -b "auto" -l 250 -f 5 -t 4
3
model_name: "Qwen/Qwen2-57B-A14B-Instruct"
4
tasks:
.buildkite/lm-eval-harness/configs/SparseLlama3.1_2of4_fp8_compressed.yaml
+1
@@ -1,3 +1,4 @@
1
+# For vllm script, with -t option (tensor parallel size).
2
# bash ./run-lm-eval-gsm-vllm-baseline.sh -m nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM -b "auto" -t 2
3
model_name: "nm-testing/SparseLlama-3.1-8B-gsm8k-pruned.2of4-chnl_wts_per_tok_dyn_act_fp8-BitM"
4
tasks: