A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
Normalize lm-eval command between baseline and correctness test (#18560)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jul 10, 2025 at 22:27 UTC
1a4f35e2eaa3ebdecb8ef9ff8302b01e289305c9
2 files changed
+4
-2
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46
done
47
48
lm_eval --model vllm \
49
- --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,distributed_executor_backend=ray,trust_remote_code=true,max_model_len=4096" \
49
+ --model_args "pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,trust_remote_code=true,max_model_len=4096" \
50
--tasks gsm8k --num_fewshot "$FEWSHOT" --limit "$LIMIT" \
51
--batch_size "$BATCH_SIZE"
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+3
-1
@@ -18,12 +18,14 @@ RTOL = 0.08
18
19
def launch_lm_eval(eval_config, tp_size):
20
trust_remote_code = eval_config.get("trust_remote_code", False)
21
+ max_model_len = eval_config.get("max_model_len", 4096)
22
model_args = (
23
f"pretrained={eval_config['model_name']},"
24
f"tensor_parallel_size={tp_size},"
25
f"enforce_eager=true,"
26
f"add_bos_token=true,"
26
- f"trust_remote_code={trust_remote_code}"
27
+ f"trust_remote_code={trust_remote_code},"
28
+ f"max_model_len={max_model_len}"
29
)
30
results = lm_eval.simple_evaluate(
31
model="vllm",