A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[Chore] Bump `lm-eval` version (#31264)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 24, 2025 at 21:39 UTC
d201807339697c6c8206ae08d2cdccfc25cb1ce1
14 files changed
+14
-15
.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh
+1
-1
@@ -2,7 +2,7 @@
2
# We can use this script to compute baseline accuracy on chartqa for vllm.
3
#
4
# Make sure you have lm-eval-harness installed:
5
-# pip install lm-eval==0.4.9
5
+# pip install "lm-eval[api]>=0.4.9.2"
6
7
usage() {
8
echo``
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
+1
-1
@@ -2,7 +2,7 @@
2
# We can use this script to compute baseline accuracy on GSM for transformers.
3
#
4
# Make sure you have lm-eval-harness installed:
5
-# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
5
+# pip install "lm-eval[api]>=0.4.9.2"
6
7
usage() {
8
echo``
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
6
+# pip install "lm-eval[api]>=0.4.9.2"
7
8
usage() {
9
echo``
.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
6
+# pip install "lm-eval[api]>=0.4.9.2"
7
8
usage() {
9
echo``
.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh
+1
-1
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
61
echo "--- Installing Python dependencies ---"
62
python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
63
&& python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
64
- && python3 -m pip install --progress-bar off "lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d" \
64
+ && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.9.2" \
65
&& python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
66
echo "--- Python dependencies installed ---"
67
.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh
+1
-1
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
61
echo "--- Installing Python dependencies ---"
62
python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
63
&& python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
64
- && python3 -m pip install --progress-bar off "lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d" \
64
+ && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.9.2" \
65
&& python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
66
echo "--- Python dependencies installed ---"
67
docs/features/quantization/fp8.md
+1
-1
@@ -84,7 +84,7 @@ Since simple RTN does not require data for weight quantization and the activatio
84
Install `vllm` and `lm-evaluation-harness` for evaluation:
85
86
```bash
87
-pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
87
+pip install vllm "lm-eval[api]>=0.4.9.2"
88
```
89
90
Load and run the model in `vllm`:
docs/features/quantization/int4.md
+1
-1
@@ -18,7 +18,7 @@ pip install llmcompressor
18
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
19
20
```bash
21
-pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
21
+pip install vllm "lm-eval[api]>=0.4.9.2"
22
```
23
24
## Quantization Process
docs/features/quantization/int8.md
+1
-1
@@ -23,7 +23,7 @@ pip install llmcompressor
23
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
24
25
```bash
26
-pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
26
+pip install vllm "lm-eval[api]>=0.4.9.2"
27
```
28
29
## Quantization Process
docs/features/quantization/quark.md
+1
-1
@@ -20,7 +20,7 @@ for more installation details.
20
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
21
22
```bash
23
-pip install vllm git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d#egg=lm-eval[api]
23
+pip install vllm "lm-eval[api]>=0.4.9.2"
24
```
25
26
## Quantization Process
requirements/nightly_torch_test.txt
+1
-1
@@ -27,7 +27,7 @@ mistral_common[image,audio] >= 1.8.5 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.11.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
30
-lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d # required for model evaluation test
30
+lm-eval[api]>=0.4.9.2 # required for model evaluation test
31
mteb>=1.38.11, <2 # required for mteb test
32
transformers==4.57.3
33
tokenizers==0.22.0
requirements/rocm-test.txt
+1
-1
@@ -58,7 +58,7 @@ schemathesis==3.39.15
58
# OpenAI schema test
59
60
# Evaluation and benchmarking
61
-lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d
61
+lm-eval[api]>=0.4.9.2
62
jiwer==4.0.0
63
64
# Required for multiprocessed tests that use spawn method, Datasets and Evaluate Test
requirements/test.in
+1
-2
@@ -34,8 +34,7 @@ num2words # required for smolvlm test
34
open_clip_torch==2.32.0 # Required for nemotron_vl test
35
opencv-python-headless >= 4.11.0 # required for video test
36
datamodel_code_generator # required for minicpm3 test
37
-# TODO: Use lm-eval[api]==0.4.10 once released
38
-lm-eval[api] @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d # required for model evaluation test
37
+lm-eval[api]>=0.4.9.2 # required for model evaluation test
38
mteb[bm25s]>=2, <3 # required for mteb test
39
transformers==4.57.3
40
tokenizers==0.22.0
requirements/test.txt
+1
-1
@@ -441,7 +441,7 @@ lightning-utilities==0.14.3
441
# torchmetrics
442
llvmlite==0.44.0
443
# via numba
444
-lm-eval @ git+https://github.com/EleutherAI/lm-evaluation-harness.git@206b7722158f58c35b7ffcd53b035fdbdda5126d
444
+lm-eval==0.4.9.2
445
# via -r requirements/test.in
446
lxml==5.3.0
447
# via