A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 05:00 UTC
Bump `lm-eval` version for Transformers v5 compatibility (#33994)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Feb 16, 2026 at 14:24 UTC
a21cedf4ff1facaee601a635e3c092fe02742290
14 files changed
+19
-31
.buildkite/lm-eval-harness/run-lm-eval-chartqa-vllm-vlm-baseline.sh
+1
-1
@@ -2,7 +2,7 @@
2
# We can use this script to compute baseline accuracy on chartqa for vllm.
3
#
4
# Make sure you have lm-eval-harness installed:
5
-# pip install "lm-eval[api]>=0.4.9.2"
5
+# pip install "lm-eval[api]>=0.4.11"
6
7
usage() {
8
echo``
.buildkite/lm-eval-harness/run-lm-eval-gsm-hf-baseline.sh
+1
-1
@@ -2,7 +2,7 @@
2
# We can use this script to compute baseline accuracy on GSM for transformers.
3
#
4
# Make sure you have lm-eval-harness installed:
5
-# pip install "lm-eval[api]>=0.4.9.2"
5
+# pip install "lm-eval[api]>=0.4.11"
6
7
usage() {
8
echo``
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install "lm-eval[api]>=0.4.9.2"
6
+# pip install "lm-eval[api]>=0.4.11"
7
8
usage() {
9
echo``
.buildkite/lm-eval-harness/run-lm-eval-mmlupro-vllm-baseline.sh
+1
-1
@@ -3,7 +3,7 @@
3
# We use this for fp8, which HF does not support.
4
#
5
# Make sure you have lm-eval-harness installed:
6
-# pip install "lm-eval[api]>=0.4.9.2"
6
+# pip install "lm-eval[api]>=0.4.11"
7
8
usage() {
9
echo``
.buildkite/scripts/hardware_ci/run-tpu-v1-test-part2.sh
+1
-1
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
61
echo "--- Installing Python dependencies ---"
62
python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
63
&& python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
64
- && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.9.2" \
64
+ && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.11" \
65
&& python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
66
echo "--- Python dependencies installed ---"
67
.buildkite/scripts/hardware_ci/run-tpu-v1-test.sh
+1
-1
@@ -61,7 +61,7 @@ echo "Results will be stored in: $RESULTS_DIR"
61
echo "--- Installing Python dependencies ---"
62
python3 -m pip install --progress-bar off git+https://github.com/thuml/depyf.git \
63
&& python3 -m pip install --progress-bar off pytest pytest-asyncio tpu-info \
64
- && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.9.2" \
64
+ && python3 -m pip install --progress-bar off "lm-eval[api]>=0.4.11" \
65
&& python3 -m pip install --progress-bar off hf-transfer tblib==3.1.0
66
echo "--- Python dependencies installed ---"
67
docs/features/quantization/fp8.md
+1
-1
@@ -84,7 +84,7 @@ Since simple RTN does not require data for weight quantization and the activatio
84
Install `vllm` and `lm-evaluation-harness` for evaluation:
85
86
```bash
87
-pip install vllm "lm-eval[api]>=0.4.9.2"
87
+pip install vllm "lm-eval[api]>=0.4.11"
88
```
89
90
Load and run the model in `vllm`:
docs/features/quantization/int4.md
+1
-1
@@ -18,7 +18,7 @@ pip install llmcompressor
18
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
19
20
```bash
21
-pip install vllm "lm-eval[api]>=0.4.9.2"
21
+pip install vllm "lm-eval[api]>=0.4.11"
22
```
23
24
## Quantization Process
docs/features/quantization/int8.md
+1
-1
@@ -23,7 +23,7 @@ pip install llmcompressor
23
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
24
25
```bash
26
-pip install vllm "lm-eval[api]>=0.4.9.2"
26
+pip install vllm "lm-eval[api]>=0.4.11"
27
```
28
29
## Quantization Process
docs/features/quantization/quark.md
+1
-1
@@ -20,7 +20,7 @@ for more installation details.
20
Additionally, install `vllm` and `lm-evaluation-harness` for evaluation:
21
22
```bash
23
-pip install vllm "lm-eval[api]>=0.4.9.2"
23
+pip install vllm "lm-eval[api]>=0.4.11"
24
```
25
26
## Quantization Process
requirements/nightly_torch_test.txt
+1
-1
@@ -27,7 +27,7 @@ mistral_common[image,audio] >= 1.9.1 # required for voxtral test
27
num2words # required for smolvlm test
28
opencv-python-headless >= 4.13.0 # required for video test
29
datamodel_code_generator # required for minicpm3 test
30
-lm-eval[api]>=0.4.9.2 # required for model evaluation test
30
+lm-eval[api]>=0.4.11 # required for model evaluation test
31
mteb>=1.38.11, <2 # required for mteb test
32
transformers==4.57.5
33
tokenizers==0.22.0
requirements/rocm-test.txt
+1
-1
@@ -58,7 +58,7 @@ schemathesis==3.39.15
58
# OpenAI schema test
59
60
# Evaluation and benchmarking
61
-lm-eval[api]==0.4.9.2
61
+lm-eval[api]==0.4.11
62
jiwer==4.0.0
63
64
# Required for multiprocessed tests that use spawn method, Datasets and Evaluate Test
requirements/test.in
+1
-1
@@ -35,7 +35,7 @@ num2words # required for smolvlm test
35
open_clip_torch==2.32.0 # Required for nemotron_vl test, Nemotron Parse in test_common.py
36
opencv-python-headless >= 4.13.0 # required for video test
37
datamodel_code_generator # required for minicpm3 test
38
-lm-eval[api]>=0.4.9.2 # required for model evaluation test
38
+lm-eval[api]>=0.4.11 # required for model evaluation test
39
mteb[bm25s]>=2, <3 # required for mteb test
40
transformers==4.57.5
41
tokenizers==0.22.0
requirements/test.txt
+6
-18
@@ -5,9 +5,7 @@ absl-py==2.1.0
5
# rouge-score
6
# tensorboard
7
accelerate==1.0.1
8
- # via
9
- # lm-eval
10
- # peft
8
+ # via peft
9
aenum==3.1.16
10
# via lightly
11
affine==2.4.0
@@ -138,7 +136,6 @@ colorama==0.4.6
136
# perceptron
137
# sacrebleu
138
# schemathesis
141
- # tqdm-multiprocess
139
colorful==0.5.6
140
# via ray
141
colorlog==6.10.1
@@ -383,6 +380,7 @@ jinja2==3.1.6
380
# via
381
# datamodel-code-generator
382
# genai-perf
383
+ # lm-eval
384
# torch
385
jiwer==3.0.5
386
# via -r requirements/test.in
@@ -448,7 +446,7 @@ lightning-utilities==0.14.3
446
# torchmetrics
447
llvmlite==0.44.0
448
# via numba
451
-lm-eval==0.4.9.2
449
+lm-eval==0.4.11
450
# via -r requirements/test.in
451
lxml==5.3.0
452
# via
@@ -513,8 +511,6 @@ numba==0.61.2
511
# via
512
# -r requirements/test.in
513
# librosa
516
-numexpr==2.10.1
517
- # via lm-eval
514
numpy==2.2.6
515
# via
516
# -r requirements/test.in
@@ -540,11 +536,11 @@ numpy==2.2.6
536
# librosa
537
# lightly
538
# lightly-utils
539
+ # lm-eval
540
# matplotlib
541
# mistral-common
542
# mteb
543
# numba
547
- # numexpr
544
# opencv-python-headless
545
# optuna
546
# pandas
@@ -707,9 +703,7 @@ pathvalidate==3.2.1
703
patsy==1.0.1
704
# via statsmodels
705
peft==0.16.0
710
- # via
711
- # -r requirements/test.in
712
- # lm-eval
706
+ # via -r requirements/test.in
707
perceptron==0.1.4
708
# via -r requirements/test.in
709
perf-analyzer==0.1.0
@@ -792,8 +786,6 @@ pyasn1==0.6.1
786
# rsa
787
pyasn1-modules==0.4.2
788
# via google-auth
795
-pybind11==2.13.6
796
- # via lm-eval
789
pycocotools==2.0.8
790
# via terratorch
791
pycountry==24.6.1
@@ -1171,7 +1163,6 @@ torch==2.10.0+cu129
1163
# kornia
1164
# lightly
1165
# lightning
1174
- # lm-eval
1166
# mteb
1167
# open-clip-torch
1168
# peft
@@ -1229,15 +1220,11 @@ tqdm==4.67.3
1220
# sentence-transformers
1221
# tacoreader
1222
# terratorch
1232
- # tqdm-multiprocess
1223
# transformers
1234
-tqdm-multiprocess==0.0.11
1235
- # via lm-eval
1224
transformers==4.57.5
1225
# via
1226
# -r requirements/test.in
1227
# genai-perf
1240
- # lm-eval
1228
# peft
1229
# sentence-transformers
1230
# transformers-stream-generator
@@ -1272,6 +1259,7 @@ typing-extensions==4.15.0
1259
# librosa
1260
# lightning
1261
# lightning-utilities
1262
+ # lm-eval
1263
# mistral-common
1264
# mteb
1265
# opentelemetry-api