A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[CI] Increase the threshold of the MTEB RERANK tests (#20615)
Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Jul 8, 2025 at 23:10 UTC
baba0389f7e810a361fff5229ce20c2d5a2b1fac
4 files changed
+3
-9
tests/models/language/pooling/mteb_utils.py
+1
-1
@@ -23,7 +23,7 @@ MTEB_EMBED_TOL = 1e-4
23
# See #19344
24
MTEB_RERANK_TASKS = ["NFCorpus"]
25
MTEB_RERANK_LANGS = ["en"]
26
-MTEB_RERANK_TOL = 1e-3
26
+MTEB_RERANK_TOL = 2e-3
27
28
29
class VllmMtebEncoder(mteb.Encoder):
tests/models/language/pooling/test_baai.py
-1
@@ -68,7 +68,6 @@ RERANK_MODELS = [
68
enable_test=False),
69
RerankModelInfo("BAAI/bge-reranker-v2-m3",
70
architecture="XLMRobertaForSequenceClassification",
71
- dtype="float32",
71
enable_test=False)
72
]
73
tests/models/language/pooling/test_jina.py
+2
-5
@@ -18,11 +18,8 @@ EMBEDDING_MODELS = [
18
]
19
20
RERANK_MODELS = [
21
- RerankModelInfo(
22
- "jinaai/jina-reranker-v2-base-multilingual",
23
- architecture="XLMRobertaForSequenceClassification",
24
- dtype="float32",
25
- )
21
+ RerankModelInfo("jinaai/jina-reranker-v2-base-multilingual",
22
+ architecture="XLMRobertaForSequenceClassification")
23
]
24
25
tests/models/language/pooling/test_qwen3_reranker.py
-2
@@ -12,11 +12,9 @@ from .mteb_utils import RerankModelInfo, mteb_test_rerank_models
12
RERANK_MODELS = [
13
RerankModelInfo("Qwen/Qwen3-Reranker-0.6B",
14
architecture="Qwen3ForSequenceClassification",
15
- dtype="float32",
15
enable_test=True),
16
RerankModelInfo("Qwen/Qwen3-Reranker-4B",
17
architecture="Qwen3ForSequenceClassification",
19
- dtype="float32",
18
enable_test=False)
19
]
20