A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Accelerate mteb test by setting SentenceTransformers mteb score to a constant (#24088)
Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Sep 3, 2025 at 17:23 UTC
51383bd472747cfbc1f8f2e40767e1becef01f0f
17 files changed
+83
-52
tests/entrypoints/openai/correctness/test_mteb_embed.py
+3
-1
@@ -37,4 +37,6 @@ def test_mteb_embed(server):
37
print("SentenceTransformer main score: ", st_main_score)
38
print("Difference: ", st_main_score - vllm_main_score)
39
40
- assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_EMBED_TOL)
40
+ # We are not concerned that the vllm mteb results are better
41
+ # than SentenceTransformers, so we only perform one-sided testing.
42
+ assert st_main_score - vllm_main_score < MTEB_EMBED_TOL
tests/entrypoints/openai/correctness/test_mteb_score.py
+15
-16
@@ -6,16 +6,19 @@ import pytest
6
7
# yapf conflicts with isort for this block
8
# yapf: disable
9
-from tests.models.language.pooling.mteb_utils import (
10
- MTEB_RERANK_LANGS, MTEB_RERANK_TASKS, MTEB_RERANK_TOL,
11
- RerankClientMtebEncoder, ScoreClientMtebEncoder,
12
- mteb_test_rerank_models_hf, run_mteb_rerank)
9
+from tests.models.language.pooling.mteb_utils import (MTEB_RERANK_LANGS,
10
+ MTEB_RERANK_TASKS,
11
+ MTEB_RERANK_TOL,
12
+ RerankClientMtebEncoder,
13
+ ScoreClientMtebEncoder,
14
+ run_mteb_rerank)
15
# yapf: enable
16
from tests.utils import RemoteOpenAIServer
17
18
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
19
20
MODEL_NAME = "cross-encoder/ms-marco-MiniLM-L-6-v2"
21
+st_main_score = 0.33457
22
23
24
@pytest.fixture(scope="module")
@@ -29,15 +32,7 @@ def server():
32
yield remote_server
33
34
32
-@pytest.fixture(scope="module")
33
-def st_main_score(hf_runner):
34
- # The main score related to the version of the dependency.
35
- # So we need to recalculate every time.
36
- main_score, st_dtype = mteb_test_rerank_models_hf(hf_runner, MODEL_NAME)
37
- return main_score
38
-
39
-
40
-def test_mteb_score(server, st_main_score):
35
+def test_mteb_score(server):
36
url = server.url_for("score")
37
encoder = ScoreClientMtebEncoder(MODEL_NAME, url)
38
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS,
@@ -47,10 +42,12 @@ def test_mteb_score(server, st_main_score):
42
print("SentenceTransformer main score: ", st_main_score)
43
print("Difference: ", st_main_score - vllm_main_score)
44
50
- assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_RERANK_TOL)
45
+ # We are not concerned that the vllm mteb results are better
46
+ # than SentenceTransformers, so we only perform one-sided testing.
47
+ assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
48
49
53
-def test_mteb_rerank(server, st_main_score):
50
+def test_mteb_rerank(server):
51
url = server.url_for("rerank")
52
encoder = RerankClientMtebEncoder(MODEL_NAME, url)
53
vllm_main_score = run_mteb_rerank(encoder, MTEB_RERANK_TASKS,
@@ -60,4 +57,6 @@ def test_mteb_rerank(server, st_main_score):
57
print("SentenceTransformer main score: ", st_main_score)
58
print("Difference: ", st_main_score - vllm_main_score)
59
63
- assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_RERANK_TOL)
60
+ # We are not concerned that the vllm mteb results are better
61
+ # than SentenceTransformers, so we only perform one-sided testing.
62
+ assert st_main_score - vllm_main_score < MTEB_RERANK_TOL
tests/models/language/pooling/embed_utils.py
+1
-4
@@ -35,10 +35,7 @@ def correctness_test_embed_models(hf_runner,
35
example_prompts,
36
vllm_extra_kwargs=None,
37
hf_model_callback=None):
38
- if not model_info.enable_test:
39
- # A model family has many models with the same architecture,
40
- # and we don't need to test each one.
41
- pytest.skip("Skipping test.")
38
+ pytest.skip("Debug only, ci prefers to use mteb test.")
39
40
# The example_prompts has ending "\n", for example:
41
# "Write a short story about a robot that dreams for the first time.\n"
tests/models/language/pooling/mteb_utils.py
+24
-12
@@ -18,7 +18,7 @@ from tests.models.utils import EmbedModelInfo, RerankModelInfo
18
# - Different model results in differences more than 1e-3
19
# 1e-4 is a good tolerance threshold
20
MTEB_EMBED_TASKS = ["STS12"]
21
-MTEB_EMBED_TOL = 0.02
21
+MTEB_EMBED_TOL = 1e-4
22
23
# See #19344
24
MTEB_RERANK_TASKS = ["NFCorpus"]
@@ -192,22 +192,28 @@ def mteb_test_embed_models(hf_runner,
192
MTEB_EMBED_TASKS)
193
vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
194
195
- with hf_runner(model_info.name,
196
- is_sentence_transformer=True,
197
- dtype="float32") as hf_model:
195
+ if model_info.mteb_score is None:
196
+ with hf_runner(model_info.name,
197
+ is_sentence_transformer=True,
198
+ dtype="float32") as hf_model:
199
199
- if hf_model_callback is not None:
200
- hf_model_callback(hf_model)
200
+ if hf_model_callback is not None:
201
+ hf_model_callback(hf_model)
202
202
- st_main_score = run_mteb_embed_task(hf_model, MTEB_EMBED_TASKS)
203
- st_dtype = next(hf_model.model.parameters()).dtype
203
+ st_main_score = run_mteb_embed_task(hf_model, MTEB_EMBED_TASKS)
204
+ st_dtype = next(hf_model.model.parameters()).dtype
205
+ else:
206
+ st_main_score = model_info.mteb_score
207
+ st_dtype = "Constant"
208
209
print("Model:", model_info.name)
210
print("VLLM:", vllm_dtype, vllm_main_score)
211
print("SentenceTransformers:", st_dtype, st_main_score)
212
print("Difference:", st_main_score - vllm_main_score)
213
210
- assert st_main_score == pytest.approx(vllm_main_score, abs=atol)
214
+ # We are not concerned that the vllm mteb results are better
215
+ # than SentenceTransformers, so we only perform one-sided testing.
216
+ assert st_main_score - vllm_main_score < atol
217
218
219
def run_mteb_rerank(cross_encoder, tasks, languages):
@@ -310,12 +316,18 @@ def mteb_test_rerank_models(hf_runner,
316
languages=MTEB_RERANK_LANGS)
317
vllm_dtype = model_config.dtype
318
313
- st_main_score, st_dtype = mteb_test_rerank_models_hf(
314
- hf_runner, model_info.name, hf_model_callback)
319
+ if model_info.mteb_score is None:
320
+ st_main_score, st_dtype = mteb_test_rerank_models_hf(
321
+ hf_runner, model_info.name, hf_model_callback)
322
+ else:
323
+ st_main_score = model_info.mteb_score
324
+ st_dtype = "Constant"
325
326
print("Model:", model_info.name)
327
print("VLLM:", vllm_dtype, vllm_main_score)
328
print("SentenceTransformers:", st_dtype, st_main_score)
329
print("Difference:", st_main_score - vllm_main_score)
330
321
- assert st_main_score == pytest.approx(vllm_main_score, abs=atol)
331
+ # We are not concerned that the vllm mteb results are better
332
+ # than SentenceTransformers, so we only perform one-sided testing.
333
+ assert st_main_score - vllm_main_score < atol
tests/models/language/pooling/test_baai.py
+4
@@ -12,6 +12,7 @@ MODELS = [
12
########## BertModel
13
CLSPoolingEmbedModelInfo("BAAI/bge-base-en",
14
architecture="BertModel",
15
+ mteb_score=0.779336792,
16
enable_test=True),
17
CLSPoolingEmbedModelInfo("BAAI/bge-base-zh",
18
architecture="BertModel",
@@ -52,10 +53,12 @@ MODELS = [
53
########## XLMRobertaModel
54
CLSPoolingEmbedModelInfo("BAAI/bge-m3",
55
architecture="XLMRobertaModel",
56
+ mteb_score=0.787343078,
57
enable_test=True),
58
########## Qwen2Model
59
LASTPoolingEmbedModelInfo("BAAI/bge-code-v1",
60
architecture="Qwen2Model",
61
+ mteb_score=0.75724465,
62
dtype="float32",
63
enable_test=True),
64
]
@@ -65,6 +68,7 @@ RERANK_MODELS = [
68
CLSPoolingRerankModelInfo(
69
"BAAI/bge-reranker-base",
70
architecture="XLMRobertaForSequenceClassification",
71
+ mteb_score=0.32398,
72
enable_test=True),
73
CLSPoolingRerankModelInfo(
74
"BAAI/bge-reranker-large",
tests/models/language/pooling/test_bge_reranker_v2_gemma.py
+1
-2
@@ -104,7 +104,6 @@ class GemmaMtebEncoder(VllmMtebEncoder):
104
105
def __init__(self, *args, **kwargs):
106
super().__init__(*args, **kwargs)
107
- self.prompt = PROMPT
107
self.query_template = "A: {query}\n"
108
self.document_template = "B: {doc}\n{prompt}"
109
@@ -119,7 +118,7 @@ class GemmaMtebEncoder(VllmMtebEncoder):
118
_sentences = []
119
for query, corpus, prompt in sentences:
120
query = self.query_template.format(query=query)
122
- corpus = self.document_template.format(doc=corpus, prompt=prompt)
121
+ corpus = self.document_template.format(doc=corpus, prompt=PROMPT)
122
_sentences.append((query, corpus, prompt))
123
124
return super().predict(_sentences, *args, **kwargs)
tests/models/language/pooling/test_cross_encoder.py
+2
@@ -8,8 +8,10 @@ from .mteb_utils import mteb_test_rerank_models
8
9
RERANK_MODELS = [
10
CLSPoolingRerankModelInfo("cross-encoder/ms-marco-TinyBERT-L-2-v2",
11
+ mteb_score=0.32898,
12
architecture="BertForSequenceClassification"),
13
LASTPoolingRerankModelInfo("tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
14
+ mteb_score=0.25736,
15
architecture="Qwen3ForSequenceClassification")
16
]
17
tests/models/language/pooling/test_embedding.py
+1
-4
@@ -7,7 +7,7 @@ import pytest
7
from vllm.config import PoolerConfig
8
from vllm.platforms import current_platform
9
10
-from ...utils import check_embeddings_close, check_transformers_version
10
+from ...utils import check_embeddings_close
11
12
13
@pytest.mark.parametrize(
@@ -30,7 +30,6 @@ from ...utils import check_embeddings_close, check_transformers_version
30
pytest.param("BAAI/bge-base-en-v1.5", marks=[pytest.mark.core_model]),
31
pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
32
pytest.param("intfloat/multilingual-e5-small"),
33
- pytest.param("Alibaba-NLP/gte-Qwen2-1.5B-instruct"),
33
# [Cross-Encoder]
34
pytest.param("sentence-transformers/stsb-roberta-base-v2"),
35
],
@@ -42,8 +41,6 @@ def test_models(
41
model,
42
monkeypatch,
43
) -> None:
45
- if model == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
46
- check_transformers_version(model, max_transformers_version="4.53.2")
44
45
if model == "BAAI/bge-multilingual-gemma2" and current_platform.is_rocm():
46
# ROCm Triton FA does not currently support sliding window attention
tests/models/language/pooling/test_gte.py
+15
-11
@@ -5,13 +5,14 @@ import pytest
5
6
from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
7
EmbedModelInfo, LASTPoolingEmbedModelInfo,
8
- RerankModelInfo, check_transformers_version)
8
+ RerankModelInfo)
9
from .embed_utils import correctness_test_embed_models
10
from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
11
12
MODELS = [
13
########## BertModel
14
CLSPoolingEmbedModelInfo("thenlper/gte-large",
15
+ mteb_score=0.76807651,
16
architecture="BertModel",
17
enable_test=True),
18
CLSPoolingEmbedModelInfo("thenlper/gte-base",
@@ -30,28 +31,37 @@ MODELS = [
31
architecture="BertModel",
32
enable_test=False),
33
########### NewModel
34
+ # These three architectures are almost the same, but not exactly the same.
35
+ # For example,
36
+ # - whether to use token_type_embeddings
37
+ # - whether to use context expansion
38
+ # So only test one (the most widely used) model
39
CLSPoolingEmbedModelInfo("Alibaba-NLP/gte-multilingual-base",
40
architecture="GteNewModel",
41
+ mteb_score=0.775074696,
42
hf_overrides={"architectures": ["GteNewModel"]},
43
enable_test=True),
44
CLSPoolingEmbedModelInfo("Alibaba-NLP/gte-base-en-v1.5",
45
architecture="GteNewModel",
46
hf_overrides={"architectures": ["GteNewModel"]},
40
- enable_test=True),
47
+ enable_test=False),
48
CLSPoolingEmbedModelInfo("Alibaba-NLP/gte-large-en-v1.5",
49
architecture="GteNewModel",
50
hf_overrides={"architectures": ["GteNewModel"]},
44
- enable_test=True),
51
+ enable_test=False),
52
########### Qwen2ForCausalLM
53
LASTPoolingEmbedModelInfo("Alibaba-NLP/gte-Qwen2-1.5B-instruct",
54
+ mteb_score=0.758473459018872,
55
architecture="Qwen2ForCausalLM",
56
enable_test=True),
57
########## ModernBertModel
58
CLSPoolingEmbedModelInfo("Alibaba-NLP/gte-modernbert-base",
59
+ mteb_score=0.748193353,
60
architecture="ModernBertModel",
61
enable_test=True),
62
########## Qwen3ForCausalLM
63
LASTPoolingEmbedModelInfo("Qwen/Qwen3-Embedding-0.6B",
64
+ mteb_score=0.771163695,
65
architecture="Qwen3ForCausalLM",
66
dtype="float32",
67
enable_test=True),
@@ -65,10 +75,12 @@ RERANK_MODELS = [
75
CLSPoolingRerankModelInfo(
76
# classifier_pooling: mean
77
"Alibaba-NLP/gte-reranker-modernbert-base",
78
+ mteb_score=0.33386,
79
architecture="ModernBertForSequenceClassification",
80
enable_test=True),
81
CLSPoolingRerankModelInfo(
82
"Alibaba-NLP/gte-multilingual-reranker-base",
83
+ mteb_score=0.33062,
84
architecture="GteNewForSequenceClassification",
85
hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
86
enable_test=True),
@@ -78,10 +90,6 @@ RERANK_MODELS = [
90
@pytest.mark.parametrize("model_info", MODELS)
91
def test_embed_models_mteb(hf_runner, vllm_runner,
92
model_info: EmbedModelInfo) -> None:
81
- if model_info.name == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
82
- check_transformers_version(model_info.name,
83
- max_transformers_version="4.53.2")
84
-
93
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
94
95
@@ -89,10 +97,6 @@ def test_embed_models_mteb(hf_runner, vllm_runner,
97
def test_embed_models_correctness(hf_runner, vllm_runner,
98
model_info: EmbedModelInfo,
99
example_prompts) -> None:
92
- if model_info.name == "Alibaba-NLP/gte-Qwen2-1.5B-instruct":
93
- check_transformers_version(model_info.name,
94
- max_transformers_version="4.53.2")
95
-
100
correctness_test_embed_models(hf_runner, vllm_runner, model_info,
101
example_prompts)
102
tests/models/language/pooling/test_intfloat.py
+3
-1
@@ -10,6 +10,7 @@ MODELS = [
10
########## BertModel
11
CLSPoolingEmbedModelInfo("intfloat/e5-small",
12
architecture="BertModel",
13
+ mteb_score=0.742285423,
14
enable_test=True),
15
CLSPoolingEmbedModelInfo("intfloat/e5-base",
16
architecture="BertModel",
@@ -23,6 +24,7 @@ MODELS = [
24
########## XLMRobertaModel
25
CLSPoolingEmbedModelInfo("intfloat/multilingual-e5-base",
26
architecture="XLMRobertaModel",
27
+ mteb_score=0.779325955,
28
enable_test=True),
29
CLSPoolingEmbedModelInfo("intfloat/multilingual-e5-large",
30
architecture="XLMRobertaModel",
@@ -36,7 +38,7 @@ MODELS = [
38
@pytest.mark.parametrize("model_info", MODELS)
39
def test_embed_models_mteb(hf_runner, vllm_runner,
40
model_info: EmbedModelInfo) -> None:
39
- mteb_test_embed_models(hf_runner, vllm_runner, model_info, atol=0.02)
41
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
42
43
44
@pytest.mark.parametrize("model_info", MODELS)
tests/models/language/pooling/test_jina.py
+2
@@ -14,6 +14,7 @@ from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
14
15
EMBEDDING_MODELS = [
16
CLSPoolingEmbedModelInfo("jinaai/jina-embeddings-v3",
17
+ mteb_score=0.824413164,
18
architecture="XLMRobertaModel",
19
is_matryoshka=True)
20
]
@@ -21,6 +22,7 @@ EMBEDDING_MODELS = [
22
RERANK_MODELS = [
23
CLSPoolingRerankModelInfo(
24
"jinaai/jina-reranker-v2-base-multilingual",
25
+ mteb_score=0.33643,
26
architecture="XLMRobertaForSequenceClassification")
27
]
28
tests/models/language/pooling/test_mxbai_rerank.py
+1
@@ -20,6 +20,7 @@ RERANK_MODELS = [
20
LASTPoolingRerankModelInfo("mixedbread-ai/mxbai-rerank-base-v2",
21
architecture="Qwen2ForSequenceClassification",
22
hf_overrides=mxbai_rerank_hf_overrides,
23
+ mteb_score=0.273,
24
enable_test=True),
25
LASTPoolingRerankModelInfo("mixedbread-ai/mxbai-rerank-large-v2",
26
architecture="Qwen2ForSequenceClassification",
tests/models/language/pooling/test_nomic.py
+2
@@ -10,6 +10,7 @@ from .mteb_utils import mteb_test_embed_models
10
MODELS = [
11
CLSPoolingEmbedModelInfo("nomic-ai/nomic-embed-text-v1",
12
architecture="NomicBertModel",
13
+ mteb_score=0.737568559,
14
enable_test=True),
15
CLSPoolingEmbedModelInfo("nomic-ai/nomic-embed-text-v1.5",
16
architecture="NomicBertModel",
@@ -19,6 +20,7 @@ MODELS = [
20
enable_test=False),
21
CLSPoolingEmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe",
22
architecture="NomicBertModel",
23
+ mteb_score=0.715488912,
24
enable_test=True)
25
]
26
tests/models/language/pooling/test_qwen3_reranker.py
+1
@@ -20,6 +20,7 @@ qwen3_reranker_hf_overrides = {
20
RERANK_MODELS = [
21
LASTPoolingRerankModelInfo("Qwen/Qwen3-Reranker-0.6B",
22
architecture="Qwen3ForSequenceClassification",
23
+ mteb_score=0.25736,
24
hf_overrides=qwen3_reranker_hf_overrides,
25
enable_test=True),
26
LASTPoolingRerankModelInfo("Qwen/Qwen3-Reranker-4B",
tests/models/language/pooling/test_snowflake_arctic_embed.py
+6
-1
@@ -11,6 +11,7 @@ MODELS = [
11
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-xs",
12
is_matryoshka=False,
13
architecture="BertModel",
14
+ mteb_score=0.714927797,
15
enable_test=True),
16
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-s",
17
is_matryoshka=False,
@@ -23,6 +24,7 @@ MODELS = [
24
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long",
25
is_matryoshka=False,
26
architecture="NomicBertModel",
27
+ mteb_score=0.681146831,
28
enable_test=True),
29
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-l",
30
is_matryoshka=False,
@@ -31,14 +33,17 @@ MODELS = [
33
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-m-v1.5",
34
is_matryoshka=True,
35
architecture="BertModel",
36
+ mteb_score=0.649088363,
37
enable_test=True),
38
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-l-v2.0",
39
is_matryoshka=True,
40
architecture="XLMRobertaModel",
41
+ mteb_score=0.712258299,
42
enable_test=True),
43
CLSPoolingEmbedModelInfo("Snowflake/snowflake-arctic-embed-m-v2.0",
44
is_matryoshka=True,
45
architecture="GteModel",
46
+ mteb_score=0.706622444,
47
enable_test=True),
48
]
49
@@ -46,7 +51,7 @@ MODELS = [
51
@pytest.mark.parametrize("model_info", MODELS)
52
def test_embed_models_mteb(hf_runner, vllm_runner,
53
model_info: EmbedModelInfo) -> None:
49
- mteb_test_embed_models(hf_runner, vllm_runner, model_info, atol=0.02)
54
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
55
56
57
@pytest.mark.parametrize("model_info", MODELS)
tests/models/language/pooling/test_st_projector.py
+1
@@ -10,6 +10,7 @@ ST_PROJECTOR_MODELS = [
10
CLSPoolingEmbedModelInfo(
11
"TencentBAC/Conan-embedding-v1",
12
architecture="BertModel",
13
+ mteb_score=0.688611955,
14
enable_test=True,
15
),
16
]
tests/models/utils.py
+1
@@ -347,6 +347,7 @@ class ModelInfo:
347
dtype: str = "auto"
348
hf_overrides: Optional[dict[str, Any]] = None
349
default_pooling_type: str = ""
350
+ mteb_score: Optional[float] = None
351
enable_test: bool = True
352
353