A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 16:30 UTC
[CI] Increase the MTEB_EMBED_TOL threshold to 5e-4. (#31797)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
wang.yuqi committed
Jan 6, 2026 at 19:30 UTC
43d384bab4b86e494cb1c3d04b14e13512b999d1
5 files changed
+2
-7
tests/models/language/pooling_mteb_test/mteb_embed_utils.py
+2
-2
@@ -19,9 +19,9 @@ from tests.models.utils import (
19
# - Model implementation and minor changes in tensor dtype
20
# results in differences less than 1e-4
21
# - Different model results in differences more than 1e-3
22
-# 1e-4 is a good tolerance threshold
22
+# 5e-4 is a good tolerance threshold
23
MTEB_EMBED_TASKS = ["STS12"]
24
-MTEB_EMBED_TOL = 1e-4
24
+MTEB_EMBED_TOL = 5e-4
25
26
27
_empty_model_meta = ModelMeta(
tests/models/language/pooling_mteb_test/test_baai.py
-1
@@ -65,7 +65,6 @@ MODELS = [
65
"BAAI/bge-code-v1",
66
architecture="Qwen2Model",
67
mteb_score=0.75724465,
68
- dtype="float32",
68
pooling_type="LAST",
69
attn_type="decoder",
70
is_prefix_caching_supported=True,
tests/models/language/pooling_mteb_test/test_gte.py
-2
@@ -89,7 +89,6 @@ MODELS = [
89
"Qwen/Qwen3-Embedding-0.6B",
90
mteb_score=0.771163695,
91
architecture="Qwen3ForCausalLM",
92
- dtype="float32",
92
pooling_type="LAST",
93
attn_type="decoder",
94
is_prefix_caching_supported=True,
@@ -99,7 +98,6 @@ MODELS = [
98
EmbedModelInfo(
99
"Qwen/Qwen3-Embedding-4B",
100
architecture="Qwen3ForCausalLM",
102
- dtype="float32",
101
enable_test=False,
102
),
103
]
tests/models/language/pooling_mteb_test/test_jina.py
-1
@@ -28,7 +28,6 @@ EMBEDDING_MODELS = [
28
attn_type="encoder_only",
29
is_prefix_caching_supported=False,
30
is_chunked_prefill_supported=False,
31
- dtype="float32",
31
)
32
]
33
tests/models/language/pooling_mteb_test/test_st_projector.py
-1
@@ -29,7 +29,6 @@ ST_PROJECTOR_MODELS = [
29
is_prefix_caching_supported=False,
30
is_chunked_prefill_supported=False,
31
enable_test=True,
32
- dtype="float32",
32
),
33
]
34