A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[CI] Split mteb test from Language Models Test (#24634)
Signed-off-by: wang.yuqi <noooop@126.com>
wang.yuqi committed
Sep 11, 2025 at 21:37 UTC
fd1ce98cdd1f4699c781d2f24b39f7b16a338886
16 files changed
+56
-40
.buildkite/test-pipeline.yaml
+10
@@ -642,6 +642,16 @@ steps:
642
commands:
643
- pytest -v -s models/language/pooling -m 'not core_model'
644
645
+- label: Language Models Test (MTEB)
646
+ timeout_in_minutes: 110
647
+ mirror_hardwares: [amdexperimental]
648
+ optional: true
649
+ source_file_dependencies:
650
+ - vllm/
651
+ - tests/models/language/pooling_mteb_test
652
+ commands:
653
+ - pytest -v -s models/language/pooling_mteb_test
654
+
655
- label: Multi-Modal Processor Test # 44min
656
timeout_in_minutes: 60
657
source_file_dependencies:
tests/entrypoints/pooling/correctness/test_mteb_embed.py
+3
-4
@@ -4,10 +4,9 @@ import os
4
5
import pytest
6
7
-from tests.models.language.pooling.mteb_utils import (MTEB_EMBED_TASKS,
8
- MTEB_EMBED_TOL,
9
- OpenAIClientMtebEncoder,
10
- run_mteb_embed_task)
7
+from tests.models.language.pooling_mteb_test.mteb_utils import (
8
+ MTEB_EMBED_TASKS, MTEB_EMBED_TOL, OpenAIClientMtebEncoder,
9
+ run_mteb_embed_task)
10
from tests.utils import RemoteOpenAIServer
11
12
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
tests/entrypoints/pooling/correctness/test_mteb_score.py
+3
-9
@@ -4,15 +4,9 @@ import os
4
5
import pytest
6
7
-# yapf conflicts with isort for this block
8
-# yapf: disable
9
-from tests.models.language.pooling.mteb_utils import (MTEB_RERANK_LANGS,
10
- MTEB_RERANK_TASKS,
11
- MTEB_RERANK_TOL,
12
- RerankClientMtebEncoder,
13
- ScoreClientMtebEncoder,
14
- run_mteb_rerank)
15
-# yapf: enable
7
+from tests.models.language.pooling_mteb_test.mteb_utils import (
8
+ MTEB_RERANK_LANGS, MTEB_RERANK_TASKS, MTEB_RERANK_TOL,
9
+ RerankClientMtebEncoder, ScoreClientMtebEncoder, run_mteb_rerank)
10
from tests.utils import RemoteOpenAIServer
11
12
os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
tests/models/language/pooling_mteb_test/__init__.py
tests/models/language/pooling_mteb_test/mteb_utils.py
renamed
tests/models/language/pooling_mteb_test/test_baai.py
renamed
+6
-4
@@ -2,10 +2,12 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
6
- EmbedModelInfo, LASTPoolingEmbedModelInfo,
7
- RerankModelInfo)
8
-from .embed_utils import correctness_test_embed_models
5
+from tests.models.language.pooling.embed_utils import (
6
+ correctness_test_embed_models)
7
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
8
+ CLSPoolingRerankModelInfo, EmbedModelInfo,
9
+ LASTPoolingEmbedModelInfo, RerankModelInfo)
10
+
11
from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
12
13
MODELS = [
tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py
renamed
+3
-3
@@ -7,9 +7,9 @@ import pytest
7
import torch
8
9
from tests.conftest import HfRunner
10
-
11
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
12
-from .mteb_utils import VllmMtebEncoder, mteb_test_rerank_models
10
+from tests.models.language.pooling_mteb_test.mteb_utils import (
11
+ VllmMtebEncoder, mteb_test_rerank_models)
12
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
13
14
RERANK_MODELS = [
15
LASTPoolingRerankModelInfo("BAAI/bge-reranker-v2-gemma",
tests/models/language/pooling_mteb_test/test_cross_encoder.py
renamed
+3
-2
@@ -2,8 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-from ...utils import (CLSPoolingRerankModelInfo, LASTPoolingRerankModelInfo,
6
- RerankModelInfo)
5
+from tests.models.utils import (CLSPoolingRerankModelInfo,
6
+ LASTPoolingRerankModelInfo, RerankModelInfo)
7
+
8
from .mteb_utils import mteb_test_rerank_models
9
10
RERANK_MODELS = [
tests/models/language/pooling_mteb_test/test_gte.py
renamed
+6
-4
@@ -3,10 +3,12 @@
3
4
import pytest
5
6
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
7
- EmbedModelInfo, LASTPoolingEmbedModelInfo,
8
- RerankModelInfo)
9
-from .embed_utils import correctness_test_embed_models
6
+from tests.models.language.pooling.embed_utils import (
7
+ correctness_test_embed_models)
8
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
9
+ CLSPoolingRerankModelInfo, EmbedModelInfo,
10
+ LASTPoolingEmbedModelInfo, RerankModelInfo)
11
+
12
from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
13
14
MODELS = [
tests/models/language/pooling_mteb_test/test_intfloat.py
renamed
+4
-2
@@ -2,8 +2,10 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
6
-from .embed_utils import correctness_test_embed_models
5
+from tests.models.language.pooling.embed_utils import (
6
+ correctness_test_embed_models)
7
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
8
+
9
from .mteb_utils import mteb_test_embed_models
10
11
MODELS = [
tests/models/language/pooling_mteb_test/test_jina.py
renamed
+5
-4
@@ -4,12 +4,13 @@ from functools import partial
4
5
import pytest
6
7
+from tests.models.language.pooling.embed_utils import (
8
+ check_embeddings_close, correctness_test_embed_models, matryoshka_fy)
9
+from tests.models.utils import (CLSPoolingEmbedModelInfo,
10
+ CLSPoolingRerankModelInfo, EmbedModelInfo,
11
+ RerankModelInfo)
12
from vllm import PoolingParams
13
9
-from ...utils import (CLSPoolingEmbedModelInfo, CLSPoolingRerankModelInfo,
10
- EmbedModelInfo, RerankModelInfo)
11
-from .embed_utils import (check_embeddings_close,
12
- correctness_test_embed_models, matryoshka_fy)
14
from .mteb_utils import mteb_test_embed_models, mteb_test_rerank_models
15
16
EMBEDDING_MODELS = [
tests/models/language/pooling_mteb_test/test_mxbai_rerank.py
renamed
+1
-1
@@ -6,8 +6,8 @@ import pytest
6
import torch
7
8
from tests.conftest import HfRunner
9
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
10
10
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
11
from .mteb_utils import mteb_test_rerank_models
12
13
mxbai_rerank_hf_overrides = {
tests/models/language/pooling_mteb_test/test_nomic.py
renamed
+4
-2
@@ -3,8 +3,10 @@
3
4
import pytest
5
6
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
7
-from .embed_utils import correctness_test_embed_models
6
+from tests.models.language.pooling.embed_utils import (
7
+ correctness_test_embed_models)
8
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
9
+
10
from .mteb_utils import mteb_test_embed_models
11
12
MODELS = [
tests/models/language/pooling_mteb_test/test_qwen3_reranker.py
renamed
+1
-1
@@ -6,9 +6,9 @@ import pytest
6
import torch
7
8
from tests.conftest import HfRunner
9
+from tests.models.utils import LASTPoolingRerankModelInfo, RerankModelInfo
10
from tests.utils import multi_gpu_test
11
11
-from ...utils import LASTPoolingRerankModelInfo, RerankModelInfo
12
from .mteb_utils import mteb_test_rerank_models
13
14
qwen3_reranker_hf_overrides = {
tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py
renamed
+4
-2
@@ -3,8 +3,10 @@
3
4
import pytest
5
6
-from ...utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
7
-from .embed_utils import correctness_test_embed_models
6
+from tests.models.language.pooling.embed_utils import (
7
+ correctness_test_embed_models)
8
+from tests.models.utils import CLSPoolingEmbedModelInfo, EmbedModelInfo
9
+
10
from .mteb_utils import mteb_test_embed_models
11
12
MODELS = [
tests/models/language/pooling_mteb_test/test_st_projector.py
renamed
+3
-2
@@ -2,8 +2,9 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import pytest
4
5
-from ...utils import (CLSPoolingEmbedModelInfo, EmbedModelInfo,
6
- LASTPoolingEmbedModelInfo)
5
+from tests.models.utils import (CLSPoolingEmbedModelInfo, EmbedModelInfo,
6
+ LASTPoolingEmbedModelInfo)
7
+
8
from .mteb_utils import mteb_test_embed_models
9
10
# ST models with projector (Dense) layers