| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | |
| 5 | from tests.models.utils import ( |
| 6 | EmbedModelInfo, |
| 7 | ) |
| 8 | |
| 9 | from .mteb_embed_utils import mteb_test_embed_models |
| 10 | |
| 11 | # ST models with projector (Dense) layers |
| 12 | ST_PROJECTOR_MODELS = [ |
| 13 | EmbedModelInfo( |
| 14 | "TencentBAC/Conan-embedding-v1", |
| 15 | architecture="BertModel", |
| 16 | mteb_score=0.688611955, |
| 17 | mteb_tol=2e-3, |
| 18 | seq_pooling_type="MEAN", |
| 19 | attn_type="encoder_only", |
| 20 | is_prefix_caching_supported=False, |
| 21 | is_chunked_prefill_supported=False, |
| 22 | enable_test=True, |
| 23 | ), |
| 24 | EmbedModelInfo( |
| 25 | "google/embeddinggemma-300m", |
| 26 | architecture="Gemma3TextModel", |
| 27 | mteb_score=0.7473819294684156, |
| 28 | mteb_tol=2e-3, |
| 29 | seq_pooling_type="MEAN", |
| 30 | attn_type="encoder_only", |
| 31 | is_prefix_caching_supported=False, |
| 32 | is_chunked_prefill_supported=False, |
| 33 | enable_test=True, |
| 34 | ), |
| 35 | ] |
| 36 | |
| 37 | |
| 38 | @pytest.mark.flaky(reruns=2) |
| 39 | @pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS) |
| 40 | def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: |
| 41 | mteb_test_embed_models(hf_runner, vllm_runner, model_info) |