| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.language.pooling.embed_utils import correctness_test_embed_models |
| 7 | from tests.models.utils import EmbedModelInfo |
| 8 | |
| 9 | from .mteb_embed_utils import mteb_test_embed_models |
| 10 | |
| 11 | MODELS = [ |
| 12 | EmbedModelInfo( |
| 13 | "Snowflake/snowflake-arctic-embed-xs", |
| 14 | is_matryoshka=False, |
| 15 | architecture="BertModel", |
| 16 | mteb_score=0.714927797, |
| 17 | mteb_tol=2e-3, |
| 18 | seq_pooling_type="CLS", |
| 19 | attn_type="encoder_only", |
| 20 | is_prefix_caching_supported=False, |
| 21 | is_chunked_prefill_supported=False, |
| 22 | enable_test=True, |
| 23 | ), |
| 24 | EmbedModelInfo( |
| 25 | "Snowflake/snowflake-arctic-embed-s", |
| 26 | is_matryoshka=False, |
| 27 | architecture="BertModel", |
| 28 | enable_test=False, |
| 29 | ), |
| 30 | EmbedModelInfo( |
| 31 | "Snowflake/snowflake-arctic-embed-m", |
| 32 | is_matryoshka=False, |
| 33 | architecture="BertModel", |
| 34 | enable_test=False, |
| 35 | ), |
| 36 | EmbedModelInfo( |
| 37 | "Snowflake/snowflake-arctic-embed-m-long", |
| 38 | is_matryoshka=False, |
| 39 | architecture="NomicBertModel", |
| 40 | mteb_score=0.681146831, |
| 41 | mteb_tol=2e-3, |
| 42 | seq_pooling_type="CLS", |
| 43 | attn_type="encoder_only", |
| 44 | is_prefix_caching_supported=False, |
| 45 | is_chunked_prefill_supported=False, |
| 46 | enable_test=True, |
| 47 | ), |
| 48 | EmbedModelInfo( |
| 49 | "Snowflake/snowflake-arctic-embed-l", |
| 50 | is_matryoshka=False, |
| 51 | architecture="BertModel", |
| 52 | enable_test=False, |
| 53 | ), |
| 54 | EmbedModelInfo( |
| 55 | "Snowflake/snowflake-arctic-embed-m-v1.5", |
| 56 | is_matryoshka=True, |
| 57 | architecture="BertModel", |
| 58 | mteb_score=0.649088363, |
| 59 | mteb_tol=2e-3, |
| 60 | seq_pooling_type="CLS", |
| 61 | attn_type="encoder_only", |
| 62 | is_prefix_caching_supported=False, |
| 63 | is_chunked_prefill_supported=False, |
| 64 | enable_test=True, |
| 65 | ), |
| 66 | EmbedModelInfo( |
| 67 | "Snowflake/snowflake-arctic-embed-l-v2.0", |
| 68 | is_matryoshka=True, |
| 69 | architecture="XLMRobertaModel", |
| 70 | mteb_score=0.712258299, |
| 71 | mteb_tol=2e-3, |
| 72 | seq_pooling_type="CLS", |
| 73 | attn_type="encoder_only", |
| 74 | is_prefix_caching_supported=False, |
| 75 | is_chunked_prefill_supported=False, |
| 76 | enable_test=True, |
| 77 | ), |
| 78 | EmbedModelInfo( |
| 79 | "Snowflake/snowflake-arctic-embed-m-v2.0", |
| 80 | is_matryoshka=True, |
| 81 | architecture="GteModel", |
| 82 | mteb_score=0.706622444, |
| 83 | mteb_tol=2e-3, |
| 84 | seq_pooling_type="CLS", |
| 85 | attn_type="encoder_only", |
| 86 | is_prefix_caching_supported=False, |
| 87 | is_chunked_prefill_supported=False, |
| 88 | enable_test=True, |
| 89 | ), |
| 90 | ] |
| 91 | |
| 92 | |
| 93 | @pytest.mark.flaky(reruns=2) |
| 94 | @pytest.mark.parametrize("model_info", MODELS) |
| 95 | def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: |
| 96 | mteb_test_embed_models(hf_runner, vllm_runner, model_info) |
| 97 | |
| 98 | |
| 99 | @pytest.mark.parametrize("model_info", MODELS) |
| 100 | def test_embed_models_correctness( |
| 101 | hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts |
| 102 | ) -> None: |
| 103 | correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) |