| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import pytest |
| 4 | |
| 5 | from tests.models.language.pooling.embed_utils import correctness_test_embed_models |
| 6 | from tests.models.utils import ( |
| 7 | EmbedModelInfo, |
| 8 | RerankModelInfo, |
| 9 | ) |
| 10 | |
| 11 | from .mteb_embed_utils import mteb_test_embed_models |
| 12 | from .mteb_score_utils import mteb_test_rerank_models |
| 13 | |
| 14 | MODELS = [ |
| 15 | ########## BertModel |
| 16 | EmbedModelInfo( |
| 17 | "BAAI/bge-base-en", |
| 18 | architecture="BertModel", |
| 19 | mteb_score=0.779336792, |
| 20 | mteb_tol=2e-3, |
| 21 | seq_pooling_type="CLS", |
| 22 | attn_type="encoder_only", |
| 23 | is_prefix_caching_supported=False, |
| 24 | is_chunked_prefill_supported=False, |
| 25 | enable_test=True, |
| 26 | ), |
| 27 | EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False), |
| 28 | EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False), |
| 29 | EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False), |
| 30 | EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False), |
| 31 | EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False), |
| 32 | EmbedModelInfo( |
| 33 | "BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False |
| 34 | ), |
| 35 | EmbedModelInfo( |
| 36 | "BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False |
| 37 | ), |
| 38 | EmbedModelInfo( |
| 39 | "BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False |
| 40 | ), |
| 41 | EmbedModelInfo( |
| 42 | "BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False |
| 43 | ), |
| 44 | EmbedModelInfo( |
| 45 | "BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False |
| 46 | ), |
| 47 | EmbedModelInfo( |
| 48 | "BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False |
| 49 | ), |
| 50 | EmbedModelInfo( |
| 51 | "BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False |
| 52 | ), |
| 53 | ########## XLMRobertaModel |
| 54 | EmbedModelInfo( |
| 55 | "BAAI/bge-m3", |
| 56 | architecture="XLMRobertaModel", |
| 57 | mteb_score=0.787343078, |
| 58 | mteb_tol=2e-3, |
| 59 | seq_pooling_type="CLS", |
| 60 | attn_type="encoder_only", |
| 61 | is_prefix_caching_supported=False, |
| 62 | is_chunked_prefill_supported=False, |
| 63 | enable_test=True, |
| 64 | ), |
| 65 | ########## Qwen2Model |
| 66 | EmbedModelInfo( |
| 67 | "BAAI/bge-code-v1", |
| 68 | architecture="Qwen2Model", |
| 69 | mteb_score=0.75724465, |
| 70 | seq_pooling_type="LAST", |
| 71 | attn_type="decoder", |
| 72 | is_prefix_caching_supported=True, |
| 73 | is_chunked_prefill_supported=True, |
| 74 | # Skip: model's custom tokenizer on HF hub is incompatible with |
| 75 | # transformers v5 (sets attrs before super().__init__, triggering |
| 76 | # AttributeError on 'verbose' in __getattr__). |
| 77 | enable_test=False, |
| 78 | ), |
| 79 | ] |
| 80 | |
| 81 | RERANK_MODELS = [ |
| 82 | ########## XLMRobertaForSequenceClassification |
| 83 | RerankModelInfo( |
| 84 | "BAAI/bge-reranker-base", |
| 85 | architecture="XLMRobertaForSequenceClassification", |
| 86 | mteb_score=0.32398, |
| 87 | mteb_tol=1e-2, |
| 88 | seq_pooling_type="CLS", |
| 89 | attn_type="encoder_only", |
| 90 | is_prefix_caching_supported=False, |
| 91 | is_chunked_prefill_supported=False, |
| 92 | enable_test=True, |
| 93 | ), |
| 94 | RerankModelInfo( |
| 95 | "BAAI/bge-reranker-large", |
| 96 | architecture="XLMRobertaForSequenceClassification", |
| 97 | enable_test=False, |
| 98 | ), |
| 99 | RerankModelInfo( |
| 100 | "BAAI/bge-reranker-v2-m3", |
| 101 | architecture="XLMRobertaForSequenceClassification", |
| 102 | enable_test=False, |
| 103 | ), |
| 104 | ] |
| 105 | |
| 106 | |
| 107 | @pytest.mark.flaky(reruns=2) |
| 108 | @pytest.mark.parametrize("model_info", MODELS) |
| 109 | def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: |
| 110 | mteb_test_embed_models(hf_runner, vllm_runner, model_info) |
| 111 | |
| 112 | |
| 113 | @pytest.mark.parametrize("model_info", MODELS) |
| 114 | def test_embed_models_correctness( |
| 115 | hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts |
| 116 | ) -> None: |
| 117 | correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts) |
| 118 | |
| 119 | |
| 120 | @pytest.mark.flaky(reruns=2) |
| 121 | @pytest.mark.parametrize("model_info", RERANK_MODELS) |
| 122 | def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: |
| 123 | mteb_test_rerank_models(vllm_runner, model_info) |