| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.language.pooling_mteb_test.mteb_embed_utils import ( |
| 7 | mteb_test_embed_models, |
| 8 | ) |
| 9 | from tests.models.language.pooling_mteb_test.mteb_score_utils import ( |
| 10 | mteb_test_rerank_models, |
| 11 | ) |
| 12 | from tests.models.utils import ( |
| 13 | EmbedModelInfo, |
| 14 | RerankModelInfo, |
| 15 | ) |
| 16 | |
| 17 | EMBEDDING_MODELS = [ |
| 18 | EmbedModelInfo( |
| 19 | "nvidia/llama-nemotron-embed-1b-v2", |
| 20 | architecture="LlamaBidirectionalModel", |
| 21 | mteb_score=0.689164662128673, |
| 22 | mteb_tol=2e-3, |
| 23 | seq_pooling_type="MEAN", |
| 24 | attn_type="encoder_only", |
| 25 | is_prefix_caching_supported=False, |
| 26 | is_chunked_prefill_supported=False, |
| 27 | ) |
| 28 | ] |
| 29 | |
| 30 | RERANK_MODELS = [ |
| 31 | RerankModelInfo( |
| 32 | "nvidia/llama-nemotron-rerank-1b-v2", |
| 33 | architecture="LlamaBidirectionalForSequenceClassification", |
| 34 | chat_template_name="nemotron-rerank.jinja", |
| 35 | mteb_score=0.33994, |
| 36 | mteb_tol=1e-2, |
| 37 | seq_pooling_type="MEAN", |
| 38 | attn_type="encoder_only", |
| 39 | is_prefix_caching_supported=False, |
| 40 | is_chunked_prefill_supported=False, |
| 41 | ), |
| 42 | ] |
| 43 | |
| 44 | |
| 45 | @pytest.mark.flaky(reruns=2) |
| 46 | @pytest.mark.parametrize("model_info", EMBEDDING_MODELS) |
| 47 | def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: |
| 48 | mteb_test_embed_models(hf_runner, vllm_runner, model_info) |
| 49 | |
| 50 | |
| 51 | @pytest.mark.flaky(reruns=2) |
| 52 | @pytest.mark.parametrize("model_info", RERANK_MODELS) |
| 53 | def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None: |
| 54 | mteb_test_rerank_models(vllm_runner, model_info) |