| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | |
| 6 | from tests.models.language.pooling.embed_utils import correctness_test_embed_models |
| 7 | from tests.models.utils import EmbedModelInfo |
| 8 | |
| 9 | from .mteb_embed_utils import mteb_test_embed_models |
| 10 | |
| 11 | MODELS = [ |
| 12 | EmbedModelInfo( |
| 13 | "voyageai/voyage-4-nano", |
| 14 | architecture="VoyageQwen3BidirectionalEmbedModel", |
| 15 | enable_test=True, |
| 16 | seq_pooling_type="MEAN", |
| 17 | attn_type="encoder_only", |
| 18 | is_prefix_caching_supported=False, |
| 19 | is_chunked_prefill_supported=False, |
| 20 | hf_overrides={ |
| 21 | "architectures": ["VoyageQwen3BidirectionalEmbedModel"], |
| 22 | "num_labels": 2048, |
| 23 | }, |
| 24 | mteb_score=0.7054, |
| 25 | mteb_tol=2e-3, |
| 26 | # === MTEB Results === |
| 27 | # STS12: 0.6613 |
| 28 | # STS13: 0.6906 |
| 29 | # STS14: 0.6556 |
| 30 | # STS15: 0.7843 |
| 31 | # STS16: 0.7340 |
| 32 | # STSBenchmark: 0.7063 |
| 33 | # Average score: 0.7054 |
| 34 | ), |
| 35 | ] |
| 36 | |
| 37 | |
| 38 | @pytest.mark.flaky(reruns=2) |
| 39 | @pytest.mark.parametrize("model_info", MODELS) |
| 40 | def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None: |
| 41 | # Encoder-only attention models need enforce_eager=True to avoid |
| 42 | # CUDA graph capture issues with piecewise compilation |
| 43 | mteb_test_embed_models( |
| 44 | hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True} |
| 45 | ) |
| 46 | |
| 47 | |
| 48 | @pytest.mark.parametrize("model_info", MODELS) |
| 49 | def test_embed_models_correctness( |
| 50 | hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts |
| 51 | ) -> None: |
| 52 | correctness_test_embed_models( |
| 53 | hf_runner, |
| 54 | vllm_runner, |
| 55 | model_info, |
| 56 | example_prompts, |
| 57 | vllm_extra_kwargs={"enforce_eager": True}, |
| 58 | ) |