A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[CI] improve embed testing (#18747)
wang.yuqi committed
May 28, 2025 at 15:16 UTC
de65fc8e1e7056e845a90bf4df4ff39c25cb03fa
13 files changed
+244
-174
tests/entrypoints/openai/correctness/test_mteb.py
+2
-1
@@ -4,6 +4,7 @@ import os
4
import pytest
5
6
from tests.models.language.pooling.mteb_utils import (MTEB_EMBED_TASKS,
7
+ MTEB_EMBED_TOL,
8
OpenAIClientMtebEncoder,
9
run_mteb_embed_task,
10
run_mteb_embed_task_st)
@@ -38,4 +39,4 @@ def test_mteb(server):
39
print("SentenceTransformer main score: ", st_main_score)
40
print("Difference: ", st_main_score - vllm_main_score)
41
41
- assert st_main_score == pytest.approx(vllm_main_score, rel=1e-4)
42
+ assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_EMBED_TOL)
tests/entrypoints/openai/test_embedding.py
+2
-1
@@ -11,7 +11,8 @@ import requests
11
from vllm.entrypoints.openai.protocol import EmbeddingResponse
12
from vllm.transformers_utils.tokenizer import get_tokenizer
13
14
-from ...models.utils import run_embedding_correctness_test
14
+from ...models.language.pooling.embed_utils import (
15
+ run_embedding_correctness_test)
16
from ...utils import RemoteOpenAIServer
17
18
MODEL_NAME = "intfloat/multilingual-e5-small"
tests/entrypoints/openai/test_embedding_dimensions.py
+3
-1
@@ -11,7 +11,9 @@ import pytest
11
from vllm.entrypoints.openai.protocol import EmbeddingResponse
12
13
from ...conftest import HfRunner
14
-from ...models.utils import EmbedModelInfo, run_embedding_correctness_test
14
+from ...models.language.pooling.embed_utils import (
15
+ run_embedding_correctness_test)
16
+from ...models.utils import EmbedModelInfo
17
from ...utils import RemoteOpenAIServer
18
19
MODELS = [
tests/models/language/pooling/embed_utils.py
new
+72
@@ -0,0 +1,72 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from collections.abc import Sequence
3
+from typing import Optional
4
+
5
+import pytest
6
+
7
+from tests.conftest import HfRunner
8
+from tests.models.utils import (EmbedModelInfo, check_embeddings_close,
9
+ matryoshka_fy)
10
+
11
+
12
+def run_embedding_correctness_test(
13
+ hf_model: "HfRunner",
14
+ inputs: list[str],
15
+ vllm_outputs: Sequence[list[float]],
16
+ dimensions: Optional[int] = None,
17
+):
18
+ hf_outputs = hf_model.encode(inputs)
19
+ if dimensions:
20
+ hf_outputs = matryoshka_fy(hf_outputs, dimensions)
21
+
22
+ check_embeddings_close(
23
+ embeddings_0_lst=hf_outputs,
24
+ embeddings_1_lst=vllm_outputs,
25
+ name_0="hf",
26
+ name_1="vllm",
27
+ tol=1e-2,
28
+ )
29
+
30
+
31
+def correctness_test_embed_models(hf_runner,
32
+ vllm_runner,
33
+ model_info: EmbedModelInfo,
34
+ example_prompts,
35
+ vllm_extra_kwargs=None,
36
+ hf_model_callback=None):
37
+ if not model_info.enable_test:
38
+ # A model family has many models with the same architecture,
39
+ # and we don't need to test each one.
40
+ pytest.skip("Skipping test.")
41
+
42
+ # The example_prompts has ending "\n", for example:
43
+ # "Write a short story about a robot that dreams for the first time.\n"
44
+ # sentence_transformers will strip the input texts, see:
45
+ # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
46
+ # This makes the input_ids different between hf_model and vllm_model.
47
+ # So we need to strip the input texts to avoid test failing.
48
+ example_prompts = [str(s).strip() for s in example_prompts]
49
+
50
+ vllm_extra_kwargs = vllm_extra_kwargs or {}
51
+ vllm_extra_kwargs["dtype"] = model_info.dtype
52
+
53
+ with vllm_runner(model_info.name,
54
+ task="embed",
55
+ max_model_len=None,
56
+ **vllm_extra_kwargs) as vllm_model:
57
+ vllm_outputs = vllm_model.encode(example_prompts)
58
+ vllm_dtype = vllm_model.model.llm_engine.model_config.dtype
59
+ model_dtype = getattr(
60
+ vllm_model.model.llm_engine.model_config.hf_config, "torch_dtype",
61
+ vllm_dtype)
62
+
63
+ with hf_runner(
64
+ model_info.name,
65
+ dtype=model_dtype,
66
+ is_sentence_transformer=True,
67
+ ) as hf_model:
68
+
69
+ if hf_model_callback is not None:
70
+ hf_model_callback(hf_model)
71
+
72
+ run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
tests/models/language/pooling/mteb_utils.py
+8
-3
@@ -80,18 +80,19 @@ def run_mteb_embed_task_st(model_name, tasks):
80
def mteb_test_embed_models(hf_runner,
81
vllm_runner,
82
model_info: EmbedModelInfo,
83
- vllm_extra_kwargs=None):
83
+ vllm_extra_kwargs=None,
84
+ hf_model_callback=None):
85
if not model_info.enable_test:
86
# A model family has many models with the same architecture,
87
# and we don't need to test each one.
88
pytest.skip("Skipping test.")
89
90
vllm_extra_kwargs = vllm_extra_kwargs or {}
91
+ vllm_extra_kwargs["dtype"] = model_info.dtype
92
93
with vllm_runner(model_info.name,
94
task="embed",
95
max_model_len=None,
94
- dtype=model_info.dtype,
96
**vllm_extra_kwargs) as vllm_model:
97
98
if model_info.architecture:
@@ -108,10 +109,14 @@ def mteb_test_embed_models(hf_runner,
109
with set_default_torch_dtype(model_dtype) and hf_runner(
110
model_info.name, is_sentence_transformer=True,
111
dtype=model_dtype) as hf_model:
112
+
113
+ if hf_model_callback is not None:
114
+ hf_model_callback(hf_model)
115
+
116
st_main_score = run_mteb_embed_task(hf_model, MTEB_EMBED_TASKS)
117
118
print("VLLM:", vllm_dtype, vllm_main_score)
119
print("SentenceTransformer:", model_dtype, st_main_score)
120
print("Difference:", st_main_score - vllm_main_score)
121
117
- assert st_main_score == pytest.approx(vllm_main_score, rel=MTEB_EMBED_TOL)
122
+ assert st_main_score == pytest.approx(vllm_main_score, abs=MTEB_EMBED_TOL)
tests/models/language/pooling/test_baai.py
new
+71
@@ -0,0 +1,71 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+import pytest
3
+
4
+from .embed_utils import EmbedModelInfo, correctness_test_embed_models
5
+from .mteb_utils import mteb_test_embed_models
6
+
7
+MODELS = [
8
+ ########## BertModel
9
+ EmbedModelInfo("BAAI/bge-base-en",
10
+ architecture="BertModel",
11
+ enable_test=True),
12
+ EmbedModelInfo("BAAI/bge-base-zh",
13
+ architecture="BertModel",
14
+ enable_test=False),
15
+ EmbedModelInfo("BAAI/bge-small-en",
16
+ architecture="BertModel",
17
+ enable_test=False),
18
+ EmbedModelInfo("BAAI/bge-small-zh",
19
+ architecture="BertModel",
20
+ enable_test=False),
21
+ EmbedModelInfo("BAAI/bge-large-en",
22
+ architecture="BertModel",
23
+ enable_test=False),
24
+ EmbedModelInfo("BAAI/bge-large-zh",
25
+ architecture="BertModel",
26
+ enable_test=False),
27
+ EmbedModelInfo("BAAI/bge-large-zh-noinstruct",
28
+ architecture="BertModel",
29
+ enable_test=False),
30
+ EmbedModelInfo("BAAI/bge-base-en-v1.5",
31
+ architecture="BertModel",
32
+ enable_test=False),
33
+ EmbedModelInfo("BAAI/bge-base-zh-v1.5",
34
+ architecture="BertModel",
35
+ enable_test=False),
36
+ EmbedModelInfo("BAAI/bge-small-en-v1.5",
37
+ architecture="BertModel",
38
+ enable_test=False),
39
+ EmbedModelInfo("BAAI/bge-small-zh-v1.5",
40
+ architecture="BertModel",
41
+ enable_test=False),
42
+ EmbedModelInfo("BAAI/bge-large-en-v1.5",
43
+ architecture="BertModel",
44
+ enable_test=False),
45
+ EmbedModelInfo("BAAI/bge-large-zh-v1.5",
46
+ architecture="BertModel",
47
+ enable_test=False),
48
+ ########## XLMRobertaModel
49
+ EmbedModelInfo("BAAI/bge-m3",
50
+ architecture="XLMRobertaModel",
51
+ enable_test=True),
52
+ ########## Qwen2Model
53
+ EmbedModelInfo("BAAI/bge-code-v1",
54
+ architecture="Qwen2Model",
55
+ dtype="float32",
56
+ enable_test=True),
57
+]
58
+
59
+
60
+@pytest.mark.parametrize("model_info", MODELS)
61
+def test_embed_models_mteb(hf_runner, vllm_runner,
62
+ model_info: EmbedModelInfo) -> None:
63
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
64
+
65
+
66
+@pytest.mark.parametrize("model_info", MODELS)
67
+def test_embed_models_correctness(hf_runner, vllm_runner,
68
+ model_info: EmbedModelInfo,
69
+ example_prompts) -> None:
70
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info,
71
+ example_prompts)
tests/models/language/pooling/test_gte.py
+9
-24
@@ -3,7 +3,8 @@ from typing import Any
3
4
import pytest
5
6
-from ...utils import EmbedModelInfo, run_embedding_correctness_test
6
+from .embed_utils import EmbedModelInfo, correctness_test_embed_models
7
+from .mteb_utils import mteb_test_embed_models
8
9
MODELS = [
10
########## BertModel
@@ -53,9 +54,8 @@ MODELS = [
54
55
56
@pytest.mark.parametrize("model_info", MODELS)
56
-def test_models_mteb(hf_runner, vllm_runner,
57
- model_info: EmbedModelInfo) -> None:
58
- from .mteb_utils import mteb_test_embed_models
57
+def test_embed_models_mteb(hf_runner, vllm_runner,
58
+ model_info: EmbedModelInfo) -> None:
59
60
vllm_extra_kwargs: dict[str, Any] = {}
61
if model_info.architecture == "GteNewModel":
@@ -66,28 +66,13 @@ def test_models_mteb(hf_runner, vllm_runner,
66
67
68
@pytest.mark.parametrize("model_info", MODELS)
69
-def test_models_correctness(hf_runner, vllm_runner, model_info: EmbedModelInfo,
70
- example_prompts) -> None:
71
- if not model_info.enable_test:
72
- pytest.skip("Skipping test.")
73
-
74
- # ST will strip the input texts, see test_embedding.py
75
- example_prompts = [str(s).strip() for s in example_prompts]
69
+def test_embed_models_correctness(hf_runner, vllm_runner,
70
+ model_info: EmbedModelInfo,
71
+ example_prompts) -> None:
72
73
vllm_extra_kwargs: dict[str, Any] = {}
74
if model_info.architecture == "GteNewModel":
75
vllm_extra_kwargs["hf_overrides"] = {"architectures": ["GteNewModel"]}
76
81
- with vllm_runner(model_info.name,
82
- task="embed",
83
- dtype=model_info.dtype,
84
- max_model_len=None,
85
- **vllm_extra_kwargs) as vllm_model:
86
- vllm_outputs = vllm_model.encode(example_prompts)
87
-
88
- with hf_runner(
89
- model_info.name,
90
- dtype=model_info.dtype,
91
- is_sentence_transformer=True,
92
- ) as hf_model:
93
- run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
77
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info,
78
+ example_prompts, vllm_extra_kwargs)
tests/models/language/pooling/test_jina.py
+42
-51
@@ -1,9 +1,13 @@
1
# SPDX-License-Identifier: Apache-2.0
2
+from functools import partial
3
+
4
import pytest
5
6
from vllm import PoolingParams
7
6
-from ...utils import check_embeddings_close, matryoshka_fy
8
+from .embed_utils import (EmbedModelInfo, check_embeddings_close,
9
+ correctness_test_embed_models, matryoshka_fy)
10
+from .mteb_utils import mteb_test_embed_models
11
12
SCORING_MODELS = [
13
"jinaai/jina-reranker-v2-base-multilingual", # Roberta
@@ -25,16 +29,10 @@ TEXTS_2 = [
29
]
30
31
EMBEDDING_MODELS = [
28
- "jinaai/jina-embeddings-v3",
29
-]
30
-
31
-EMBEDDING_PROMPTS = [
32
- "Follow the white rabbit.", # English
33
- "Sigue al conejo blanco.", # Spanish
34
- "Suis le lapin blanc.", # French
35
- "跟着白兔走。", # Chinese
36
- "اتبع الأرنب الأبيض.", # Arabic
37
- "Folge dem weißen Kaninchen.", # German
32
+ EmbedModelInfo("jinaai/jina-embeddings-v3",
33
+ architecture="XLMRobertaModel",
34
+ is_matryoshka=True,
35
+ dtype="float32")
36
]
37
38
@@ -80,73 +78,66 @@ def test_llm_1_to_N(vllm_runner, hf_runner, model_name, dtype: str):
78
assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
79
80
83
-@pytest.fixture(scope="module", params=EMBEDDING_MODELS)
84
-def emb_model_name(request):
85
- yield request.param
81
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
82
+def test_embed_models_mteb(hf_runner, vllm_runner,
83
+ model_info: EmbedModelInfo) -> None:
84
85
+ def hf_model_callback(model):
86
+ model.encode = partial(model.encode, task="text-matching")
87
88
-def test_is_matryoshka(vllm_runner, emb_model_name):
89
- with vllm_runner(emb_model_name, task="embed",
90
- max_model_len=None) as vllm_model:
91
- assert vllm_model.model.llm_engine.model_config.is_matryoshka
92
-
93
-
94
-@pytest.mark.parametrize("model", EMBEDDING_MODELS)
95
-@pytest.mark.parametrize("dtype", ["half"])
96
-def test_embeddings(
97
- hf_runner,
98
- vllm_runner,
99
- model,
100
- dtype: str,
101
- monkeypatch,
102
-) -> None:
88
+ mteb_test_embed_models(hf_runner,
89
+ vllm_runner,
90
+ model_info,
91
+ hf_model_callback=hf_model_callback)
92
104
- example_prompts = EMBEDDING_PROMPTS
93
106
- with hf_runner(
107
- model,
108
- dtype=dtype,
109
- is_sentence_transformer=True,
110
- ) as hf_model:
111
- hf_outputs = hf_model.encode(example_prompts, task="text-matching")
94
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
95
+def test_embed_models_correctness(hf_runner, vllm_runner,
96
+ model_info: EmbedModelInfo,
97
+ example_prompts) -> None:
98
113
- with vllm_runner(model, task="embed", dtype=dtype,
114
- max_model_len=None) as vllm_model:
115
- vllm_outputs = vllm_model.encode(example_prompts)
99
+ def hf_model_callback(model):
100
+ model.encode = partial(model.encode, task="text-matching")
101
117
- check_embeddings_close(
118
- embeddings_0_lst=hf_outputs,
119
- embeddings_1_lst=vllm_outputs,
120
- name_0="hf",
121
- name_1="vllm",
122
- tol=1e-2,
123
- )
102
+ correctness_test_embed_models(hf_runner,
103
+ vllm_runner,
104
+ model_info,
105
+ example_prompts,
106
+ hf_model_callback=hf_model_callback)
107
108
126
-@pytest.mark.parametrize("model", EMBEDDING_MODELS)
109
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
110
@pytest.mark.parametrize("dtype", ["half"])
111
@pytest.mark.parametrize("dimensions", [16, 32])
112
def test_matryoshka(
113
hf_runner,
114
vllm_runner,
132
- model,
115
+ model_info,
116
dtype: str,
117
dimensions: int,
118
+ example_prompts,
119
monkeypatch,
120
) -> None:
121
+ if not model_info.is_matryoshka:
122
+ pytest.skip("Model is not matryoshka")
123
138
- example_prompts = EMBEDDING_PROMPTS
124
+ # ST will strip the input texts, see test_embedding.py
125
+ example_prompts = [str(s).strip() for s in example_prompts]
126
127
with hf_runner(
141
- model,
128
+ model_info.name,
129
dtype=dtype,
130
is_sentence_transformer=True,
131
) as hf_model:
132
hf_outputs = hf_model.encode(example_prompts, task="text-matching")
133
hf_outputs = matryoshka_fy(hf_outputs, dimensions)
134
148
- with vllm_runner(model, task="embed", dtype=dtype,
135
+ with vllm_runner(model_info.name,
136
+ task="embed",
137
+ dtype=dtype,
138
max_model_len=None) as vllm_model:
139
+ assert vllm_model.model.llm_engine.model_config.is_matryoshka
140
+
141
matryoshka_dimensions = (
142
vllm_model.model.llm_engine.model_config.matryoshka_dimensions)
143
assert matryoshka_dimensions is not None
tests/models/language/pooling/test_nomic.py
+12
-24
@@ -2,7 +2,8 @@
2
3
import pytest
4
5
-from ...utils import EmbedModelInfo, run_embedding_correctness_test
5
+from .embed_utils import EmbedModelInfo, correctness_test_embed_models
6
+from .mteb_utils import mteb_test_embed_models
7
8
MODELS = [
9
EmbedModelInfo("nomic-ai/nomic-embed-text-v1",
@@ -13,6 +14,9 @@ MODELS = [
14
architecture="NomicBertModel",
15
dtype="float32",
16
enable_test=False),
17
+ EmbedModelInfo("nomic-ai/CodeRankEmbed",
18
+ architecture="NomicBertModel",
19
+ enable_test=False),
20
EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe",
21
architecture="NomicBertModel",
22
dtype="float32",
@@ -21,30 +25,14 @@ MODELS = [
25
26
27
@pytest.mark.parametrize("model_info", MODELS)
24
-def test_models_mteb(hf_runner, vllm_runner,
25
- model_info: EmbedModelInfo) -> None:
26
- from .mteb_utils import mteb_test_embed_models
28
+def test_embed_models_mteb(hf_runner, vllm_runner,
29
+ model_info: EmbedModelInfo) -> None:
30
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
31
32
33
@pytest.mark.parametrize("model_info", MODELS)
31
-def test_models_correctness(hf_runner, vllm_runner, model_info: EmbedModelInfo,
32
- example_prompts) -> None:
33
- if not model_info.enable_test:
34
- pytest.skip("Skipping test.")
35
-
36
- # ST will strip the input texts, see test_embedding.py
37
- example_prompts = [str(s).strip() for s in example_prompts]
38
-
39
- with vllm_runner(model_info.name,
40
- task="embed",
41
- dtype=model_info.dtype,
42
- max_model_len=None) as vllm_model:
43
- vllm_outputs = vllm_model.encode(example_prompts)
44
-
45
- with hf_runner(
46
- model_info.name,
47
- dtype=model_info.dtype,
48
- is_sentence_transformer=True,
49
- ) as hf_model:
50
- run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
34
+def test_embed_models_correctness(hf_runner, vllm_runner,
35
+ model_info: EmbedModelInfo,
36
+ example_prompts) -> None:
37
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info,
38
+ example_prompts)
tests/models/language/pooling/test_snowflake_arctic_embed.py
+9
-31
@@ -2,7 +2,8 @@
2
3
import pytest
4
5
-from ...utils import EmbedModelInfo, run_embedding_correctness_test
5
+from .embed_utils import EmbedModelInfo, correctness_test_embed_models
6
+from .mteb_utils import mteb_test_embed_models
7
8
MODELS = [
9
EmbedModelInfo("Snowflake/snowflake-arctic-embed-xs",
@@ -41,37 +42,14 @@ MODELS = [
42
43
44
@pytest.mark.parametrize("model_info", MODELS)
44
-def test_models_mteb(
45
- hf_runner,
46
- vllm_runner,
47
- model_info: EmbedModelInfo,
48
-) -> None:
49
- from .mteb_utils import mteb_test_embed_models
45
+def test_embed_models_mteb(hf_runner, vllm_runner,
46
+ model_info: EmbedModelInfo) -> None:
47
mteb_test_embed_models(hf_runner, vllm_runner, model_info)
48
49
50
@pytest.mark.parametrize("model_info", MODELS)
54
-def test_models_correctness(
55
- hf_runner,
56
- vllm_runner,
57
- model_info: EmbedModelInfo,
58
- example_prompts,
59
-) -> None:
60
- if not model_info.enable_test:
61
- pytest.skip("Skipping test.")
62
-
63
- # ST will strip the input texts, see test_embedding.py
64
- example_prompts = [str(s).strip() for s in example_prompts]
65
-
66
- with vllm_runner(model_info.name,
67
- task="embed",
68
- dtype=model_info.dtype,
69
- max_model_len=None) as vllm_model:
70
- vllm_outputs = vllm_model.encode(example_prompts)
71
-
72
- with hf_runner(
73
- model_info.name,
74
- dtype=model_info.dtype,
75
- is_sentence_transformer=True,
76
- ) as hf_model:
77
- run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
51
+def test_embed_models_correctness(hf_runner, vllm_runner,
52
+ model_info: EmbedModelInfo,
53
+ example_prompts) -> None:
54
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info,
55
+ example_prompts)
tests/models/registry.py
+1
-1
@@ -283,7 +283,7 @@ _EMBEDDING_EXAMPLE_MODELS = {
283
"MistralModel": _HfExamplesInfo("intfloat/e5-mistral-7b-instruct"),
284
"ModernBertModel": _HfExamplesInfo("Alibaba-NLP/gte-modernbert-base",
285
trust_remote_code=True),
286
- "NomicBertModel": _HfExamplesInfo("Snowflake/snowflake-arctic-embed-m-long", # noqa: E501
286
+ "NomicBertModel": _HfExamplesInfo("nomic-ai/nomic-embed-text-v2-moe",
287
trust_remote_code=True),
288
"Qwen2Model": _HfExamplesInfo("ssmits/Qwen2-7B-Instruct-embed-base"),
289
"Qwen2ForRewardModel": _HfExamplesInfo("Qwen/Qwen2.5-Math-RM-72B"),
tests/models/utils.py
+1
-23
@@ -2,7 +2,7 @@
2
3
import warnings
4
from collections.abc import Sequence
5
-from typing import TYPE_CHECKING, Any, NamedTuple, Optional, Union
5
+from typing import Any, NamedTuple, Optional, Union
6
7
import torch
8
import torch.nn.functional as F
@@ -13,9 +13,6 @@ from vllm.sequence import Logprob, PromptLogprobs, SampleLogprobs
13
14
from .registry import HF_EXAMPLE_MODELS
15
16
-if TYPE_CHECKING:
17
- from ..conftest import HfRunner
18
-
16
TokensText = tuple[list[int], str]
17
18
@@ -337,22 +334,3 @@ class EmbedModelInfo(NamedTuple):
334
architecture: str = ""
335
dtype: str = "auto"
336
enable_test: bool = True
340
-
341
-
342
-def run_embedding_correctness_test(
343
- hf_model: "HfRunner",
344
- inputs: list[str],
345
- vllm_outputs: Sequence[list[float]],
346
- dimensions: Optional[int] = None,
347
-):
348
- hf_outputs = hf_model.encode(inputs)
349
- if dimensions:
350
- hf_outputs = matryoshka_fy(hf_outputs, dimensions)
351
-
352
- check_embeddings_close(
353
- embeddings_0_lst=hf_outputs,
354
- embeddings_1_lst=vllm_outputs,
355
- name_0="hf",
356
- name_1="vllm",
357
- tol=1e-2,
358
- )
vllm/config.py
+12
-14
@@ -572,13 +572,7 @@ class ModelConfig:
572
sliding_window = None
573
574
self.original_max_model_len = self.max_model_len
575
- self.max_model_len = _get_and_verify_max_len(
576
- hf_config=self.hf_text_config,
577
- max_model_len=self.max_model_len,
578
- disable_sliding_window=self.disable_sliding_window,
579
- sliding_window_len=self.get_hf_config_sliding_window(),
580
- spec_target_max_model_len=self.spec_target_max_model_len,
581
- encoder_config=self.encoder_config)
575
+ self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
576
self.served_model_name = get_served_model_name(self.model,
577
self.served_model_name)
578
self.multimodal_config = self._init_multimodal_config()
@@ -1382,6 +1376,16 @@ class ModelConfig:
1376
def matryoshka_dimensions(self):
1377
return getattr(self.hf_config, "matryoshka_dimensions", None)
1378
1379
+ def get_and_verify_max_len(self, max_model_len: int):
1380
+ max_model_len = _get_and_verify_max_len(
1381
+ hf_config=self.hf_text_config,
1382
+ max_model_len=max_model_len,
1383
+ disable_sliding_window=self.disable_sliding_window,
1384
+ sliding_window_len=self.get_hf_config_sliding_window(),
1385
+ spec_target_max_model_len=self.spec_target_max_model_len,
1386
+ encoder_config=self.encoder_config)
1387
+ return max_model_len
1388
+
1389
1390
BlockSize = Literal[1, 8, 16, 32, 64, 128]
1391
CacheDType = Literal["auto", "fp8", "fp8_e4m3", "fp8_e5m2"]
@@ -4469,13 +4473,7 @@ class VllmConfig:
4473
4474
def recalculate_max_model_len(self, max_model_len: int):
4475
model_config = self.model_config
4472
- max_model_len = _get_and_verify_max_len(
4473
- hf_config=model_config.hf_text_config,
4474
- max_model_len=max_model_len,
4475
- disable_sliding_window=model_config.disable_sliding_window,
4476
- sliding_window_len=model_config.get_hf_config_sliding_window(),
4477
- spec_target_max_model_len=model_config.spec_target_max_model_len,
4478
- encoder_config=model_config.encoder_config)
4476
+ max_model_len = model_config.get_and_verify_max_len(max_model_len)
4477
self.model_config.max_model_len = max_model_len
4478
self.scheduler_config.max_model_len = max_model_len
4479
self.compute_hash()