@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dcdc145893
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 5, 23:30 UTC

[CI] Reorganize scoring tests (#38207)

Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>

wang.yuqi committed Mar 26, 2026 at 20:07 UTC dcdc1458931a1a57cbcadb92b2dfc10d225d675f
20 files changed +1595 -975
tests/entrypoints/pooling/basic/test_truncation.py
+1 -1
@@ -85,7 +85,7 @@ async def test_bigger_truncation_size(client: openai.AsyncOpenAI):
85 expected_message = (
86 "truncate_prompt_tokens value is "
87 "greater than max_model_len."
88 - " Please, select a smaller truncation size."
88 + " Please request a smaller truncation size."
89 )
90 assert error_details["message"] == expected_message
91
tests/entrypoints/pooling/embed/test_online.py
+1 -1
@@ -288,7 +288,7 @@ async def test_truncate_prompt_tokens(client: openai.AsyncOpenAI, model_name: st
288 assert "error" in response.object
289 assert (
290 "truncate_prompt_tokens value is greater than max_model_len. "
291 - "Please, select a smaller truncation size." in response.message
291 + "Please request a smaller truncation size." in response.message
292 )
293
294
tests/entrypoints/pooling/score/test_offline.py deleted
-69
@@ -1,69 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import weakref
5 -
6 -import pytest
7 -import torch
8 -
9 -from tests.models.utils import softmax
10 -from vllm import LLM, PoolingParams
11 -from vllm.distributed import cleanup_dist_env_and_memory
12 -from vllm.platforms import current_platform
13 -
14 -MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
15 -
16 -
17 -@pytest.fixture(scope="module")
18 -def llm():
19 - # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend
20 - # that supports encoder-only models on ROCm.
21 - attention_config = None
22 - if current_platform.is_rocm():
23 - attention_config = {"backend": "FLEX_ATTENTION"}
24 -
25 - # pytest caches the fixture so we use weakref.proxy to
26 - # enable garbage collection
27 - llm = LLM(
28 - model=MODEL_NAME,
29 - max_num_batched_tokens=32768,
30 - tensor_parallel_size=1,
31 - gpu_memory_utilization=0.75,
32 - enforce_eager=True,
33 - seed=0,
34 - attention_config=attention_config,
35 - )
36 -
37 - yield weakref.proxy(llm)
38 -
39 - del llm
40 -
41 - cleanup_dist_env_and_memory()
42 -
43 -
44 -def test_pooling_params(llm: LLM):
45 - def get_outputs(use_activation):
46 - queries = "What is the capital of France?"
47 - documents = "The capital of France is Paris."
48 -
49 - outputs = llm.score(
50 - queries,
51 - documents,
52 - pooling_params=PoolingParams(use_activation=use_activation),
53 - use_tqdm=False,
54 - )
55 - return torch.tensor([x.outputs.score for x in outputs])
56 -
57 - default = get_outputs(use_activation=None)
58 - w_activation = get_outputs(use_activation=True)
59 - wo_activation = get_outputs(use_activation=False)
60 -
61 - assert torch.allclose(default, w_activation, atol=1e-2), (
62 - "Default should use activation."
63 - )
64 - assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
65 - "wo_activation should not use activation."
66 - )
67 - assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
68 - "w_activation should be close to activation(wo_activation)."
69 - )
tests/entrypoints/pooling/score/test_online_colbert.py deleted
-142
@@ -1,142 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -"""Online API tests for ColBERT late interaction scoring."""
4 -
5 -import pytest
6 -import requests
7 -
8 -from tests.utils import RemoteOpenAIServer
9 -from vllm.entrypoints.pooling.score.protocol import RerankResponse, ScoreResponse
10 -
11 -MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
12 -COLBERT_DIM = 96
13 -MAX_MODEL_LEN = 512
14 -
15 -
16 -@pytest.fixture(scope="module")
17 -def server():
18 - args = [
19 - "--max-model-len",
20 - str(MAX_MODEL_LEN),
21 - ]
22 -
23 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
24 - yield remote_server
25 -
26 -
27 -class TestColBERTOnline:
28 - def test_rerank(self, server: RemoteOpenAIServer):
29 - """Test ColBERT rerank endpoint."""
30 - query = "What is the capital of France?"
31 - documents = [
32 - "The capital of Brazil is Brasilia.",
33 - "The capital of France is Paris.",
34 - ]
35 -
36 - rerank_response = requests.post(
37 - server.url_for("rerank"),
38 - json={
39 - "model": MODEL_NAME,
40 - "query": query,
41 - "documents": documents,
42 - },
43 - )
44 - rerank_response.raise_for_status()
45 - rerank = RerankResponse.model_validate(rerank_response.json())
46 -
47 - assert rerank.id is not None
48 - assert rerank.results is not None
49 - assert len(rerank.results) == 2
50 -
51 - paris_result = next(r for r in rerank.results if r.index == 1)
52 - brazil_result = next(r for r in rerank.results if r.index == 0)
53 -
54 - assert paris_result.relevance_score > brazil_result.relevance_score
55 -
56 - def test_rerank_top_n(self, server: RemoteOpenAIServer):
57 - """Test ColBERT rerank with top_n parameter."""
58 - query = "What is the capital of France?"
59 - documents = [
60 - "The capital of Brazil is Brasilia.",
61 - "The capital of France is Paris.",
62 - "Machine learning is a field of AI.",
63 - ]
64 -
65 - rerank_response = requests.post(
66 - server.url_for("rerank"),
67 - json={
68 - "model": MODEL_NAME,
69 - "query": query,
70 - "documents": documents,
71 - "top_n": 2,
72 - },
73 - )
74 - rerank_response.raise_for_status()
75 - rerank = RerankResponse.model_validate(rerank_response.json())
76 -
77 - assert len(rerank.results) == 2
78 - assert rerank.results[0].index == 1
79 -
80 - def test_score(self, server: RemoteOpenAIServer):
81 - """Test ColBERT score endpoint."""
82 - text_1 = "What is the capital of France?"
83 - text_2 = ["The capital of France is Paris.", "Python is a language."]
84 -
85 - score_response = requests.post(
86 - server.url_for("score"),
87 - json={
88 - "model": MODEL_NAME,
89 - "text_1": text_1,
90 - "text_2": text_2,
91 - },
92 - )
93 - score_response.raise_for_status()
94 - score = ScoreResponse.model_validate(score_response.json())
95 -
96 - assert score.id is not None
97 - assert score.data is not None
98 - assert len(score.data) == 2
99 -
100 - assert score.data[0].score > score.data[1].score
101 -
102 - def test_token_embed(self, server: RemoteOpenAIServer):
103 - """Test ColBERT token_embed task via pooling endpoint."""
104 - text = "What is the capital of France?"
105 -
106 - pooling_response = requests.post(
107 - server.url_for("pooling"),
108 - json={
109 - "model": MODEL_NAME,
110 - "input": text,
111 - "task": "token_embed",
112 - },
113 - )
114 - pooling_response.raise_for_status()
115 - pooling = pooling_response.json()
116 -
117 - assert "data" in pooling
118 - assert len(pooling["data"]) == 1
119 -
120 - embeddings = pooling["data"][0]["data"]
121 - assert isinstance(embeddings, list)
122 - assert len(embeddings) > 0
123 - assert len(embeddings[0]) == COLBERT_DIM
124 -
125 - def test_embed_not_supported(self, server: RemoteOpenAIServer):
126 - """Test that ColBERT model does not support 'embed' task."""
127 - task = "embed"
128 - text = "What is the capital of France?"
129 -
130 - response = requests.post(
131 - server.url_for("pooling"),
132 - json={
133 - "model": MODEL_NAME,
134 - "input": text,
135 - "task": task,
136 - },
137 - )
138 -
139 - assert response.json()["error"]["type"] == "BadRequestError"
140 - assert response.json()["error"]["message"].startswith(
141 - f"Unsupported task: {task!r}"
142 - )
tests/entrypoints/pooling/score/test_online_rerank.py deleted
-240
@@ -1,240 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -
4 -import pytest
5 -import requests
6 -import torch
7 -import torch.nn.functional as F
8 -
9 -from tests.utils import RemoteOpenAIServer
10 -from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
11 -from vllm.entrypoints.pooling.score.protocol import RerankResponse
12 -from vllm.platforms import current_platform
13 -
14 -MODEL_NAME = "BAAI/bge-reranker-base"
15 -DTYPE = "bfloat16"
16 -input_text = "This product was excellent and exceeded my expectations"
17 -input_tokens = [0, 3293, 12996, 509, 40881, 136, 204839, 297, 759, 202702, 2]
18 -
19 -
20 -@pytest.fixture(scope="module")
21 -def server():
22 - args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
23 -
24 - # ROCm: Use Flex Attention to support encoder-only self-attention.
25 - if current_platform.is_rocm():
26 - args.extend(["--attention-backend", "FLEX_ATTENTION"])
27 -
28 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
29 - yield remote_server
30 -
31 -
32 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
33 -def test_basic(server: RemoteOpenAIServer, model_name: str):
34 - # test /v1/models
35 - response = requests.get(server.url_for("/v1/models"))
36 - served_model = response.json()["data"][0]["id"]
37 - assert served_model == MODEL_NAME
38 -
39 - # test /tokenize
40 - response = requests.post(
41 - server.url_for("/tokenize"),
42 - json={"model": model_name, "prompt": input_text},
43 - )
44 - assert response.json()["tokens"] == input_tokens
45 -
46 -
47 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
48 -def test_rerank_texts(server: RemoteOpenAIServer, model_name: str):
49 - query = "What is the capital of France?"
50 - documents = [
51 - "The capital of Brazil is Brasilia.",
52 - "The capital of France is Paris.",
53 - ]
54 -
55 - rerank_response = requests.post(
56 - server.url_for("rerank"),
57 - json={
58 - "model": model_name,
59 - "query": query,
60 - "documents": documents,
61 - },
62 - )
63 - rerank_response.raise_for_status()
64 - rerank = RerankResponse.model_validate(rerank_response.json())
65 -
66 - assert rerank.id is not None
67 - assert rerank.results is not None
68 - assert len(rerank.results) == 2
69 - assert rerank.results[0].relevance_score >= 0.9
70 - assert rerank.results[1].relevance_score <= 0.01
71 -
72 -
73 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
74 -def test_top_n(server: RemoteOpenAIServer, model_name: str):
75 - query = "What is the capital of France?"
76 - documents = [
77 - "The capital of Brazil is Brasilia.",
78 - "The capital of France is Paris.",
79 - "Cross-encoder models are neat",
80 - ]
81 -
82 - rerank_response = requests.post(
83 - server.url_for("rerank"),
84 - json={"model": model_name, "query": query, "documents": documents, "top_n": 2},
85 - )
86 - rerank_response.raise_for_status()
87 - rerank = RerankResponse.model_validate(rerank_response.json())
88 -
89 - assert rerank.id is not None
90 - assert rerank.results is not None
91 - assert len(rerank.results) == 2
92 - assert rerank.results[0].relevance_score >= 0.9
93 - assert rerank.results[1].relevance_score <= 0.01
94 -
95 -
96 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
97 -def test_rerank_max_model_len(server: RemoteOpenAIServer, model_name: str):
98 - query = "What is the capital of France?" * 100
99 - documents = [
100 - "The capital of Brazil is Brasilia.",
101 - "The capital of France is Paris.",
102 - ]
103 -
104 - rerank_response = requests.post(
105 - server.url_for("rerank"),
106 - json={"model": model_name, "query": query, "documents": documents},
107 - )
108 - assert rerank_response.status_code == 400
109 - # Assert just a small fragments of the response
110 - assert "Please reduce the length of the input." in rerank_response.text
111 -
112 -
113 -def test_invocations(server: RemoteOpenAIServer):
114 - query = "What is the capital of France?"
115 - documents = [
116 - "The capital of Brazil is Brasilia.",
117 - "The capital of France is Paris.",
118 - ]
119 -
120 - request_args = {
121 - "model": MODEL_NAME,
122 - "query": query,
123 - "documents": documents,
124 - }
125 -
126 - rerank_response = requests.post(server.url_for("rerank"), json=request_args)
127 - rerank_response.raise_for_status()
128 -
129 - invocation_response = requests.post(
130 - server.url_for("invocations"), json=request_args
131 - )
132 - invocation_response.raise_for_status()
133 -
134 - rerank_output = rerank_response.json()
135 - invocation_output = invocation_response.json()
136 -
137 - assert rerank_output.keys() == invocation_output.keys()
138 - for rerank_result, invocations_result in zip(
139 - rerank_output["results"], invocation_output["results"]
140 - ):
141 - assert rerank_result.keys() == invocations_result.keys()
142 - assert rerank_result["relevance_score"] == pytest.approx(
143 - invocations_result["relevance_score"], rel=0.05
144 - )
145 - # TODO: reset this tolerance to 0.01 once we find
146 - # an alternative to flash_attn with bfloat16
147 -
148 -
149 -@pytest.mark.asyncio
150 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
151 -async def test_use_activation(server: RemoteOpenAIServer, model_name: str):
152 - async def get_outputs(use_activation):
153 - query = "What is the capital of France?"
154 - documents = [
155 - "The capital of Brazil is Brasilia.",
156 - "The capital of France is Paris.",
157 - ]
158 -
159 - response = requests.post(
160 - server.url_for("rerank"),
161 - json={
162 - "model": model_name,
163 - "query": query,
164 - "documents": documents,
165 - "use_activation": use_activation,
166 - },
167 - )
168 - outputs = response.json()
169 -
170 - return torch.tensor([x["relevance_score"] for x in outputs["results"]])
171 -
172 - default = await get_outputs(use_activation=None)
173 - w_activation = await get_outputs(use_activation=True)
174 - wo_activation = await get_outputs(use_activation=False)
175 -
176 - assert torch.allclose(default, w_activation, atol=1e-2), (
177 - "Default should use activation."
178 - )
179 - assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
180 - "wo_activation should not use activation."
181 - )
182 - assert torch.allclose(F.sigmoid(wo_activation), w_activation, atol=1e-2), (
183 - "w_activation should be close to activation(wo_activation)."
184 - )
185 -
186 -
187 -@pytest.mark.asyncio
188 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
189 -async def test_pooling_classify(server: RemoteOpenAIServer, model_name: str):
190 - response = requests.post(
191 - server.url_for("pooling"),
192 - json={
193 - "model": model_name,
194 - "input": input_text,
195 - "encoding_format": "float",
196 - "task": "classify",
197 - },
198 - )
199 - poolings = PoolingResponse.model_validate(response.json())
200 - assert len(poolings.data) == 1
201 - assert len(poolings.data[0].data) == 1
202 -
203 -
204 -@pytest.mark.asyncio
205 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
206 -async def test_pooling_token_classify(server: RemoteOpenAIServer, model_name: str):
207 - response = requests.post(
208 - server.url_for("pooling"),
209 - json={
210 - "model": model_name,
211 - "task": "token_classify",
212 - "input": input_text,
213 - "encoding_format": "float",
214 - },
215 - )
216 -
217 - poolings = PoolingResponse.model_validate(response.json())
218 -
219 - assert len(poolings.data) == 1
220 - assert len(poolings.data[0].data) == len(input_tokens)
221 - assert len(poolings.data[0].data[0]) == 1
222 -
223 -
224 -@pytest.mark.asyncio
225 -@pytest.mark.parametrize("model_name", [MODEL_NAME])
226 -@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
227 -async def test_pooling_not_supported(
228 - server: RemoteOpenAIServer, model_name: str, task: str
229 -):
230 - response = requests.post(
231 - server.url_for("pooling"),
232 - json={
233 - "model": model_name,
234 - "input": input_text,
235 - "encoding_format": "float",
236 - "task": task,
237 - },
238 - )
239 - assert response.json()["error"]["type"] == "BadRequestError"
240 - assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
tests/entrypoints/pooling/score/test_online_score.py deleted
-342
@@ -1,342 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -from typing import Any
4 -
5 -import pytest
6 -import requests
7 -import torch
8 -import torch.nn.functional as F
9 -from torch import tensor
10 -
11 -from tests.utils import RemoteOpenAIServer
12 -from vllm.entrypoints.pooling.score.protocol import ScoreResponse
13 -from vllm.platforms import current_platform
14 -
15 -MODELS = [
16 - {"name": "BAAI/bge-reranker-v2-m3", "is_cross_encoder": True},
17 - {"name": "BAAI/bge-base-en-v1.5", "is_cross_encoder": False},
18 -]
19 -DTYPE = "half"
20 -
21 -
22 -def run_transformers(hf_model, model, text_pairs):
23 - if model["is_cross_encoder"]:
24 - return hf_model.predict(text_pairs).tolist()
25 - else:
26 - hf_embeddings = [hf_model.encode(text_pair) for text_pair in text_pairs]
27 - return [
28 - F.cosine_similarity(tensor(pair[0]), tensor(pair[1]), dim=0)
29 - for pair in hf_embeddings
30 - ]
31 -
32 -
33 -@pytest.fixture(scope="class", params=MODELS)
34 -def model(request):
35 - yield request.param
36 -
37 -
38 -@pytest.fixture(scope="class")
39 -def server(model: dict[str, Any]):
40 - args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
41 -
42 - # ROCm: Use Flex Attention to support encoder-only self-attention.
43 - if current_platform.is_rocm():
44 - args.extend(["--attention-backend", "FLEX_ATTENTION"])
45 -
46 - with RemoteOpenAIServer(model["name"], args) as remote_server:
47 - yield remote_server
48 -
49 -
50 -@pytest.fixture(scope="class")
51 -def runner(model: dict[str, Any], hf_runner):
52 - kwargs = {
53 - "dtype": DTYPE,
54 - "is_cross_encoder"
55 - if model["is_cross_encoder"]
56 - else "is_sentence_transformer": True,
57 - }
58 -
59 - with hf_runner(model["name"], **kwargs) as hf_model:
60 - yield hf_model
61 -
62 -
63 -class TestModel:
64 - def test_queries_str_documents_str(
65 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
66 - ):
67 - queries = "What is the capital of France?"
68 - documents = "The capital of France is Paris."
69 -
70 - score_response = requests.post(
71 - server.url_for("score"),
72 - json={
73 - "model": model["name"],
74 - "queries": queries,
75 - "documents": documents,
76 - },
77 - )
78 - score_response.raise_for_status()
79 - score = ScoreResponse.model_validate(score_response.json())
80 -
81 - assert score.id is not None
82 - assert score.data is not None
83 - assert len(score.data) == 1
84 -
85 - vllm_outputs = [d.score for d in score.data]
86 -
87 - text_pairs = [[queries, documents]]
88 - hf_outputs = run_transformers(runner, model, text_pairs)
89 -
90 - for i in range(len(vllm_outputs)):
91 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
92 -
93 - def test_queries_str_items_str(
94 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
95 - ):
96 - queries = "What is the capital of France?"
97 - items = "The capital of France is Paris."
98 -
99 - score_response = requests.post(
100 - server.url_for("score"),
101 - json={
102 - "model": model["name"],
103 - "queries": queries,
104 - "items": items,
105 - },
106 - )
107 - score_response.raise_for_status()
108 - score = ScoreResponse.model_validate(score_response.json())
109 -
110 - assert score.id is not None
111 - assert score.data is not None
112 - assert len(score.data) == 1
113 -
114 - vllm_outputs = [d.score for d in score.data]
115 -
116 - text_pairs = [[queries, items]]
117 - hf_outputs = run_transformers(runner, model, text_pairs)
118 -
119 - for i in range(len(vllm_outputs)):
120 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
121 -
122 - def test_text_1_str_text_2_str(
123 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
124 - ):
125 - text_1 = "What is the capital of France?"
126 - text_2 = "The capital of France is Paris."
127 -
128 - score_response = requests.post(
129 - server.url_for("score"),
130 - json={
131 - "model": model["name"],
132 - "text_1": text_1,
133 - "text_2": text_2,
134 - },
135 - )
136 - score_response.raise_for_status()
137 - score = ScoreResponse.model_validate(score_response.json())
138 -
139 - assert score.id is not None
140 - assert score.data is not None
141 - assert len(score.data) == 1
142 -
143 - vllm_outputs = [d.score for d in score.data]
144 -
145 - text_pairs = [[text_1, text_2]]
146 - hf_outputs = run_transformers(runner, model, text_pairs)
147 -
148 - for i in range(len(vllm_outputs)):
149 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
150 -
151 - def test_data_1_str_data_2_str(
152 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
153 - ):
154 - data_1 = "What is the capital of France?"
155 - data_2 = "The capital of France is Paris."
156 -
157 - score_response = requests.post(
158 - server.url_for("score"),
159 - json={
160 - "model": model["name"],
161 - "data_1": data_1,
162 - "data_2": data_2,
163 - },
164 - )
165 - score_response.raise_for_status()
166 - score = ScoreResponse.model_validate(score_response.json())
167 -
168 - assert score.id is not None
169 - assert score.data is not None
170 - assert len(score.data) == 1
171 -
172 - vllm_outputs = [d.score for d in score.data]
173 -
174 - text_pairs = [[data_1, data_2]]
175 - hf_outputs = run_transformers(runner, model, text_pairs)
176 -
177 - for i in range(len(vllm_outputs)):
178 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
179 -
180 - def test_queries_str_documents_list(
181 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
182 - ):
183 - queries = "What is the capital of France?"
184 - documents = [
185 - "The capital of Brazil is Brasilia.",
186 - "The capital of France is Paris.",
187 - ]
188 -
189 - score_response = requests.post(
190 - server.url_for("score"),
191 - json={
192 - "model": model["name"],
193 - "queries": queries,
194 - "documents": documents,
195 - },
196 - )
197 - score_response.raise_for_status()
198 - score = ScoreResponse.model_validate(score_response.json())
199 -
200 - assert score.id is not None
201 - assert score.data is not None
202 - assert len(score.data) == 2
203 -
204 - vllm_outputs = [d.score for d in score.data]
205 -
206 - text_pairs = [[queries, documents[0]], [queries, documents[1]]]
207 - hf_outputs = run_transformers(runner, model, text_pairs)
208 -
209 - for i in range(len(vllm_outputs)):
210 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
211 -
212 - def test_queries_list_documents_list(
213 - self, server: RemoteOpenAIServer, model: dict[str, Any], runner
214 - ):
215 - queries = [
216 - "What is the capital of the United States?",
217 - "What is the capital of France?",
218 - ]
219 - documents = [
220 - "The capital of Brazil is Brasilia.",
221 - "The capital of France is Paris.",
222 - ]
223 -
224 - score_response = requests.post(
225 - server.url_for("score"),
226 - json={
227 - "model": model["name"],
228 - "queries": queries,
229 - "documents": documents,
230 - },
231 - )
232 - score_response.raise_for_status()
233 - score = ScoreResponse.model_validate(score_response.json())
234 -
235 - assert score.id is not None
236 - assert score.data is not None
237 - assert len(score.data) == 2
238 -
239 - vllm_outputs = [d.score for d in score.data]
240 -
241 - text_pairs = [[queries[0], documents[0]], [queries[1], documents[1]]]
242 - hf_outputs = run_transformers(runner, model, text_pairs)
243 -
244 - for i in range(len(vllm_outputs)):
245 - assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
246 -
247 - def test_score_max_model_len(
248 - self, server: RemoteOpenAIServer, model: dict[str, Any]
249 - ):
250 - queries = "What is the capital of France?" * 20
251 - documents = [
252 - "The capital of Brazil is Brasilia.",
253 - "The capital of France is Paris.",
254 - ]
255 -
256 - score_response = requests.post(
257 - server.url_for("score"),
258 - json={
259 - "model": model["name"],
260 - "queries": queries,
261 - "documents": documents,
262 - },
263 - )
264 - assert score_response.status_code == 400
265 - # Assert just a small fragments of the response
266 - assert "Please reduce the length of the input." in score_response.text
267 -
268 - # Test truncation
269 - score_response = requests.post(
270 - server.url_for("score"),
271 - json={
272 - "model": model["name"],
273 - "queries": queries,
274 - "documents": documents,
275 - "truncate_prompt_tokens": 101,
276 - },
277 - )
278 - assert score_response.status_code == 400
279 - assert "Please request a smaller truncation size." in score_response.text
280 -
281 - def test_invocations(self, server: RemoteOpenAIServer, model: dict[str, Any]):
282 - queries = "What is the capital of France?"
283 - documents = "The capital of France is Paris."
284 -
285 - request_args = {
286 - "model": model["name"],
287 - "queries": queries,
288 - "documents": documents,
289 - }
290 -
291 - score_response = requests.post(server.url_for("score"), json=request_args)
292 - score_response.raise_for_status()
293 -
294 - invocation_response = requests.post(
295 - server.url_for("invocations"), json=request_args
296 - )
297 - invocation_response.raise_for_status()
298 -
299 - score_output = score_response.json()
300 - invocation_output = invocation_response.json()
301 -
302 - assert score_output.keys() == invocation_output.keys()
303 - for score_data, invocation_data in zip(
304 - score_output["data"], invocation_output["data"]
305 - ):
306 - assert score_data.keys() == invocation_data.keys()
307 - assert score_data["score"] == pytest.approx(
308 - invocation_data["score"], rel=0.05
309 - )
310 - # TODO: reset this tolerance to 0.01 once we find
311 - # an alternative to flash_attn with bfloat16
312 -
313 - def test_use_activation(self, server: RemoteOpenAIServer, model: dict[str, Any]):
314 - def get_outputs(use_activation):
315 - queries = "What is the capital of France?"
316 - documents = "The capital of France is Paris."
317 - response = requests.post(
318 - server.url_for("score"),
319 - json={
320 - "model": model["name"],
321 - "queries": queries,
322 - "documents": documents,
323 - "use_activation": use_activation,
324 - },
325 - )
326 - outputs = response.json()
327 - return torch.tensor([x["score"] for x in outputs["data"]])
328 -
329 - default = get_outputs(use_activation=None)
330 - w_activation = get_outputs(use_activation=True)
331 - wo_activation = get_outputs(use_activation=False)
332 -
333 - if model["is_cross_encoder"]:
334 - assert torch.allclose(default, w_activation, atol=1e-2), (
335 - "Default should use activation."
336 - )
337 - assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
338 - "wo_activation should not use activation."
339 - )
340 - assert torch.allclose(F.sigmoid(wo_activation), w_activation, atol=1e-2), (
341 - "w_activation should be close to activation(wo_activation)."
342 - )
tests/entrypoints/pooling/scoring/__init__.py renamed
tests/entrypoints/pooling/scoring/test_bi_encoder_offline.py new
+114
@@ -0,0 +1,114 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import weakref
5 +
6 +import pytest
7 +
8 +from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
9 +from vllm import LLM
10 +from vllm.distributed import cleanup_dist_env_and_memory
11 +from vllm.platforms import current_platform
12 +
13 +MODEL_NAME = "intfloat/multilingual-e5-small"
14 +PROMPT = "The chef prepared a delicious meal."
15 +EMBEDDING_SIZE = 384
16 +
17 +TEXTS_1 = [
18 + "What is the capital of France?",
19 + "What is the capital of Germany?",
20 +]
21 +
22 +TEXTS_2 = [
23 + "The capital of France is Paris.",
24 + "The capital of Germany is Berlin.",
25 +]
26 +
27 +DTYPE = "half"
28 +
29 +
30 +@pytest.fixture(scope="module")
31 +def llm():
32 + # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend
33 + # that supports encoder-only models on ROCm.
34 + attention_config = None
35 + if current_platform.is_rocm():
36 + attention_config = {"backend": "FLEX_ATTENTION"}
37 +
38 + # pytest caches the fixture so we use weakref.proxy to
39 + # enable garbage collection
40 + llm = LLM(
41 + model=MODEL_NAME,
42 + max_num_batched_tokens=32768,
43 + tensor_parallel_size=1,
44 + gpu_memory_utilization=0.75,
45 + enforce_eager=True,
46 + seed=0,
47 + attention_config=attention_config,
48 + )
49 +
50 + yield weakref.proxy(llm)
51 +
52 + del llm
53 +
54 + cleanup_dist_env_and_memory()
55 +
56 +
57 +@pytest.fixture(scope="module")
58 +def hf_model():
59 + return EncoderScoringHfRunner(MODEL_NAME)
60 +
61 +
62 +@pytest.mark.skip_global_cleanup
63 +def test_1_to_1(llm, hf_model):
64 + text_pair = [TEXTS_1[0], TEXTS_2[0]]
65 +
66 + hf_outputs = hf_model.predict([text_pair]).tolist()
67 + vllm_outputs = [
68 + output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
69 + ]
70 +
71 + assert len(vllm_outputs) == 1
72 + assert len(hf_outputs) == 1
73 +
74 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
75 +
76 +
77 +@pytest.mark.skip_global_cleanup
78 +def test_1_to_n(llm, hf_model):
79 + text_pairs = [
80 + [TEXTS_1[0], TEXTS_2[0]],
81 + [TEXTS_1[0], TEXTS_2[1]],
82 + ]
83 +
84 + hf_outputs = hf_model.predict(text_pairs).tolist()
85 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
86 +
87 + assert len(vllm_outputs) == 2
88 + assert len(hf_outputs) == 2
89 +
90 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
91 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
92 +
93 +
94 +@pytest.mark.skip_global_cleanup
95 +def test_n_to_n(llm, hf_model):
96 + text_pairs = [
97 + [TEXTS_1[0], TEXTS_2[0]],
98 + [TEXTS_1[1], TEXTS_2[1]],
99 + ]
100 +
101 + hf_outputs = hf_model.predict(text_pairs).tolist()
102 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
103 +
104 + assert len(vllm_outputs) == 2
105 + assert len(hf_outputs) == 2
106 +
107 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
108 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
109 +
110 +
111 +def test_embed(llm):
112 + outputs = llm.encode(PROMPT, pooling_task="embed", use_tqdm=False)
113 + assert len(outputs) == 1
114 + assert len(outputs[0].outputs.data) == EMBEDDING_SIZE
tests/entrypoints/pooling/scoring/test_bi_encoder_online.py new
+414
@@ -0,0 +1,414 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +import requests
6 +
7 +from tests.entrypoints.pooling.scoring.util import EncoderScoringHfRunner
8 +from tests.utils import RemoteOpenAIServer
9 +from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
10 +from vllm.entrypoints.pooling.score.protocol import RerankResponse, ScoreResponse
11 +from vllm.platforms import current_platform
12 +
13 +MODEL_NAME = "BAAI/bge-base-en-v1.5"
14 +input_text = "This product was excellent and exceeded my expectations"
15 +DTYPE = "half"
16 +EMBEDDING_SIZE = 768
17 +
18 +TEXTS_1 = [
19 + "What is the capital of France?",
20 + "What is the capital of Germany?",
21 +]
22 +
23 +TEXTS_2 = [
24 + "The capital of France is Paris.",
25 + "The capital of Germany is Berlin.",
26 +]
27 +
28 +
29 +@pytest.fixture(scope="module")
30 +def server():
31 + args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
32 +
33 + # ROCm: Use Flex Attention to support encoder-only self-attention.
34 + if current_platform.is_rocm():
35 + args.extend(["--attention-backend", "FLEX_ATTENTION"])
36 +
37 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
38 + yield remote_server
39 +
40 +
41 +@pytest.fixture(scope="module")
42 +def hf_model():
43 + return EncoderScoringHfRunner(MODEL_NAME)
44 +
45 +
46 +@pytest.mark.asyncio
47 +async def test_score_api_queries_str_1_documents_str_1(
48 + hf_model, server: RemoteOpenAIServer
49 +):
50 + score_response = requests.post(
51 + server.url_for("score"),
52 + json={
53 + "model": MODEL_NAME,
54 + "queries": TEXTS_1[0],
55 + "documents": TEXTS_2[0],
56 + },
57 + )
58 + score_response.raise_for_status()
59 + score = ScoreResponse.model_validate(score_response.json())
60 +
61 + assert score.id is not None
62 + assert score.data is not None
63 + assert len(score.data) == 1
64 +
65 + vllm_outputs = [d.score for d in score.data]
66 + hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
67 +
68 + for i in range(len(vllm_outputs)):
69 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
70 +
71 +
72 +@pytest.mark.asyncio
73 +async def test_score_api_queries_str_1_documents_str_n(
74 + hf_model, server: RemoteOpenAIServer
75 +):
76 + text_pairs = [
77 + [TEXTS_1[0], TEXTS_2[0]],
78 + [TEXTS_1[0], TEXTS_2[1]],
79 + ]
80 +
81 + score_response = requests.post(
82 + server.url_for("score"),
83 + json={
84 + "model": MODEL_NAME,
85 + "queries": TEXTS_1[0],
86 + "documents": TEXTS_2,
87 + },
88 + )
89 + score_response.raise_for_status()
90 + score = ScoreResponse.model_validate(score_response.json())
91 +
92 + assert score.id is not None
93 + assert score.data is not None
94 + assert len(score.data) == 2
95 +
96 + vllm_outputs = [d.score for d in score.data]
97 + hf_outputs = hf_model.predict(text_pairs).tolist()
98 +
99 + for i in range(len(vllm_outputs)):
100 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
101 +
102 +
103 +@pytest.mark.asyncio
104 +async def test_score_api_queries_str_n_documents_str_n(
105 + hf_model, server: RemoteOpenAIServer
106 +):
107 + text_pairs = [
108 + [TEXTS_1[0], TEXTS_2[0]],
109 + [TEXTS_1[1], TEXTS_2[1]],
110 + ]
111 +
112 + score_response = requests.post(
113 + server.url_for("score"),
114 + json={
115 + "model": MODEL_NAME,
116 + "queries": TEXTS_1,
117 + "documents": TEXTS_2,
118 + },
119 + )
120 + score_response.raise_for_status()
121 + score = ScoreResponse.model_validate(score_response.json())
122 +
123 + assert score.id is not None
124 + assert score.data is not None
125 + assert len(score.data) == 2
126 +
127 + vllm_outputs = [d.score for d in score.data]
128 + hf_outputs = hf_model.predict(text_pairs).tolist()
129 +
130 + for i in range(len(vllm_outputs)):
131 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
132 +
133 +
134 +@pytest.mark.asyncio
135 +async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer):
136 + text_pairs = [
137 + [TEXTS_1[0], TEXTS_2[0]],
138 + [TEXTS_1[1], TEXTS_2[1]],
139 + ]
140 +
141 + score_response = requests.post(
142 + server.url_for("score"),
143 + json={
144 + "model": MODEL_NAME,
145 + "queries": TEXTS_1,
146 + "documents": TEXTS_2,
147 + },
148 + )
149 + score_response.raise_for_status()
150 + score = ScoreResponse.model_validate(score_response.json())
151 +
152 + assert score.id is not None
153 + assert score.data is not None
154 + assert len(score.data) == 2
155 +
156 + vllm_outputs = [d.score for d in score.data]
157 + hf_outputs = hf_model.predict(text_pairs).tolist()
158 +
159 + for i in range(len(vllm_outputs)):
160 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
161 +
162 +
163 +@pytest.mark.asyncio
164 +async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
165 + text_pairs = [
166 + [TEXTS_1[0], TEXTS_2[0]],
167 + [TEXTS_1[1], TEXTS_2[1]],
168 + ]
169 +
170 + score_response = requests.post(
171 + server.url_for("score"),
172 + json={
173 + "model": MODEL_NAME,
174 + "queries": TEXTS_1,
175 + "items": TEXTS_2,
176 + },
177 + )
178 + score_response.raise_for_status()
179 + score = ScoreResponse.model_validate(score_response.json())
180 +
181 + assert score.id is not None
182 + assert score.data is not None
183 + assert len(score.data) == 2
184 +
185 + vllm_outputs = [d.score for d in score.data]
186 + hf_outputs = hf_model.predict(text_pairs).tolist()
187 +
188 + for i in range(len(vllm_outputs)):
189 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
190 +
191 +
192 +@pytest.mark.asyncio
193 +async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
194 + text_pairs = [
195 + [TEXTS_1[0], TEXTS_2[0]],
196 + [TEXTS_1[1], TEXTS_2[1]],
197 + ]
198 +
199 + score_response = requests.post(
200 + server.url_for("score"),
201 + json={
202 + "model": MODEL_NAME,
203 + "text_1": TEXTS_1,
204 + "text_2": TEXTS_2,
205 + },
206 + )
207 + score_response.raise_for_status()
208 + score = ScoreResponse.model_validate(score_response.json())
209 +
210 + assert score.id is not None
211 + assert score.data is not None
212 + assert len(score.data) == 2
213 +
214 + vllm_outputs = [d.score for d in score.data]
215 + hf_outputs = hf_model.predict(text_pairs).tolist()
216 +
217 + for i in range(len(vllm_outputs)):
218 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
219 +
220 +
221 +@pytest.mark.asyncio
222 +async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
223 + text_pairs = [
224 + [TEXTS_1[0], TEXTS_2[0]],
225 + [TEXTS_1[1], TEXTS_2[1]],
226 + ]
227 +
228 + score_response = requests.post(
229 + server.url_for("score"),
230 + json={
231 + "model": MODEL_NAME,
232 + "data_1": TEXTS_1,
233 + "data_2": TEXTS_2,
234 + },
235 + )
236 + score_response.raise_for_status()
237 + score = ScoreResponse.model_validate(score_response.json())
238 +
239 + assert score.id is not None
240 + assert score.data is not None
241 + assert len(score.data) == 2
242 +
243 + vllm_outputs = [d.score for d in score.data]
244 + hf_outputs = hf_model.predict(text_pairs).tolist()
245 +
246 + for i in range(len(vllm_outputs)):
247 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
248 +
249 +
250 +@pytest.mark.asyncio
251 +async def test_rerank_api_texts(server: RemoteOpenAIServer):
252 + query = "What is the capital of France?"
253 + documents = [
254 + "The capital of Brazil is Brasilia.",
255 + "The capital of France is Paris.",
256 + ]
257 +
258 + rerank_response = requests.post(
259 + server.url_for("rerank"),
260 + json={
261 + "model": MODEL_NAME,
262 + "query": query,
263 + "documents": documents,
264 + },
265 + )
266 + rerank_response.raise_for_status()
267 + rerank = RerankResponse.model_validate(rerank_response.json())
268 +
269 + assert rerank.id is not None
270 + assert rerank.results is not None
271 + assert len(rerank.results) == 2
272 + paris_result = next(r for r in rerank.results if r.index == 1)
273 + brazil_result = next(r for r in rerank.results if r.index == 0)
274 + assert paris_result.relevance_score > brazil_result.relevance_score
275 +
276 +
277 +@pytest.mark.asyncio
278 +async def test_rerank_api_top_n(server: RemoteOpenAIServer):
279 + query = "What is the capital of France?"
280 + documents = [
281 + "The capital of Brazil is Brasilia.",
282 + "The capital of France is Paris.",
283 + "Cross-encoder models are neat",
284 + ]
285 +
286 + rerank_response = requests.post(
287 + server.url_for("rerank"),
288 + json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
289 + )
290 + rerank_response.raise_for_status()
291 + rerank = RerankResponse.model_validate(rerank_response.json())
292 +
293 + assert rerank.id is not None
294 + assert rerank.results is not None
295 + assert len(rerank.results) == 2
296 + assert rerank.results[0].index == 1
297 +
298 +
299 +@pytest.mark.asyncio
300 +async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
301 + query = "What is the capital of France?" * 100
302 + documents = [
303 + "The capital of Brazil is Brasilia.",
304 + "The capital of France is Paris.",
305 + ]
306 +
307 + rerank_response = requests.post(
308 + server.url_for("rerank"),
309 + json={"model": MODEL_NAME, "query": query, "documents": documents},
310 + )
311 + assert rerank_response.status_code == 400
312 + # Assert just a small fragments of the response
313 + assert "Please reduce the length of the input prompt" in rerank_response.text
314 +
315 +
316 +@pytest.mark.asyncio
317 +async def test_score_api_max_model_len(server: RemoteOpenAIServer):
318 + queries = "What is the capital of France?" * 20
319 + documents = [
320 + "The capital of Brazil is Brasilia.",
321 + "The capital of France is Paris.",
322 + ]
323 +
324 + score_response = requests.post(
325 + server.url_for("score"),
326 + json={
327 + "model": MODEL_NAME,
328 + "queries": queries,
329 + "documents": documents,
330 + },
331 + )
332 + assert score_response.status_code == 400
333 + # Assert just a small fragments of the response
334 + assert "Please reduce the length of the input prompt" in score_response.text
335 +
336 + # Test truncation
337 + score_response = requests.post(
338 + server.url_for("score"),
339 + json={
340 + "model": MODEL_NAME,
341 + "queries": queries,
342 + "documents": documents,
343 + "truncate_prompt_tokens": 101,
344 + },
345 + )
346 + assert score_response.status_code == 400
347 + assert "Please request a smaller truncation size." in score_response.text
348 +
349 +
350 +@pytest.mark.asyncio
351 +async def test_invocations(server: RemoteOpenAIServer):
352 + query = "What is the capital of France?"
353 + documents = [
354 + "The capital of Brazil is Brasilia.",
355 + "The capital of France is Paris.",
356 + ]
357 +
358 + request_args = {
359 + "model": MODEL_NAME,
360 + "query": query,
361 + "documents": documents,
362 + }
363 +
364 + rerank_response = requests.post(server.url_for("rerank"), json=request_args)
365 + rerank_response.raise_for_status()
366 +
367 + invocation_response = requests.post(
368 + server.url_for("invocations"), json=request_args
369 + )
370 + invocation_response.raise_for_status()
371 +
372 + rerank_output = rerank_response.json()
373 + invocation_output = invocation_response.json()
374 +
375 + assert rerank_output.keys() == invocation_output.keys()
376 + for rerank_result, invocations_result in zip(
377 + rerank_output["results"], invocation_output["results"]
378 + ):
379 + assert rerank_result.keys() == invocations_result.keys()
380 + assert rerank_result["relevance_score"] == pytest.approx(
381 + invocations_result["relevance_score"], rel=0.01
382 + )
383 +
384 +
385 +@pytest.mark.asyncio
386 +async def test_pooling_embed(server: RemoteOpenAIServer):
387 + response = requests.post(
388 + server.url_for("pooling"),
389 + json={
390 + "model": MODEL_NAME,
391 + "input": input_text,
392 + "encoding_format": "float",
393 + "task": "embed",
394 + },
395 + )
396 + poolings = PoolingResponse.model_validate(response.json())
397 + assert len(poolings.data) == 1
398 + assert len(poolings.data[0].data) == EMBEDDING_SIZE
399 +
400 +
401 +@pytest.mark.asyncio
402 +@pytest.mark.parametrize("task", ["classify", "token_classify", "plugin"])
403 +async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
404 + response = requests.post(
405 + server.url_for("pooling"),
406 + json={
407 + "model": MODEL_NAME,
408 + "input": input_text,
409 + "encoding_format": "float",
410 + "task": task,
411 + },
412 + )
413 + assert response.json()["error"]["type"] == "BadRequestError"
414 + assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
tests/entrypoints/pooling/scoring/test_cross_encoder_correctness_mteb.py renamed
tests/entrypoints/pooling/scoring/test_cross_encoder_offline.py new
+137
@@ -0,0 +1,137 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import weakref
5 +
6 +import pytest
7 +import torch
8 +
9 +from tests.models.utils import softmax
10 +from vllm import LLM, PoolingParams
11 +from vllm.distributed import cleanup_dist_env_and_memory
12 +from vllm.platforms import current_platform
13 +
14 +MODEL_NAME = "tomaarsen/Qwen3-Reranker-0.6B-seq-cls"
15 +PROMPT = "The chef prepared a delicious meal."
16 +TEXTS_1 = [
17 + "What is the capital of France?",
18 + "What is the capital of Germany?",
19 +]
20 +
21 +TEXTS_2 = [
22 + "The capital of France is Paris.",
23 + "The capital of Germany is Berlin.",
24 +]
25 +
26 +
27 +@pytest.fixture(scope="module")
28 +def llm():
29 + # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend
30 + # that supports encoder-only models on ROCm.
31 + attention_config = None
32 + if current_platform.is_rocm():
33 + attention_config = {"backend": "FLEX_ATTENTION"}
34 +
35 + # pytest caches the fixture so we use weakref.proxy to
36 + # enable garbage collection
37 + llm = LLM(
38 + model=MODEL_NAME,
39 + max_num_batched_tokens=32768,
40 + tensor_parallel_size=1,
41 + gpu_memory_utilization=0.75,
42 + enforce_eager=True,
43 + seed=0,
44 + attention_config=attention_config,
45 + )
46 +
47 + yield weakref.proxy(llm)
48 +
49 + del llm
50 +
51 + cleanup_dist_env_and_memory()
52 +
53 +
54 +@pytest.fixture(scope="module")
55 +def hf_model(hf_runner):
56 + return hf_runner(MODEL_NAME, is_cross_encoder=True)
57 +
58 +
59 +@pytest.mark.skip_global_cleanup
60 +def test_1_to_1(llm, hf_model):
61 + text_pair = [TEXTS_1[0], TEXTS_2[0]]
62 +
63 + hf_outputs = hf_model.predict([text_pair]).tolist()
64 + vllm_outputs = [
65 + output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
66 + ]
67 +
68 + assert len(vllm_outputs) == 1
69 + assert len(hf_outputs) == 1
70 +
71 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
72 +
73 +
74 +@pytest.mark.skip_global_cleanup
75 +def test_1_to_n(llm, hf_model):
76 + text_pairs = [
77 + [TEXTS_1[0], TEXTS_2[0]],
78 + [TEXTS_1[0], TEXTS_2[1]],
79 + ]
80 +
81 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
82 + hf_outputs = hf_model.predict(text_pairs).tolist()
83 +
84 + assert len(vllm_outputs) == 2
85 + assert len(hf_outputs) == 2
86 +
87 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
88 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
89 +
90 +
91 +@pytest.mark.skip_global_cleanup
92 +def test_n_to_n(llm, hf_model):
93 + text_pairs = [
94 + [TEXTS_1[0], TEXTS_2[0]],
95 + [TEXTS_1[1], TEXTS_2[1]],
96 + ]
97 +
98 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
99 + hf_outputs = hf_model.predict(text_pairs).tolist()
100 +
101 + assert len(vllm_outputs) == 2
102 + assert len(hf_outputs) == 2
103 +
104 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
105 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
106 +
107 +
108 +@pytest.mark.skip_global_cleanup
109 +def test_classify(llm):
110 + outputs = llm.encode(PROMPT, pooling_task="classify", use_tqdm=False)
111 + assert len(outputs) == 1
112 + assert len(outputs[0].outputs.data) == 1
113 +
114 +
115 +def test_pooling_params(llm: LLM):
116 + def get_outputs(use_activation):
117 + outputs = llm.score(
118 + TEXTS_1[0],
119 + TEXTS_2[0],
120 + pooling_params=PoolingParams(use_activation=use_activation),
121 + use_tqdm=False,
122 + )
123 + return torch.tensor([x.outputs.score for x in outputs])
124 +
125 + default = get_outputs(use_activation=None)
126 + w_activation = get_outputs(use_activation=True)
127 + wo_activation = get_outputs(use_activation=False)
128 +
129 + assert torch.allclose(default, w_activation, atol=1e-2), (
130 + "Default should use activation."
131 + )
132 + assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
133 + "wo_activation should not use activation."
134 + )
135 + assert torch.allclose(softmax(wo_activation), w_activation, atol=1e-2), (
136 + "w_activation should be close to activation(wo_activation)."
137 + )
tests/entrypoints/pooling/scoring/test_cross_encoder_online.py new
+487
@@ -0,0 +1,487 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import pytest
5 +import requests
6 +import torch
7 +import torch.nn.functional as F
8 +
9 +from tests.utils import RemoteOpenAIServer
10 +from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
11 +from vllm.entrypoints.pooling.score.protocol import RerankResponse, ScoreResponse
12 +from vllm.platforms import current_platform
13 +
14 +MODEL_NAME = "BAAI/bge-reranker-base"
15 +DTYPE = "half"
16 +input_text = "This product was excellent and exceeded my expectations"
17 +input_tokens = [0, 3293, 12996, 509, 40881, 136, 204839, 297, 759, 202702, 2]
18 +
19 +
20 +TEXTS_1 = [
21 + "What is the capital of France?",
22 + "What is the capital of Germany?",
23 +]
24 +
25 +TEXTS_2 = [
26 + "The capital of France is Paris.",
27 + "The capital of Germany is Berlin.",
28 +]
29 +
30 +
31 +@pytest.fixture(scope="module")
32 +def server():
33 + args = ["--enforce-eager", "--max-model-len", "100", "--dtype", DTYPE]
34 +
35 + # ROCm: Use Flex Attention to support encoder-only self-attention.
36 + if current_platform.is_rocm():
37 + args.extend(["--attention-backend", "FLEX_ATTENTION"])
38 +
39 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
40 + yield remote_server
41 +
42 +
43 +@pytest.fixture(scope="module")
44 +def hf_model(hf_runner):
45 + return hf_runner(MODEL_NAME, is_cross_encoder=True)
46 +
47 +
48 +@pytest.mark.asyncio
49 +async def test_basic(server: RemoteOpenAIServer):
50 + # test /v1/models
51 + response = requests.get(server.url_for("/v1/models"))
52 + served_model = response.json()["data"][0]["id"]
53 + assert served_model == MODEL_NAME
54 +
55 + # test /tokenize
56 + response = requests.post(
57 + server.url_for("/tokenize"),
58 + json={"model": MODEL_NAME, "prompt": input_text},
59 + )
60 + assert response.json()["tokens"] == input_tokens
61 +
62 +
63 +@pytest.mark.asyncio
64 +async def test_score_api_queries_str_1_documents_str_1(
65 + hf_model, server: RemoteOpenAIServer
66 +):
67 + score_response = requests.post(
68 + server.url_for("score"),
69 + json={
70 + "model": MODEL_NAME,
71 + "queries": TEXTS_1[0],
72 + "documents": TEXTS_2[0],
73 + },
74 + )
75 + score_response.raise_for_status()
76 + score = ScoreResponse.model_validate(score_response.json())
77 +
78 + assert score.id is not None
79 + assert score.data is not None
80 + assert len(score.data) == 1
81 +
82 + vllm_outputs = [d.score for d in score.data]
83 + hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
84 +
85 + for i in range(len(vllm_outputs)):
86 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
87 +
88 +
89 +@pytest.mark.asyncio
90 +async def test_score_api_queries_str_1_documents_str_n(
91 + hf_model, server: RemoteOpenAIServer
92 +):
93 + text_pairs = [
94 + [TEXTS_1[0], TEXTS_2[0]],
95 + [TEXTS_1[0], TEXTS_2[1]],
96 + ]
97 +
98 + score_response = requests.post(
99 + server.url_for("score"),
100 + json={
101 + "model": MODEL_NAME,
102 + "queries": TEXTS_1[0],
103 + "documents": TEXTS_2,
104 + },
105 + )
106 + score_response.raise_for_status()
107 + score = ScoreResponse.model_validate(score_response.json())
108 +
109 + assert score.id is not None
110 + assert score.data is not None
111 + assert len(score.data) == 2
112 +
113 + vllm_outputs = [d.score for d in score.data]
114 + hf_outputs = hf_model.predict(text_pairs).tolist()
115 +
116 + for i in range(len(vllm_outputs)):
117 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
118 +
119 +
120 +@pytest.mark.asyncio
121 +async def test_score_api_queries_str_n_documents_str_n(
122 + hf_model, server: RemoteOpenAIServer
123 +):
124 + text_pairs = [
125 + [TEXTS_1[0], TEXTS_2[0]],
126 + [TEXTS_1[1], TEXTS_2[1]],
127 + ]
128 +
129 + score_response = requests.post(
130 + server.url_for("score"),
131 + json={
132 + "model": MODEL_NAME,
133 + "queries": TEXTS_1,
134 + "documents": TEXTS_2,
135 + },
136 + )
137 + score_response.raise_for_status()
138 + score = ScoreResponse.model_validate(score_response.json())
139 +
140 + assert score.id is not None
141 + assert score.data is not None
142 + assert len(score.data) == 2
143 +
144 + vllm_outputs = [d.score for d in score.data]
145 + hf_outputs = hf_model.predict(text_pairs).tolist()
146 +
147 + for i in range(len(vllm_outputs)):
148 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
149 +
150 +
151 +@pytest.mark.asyncio
152 +async def test_score_api_queries_vs_documents(hf_model, server: RemoteOpenAIServer):
153 + text_pairs = [
154 + [TEXTS_1[0], TEXTS_2[0]],
155 + [TEXTS_1[1], TEXTS_2[1]],
156 + ]
157 +
158 + score_response = requests.post(
159 + server.url_for("score"),
160 + json={
161 + "model": MODEL_NAME,
162 + "queries": TEXTS_1,
163 + "documents": TEXTS_2,
164 + },
165 + )
166 + score_response.raise_for_status()
167 + score = ScoreResponse.model_validate(score_response.json())
168 +
169 + assert score.id is not None
170 + assert score.data is not None
171 + assert len(score.data) == 2
172 +
173 + vllm_outputs = [d.score for d in score.data]
174 + hf_outputs = hf_model.predict(text_pairs).tolist()
175 +
176 + for i in range(len(vllm_outputs)):
177 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
178 +
179 +
180 +@pytest.mark.asyncio
181 +async def test_score_api_queries_vs_items(hf_model, server: RemoteOpenAIServer):
182 + text_pairs = [
183 + [TEXTS_1[0], TEXTS_2[0]],
184 + [TEXTS_1[1], TEXTS_2[1]],
185 + ]
186 +
187 + score_response = requests.post(
188 + server.url_for("score"),
189 + json={
190 + "model": MODEL_NAME,
191 + "queries": TEXTS_1,
192 + "items": TEXTS_2,
193 + },
194 + )
195 + score_response.raise_for_status()
196 + score = ScoreResponse.model_validate(score_response.json())
197 +
198 + assert score.id is not None
199 + assert score.data is not None
200 + assert len(score.data) == 2
201 +
202 + vllm_outputs = [d.score for d in score.data]
203 + hf_outputs = hf_model.predict(text_pairs).tolist()
204 +
205 + for i in range(len(vllm_outputs)):
206 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
207 +
208 +
209 +@pytest.mark.asyncio
210 +async def test_score_api_text_1_vs_text_2(hf_model, server: RemoteOpenAIServer):
211 + text_pairs = [
212 + [TEXTS_1[0], TEXTS_2[0]],
213 + [TEXTS_1[1], TEXTS_2[1]],
214 + ]
215 +
216 + score_response = requests.post(
217 + server.url_for("score"),
218 + json={
219 + "model": MODEL_NAME,
220 + "text_1": TEXTS_1,
221 + "text_2": TEXTS_2,
222 + },
223 + )
224 + score_response.raise_for_status()
225 + score = ScoreResponse.model_validate(score_response.json())
226 +
227 + assert score.id is not None
228 + assert score.data is not None
229 + assert len(score.data) == 2
230 +
231 + vllm_outputs = [d.score for d in score.data]
232 + hf_outputs = hf_model.predict(text_pairs).tolist()
233 +
234 + for i in range(len(vllm_outputs)):
235 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
236 +
237 +
238 +@pytest.mark.asyncio
239 +async def test_score_api_data_1_vs_data_2(hf_model, server: RemoteOpenAIServer):
240 + text_pairs = [
241 + [TEXTS_1[0], TEXTS_2[0]],
242 + [TEXTS_1[1], TEXTS_2[1]],
243 + ]
244 +
245 + score_response = requests.post(
246 + server.url_for("score"),
247 + json={
248 + "model": MODEL_NAME,
249 + "data_1": TEXTS_1,
250 + "data_2": TEXTS_2,
251 + },
252 + )
253 + score_response.raise_for_status()
254 + score = ScoreResponse.model_validate(score_response.json())
255 +
256 + assert score.id is not None
257 + assert score.data is not None
258 + assert len(score.data) == 2
259 +
260 + vllm_outputs = [d.score for d in score.data]
261 + hf_outputs = hf_model.predict(text_pairs).tolist()
262 +
263 + for i in range(len(vllm_outputs)):
264 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
265 +
266 +
267 +@pytest.mark.asyncio
268 +async def test_rerank_api_texts(server: RemoteOpenAIServer):
269 + query = "What is the capital of France?"
270 + documents = [
271 + "The capital of Brazil is Brasilia.",
272 + "The capital of France is Paris.",
273 + ]
274 +
275 + rerank_response = requests.post(
276 + server.url_for("rerank"),
277 + json={
278 + "model": MODEL_NAME,
279 + "query": query,
280 + "documents": documents,
281 + },
282 + )
283 + rerank_response.raise_for_status()
284 + rerank = RerankResponse.model_validate(rerank_response.json())
285 +
286 + assert rerank.id is not None
287 + assert rerank.results is not None
288 + assert len(rerank.results) == 2
289 + assert rerank.results[0].relevance_score >= 0.9
290 + assert rerank.results[1].relevance_score <= 0.01
291 +
292 +
293 +@pytest.mark.asyncio
294 +async def test_rerank_api_top_n(server: RemoteOpenAIServer):
295 + query = "What is the capital of France?"
296 + documents = [
297 + "The capital of Brazil is Brasilia.",
298 + "The capital of France is Paris.",
299 + "Cross-encoder models are neat",
300 + ]
301 +
302 + rerank_response = requests.post(
303 + server.url_for("rerank"),
304 + json={"model": MODEL_NAME, "query": query, "documents": documents, "top_n": 2},
305 + )
306 + rerank_response.raise_for_status()
307 + rerank = RerankResponse.model_validate(rerank_response.json())
308 +
309 + assert rerank.id is not None
310 + assert rerank.results is not None
311 + assert len(rerank.results) == 2
312 + assert rerank.results[0].relevance_score >= 0.9
313 + assert rerank.results[1].relevance_score <= 0.01
314 +
315 +
316 +@pytest.mark.asyncio
317 +async def test_rerank_api_max_model_len(server: RemoteOpenAIServer):
318 + query = "What is the capital of France?" * 100
319 + documents = [
320 + "The capital of Brazil is Brasilia.",
321 + "The capital of France is Paris.",
322 + ]
323 +
324 + rerank_response = requests.post(
325 + server.url_for("rerank"),
326 + json={"model": MODEL_NAME, "query": query, "documents": documents},
327 + )
328 + assert rerank_response.status_code == 400
329 + # Assert just a small fragments of the response
330 + assert "Please reduce the length of the input prompt" in rerank_response.text
331 +
332 +
333 +@pytest.mark.asyncio
334 +async def test_score_api_max_model_len(server: RemoteOpenAIServer):
335 + queries = "What is the capital of France?" * 20
336 + documents = [
337 + "The capital of Brazil is Brasilia.",
338 + "The capital of France is Paris.",
339 + ]
340 +
341 + score_response = requests.post(
342 + server.url_for("score"),
343 + json={
344 + "model": MODEL_NAME,
345 + "queries": queries,
346 + "documents": documents,
347 + },
348 + )
349 + assert score_response.status_code == 400
350 + # Assert just a small fragments of the response
351 + assert "Please reduce the length of the input prompt" in score_response.text
352 +
353 + # Test truncation
354 + score_response = requests.post(
355 + server.url_for("score"),
356 + json={
357 + "model": MODEL_NAME,
358 + "queries": queries,
359 + "documents": documents,
360 + "truncate_prompt_tokens": 101,
361 + },
362 + )
363 + assert score_response.status_code == 400
364 + assert "Please request a smaller truncation size." in score_response.text
365 +
366 +
367 +@pytest.mark.asyncio
368 +async def test_invocations(server: RemoteOpenAIServer):
369 + query = "What is the capital of France?"
370 + documents = [
371 + "The capital of Brazil is Brasilia.",
372 + "The capital of France is Paris.",
373 + ]
374 +
375 + request_args = {
376 + "model": MODEL_NAME,
377 + "query": query,
378 + "documents": documents,
379 + }
380 +
381 + rerank_response = requests.post(server.url_for("rerank"), json=request_args)
382 + rerank_response.raise_for_status()
383 +
384 + invocation_response = requests.post(
385 + server.url_for("invocations"), json=request_args
386 + )
387 + invocation_response.raise_for_status()
388 +
389 + rerank_output = rerank_response.json()
390 + invocation_output = invocation_response.json()
391 +
392 + assert rerank_output.keys() == invocation_output.keys()
393 + for rerank_result, invocations_result in zip(
394 + rerank_output["results"], invocation_output["results"]
395 + ):
396 + assert rerank_result.keys() == invocations_result.keys()
397 + assert rerank_result["relevance_score"] == pytest.approx(
398 + invocations_result["relevance_score"], rel=0.01
399 + )
400 +
401 +
402 +@pytest.mark.asyncio
403 +async def test_use_activation(server: RemoteOpenAIServer):
404 + async def get_outputs(use_activation):
405 + query = "What is the capital of France?"
406 + documents = [
407 + "The capital of Brazil is Brasilia.",
408 + "The capital of France is Paris.",
409 + ]
410 +
411 + response = requests.post(
412 + server.url_for("rerank"),
413 + json={
414 + "model": MODEL_NAME,
415 + "query": query,
416 + "documents": documents,
417 + "use_activation": use_activation,
418 + },
419 + )
420 + outputs = response.json()
421 +
422 + return torch.tensor([x["relevance_score"] for x in outputs["results"]])
423 +
424 + default = await get_outputs(use_activation=None)
425 + w_activation = await get_outputs(use_activation=True)
426 + wo_activation = await get_outputs(use_activation=False)
427 +
428 + assert torch.allclose(default, w_activation, atol=1e-2), (
429 + "Default should use activation."
430 + )
431 + assert not torch.allclose(w_activation, wo_activation, atol=1e-2), (
432 + "wo_activation should not use activation."
433 + )
434 + assert torch.allclose(F.sigmoid(wo_activation), w_activation, atol=1e-2), (
435 + "w_activation should be close to activation(wo_activation)."
436 + )
437 +
438 +
439 +@pytest.mark.asyncio
440 +async def test_pooling_classify(server: RemoteOpenAIServer):
441 + response = requests.post(
442 + server.url_for("pooling"),
443 + json={
444 + "model": MODEL_NAME,
445 + "input": input_text,
446 + "encoding_format": "float",
447 + "task": "classify",
448 + },
449 + )
450 + poolings = PoolingResponse.model_validate(response.json())
451 + assert len(poolings.data) == 1
452 + assert len(poolings.data[0].data) == 1
453 +
454 +
455 +@pytest.mark.asyncio
456 +async def test_pooling_token_classify(server: RemoteOpenAIServer):
457 + response = requests.post(
458 + server.url_for("pooling"),
459 + json={
460 + "model": MODEL_NAME,
461 + "task": "token_classify",
462 + "input": input_text,
463 + "encoding_format": "float",
464 + },
465 + )
466 +
467 + poolings = PoolingResponse.model_validate(response.json())
468 +
469 + assert len(poolings.data) == 1
470 + assert len(poolings.data[0].data) == len(input_tokens)
471 + assert len(poolings.data[0].data[0]) == 1
472 +
473 +
474 +@pytest.mark.asyncio
475 +@pytest.mark.parametrize("task", ["embed", "token_embed", "plugin"])
476 +async def test_pooling_not_supported(server: RemoteOpenAIServer, task: str):
477 + response = requests.post(
478 + server.url_for("pooling"),
479 + json={
480 + "model": MODEL_NAME,
481 + "input": input_text,
482 + "encoding_format": "float",
483 + "task": task,
484 + },
485 + )
486 + assert response.json()["error"]["type"] == "BadRequestError"
487 + assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
tests/entrypoints/pooling/scoring/test_cross_encoder_online_vision.py renamed
+18 -8
@@ -123,7 +123,10 @@ def server(request):
123 yield remote_server, backend
124
125
126 -def test_score_api_queries_str_documents_str(server: tuple[RemoteOpenAIServer, str]):
126 +@pytest.mark.asyncio
127 +async def test_score_api_queries_str_documents_str(
128 + server: tuple[RemoteOpenAIServer, str],
129 +):
130 remote_server, backend = server
131 score_response = requests.post(
132 remote_server.url_for("score"),
@@ -143,7 +146,8 @@ def test_score_api_queries_str_documents_str(server: tuple[RemoteOpenAIServer, s
146 assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
147
148
146 -def test_score_api_queries_str_documents_text_content(
149 +@pytest.mark.asyncio
150 +async def test_score_api_queries_str_documents_text_content(
151 server: tuple[RemoteOpenAIServer, str],
152 ):
153 remote_server, backend = server
@@ -165,7 +169,8 @@ def test_score_api_queries_str_documents_text_content(
169 assert_score(score.data[0].score, TEXT_VS_TEXT, backend, "text_vs_text")
170
171
168 -def test_score_api_queries_str_documents_image_url_content(
172 +@pytest.mark.asyncio
173 +async def test_score_api_queries_str_documents_image_url_content(
174 server: tuple[RemoteOpenAIServer, str],
175 ):
176 remote_server, backend = server
@@ -187,7 +192,8 @@ def test_score_api_queries_str_documents_image_url_content(
192 assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image")
193
194
190 -def test_score_api_queries_str_documents_image_base64_content(
195 +@pytest.mark.asyncio
196 +async def test_score_api_queries_str_documents_image_base64_content(
197 server: tuple[RemoteOpenAIServer, str],
198 ):
199 remote_server, backend = server
@@ -209,7 +215,8 @@ def test_score_api_queries_str_documents_image_base64_content(
215 assert_score(score.data[0].score, TEXT_VS_IMAGE, backend, "text_vs_image_base64")
216
217
212 -def test_score_api_queries_str_documents_image_url_plus_text_content(
218 +@pytest.mark.asyncio
219 +async def test_score_api_queries_str_documents_image_url_plus_text_content(
220 server: tuple[RemoteOpenAIServer, str],
221 ):
222 remote_server, backend = server
@@ -233,7 +240,8 @@ def test_score_api_queries_str_documents_image_url_plus_text_content(
240 )
241
242
236 -def test_score_api_queries_str_documents_list(
243 +@pytest.mark.asyncio
244 +async def test_score_api_queries_str_documents_list(
245 server: tuple[RemoteOpenAIServer, str],
246 ):
247 remote_server, backend = server
@@ -268,7 +276,8 @@ def test_score_api_queries_str_documents_list(
276 )
277
278
271 -def test_rerank_api_queries_str_documents_list(
279 +@pytest.mark.asyncio
280 +async def test_rerank_api_queries_str_documents_list(
281 server: tuple[RemoteOpenAIServer, str],
282 ):
283 remote_server, backend = server
@@ -320,7 +329,8 @@ def test_rerank_api_queries_str_documents_list(
329 )
330
331
323 -def test_score_api_queries_list_documents_list(
332 +@pytest.mark.asyncio
333 +async def test_score_api_queries_list_documents_list(
334 server: tuple[RemoteOpenAIServer, str],
335 ):
336 remote_server, backend = server
tests/entrypoints/pooling/scoring/test_late_interaction_offline.py new
+119
@@ -0,0 +1,119 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import weakref
5 +
6 +import pytest
7 +
8 +from vllm import LLM
9 +from vllm.distributed import cleanup_dist_env_and_memory
10 +from vllm.platforms import current_platform
11 +
12 +from .util import ColBERTScoringHfRunner
13 +
14 +MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
15 +COLBERT_DIM = 96
16 +
17 +LINEAR_WEIGHTS_KEY = "linear.weight"
18 +PROMPT = "The chef prepared a delicious meal."
19 +
20 +TEXTS_1 = [
21 + "What is the capital of France?",
22 + "What is the capital of Germany?",
23 +]
24 +
25 +TEXTS_2 = [
26 + "The capital of France is Paris.",
27 + "The capital of Germany is Berlin.",
28 +]
29 +
30 +DTYPE = "half"
31 +
32 +
33 +@pytest.fixture(scope="module")
34 +def llm():
35 + # ROCm: Use FLEX_ATTENTION backend as it's the only attention backend
36 + # that supports encoder-only models on ROCm.
37 + attention_config = None
38 + if current_platform.is_rocm():
39 + attention_config = {"backend": "FLEX_ATTENTION"}
40 +
41 + # pytest caches the fixture so we use weakref.proxy to
42 + # enable garbage collection
43 + llm = LLM(
44 + model=MODEL_NAME,
45 + max_num_batched_tokens=32768,
46 + tensor_parallel_size=1,
47 + gpu_memory_utilization=0.75,
48 + enforce_eager=True,
49 + seed=0,
50 + attention_config=attention_config,
51 + )
52 +
53 + yield weakref.proxy(llm)
54 +
55 + del llm
56 +
57 + cleanup_dist_env_and_memory()
58 +
59 +
60 +@pytest.fixture(scope="module")
61 +def hf_model():
62 + return ColBERTScoringHfRunner(
63 + model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
64 + )
65 +
66 +
67 +@pytest.mark.skip_global_cleanup
68 +def test_1_to_1(llm, hf_model):
69 + text_pair = [TEXTS_1[0], TEXTS_2[0]]
70 +
71 + hf_outputs = hf_model.predict([text_pair]).tolist()
72 + vllm_outputs = [
73 + output.outputs.score for output in llm.score(text_pair[0], text_pair[1])
74 + ]
75 +
76 + assert len(vllm_outputs) == 1
77 + assert len(hf_outputs) == 1
78 +
79 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
80 +
81 +
82 +@pytest.mark.skip_global_cleanup
83 +def test_1_to_n(llm, hf_model):
84 + text_pairs = [
85 + [TEXTS_1[0], TEXTS_2[0]],
86 + [TEXTS_1[0], TEXTS_2[1]],
87 + ]
88 +
89 + hf_outputs = hf_model.predict(text_pairs).tolist()
90 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1[0], TEXTS_2)]
91 +
92 + assert len(vllm_outputs) == 2
93 + assert len(hf_outputs) == 2
94 +
95 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
96 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
97 +
98 +
99 +@pytest.mark.skip_global_cleanup
100 +def test_n_to_n(llm, hf_model):
101 + text_pairs = [
102 + [TEXTS_1[0], TEXTS_2[0]],
103 + [TEXTS_1[1], TEXTS_2[1]],
104 + ]
105 +
106 + hf_outputs = hf_model.predict(text_pairs).tolist()
107 + vllm_outputs = [output.outputs.score for output in llm.score(TEXTS_1, TEXTS_2)]
108 +
109 + assert len(vllm_outputs) == 2
110 + assert len(hf_outputs) == 2
111 +
112 + assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
113 + assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
114 +
115 +
116 +def test_token_embed(llm):
117 + outputs = llm.encode(PROMPT, pooling_task="token_embed", use_tqdm=False)
118 + assert len(outputs) == 1
119 + assert outputs[0].outputs.data.shape == (9, COLBERT_DIM)
tests/entrypoints/pooling/scoring/test_late_interaction_online.py new
+232
@@ -0,0 +1,232 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +"""Online API tests for ColBERT late interaction scoring."""
4 +
5 +import pytest
6 +import requests
7 +
8 +from tests.utils import RemoteOpenAIServer
9 +from vllm.entrypoints.pooling.score.protocol import RerankResponse, ScoreResponse
10 +
11 +from .util import ColBERTScoringHfRunner
12 +
13 +MODEL_NAME = "answerdotai/answerai-colbert-small-v1"
14 +COLBERT_DIM = 96
15 +MAX_MODEL_LEN = 512
16 +LINEAR_WEIGHTS_KEY = "linear.weight"
17 +
18 +TEXTS_1 = [
19 + "What is the capital of France?",
20 + "What is the capital of Germany?",
21 +]
22 +
23 +TEXTS_2 = [
24 + "The capital of France is Paris.",
25 + "The capital of Germany is Berlin.",
26 +]
27 +
28 +
29 +@pytest.fixture(scope="module")
30 +def server():
31 + args = [
32 + "--max-model-len",
33 + str(MAX_MODEL_LEN),
34 + ]
35 +
36 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
37 + yield remote_server
38 +
39 +
40 +@pytest.fixture(scope="module")
41 +def hf_model():
42 + return ColBERTScoringHfRunner(
43 + model_name=MODEL_NAME, linear_weights_key=LINEAR_WEIGHTS_KEY
44 + )
45 +
46 +
47 +@pytest.mark.asyncio
48 +async def test_score_api_queries_str_1_documents_str_1(
49 + hf_model, server: RemoteOpenAIServer
50 +):
51 + score_response = requests.post(
52 + server.url_for("score"),
53 + json={
54 + "model": MODEL_NAME,
55 + "queries": TEXTS_1[0],
56 + "documents": TEXTS_2[0],
57 + },
58 + )
59 + score_response.raise_for_status()
60 + score = ScoreResponse.model_validate(score_response.json())
61 +
62 + assert score.id is not None
63 + assert score.data is not None
64 + assert len(score.data) == 1
65 +
66 + vllm_outputs = [d.score for d in score.data]
67 + hf_outputs = hf_model.predict([[TEXTS_1[0], TEXTS_2[0]]]).tolist()
68 +
69 + for i in range(len(vllm_outputs)):
70 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
71 +
72 +
73 +@pytest.mark.asyncio
74 +async def test_score_api_queries_str_1_documents_str_n(
75 + hf_model, server: RemoteOpenAIServer
76 +):
77 + text_pairs = [
78 + [TEXTS_1[0], TEXTS_2[0]],
79 + [TEXTS_1[0], TEXTS_2[1]],
80 + ]
81 +
82 + score_response = requests.post(
83 + server.url_for("score"),
84 + json={
85 + "model": MODEL_NAME,
86 + "queries": TEXTS_1[0],
87 + "documents": TEXTS_2,
88 + },
89 + )
90 + score_response.raise_for_status()
91 + score = ScoreResponse.model_validate(score_response.json())
92 +
93 + assert score.id is not None
94 + assert score.data is not None
95 + assert len(score.data) == 2
96 +
97 + vllm_outputs = [d.score for d in score.data]
98 + hf_outputs = hf_model.predict(text_pairs).tolist()
99 +
100 + for i in range(len(vllm_outputs)):
101 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
102 +
103 +
104 +@pytest.mark.asyncio
105 +async def test_score_api_queries_str_n_documents_str_n(
106 + hf_model, server: RemoteOpenAIServer
107 +):
108 + text_pairs = [
109 + [TEXTS_1[0], TEXTS_2[0]],
110 + [TEXTS_1[1], TEXTS_2[1]],
111 + ]
112 +
113 + score_response = requests.post(
114 + server.url_for("score"),
115 + json={
116 + "model": MODEL_NAME,
117 + "queries": TEXTS_1,
118 + "documents": TEXTS_2,
119 + },
120 + )
121 + score_response.raise_for_status()
122 + score = ScoreResponse.model_validate(score_response.json())
123 +
124 + assert score.id is not None
125 + assert score.data is not None
126 + assert len(score.data) == 2
127 +
128 + vllm_outputs = [d.score for d in score.data]
129 + hf_outputs = hf_model.predict(text_pairs).tolist()
130 +
131 + for i in range(len(vllm_outputs)):
132 + assert hf_outputs[i] == pytest.approx(vllm_outputs[i], rel=0.01)
133 +
134 +
135 +@pytest.mark.asyncio
136 +async def test_rerank_api_texts(server: RemoteOpenAIServer):
137 + """Test ColBERT rerank endpoint."""
138 + query = "What is the capital of France?"
139 + documents = [
140 + "The capital of Brazil is Brasilia.",
141 + "The capital of France is Paris.",
142 + ]
143 +
144 + rerank_response = requests.post(
145 + server.url_for("rerank"),
146 + json={
147 + "model": MODEL_NAME,
148 + "query": query,
149 + "documents": documents,
150 + },
151 + )
152 + rerank_response.raise_for_status()
153 + rerank = RerankResponse.model_validate(rerank_response.json())
154 +
155 + assert rerank.id is not None
156 + assert rerank.results is not None
157 + assert len(rerank.results) == 2
158 +
159 + paris_result = next(r for r in rerank.results if r.index == 1)
160 + brazil_result = next(r for r in rerank.results if r.index == 0)
161 +
162 + assert paris_result.relevance_score > brazil_result.relevance_score
163 +
164 +
165 +@pytest.mark.asyncio
166 +async def test_rerank_api_top_n(server: RemoteOpenAIServer):
167 + """Test ColBERT rerank with top_n parameter."""
168 + query = "What is the capital of France?"
169 + documents = [
170 + "The capital of Brazil is Brasilia.",
171 + "The capital of France is Paris.",
172 + "Machine learning is a field of AI.",
173 + ]
174 +
175 + rerank_response = requests.post(
176 + server.url_for("rerank"),
177 + json={
178 + "model": MODEL_NAME,
179 + "query": query,
180 + "documents": documents,
181 + "top_n": 2,
182 + },
183 + )
184 + rerank_response.raise_for_status()
185 + rerank = RerankResponse.model_validate(rerank_response.json())
186 +
187 + assert len(rerank.results) == 2
188 + assert rerank.results[0].index == 1
189 +
190 +
191 +@pytest.mark.asyncio
192 +async def test_token_embed(server: RemoteOpenAIServer):
193 + """Test ColBERT token_embed task via pooling endpoint."""
194 + text = "What is the capital of France?"
195 +
196 + pooling_response = requests.post(
197 + server.url_for("pooling"),
198 + json={
199 + "model": MODEL_NAME,
200 + "input": text,
201 + "task": "token_embed",
202 + },
203 + )
204 + pooling_response.raise_for_status()
205 + pooling = pooling_response.json()
206 +
207 + assert "data" in pooling
208 + assert len(pooling["data"]) == 1
209 +
210 + embeddings = pooling["data"][0]["data"]
211 + assert isinstance(embeddings, list)
212 + assert len(embeddings) > 0
213 + assert len(embeddings[0]) == COLBERT_DIM
214 +
215 +
216 +@pytest.mark.asyncio
217 +async def test_embed_not_supported(server: RemoteOpenAIServer):
218 + """Test that ColBERT model does not support 'embed' task."""
219 + task = "embed"
220 + text = "What is the capital of France?"
221 +
222 + response = requests.post(
223 + server.url_for("pooling"),
224 + json={
225 + "model": MODEL_NAME,
226 + "input": text,
227 + "task": task,
228 + },
229 + )
230 +
231 + assert response.json()["error"]["type"] == "BadRequestError"
232 + assert response.json()["error"]["message"].startswith(f"Unsupported task: {task!r}")
tests/entrypoints/pooling/scoring/test_utils.py renamed
tests/entrypoints/pooling/scoring/util.py new
+69
@@ -0,0 +1,69 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +
4 +import torch
5 +import torch.nn.functional as F
6 +from huggingface_hub import hf_hub_download
7 +from safetensors.torch import load_file
8 +from transformers import AutoModel, AutoTokenizer
9 +
10 +from tests.conftest import HfRunner
11 +from vllm.entrypoints.pooling.score.utils import compute_maxsim_score
12 +
13 +
14 +class ColBERTScoringHfRunner(torch.nn.Module):
15 + def __init__(self, model_name, linear_weights_key):
16 + super().__init__()
17 +
18 + self.tokenizer = AutoTokenizer.from_pretrained(model_name)
19 +
20 + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
21 +
22 + extra = {}
23 + if self.device.type == "cpu":
24 + extra["attn_implementation"] = "eager"
25 +
26 + self.model = AutoModel.from_pretrained(
27 + model_name,
28 + **extra,
29 + ).to(self.device)
30 + self.model.eval()
31 +
32 + path = hf_hub_download(model_name, filename="model.safetensors")
33 + weights = load_file(path)
34 +
35 + self.linear_weight = weights[linear_weights_key].to(self.device).float()
36 +
37 + @torch.inference_mode()
38 + def forward(self, texts):
39 + embeddings = []
40 + for text in texts:
41 + inputs = self.tokenizer(text, return_tensors="pt").to(self.device)
42 + hidden = self.model(**inputs).last_hidden_state.float()
43 + projected = F.linear(hidden, self.linear_weight.float())
44 + normalised = F.normalize(projected, p=2, dim=-1)
45 + embeddings.append(normalised.squeeze(0).cpu())
46 + return embeddings
47 +
48 + @torch.inference_mode()
49 + def predict(self, prompts: list[list[str]], *args, **kwargs):
50 + hf_embeddings = [self(prompt) for prompt in prompts]
51 + hf_outputs = [
52 + compute_maxsim_score(*map(torch.tensor, pair)).item()
53 + for pair in hf_embeddings
54 + ]
55 + return torch.as_tensor(hf_outputs)
56 +
57 +
58 +class EncoderScoringHfRunner(HfRunner):
59 + def __init__(self, *args, **kwargs):
60 + super().__init__(*args, **kwargs, is_sentence_transformer=True)
61 +
62 + @torch.inference_mode()
63 + def predict(self, prompts: list[list[str]], *args, **kwargs):
64 + hf_embeddings = [self.encode(prompt) for prompt in prompts]
65 + hf_outputs = [
66 + F.cosine_similarity(*map(torch.tensor, pair), dim=0)
67 + for pair in hf_embeddings
68 + ]
69 + return torch.as_tensor(hf_outputs)
tests/models/language/pooling/test_scoring.py deleted
-169
@@ -1,169 +0,0 @@
1 -# SPDX-License-Identifier: Apache-2.0
2 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 -import pytest
4 -import torch
5 -import torch.nn.functional as F
6 -
7 -CROSS_ENCODER_MODELS = [
8 - "cross-encoder/ms-marco-MiniLM-L-6-v2", # Bert
9 - "BAAI/bge-reranker-v2-m3", # Roberta
10 -]
11 -
12 -EMBEDDING_MODELS = [
13 - "sentence-transformers/all-MiniLM-L12-v2",
14 -]
15 -
16 -TEXTS_1 = [
17 - "What is the capital of France?",
18 - "What is the capital of Germany?",
19 -]
20 -
21 -TEXTS_2 = [
22 - "The capital of France is Paris.",
23 - "The capital of Germany is Berlin.",
24 -]
25 -
26 -DTYPE = "half"
27 -
28 -
29 -@pytest.fixture(scope="module", params=CROSS_ENCODER_MODELS)
30 -def model_name(request):
31 - yield request.param
32 -
33 -
34 -def test_cross_encoder_1_to_1(vllm_runner, hf_runner, model_name):
35 - text_pair = [TEXTS_1[0], TEXTS_2[0]]
36 -
37 - with hf_runner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
38 - hf_outputs = hf_model.predict([text_pair]).tolist()
39 -
40 - with vllm_runner(
41 - model_name, runner="pooling", dtype=DTYPE, max_model_len=None
42 - ) as vllm_model:
43 - vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
44 -
45 - assert len(vllm_outputs) == 1
46 - assert len(hf_outputs) == 1
47 -
48 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
49 -
50 -
51 -def test_cross_encoder_1_to_N(vllm_runner, hf_runner, model_name):
52 - text_pairs = [
53 - [TEXTS_1[0], TEXTS_2[0]],
54 - [TEXTS_1[0], TEXTS_2[1]],
55 - ]
56 -
57 - with hf_runner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
58 - hf_outputs = hf_model.predict(text_pairs).tolist()
59 -
60 - with vllm_runner(
61 - model_name, runner="pooling", dtype=DTYPE, max_model_len=None
62 - ) as vllm_model:
63 - vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
64 -
65 - assert len(vllm_outputs) == 2
66 - assert len(hf_outputs) == 2
67 -
68 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
69 - assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
70 -
71 -
72 -def test_cross_encoder_N_to_N(vllm_runner, hf_runner, model_name):
73 - text_pairs = [
74 - [TEXTS_1[0], TEXTS_2[0]],
75 - [TEXTS_1[1], TEXTS_2[1]],
76 - ]
77 -
78 - with hf_runner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
79 - hf_outputs = hf_model.predict(text_pairs).tolist()
80 -
81 - with vllm_runner(
82 - model_name, runner="pooling", dtype=DTYPE, max_model_len=None
83 - ) as vllm_model:
84 - vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
85 -
86 - assert len(vllm_outputs) == 2
87 - assert len(hf_outputs) == 2
88 -
89 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
90 - assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
91 -
92 -
93 -@pytest.fixture(scope="module", params=EMBEDDING_MODELS)
94 -def emb_model_name(request):
95 - yield request.param
96 -
97 -
98 -def test_embedding_1_to_1(vllm_runner, hf_runner, emb_model_name):
99 - text_pair = [TEXTS_1[0], TEXTS_2[0]]
100 -
101 - with hf_runner(
102 - emb_model_name, dtype=DTYPE, is_sentence_transformer=True
103 - ) as hf_model:
104 - hf_embeddings = hf_model.encode(text_pair)
105 - hf_outputs = [F.cosine_similarity(*map(torch.tensor, hf_embeddings), dim=0)]
106 -
107 - with vllm_runner(
108 - emb_model_name, runner="pooling", dtype=DTYPE, max_model_len=None
109 - ) as vllm_model:
110 - vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
111 -
112 - assert len(vllm_outputs) == 1
113 - assert len(hf_outputs) == 1
114 -
115 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
116 -
117 -
118 -def test_embedding_1_to_N(vllm_runner, hf_runner, emb_model_name):
119 - text_pairs = [
120 - [TEXTS_1[0], TEXTS_2[0]],
121 - [TEXTS_1[0], TEXTS_2[1]],
122 - ]
123 -
124 - with hf_runner(
125 - emb_model_name, dtype=DTYPE, is_sentence_transformer=True
126 - ) as hf_model:
127 - hf_embeddings = [hf_model.encode(text_pair) for text_pair in text_pairs]
128 - hf_outputs = [
129 - F.cosine_similarity(*map(torch.tensor, pair), dim=0)
130 - for pair in hf_embeddings
131 - ]
132 -
133 - with vllm_runner(
134 - emb_model_name, runner="pooling", dtype=DTYPE, max_model_len=None
135 - ) as vllm_model:
136 - vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
137 -
138 - assert len(vllm_outputs) == 2
139 - assert len(hf_outputs) == 2
140 -
141 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
142 - assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
143 -
144 -
145 -def test_embedding_N_to_N(vllm_runner, hf_runner, emb_model_name):
146 - text_pairs = [
147 - [TEXTS_1[0], TEXTS_2[0]],
148 - [TEXTS_1[1], TEXTS_2[1]],
149 - ]
150 -
151 - with hf_runner(
152 - emb_model_name, dtype=DTYPE, is_sentence_transformer=True
153 - ) as hf_model:
154 - hf_embeddings = [hf_model.encode(text_pair) for text_pair in text_pairs]
155 - hf_outputs = [
156 - F.cosine_similarity(*map(torch.tensor, pair), dim=0)
157 - for pair in hf_embeddings
158 - ]
159 -
160 - with vllm_runner(
161 - emb_model_name, runner="pooling", dtype=DTYPE, max_model_len=None
162 - ) as vllm_model:
163 - vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
164 -
165 - assert len(vllm_outputs) == 2
166 - assert len(hf_outputs) == 2
167 -
168 - assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
169 - assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
vllm/entrypoints/openai/engine/serving.py
+2 -2
@@ -461,7 +461,7 @@ class OpenAIServing:
461 return self.create_error_response(
462 "truncate_prompt_tokens value is "
463 "greater than max_model_len."
464 - " Please, select a smaller truncation size."
464 + " Please request a smaller truncation size."
465 )
466 return None
467
@@ -724,7 +724,7 @@ class OpenAIServing:
724 f"This model's maximum context length is "
725 f"{max_model_len} tokens. However, you requested "
726 f"{token_num} tokens in the input for {operation}. "
727 - f"Please reduce the length of the input.",
727 + f"Please reduce the length of the input prompt.",
728 parameter="input_tokens",
729 value=token_num,
730 )
vllm/entrypoints/pooling/base/serving.py
+1 -1
@@ -224,7 +224,7 @@ class PoolingServing:
224 raise ValueError(
225 "truncate_prompt_tokens value is "
226 "greater than max_model_len."
227 - " Please, select a smaller truncation size."
227 + " Please request a smaller truncation size."
228 )
229 return None
230