A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 18:15 UTC
[Frontend][3/n] Make pooling entrypoints request schema consensus | EmbedRequest & ClassifyRequest (#32905)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io> Signed-off-by: wang.yuqi <noooop@126.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
wang.yuqi committed
Jan 23, 2026 at 20:03 UTC
05f3d714dbec13542ddeaf2f7576d6fc70ee9068
11 files changed
+326
-261
docs/serving/openai_compatible_server.md
+100
-17
@@ -197,7 +197,7 @@ The following [sampling parameters](../api/README.md#inference-parameters) are s
197
??? code
198
199
```python
200
- --8<-- "vllm/entrypoints/openai/protocol.py:completion-sampling-params"
200
+ --8<-- "vllm/entrypoints/openai/completion/protocol.py:completion-sampling-params"
201
```
202
203
The following extra parameters are supported:
@@ -205,7 +205,7 @@ The following extra parameters are supported:
205
??? code
206
207
```python
208
- --8<-- "vllm/entrypoints/openai/protocol.py:completion-extra-params"
208
+ --8<-- "vllm/entrypoints/openai/completion/protocol.py:completion-extra-params"
209
```
210
211
### Chat API
@@ -228,7 +228,7 @@ The following [sampling parameters](../api/README.md#inference-parameters) are s
228
??? code
229
230
```python
231
- --8<-- "vllm/entrypoints/openai/protocol.py:chat-completion-sampling-params"
231
+ --8<-- "vllm/entrypoints/openai/chat_completion/protocol.py:chat-completion-sampling-params"
232
```
233
234
The following extra parameters are supported:
@@ -236,7 +236,7 @@ The following extra parameters are supported:
236
??? code
237
238
```python
239
- --8<-- "vllm/entrypoints/openai/protocol.py:chat-completion-extra-params"
239
+ --8<-- "vllm/entrypoints/openai/chat_completion/protocol.py:chat-completion-extra-params"
240
```
241
242
### Responses API
@@ -253,7 +253,7 @@ The following extra parameters in the request object are supported:
253
??? code
254
255
```python
256
- --8<-- "vllm/entrypoints/openai/protocol.py:responses-extra-params"
256
+ --8<-- "vllm/entrypoints/openai/responses/protocol.py:responses-extra-params"
257
```
258
259
The following extra parameters in the response object are supported:
@@ -261,7 +261,7 @@ The following extra parameters in the response object are supported:
261
??? code
262
263
```python
264
- --8<-- "vllm/entrypoints/openai/protocol.py:responses-response-extra-params"
264
+ --8<-- "vllm/entrypoints/openai/responses/protocol.py:responses-response-extra-params"
265
```
266
267
### Embeddings API
@@ -378,23 +378,53 @@ The following [pooling parameters][vllm.PoolingParams] are supported.
378
379
```python
380
--8<-- "vllm/pooling_params.py:common-pooling-params"
381
---8<-- "vllm/pooling_params.py:embedding-pooling-params"
381
+--8<-- "vllm/pooling_params.py:embed-pooling-params"
382
```
383
384
-The following extra parameters are supported by default:
384
+The following Embeddings API parameters are supported:
385
386
??? code
387
388
```python
389
- --8<-- "vllm/entrypoints/pooling/embed/protocol.py:embedding-extra-params"
389
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
390
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
391
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
392
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
393
```
394
392
-For chat-like input (i.e. if `messages` is passed), these extra parameters are supported instead:
395
+The following extra parameters are supported:
396
+
397
+??? code
398
+
399
+ ```python
400
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
401
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
402
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
403
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
404
+ ```
405
+
406
+For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
407
+
408
+The following parameters are supported by default:
409
+
410
+??? code
411
+
412
+ ```python
413
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
414
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
415
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-params"
416
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-params"
417
+ ```
418
+
419
+these extra parameters are supported instead:
420
421
??? code
422
423
```python
397
- --8<-- "vllm/entrypoints/pooling/embed/protocol.py:chat-embedding-extra-params"
424
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
425
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
426
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:encoding-extra-params"
427
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:embed-extra-params"
428
```
429
430
### Transcriptions API
@@ -659,14 +689,48 @@ The following [pooling parameters][vllm.PoolingParams] are supported.
689
690
```python
691
--8<-- "vllm/pooling_params.py:common-pooling-params"
662
---8<-- "vllm/pooling_params.py:classification-pooling-params"
692
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
693
```
694
695
+The following Classification API parameters are supported:
696
+
697
+??? code
698
+
699
+ ```python
700
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
701
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-params"
702
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
703
+ ```
704
+
705
The following extra parameters are supported:
706
667
-```python
668
---8<-- "vllm/entrypoints/pooling/classify/protocol.py:classification-extra-params"
669
-```
707
+??? code
708
+
709
+ ```python
710
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
711
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:completion-extra-params"
712
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
713
+ ```
714
+
715
+For chat-like input (i.e. if `messages` is passed), the following parameters are supported:
716
+
717
+??? code
718
+
719
+ ```python
720
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
721
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-params"
722
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-params"
723
+ ```
724
+
725
+these extra parameters are supported instead:
726
+
727
+??? code
728
+
729
+ ```python
730
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
731
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:chat-extra-params"
732
+ --8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
733
+ ```
734
735
### Score API
736
@@ -882,12 +946,21 @@ The following [pooling parameters][vllm.PoolingParams] are supported.
946
947
```python
948
--8<-- "vllm/pooling_params.py:common-pooling-params"
885
---8<-- "vllm/pooling_params.py:classification-pooling-params"
949
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
950
+```
951
+
952
+The following Score API parameters are supported:
953
+
954
+```python
955
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
956
+--8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
957
```
958
959
The following extra parameters are supported:
960
961
```python
962
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
963
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
964
--8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
965
```
966
@@ -963,12 +1036,22 @@ The following [pooling parameters][vllm.PoolingParams] are supported.
1036
1037
```python
1038
--8<-- "vllm/pooling_params.py:common-pooling-params"
966
---8<-- "vllm/pooling_params.py:classification-pooling-params"
1039
+--8<-- "vllm/pooling_params.py:classify-pooling-params"
1040
+```
1041
+
1042
+The following Re-rank API parameters are supported:
1043
+
1044
+```python
1045
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-params"
1046
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
1047
+--8<-- "vllm/entrypoints/pooling/score/protocol.py:score-extra-params"
1048
```
1049
1050
The following extra parameters are supported:
1051
1052
```python
1053
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:pooling-common-extra-params"
1054
+--8<-- "vllm/entrypoints/pooling/base/protocol.py:classify-extra-params"
1055
--8<-- "vllm/entrypoints/pooling/score/protocol.py:rerank-extra-params"
1056
```
1057
examples/pooling/score/convert_model_to_seq_cls.py
+2
-2
@@ -183,9 +183,9 @@ def parse_args():
183
help="Conversion method to use",
184
)
185
parser.add_argument(
186
- "--use-pad-token",
186
+ "--use-sep-token",
187
action="store_true",
188
- help="Enable padding token in the sequence classification model",
188
+ help="Enable separating token in the sequence classification model",
189
)
190
parser.add_argument(
191
"--path",
tests/entrypoints/pooling/classify/test_online_vision.py
+73
-18
@@ -1,27 +1,30 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
import json
4
5
import pytest
6
import requests
7
8
+from tests.entrypoints.test_utils import encode_base64_content_from_url
9
from tests.utils import RemoteOpenAIServer
10
from vllm.entrypoints.pooling.classify.protocol import ClassificationResponse
11
12
-VLM_MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
12
+MODEL_NAME = "muziyongshixin/Qwen2.5-VL-7B-for-VideoCls"
13
MAXIMUM_VIDEOS = 1
14
-TEST_VIDEO_URL = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4"
14
15
HF_OVERRIDES = {
16
"text_config": {
17
"architectures": ["Qwen2_5_VLForSequenceClassification"],
18
},
19
}
20
+input_text = "This product was excellent and exceeded my expectations"
21
+image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/cat_snow.jpg"
22
+image_base64 = encode_base64_content_from_url(image_url)
23
+video_url = "https://www.bogotobogo.com/python/OpenCV_Python/images/mean_shift_tracking/slow_traffic_small.mp4"
24
25
26
@pytest.fixture(scope="module")
24
-def server_vlm_classify():
27
+def server():
28
args = [
29
"--runner",
30
"pooling",
@@ -33,26 +36,80 @@ def server_vlm_classify():
36
]
37
38
with RemoteOpenAIServer(
36
- VLM_MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
39
+ MODEL_NAME, args, override_hf_configs=HF_OVERRIDES
40
) as remote_server:
41
yield remote_server
42
43
41
-@pytest.mark.parametrize("model_name", [VLM_MODEL_NAME])
42
-def test_classify_accepts_chat_text_only(
43
- server_vlm_classify: RemoteOpenAIServer, model_name: str
44
-) -> None:
44
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
45
+def test_chat_text_request(server: RemoteOpenAIServer, model_name: str):
46
+ messages = [
47
+ {
48
+ "role": "assistant",
49
+ "content": "Please classify this text request.",
50
+ },
51
+ {
52
+ "role": "user",
53
+ "content": input_text,
54
+ },
55
+ ]
56
+
57
+ response = requests.post(
58
+ server.url_for("classify"),
59
+ json={"model": model_name, "messages": messages},
60
+ )
61
+ response.raise_for_status()
62
+
63
+ output = ClassificationResponse.model_validate(response.json())
64
+
65
+ assert output.object == "list"
66
+ assert output.model == model_name
67
+ assert len(output.data) == 1
68
+ assert len(output.data[0].probs) == 2
69
+ assert output.usage.prompt_tokens == 35
70
+
71
+
72
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
73
+def test_chat_image_url_request(server: RemoteOpenAIServer, model_name: str):
74
+ messages = [
75
+ {
76
+ "role": "user",
77
+ "content": [
78
+ {"type": "text", "text": "Please classify this image."},
79
+ {"type": "image_url", "image_url": {"url": image_url}},
80
+ ],
81
+ }
82
+ ]
83
+
84
+ response = requests.post(
85
+ server.url_for("classify"),
86
+ json={"model": model_name, "messages": messages},
87
+ )
88
+ response.raise_for_status()
89
+
90
+ output = ClassificationResponse.model_validate(response.json())
91
+
92
+ assert output.object == "list"
93
+ assert output.model == model_name
94
+ assert len(output.data) == 1
95
+ assert len(output.data[0].probs) == 2
96
+ assert output.usage.prompt_tokens == 47
97
+
98
+
99
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
100
+def test_chat_image_base64_request(server: RemoteOpenAIServer, model_name: str):
101
messages = [
102
{
103
"role": "user",
104
"content": [
49
- {"type": "text", "text": "Please classify this text request."},
105
+ {"type": "text", "text": "Please classify this image."},
106
+ {"type": "image_url", "image_url": image_base64},
107
],
108
}
109
]
110
111
response = requests.post(
55
- server_vlm_classify.url_for("classify"),
112
+ server.url_for("classify"),
113
json={"model": model_name, "messages": messages},
114
)
115
response.raise_for_status()
@@ -63,25 +120,23 @@ def test_classify_accepts_chat_text_only(
120
assert output.model == model_name
121
assert len(output.data) == 1
122
assert len(output.data[0].probs) == 2
66
- assert output.usage.prompt_tokens == 22
123
+ assert output.usage.prompt_tokens == 47
124
125
69
-@pytest.mark.parametrize("model_name", [VLM_MODEL_NAME])
70
-def test_classify_accepts_chat_video_url(
71
- server_vlm_classify: RemoteOpenAIServer, model_name: str
72
-) -> None:
126
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
127
+def test_chat_video_url_request(server: RemoteOpenAIServer, model_name: str):
128
messages = [
129
{
130
"role": "user",
131
"content": [
132
{"type": "text", "text": "Please classify this video."},
78
- {"type": "video_url", "video_url": {"url": TEST_VIDEO_URL}},
133
+ {"type": "video_url", "video_url": {"url": video_url}},
134
],
135
}
136
]
137
138
response = requests.post(
84
- server_vlm_classify.url_for("classify"),
139
+ server.url_for("classify"),
140
json={"model": model_name, "messages": messages},
141
)
142
response.raise_for_status()
tests/entrypoints/test_utils.py
+12
@@ -1,5 +1,9 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import base64
4
+
5
+import requests
6
+
7
from vllm.entrypoints.utils import sanitize_message
8
9
@@ -8,3 +12,11 @@ def test_sanitize_message():
12
sanitize_message("<_io.BytesIO object at 0x7a95e299e750>")
13
== "<_io.BytesIO object>"
14
)
15
+
16
+
17
+def encode_base64_content_from_url(content_url: str) -> dict[str, str]:
18
+ with requests.get(content_url) as response:
19
+ response.raise_for_status()
20
+ result = base64.b64encode(response.content).decode("utf-8")
21
+
22
+ return {"url": f"data:image/jpeg;base64,{result}"}
vllm/entrypoints/openai/engine/serving.py
+7
-3
@@ -75,6 +75,8 @@ from vllm.entrypoints.pooling.embed.protocol import (
75
)
76
from vllm.entrypoints.pooling.pooling.protocol import (
77
IOProcessorRequest,
78
+ PoolingChatRequest,
79
+ PoolingCompletionRequest,
80
PoolingResponse,
81
)
82
from vllm.entrypoints.pooling.score.protocol import (
@@ -138,19 +140,21 @@ logger = init_logger(__name__)
140
141
CompletionLikeRequest: TypeAlias = (
142
CompletionRequest
143
+ | TokenizeCompletionRequest
144
| DetokenizeRequest
145
| EmbeddingCompletionRequest
143
- | RerankRequest
146
| ClassificationCompletionRequest
147
+ | RerankRequest
148
| ScoreRequest
146
- | TokenizeCompletionRequest
149
+ | PoolingCompletionRequest
150
)
151
152
ChatLikeRequest: TypeAlias = (
153
ChatCompletionRequest
151
- | EmbeddingChatRequest
154
| TokenizeChatRequest
155
+ | EmbeddingChatRequest
156
| ClassificationChatRequest
157
+ | PoolingChatRequest
158
)
159
SpeechToTextRequest: TypeAlias = TranscriptionRequest | TranslationRequest
160
AnyRequest: TypeAlias = (
vllm/entrypoints/pooling/base/protocol.py
+85
-6
@@ -6,16 +6,22 @@ from typing import Annotated, Any
6
7
from pydantic import Field, model_validator
8
9
+from vllm import PoolingParams
10
+from vllm.config.pooler import get_use_activation
11
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
12
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel
13
from vllm.utils import random_uuid
14
+from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
15
16
17
class PoolingBasicRequestMixin(OpenAIBaseModel):
18
+ # --8<-- [start:pooling-common-params]
19
model: str | None = None
20
user: str | None = None
17
- truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
21
+ # --8<-- [end:pooling-common-params]
22
23
+ # --8<-- [start:pooling-common-extra-params]
24
+ truncate_prompt_tokens: Annotated[int, Field(ge=-1)] | None = None
25
request_id: str = Field(
26
default_factory=random_uuid,
27
description=(
@@ -24,7 +30,6 @@ class PoolingBasicRequestMixin(OpenAIBaseModel):
30
"through out the inference process and return in response."
31
),
32
)
27
-
33
priority: int = Field(
34
default=0,
35
description=(
@@ -33,11 +38,15 @@ class PoolingBasicRequestMixin(OpenAIBaseModel):
38
"if the served model does not use priority scheduling."
39
),
40
)
41
+ # --8<-- [end:pooling-common-extra-params]
42
43
44
class CompletionRequestMixin(OpenAIBaseModel):
45
+ # --8<-- [start:completion-params]
46
input: list[int] | list[list[int]] | str | list[str]
47
+ # --8<-- [end:completion-params]
48
49
+ # --8<-- [start:completion-extra-params]
50
add_special_tokens: bool = Field(
51
default=True,
52
description=(
@@ -45,11 +54,15 @@ class CompletionRequestMixin(OpenAIBaseModel):
54
"the prompt."
55
),
56
)
57
+ # --8<-- [end:completion-extra-params]
58
59
60
class ChatRequestMixin(OpenAIBaseModel):
61
+ # --8<-- [start:chat-params]
62
messages: list[ChatCompletionMessageParam]
63
+ # --8<-- [end:chat-params]
64
65
+ # --8<-- [start:chat-extra-params]
66
add_generation_prompt: bool = Field(
67
default=False,
68
description=(
@@ -58,7 +71,6 @@ class ChatRequestMixin(OpenAIBaseModel):
71
"model."
72
),
73
)
61
-
74
continue_final_message: bool = Field(
75
default=False,
76
description=(
@@ -69,7 +81,6 @@ class ChatRequestMixin(OpenAIBaseModel):
81
"Cannot be used at the same time as `add_generation_prompt`."
82
),
83
)
72
-
84
add_special_tokens: bool = Field(
85
default=False,
86
description=(
@@ -80,7 +91,6 @@ class ChatRequestMixin(OpenAIBaseModel):
91
"default)."
92
),
93
)
83
-
94
chat_template: str | None = Field(
95
default=None,
96
description=(
@@ -90,7 +100,6 @@ class ChatRequestMixin(OpenAIBaseModel):
100
"does not define one."
101
),
102
)
93
-
103
chat_template_kwargs: dict[str, Any] | None = Field(
104
default=None,
105
description=(
@@ -98,6 +107,7 @@ class ChatRequestMixin(OpenAIBaseModel):
107
"Will be accessible by the chat template."
108
),
109
)
110
+ # --8<-- [end:chat-extra-params]
111
112
@model_validator(mode="before")
113
@classmethod
@@ -108,3 +118,72 @@ class ChatRequestMixin(OpenAIBaseModel):
118
"`add_generation_prompt` to True."
119
)
120
return data
121
+
122
+
123
+class EncodingRequestMixin(OpenAIBaseModel):
124
+ # --8<-- [start:encoding-params]
125
+ encoding_format: EncodingFormat = "float"
126
+ # --8<-- [end:encoding-params]
127
+
128
+ # --8<-- [start:encoding-extra-params]
129
+ embed_dtype: EmbedDType = Field(
130
+ default="float32",
131
+ description=(
132
+ "What dtype to use for encoding. Default to using float32 for base64 "
133
+ "encoding to match the OpenAI python client behavior. "
134
+ "This parameter will affect base64 and binary_response."
135
+ ),
136
+ )
137
+ endianness: Endianness = Field(
138
+ default="native",
139
+ description=(
140
+ "What endianness to use for encoding. Default to using native for "
141
+ "base64 encoding to match the OpenAI python client behavior."
142
+ "This parameter will affect base64 and binary_response."
143
+ ),
144
+ )
145
+ # --8<-- [end:encoding-extra-params]
146
+
147
+
148
+class EmbedRequestMixin(EncodingRequestMixin):
149
+ # --8<-- [start:embed-params]
150
+ dimensions: int | None = None
151
+ # --8<-- [end:embed-params]
152
+
153
+ # --8<-- [start:embed-extra-params]
154
+ normalize: bool | None = Field(
155
+ default=None,
156
+ description="Whether to normalize the embeddings outputs. Default is True.",
157
+ )
158
+ # --8<-- [end:embed-extra-params]
159
+
160
+ def to_pooling_params(self):
161
+ return PoolingParams(
162
+ dimensions=self.dimensions,
163
+ use_activation=self.normalize,
164
+ truncate_prompt_tokens=getattr(self, "truncate_prompt_tokens", None),
165
+ )
166
+
167
+
168
+class ClassifyRequestMixin(OpenAIBaseModel):
169
+ # --8<-- [start:classify-extra-params]
170
+ softmax: bool | None = Field(
171
+ default=None,
172
+ description="softmax will be deprecated, please use use_activation instead.",
173
+ )
174
+ activation: bool | None = Field(
175
+ default=None,
176
+ description="activation will be deprecated, please use use_activation instead.",
177
+ )
178
+ use_activation: bool | None = Field(
179
+ default=None,
180
+ description="Whether to use activation for classification outputs. "
181
+ "Default is True.",
182
+ )
183
+ # --8<-- [end:classify-extra-params]
184
+
185
+ def to_pooling_params(self):
186
+ return PoolingParams(
187
+ use_activation=get_use_activation(self),
188
+ truncate_prompt_tokens=getattr(self, "truncate_prompt_tokens", None),
189
+ )
vllm/entrypoints/pooling/classify/protocol.py
+8
-50
@@ -8,73 +8,31 @@ from pydantic import (
8
Field,
9
)
10
11
-from vllm import PoolingParams
12
-from vllm.config.pooler import get_use_activation
11
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
12
from vllm.entrypoints.pooling.base.protocol import (
13
ChatRequestMixin,
14
+ ClassifyRequestMixin,
15
CompletionRequestMixin,
16
PoolingBasicRequestMixin,
17
)
18
from vllm.utils import random_uuid
19
20
22
-class ClassificationCompletionRequest(PoolingBasicRequestMixin, CompletionRequestMixin):
23
- # --8<-- [start:classification-extra-params]
24
- softmax: bool | None = Field(
25
- default=None,
26
- description="softmax will be deprecated, please use use_activation instead.",
27
- )
28
-
29
- activation: bool | None = Field(
30
- default=None,
31
- description="activation will be deprecated, please use use_activation instead.",
32
- )
21
+class ClassificationCompletionRequest(
22
+ PoolingBasicRequestMixin, CompletionRequestMixin, ClassifyRequestMixin
23
+):
24
+ pass
25
34
- use_activation: bool | None = Field(
35
- default=None,
36
- description="Whether to use activation for classification outputs. "
37
- "Default is True.",
38
- )
39
- # --8<-- [end:classification-extra-params]
40
-
41
- def to_pooling_params(self):
42
- return PoolingParams(
43
- truncate_prompt_tokens=self.truncate_prompt_tokens,
44
- use_activation=get_use_activation(self),
45
- )
26
47
-
48
-class ClassificationChatRequest(PoolingBasicRequestMixin, ChatRequestMixin):
27
+class ClassificationChatRequest(
28
+ PoolingBasicRequestMixin, ChatRequestMixin, ClassifyRequestMixin
29
+):
30
# --8<-- [start:chat-classification-extra-params]
31
mm_processor_kwargs: dict[str, Any] | None = Field(
32
default=None,
33
description=("Additional kwargs to pass to the HF processor."),
34
)
35
55
- softmax: bool | None = Field(
56
- default=None,
57
- description="softmax will be deprecated, please use use_activation instead.",
58
- )
59
-
60
- activation: bool | None = Field(
61
- default=None,
62
- description="activation will be deprecated, please use use_activation instead.",
63
- )
64
-
65
- use_activation: bool | None = Field(
66
- default=None,
67
- description="Whether to use activation for classification outputs. "
68
- "Default is True.",
69
- )
70
- # --8<-- [end:chat-classification-extra-params]
71
-
72
- def to_pooling_params(self):
73
- return PoolingParams(
74
- truncate_prompt_tokens=self.truncate_prompt_tokens,
75
- use_activation=get_use_activation(self),
76
- )
77
-
36
37
ClassificationRequest: TypeAlias = (
38
ClassificationCompletionRequest | ClassificationChatRequest
vllm/entrypoints/pooling/embed/protocol.py
+8
-69
@@ -7,92 +7,31 @@ from pydantic import (
7
Field,
8
)
9
10
-from vllm import PoolingParams
10
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
11
from vllm.entrypoints.pooling.base.protocol import (
12
ChatRequestMixin,
13
CompletionRequestMixin,
14
+ EmbedRequestMixin,
15
PoolingBasicRequestMixin,
16
)
17
from vllm.utils import random_uuid
18
-from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
18
19
21
-class EmbeddingCompletionRequest(PoolingBasicRequestMixin, CompletionRequestMixin):
20
+class EmbeddingCompletionRequest(
21
+ PoolingBasicRequestMixin, CompletionRequestMixin, EmbedRequestMixin
22
+):
23
# Ordered by official OpenAI API documentation
24
# https://platform.openai.com/docs/api-reference/embeddings
25
+ pass
26
25
- encoding_format: EncodingFormat = "float"
26
- dimensions: int | None = None
27
28
- # --8<-- [start:embedding-extra-params]
29
- normalize: bool | None = Field(
30
- default=None,
31
- description="Whether to normalize the embeddings outputs. Default is True.",
32
- )
33
- embed_dtype: EmbedDType = Field(
34
- default="float32",
35
- description=(
36
- "What dtype to use for encoding. Default to using float32 for base64 "
37
- "encoding to match the OpenAI python client behavior. "
38
- "This parameter will affect base64 and binary_response."
39
- ),
40
- )
41
- endianness: Endianness = Field(
42
- default="native",
43
- description=(
44
- "What endianness to use for encoding. Default to using native for "
45
- "base64 encoding to match the OpenAI python client behavior."
46
- "This parameter will affect base64 and binary_response."
47
- ),
48
- )
49
- # --8<-- [end:embedding-extra-params]
50
-
51
- def to_pooling_params(self):
52
- return PoolingParams(
53
- dimensions=self.dimensions,
54
- use_activation=self.normalize,
55
- truncate_prompt_tokens=self.truncate_prompt_tokens,
56
- )
57
-
58
-
59
-class EmbeddingChatRequest(PoolingBasicRequestMixin, ChatRequestMixin):
60
- encoding_format: EncodingFormat = "float"
61
- dimensions: int | None = None
62
-
63
- # --8<-- [start:chat-embedding-extra-params]
28
+class EmbeddingChatRequest(
29
+ PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin
30
+):
31
mm_processor_kwargs: dict[str, Any] | None = Field(
32
default=None,
33
description=("Additional kwargs to pass to the HF processor."),
34
)
68
- normalize: bool | None = Field(
69
- default=None,
70
- description="Whether to normalize the embeddings outputs. Default is True.",
71
- )
72
- embed_dtype: EmbedDType = Field(
73
- default="float32",
74
- description=(
75
- "What dtype to use for encoding. Default to using float32 for base64 "
76
- "encoding to match the OpenAI python client behavior. "
77
- "This parameter will affect base64 and binary_response."
78
- ),
79
- )
80
- endianness: Endianness = Field(
81
- default="native",
82
- description=(
83
- "What endianness to use for encoding. Default to using native for "
84
- "base64 encoding to match the OpenAI python client behavior."
85
- "This parameter will affect base64 and binary_response."
86
- ),
87
- )
88
- # --8<-- [end:chat-embedding-extra-params]
89
-
90
- def to_pooling_params(self):
91
- return PoolingParams(
92
- truncate_prompt_tokens=self.truncate_prompt_tokens,
93
- dimensions=self.dimensions,
94
- use_activation=self.normalize,
95
- )
35
36
37
EmbeddingRequest: TypeAlias = EmbeddingCompletionRequest | EmbeddingChatRequest
vllm/entrypoints/pooling/pooling/protocol.py
+21
-52
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import time
4
-from typing import Generic, TypeAlias, TypeVar
4
+from typing import Any, Generic, TypeAlias, TypeVar
5
6
from pydantic import (
7
Field,
@@ -10,32 +10,25 @@ from pydantic import (
10
from vllm import PoolingParams
11
from vllm.config.pooler import get_use_activation
12
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
13
-from vllm.entrypoints.pooling.base.protocol import PoolingBasicRequestMixin
14
-from vllm.entrypoints.pooling.embed.protocol import (
15
- EmbeddingChatRequest,
16
- EmbeddingCompletionRequest,
13
+from vllm.entrypoints.pooling.base.protocol import (
14
+ ChatRequestMixin,
15
+ ClassifyRequestMixin,
16
+ CompletionRequestMixin,
17
+ EmbedRequestMixin,
18
+ EncodingRequestMixin,
19
+ PoolingBasicRequestMixin,
20
)
21
from vllm.tasks import PoolingTask
22
from vllm.utils import random_uuid
20
-from vllm.utils.serial_utils import EmbedDType, EncodingFormat, Endianness
23
24
23
-class PoolingCompletionRequest(EmbeddingCompletionRequest):
25
+class PoolingCompletionRequest(
26
+ PoolingBasicRequestMixin,
27
+ CompletionRequestMixin,
28
+ EmbedRequestMixin,
29
+ ClassifyRequestMixin,
30
+):
31
task: PoolingTask | None = None
25
- softmax: bool | None = Field(
26
- default=None,
27
- description="softmax will be deprecated, please use use_activation instead.",
28
- )
29
- activation: bool | None = Field(
30
- default=None,
31
- description="activation will be deprecated, please use use_activation instead.",
32
- )
33
- use_activation: bool | None = Field(
34
- default=None,
35
- description="Whether to use activation for classification outputs. "
36
- "If it is a classify or token_classify task, the default is True; "
37
- "for other tasks, this value should be None.",
38
- )
32
33
def to_pooling_params(self):
34
return PoolingParams(
@@ -45,21 +38,14 @@ class PoolingCompletionRequest(EmbeddingCompletionRequest):
38
)
39
40
48
-class PoolingChatRequest(EmbeddingChatRequest):
41
+class PoolingChatRequest(
42
+ PoolingBasicRequestMixin, ChatRequestMixin, EmbedRequestMixin, ClassifyRequestMixin
43
+):
44
task: PoolingTask | None = None
50
- softmax: bool | None = Field(
51
- default=None,
52
- description="softmax will be deprecated, please use use_activation instead.",
53
- )
54
- activation: bool | None = Field(
55
- default=None,
56
- description="activation will be deprecated, please use use_activation instead.",
57
- )
58
- use_activation: bool | None = Field(
45
+
46
+ mm_processor_kwargs: dict[str, Any] | None = Field(
47
default=None,
60
- description="Whether to use activation for classification outputs. "
61
- "If it is a classify or token_classify task, the default is True; "
62
- "for other tasks, this value should be None.",
48
+ description=("Additional kwargs to pass to the HF processor."),
49
)
50
51
def to_pooling_params(self):
@@ -73,26 +59,9 @@ class PoolingChatRequest(EmbeddingChatRequest):
59
T = TypeVar("T")
60
61
76
-class IOProcessorRequest(PoolingBasicRequestMixin, Generic[T]):
62
+class IOProcessorRequest(PoolingBasicRequestMixin, EncodingRequestMixin, Generic[T]):
63
data: T
64
task: PoolingTask = "plugin"
79
- encoding_format: EncodingFormat = "float"
80
- embed_dtype: EmbedDType = Field(
81
- default="float32",
82
- description=(
83
- "What dtype to use for encoding. Default to using float32 for base64 "
84
- "encoding to match the OpenAI python client behavior. "
85
- "This parameter will affect base64 and binary_response."
86
- ),
87
- )
88
- endianness: Endianness = Field(
89
- default="native",
90
- description=(
91
- "What endianness to use for encoding. Default to using native for "
92
- "base64 encoding to match the OpenAI python client behavior."
93
- "This parameter will affect base64 and binary_response."
94
- ),
95
- )
65
66
def to_pooling_params(self):
67
return PoolingParams()
vllm/entrypoints/pooling/score/protocol.py
+6
-40
@@ -11,7 +11,10 @@ from pydantic import (
11
from vllm import PoolingParams
12
from vllm.config.pooler import get_use_activation
13
from vllm.entrypoints.openai.engine.protocol import OpenAIBaseModel, UsageInfo
14
-from vllm.entrypoints.pooling.base.protocol import PoolingBasicRequestMixin
14
+from vllm.entrypoints.pooling.base.protocol import (
15
+ ClassifyRequestMixin,
16
+ PoolingBasicRequestMixin,
17
+)
18
from vllm.entrypoints.pooling.score.utils import (
19
ScoreContentPartParam,
20
ScoreMultiModalParam,
@@ -19,28 +22,12 @@ from vllm.entrypoints.pooling.score.utils import (
22
from vllm.utils import random_uuid
23
24
22
-class ScoreRequestMixin(PoolingBasicRequestMixin):
25
+class ScoreRequestMixin(PoolingBasicRequestMixin, ClassifyRequestMixin):
26
# --8<-- [start:score-extra-params]
27
mm_processor_kwargs: dict[str, Any] | None = Field(
28
default=None,
29
description=("Additional kwargs to pass to the HF processor."),
30
)
28
-
29
- softmax: bool | None = Field(
30
- default=None,
31
- description="softmax will be deprecated, please use use_activation instead.",
32
- )
33
-
34
- activation: bool | None = Field(
35
- default=None,
36
- description="activation will be deprecated, please use use_activation instead.",
37
- )
38
-
39
- use_activation: bool | None = Field(
40
- default=None,
41
- description="Whether to use activation for classification outputs. "
42
- "Default is True.",
43
- )
31
# --8<-- [end:score-extra-params]
32
33
def to_pooling_params(self):
@@ -86,7 +73,7 @@ ScoreRequest: TypeAlias = (
73
)
74
75
89
-class RerankRequest(PoolingBasicRequestMixin):
76
+class RerankRequest(PoolingBasicRequestMixin, ClassifyRequestMixin):
77
query: str | ScoreMultiModalParam
78
documents: list[str] | ScoreMultiModalParam
79
top_n: int = Field(default_factory=lambda: 0)
@@ -96,29 +83,8 @@ class RerankRequest(PoolingBasicRequestMixin):
83
default=None,
84
description=("Additional kwargs to pass to the HF processor."),
85
)
99
- softmax: bool | None = Field(
100
- default=None,
101
- description="softmax will be deprecated, please use use_activation instead.",
102
- )
103
-
104
- activation: bool | None = Field(
105
- default=None,
106
- description="activation will be deprecated, please use use_activation instead.",
107
- )
108
-
109
- use_activation: bool | None = Field(
110
- default=None,
111
- description="Whether to use activation for classification outputs. "
112
- "Default is True.",
113
- )
86
# --8<-- [end:rerank-extra-params]
87
116
- def to_pooling_params(self):
117
- return PoolingParams(
118
- truncate_prompt_tokens=self.truncate_prompt_tokens,
119
- use_activation=get_use_activation(self),
120
- )
121
-
88
89
class RerankDocument(BaseModel):
90
text: str | None = None
vllm/pooling_params.py
+4
-4
@@ -38,17 +38,17 @@ class PoolingParams(
38
# --8<-- [end:common-pooling-params]
39
40
## for embeddings models
41
- # --8<-- [start:embedding-pooling-params]
41
+ # --8<-- [start:embed-pooling-params]
42
dimensions: int | None = None
43
normalize: bool | None = None
44
- # --8<-- [end:embedding-pooling-params]
44
+ # --8<-- [end:embed-pooling-params]
45
46
## for classification, scoring and rerank
47
- # --8<-- [start:classification-pooling-params]
47
+ # --8<-- [start:classify-pooling-params]
48
softmax: bool | None = None
49
activation: bool | None = None
50
use_activation: bool | None = None
51
- # --8<-- [end:classification-pooling-params]
51
+ # --8<-- [end:classify-pooling-params]
52
53
## for step pooling models
54
step_tag_id: int | None = None