A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Frontend] Chat-based Embeddings API (#9759)
Cyrus Leung committed
Nov 1, 2024 at 16:13 UTC
06386a64dd706cf3fdab82510124ca2c2f9eee9d
21 files changed
+853
-415
docs/requirements-docs.txt
+2
@@ -13,5 +13,7 @@ torch
13
py-cpuinfo
14
transformers
15
mistral_common >= 1.3.4
16
+aiohttp
17
+starlette
18
openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
19
partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/conf.py
+1
-1
@@ -96,7 +96,6 @@ def setup(app):
96
97
# Mock out external dependencies here, otherwise the autodoc pages may be blank.
98
autodoc_mock_imports = [
99
- "aiohttp",
99
"compressed_tensors",
100
"cpuinfo",
101
"cv2",
@@ -143,6 +142,7 @@ intersphinx_mapping = {
142
"python": ("https://docs.python.org/3", None),
143
"typing_extensions":
144
("https://typing-extensions.readthedocs.io/en/latest", None),
145
+ "aiohttp": ("https://docs.aiohttp.org/en/stable", None),
146
"pillow": ("https://pillow.readthedocs.io/en/stable", None),
147
"numpy": ("https://numpy.org/doc/stable", None),
148
"torch": ("https://pytorch.org/docs/stable", None),
docs/source/dev/pooling_params.rst
new
+5
@@ -0,0 +1,5 @@
1
+Pooling Parameters
2
+==================
3
+
4
+.. autoclass:: vllm.PoolingParams
5
+ :members:
docs/source/getting_started/quickstart.rst
+4
-4
@@ -138,10 +138,10 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep
138
139
A more detailed client example can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`__.
140
141
-OpenAI Chat API with vLLM
142
-~~~~~~~~~~~~~~~~~~~~~~~~~~
141
+OpenAI Chat Completions API with vLLM
142
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
143
144
-vLLM is designed to also support the OpenAI Chat API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
144
+vLLM is designed to also support the OpenAI Chat Completions API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
145
146
You can use the `create chat completion <https://platform.openai.com/docs/api-reference/chat/completions/create>`_ endpoint to interact with the model:
147
@@ -157,7 +157,7 @@ You can use the `create chat completion <https://platform.openai.com/docs/api-re
157
$ ]
158
$ }'
159
160
-Alternatively, you can use the `openai` python package:
160
+Alternatively, you can use the ``openai`` python package:
161
162
.. code-block:: python
163
docs/source/index.rst
+1
@@ -134,6 +134,7 @@ Documentation
134
:caption: Developer Documentation
135
136
dev/sampling_params
137
+ dev/pooling_params
138
dev/offline_inference/offline_index
139
dev/engine/engine_index
140
dev/kernel/paged_attention
docs/source/models/vlm.rst
+51
-3
@@ -185,7 +185,7 @@ Below is an example on how to launch the same ``microsoft/Phi-3.5-vision-instruc
185
--trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
186
187
.. important::
188
- Since OpenAI Vision API is based on `Chat Completions <https://platform.openai.com/docs/api-reference/chat>`_ API,
188
+ Since OpenAI Vision API is based on `Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`_,
189
a chat template is **required** to launch the API server.
190
191
Although Phi-3.5-Vision comes with a chat template, for other models you may have to provide one if the model's tokenizer does not come with it.
@@ -243,6 +243,10 @@ To consume the server, you can use the OpenAI client like in the example below:
243
244
A full code example can be found in `examples/openai_api_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_api_client_for_multimodal.py>`_.
245
246
+.. tip::
247
+ There is no need to place image placeholders in the text content of the API request - they are already represented by the image content.
248
+ In fact, you can place image placeholders in the middle of the text by interleaving text and image content.
249
+
250
.. note::
251
252
By default, the timeout for fetching images through http url is ``5`` seconds. You can override this by setting the environment variable:
@@ -251,5 +255,49 @@ A full code example can be found in `examples/openai_api_client_for_multimodal.p
255
256
$ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
257
254
-.. note::
255
- There is no need to format the prompt in the API request since it will be handled by the server.
258
+Chat Embeddings API
259
+^^^^^^^^^^^^^^^^^^^
260
+
261
+vLLM's Chat Embeddings API is a superset of OpenAI's `Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`_,
262
+where a list of ``messages`` can be passed instead of batched ``inputs``. This enables multi-modal inputs to be passed to embedding models.
263
+
264
+.. tip::
265
+ The schema of ``messages`` is exactly the same as in Chat Completions API.
266
+
267
+In this example, we will serve the ``TIGER-Lab/VLM2Vec-Full`` model.
268
+
269
+.. code-block:: bash
270
+
271
+ vllm serve TIGER-Lab/VLM2Vec-Full --task embedding \
272
+ --trust-remote-code --max-model-len 4096
273
+
274
+.. important::
275
+
276
+ Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embedding``
277
+ to run this model in embedding mode instead of text generation mode.
278
+
279
+Since this schema is not defined by OpenAI client, we post a request to the server using the lower-level ``requests`` library:
280
+
281
+.. code-block:: python
282
+
283
+ import requests
284
+
285
+ image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
286
+
287
+ response = requests.post(
288
+ "http://localhost:8000/v1/embeddings",
289
+ json={
290
+ "model": "TIGER-Lab/VLM2Vec-Full",
291
+ "messages": [{
292
+ "role": "user",
293
+ "content": [
294
+ {"type": "image_url", "image_url": {"url": image_url}},
295
+ {"type": "text", "text": "Represent the given image."},
296
+ ],
297
+ }],
298
+ "encoding_format": "float",
299
+ },
300
+ )
301
+ response.raise_for_status()
302
+ response_json = response.json()
303
+ print("Embedding output:", response_json["data"][0]["embedding"])
docs/source/serving/openai_compatible_server.md
+44
-11
@@ -26,13 +26,26 @@ print(completion.choices[0].message)
26
```
27
28
## API Reference
29
-Please see the [OpenAI API Reference](https://platform.openai.com/docs/api-reference) for more information on the API. We support all parameters except:
30
-- Chat: `tools`, and `tool_choice`.
31
-- Completions: `suffix`.
29
33
-vLLM also provides experimental support for OpenAI Vision API compatible inference. See more details in [Using VLMs](../models/vlm.rst).
30
+We currently support the following OpenAI APIs:
31
+
32
+- [Completions API](https://platform.openai.com/docs/api-reference/completions)
33
+ - *Note: `suffix` parameter is not supported.*
34
+- [Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
35
+ - [Vision](https://platform.openai.com/docs/guides/vision)-related parameters are supported; see [Using VLMs](../models/vlm.rst).
36
+ - *Note: `image_url.detail` parameter is not supported.*
37
+ - We also support `audio_url` content type for audio files.
38
+ - Refer to [vllm.entrypoints.chat_utils](https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/chat_utils.py) for the exact schema.
39
+ - *TODO: Support `input_audio` content type as defined [here](https://github.com/openai/openai-python/blob/v1.52.2/src/openai/types/chat/chat_completion_content_part_input_audio_param.py).*
40
+ - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
41
+- [Embeddings API](https://platform.openai.com/docs/api-reference/embeddings)
42
+ - Instead of `inputs`, you can pass in a list of `messages` (same schema as Chat Completions API),
43
+ which will be treated as a single prompt to the model according to its chat template.
44
+ - This enables multi-modal inputs to be passed to embedding models, see [Using VLMs](../models/vlm.rst).
45
+ - *Note: You should run `vllm serve` with `--task embedding` to ensure that the model is being run in embedding mode.*
46
47
## Extra Parameters
48
+
49
vLLM supports a set of parameters that are not part of the OpenAI API.
50
In order to use them, you can pass them as extra parameters in the OpenAI client.
51
Or directly merge them into the JSON payload if you are using HTTP call directly.
@@ -49,7 +62,26 @@ completion = client.chat.completions.create(
62
)
63
```
64
52
-### Extra Parameters for Chat API
65
+### Extra Parameters for Completions API
66
+
67
+The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
68
+
69
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
70
+:language: python
71
+:start-after: begin-completion-sampling-params
72
+:end-before: end-completion-sampling-params
73
+```
74
+
75
+The following extra parameters are supported:
76
+
77
+```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
78
+:language: python
79
+:start-after: begin-completion-extra-params
80
+:end-before: end-completion-extra-params
81
+```
82
+
83
+### Extra Parameters for Chat Completions API
84
+
85
The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
86
87
```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
@@ -66,21 +98,22 @@ The following extra parameters are supported:
98
:end-before: end-chat-completion-extra-params
99
```
100
69
-### Extra Parameters for Completions API
70
-The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
101
+### Extra Parameters for Embeddings API
102
+
103
+The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
104
105
```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
106
:language: python
74
-:start-after: begin-completion-sampling-params
75
-:end-before: end-completion-sampling-params
107
+:start-after: begin-embedding-pooling-params
108
+:end-before: end-embedding-pooling-params
109
```
110
111
The following extra parameters are supported:
112
113
```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
114
:language: python
82
-:start-after: begin-completion-extra-params
83
-:end-before: end-completion-extra-params
115
+:start-after: begin-embedding-extra-params
116
+:end-before: end-embedding-extra-params
117
```
118
119
## Chat Template
tests/entrypoints/openai/test_basic.py
+4
-9
@@ -1,7 +1,6 @@
1
from http import HTTPStatus
2
from typing import List
3
4
-import openai
4
import pytest
5
import pytest_asyncio
6
import requests
@@ -83,10 +82,8 @@ async def client(server):
82
indirect=True,
83
)
84
@pytest.mark.asyncio
86
-async def test_show_version(client: openai.AsyncOpenAI):
87
- base_url = str(client.base_url)[:-3].strip("/")
88
-
89
- response = requests.get(base_url + "/version")
85
+async def test_show_version(server: RemoteOpenAIServer):
86
+ response = requests.get(server.url_for("version"))
87
response.raise_for_status()
88
89
assert response.json() == {"version": VLLM_VERSION}
@@ -102,9 +99,7 @@ async def test_show_version(client: openai.AsyncOpenAI):
99
indirect=True,
100
)
101
@pytest.mark.asyncio
105
-async def test_check_health(client: openai.AsyncOpenAI):
106
- base_url = str(client.base_url)[:-3].strip("/")
107
-
108
- response = requests.get(base_url + "/health")
102
+async def test_check_health(server: RemoteOpenAIServer):
103
+ response = requests.get(server.url_for("health"))
104
105
assert response.status_code == HTTPStatus.OK
tests/entrypoints/openai/test_embedding.py
+90
-47
@@ -4,14 +4,18 @@ import numpy as np
4
import openai
5
import pytest
6
import pytest_asyncio
7
+import requests
8
+
9
+from vllm.transformers_utils.tokenizer import get_tokenizer
10
11
from ...utils import RemoteOpenAIServer
12
10
-EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
13
+MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
14
+DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
15
16
17
@pytest.fixture(scope="module")
14
-def embedding_server():
18
+def server():
19
args = [
20
# use half precision for speed and memory savings in CI environment
21
"--dtype",
@@ -19,31 +23,29 @@ def embedding_server():
23
"--enforce-eager",
24
"--max-model-len",
25
"8192",
26
+ "--chat-template",
27
+ DUMMY_CHAT_TEMPLATE,
28
]
29
24
- with RemoteOpenAIServer(EMBEDDING_MODEL_NAME, args) as remote_server:
30
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31
yield remote_server
32
33
34
@pytest_asyncio.fixture
29
-async def embedding_client(embedding_server):
30
- async with embedding_server.get_async_client() as async_client:
35
+async def client(server):
36
+ async with server.get_async_client() as async_client:
37
yield async_client
38
39
40
@pytest.mark.asyncio
35
-@pytest.mark.parametrize(
36
- "model_name",
37
- [EMBEDDING_MODEL_NAME],
38
-)
39
-async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
40
- model_name: str):
41
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
42
+async def test_single_embedding(client: openai.AsyncOpenAI, model_name: str):
43
input_texts = [
44
"The chef prepared a delicious meal.",
45
]
46
47
# test single embedding
46
- embeddings = await embedding_client.embeddings.create(
48
+ embeddings = await client.embeddings.create(
49
model=model_name,
50
input=input_texts,
51
encoding_format="float",
@@ -57,7 +59,7 @@ async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
59
60
# test using token IDs
61
input_tokens = [1, 1, 1, 1, 1]
60
- embeddings = await embedding_client.embeddings.create(
62
+ embeddings = await client.embeddings.create(
63
model=model_name,
64
input=input_tokens,
65
encoding_format="float",
@@ -71,18 +73,14 @@ async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
73
74
75
@pytest.mark.asyncio
74
-@pytest.mark.parametrize(
75
- "model_name",
76
- [EMBEDDING_MODEL_NAME],
77
-)
78
-async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
79
- model_name: str):
76
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
77
+async def test_batch_embedding(client: openai.AsyncOpenAI, model_name: str):
78
# test List[str]
79
input_texts = [
80
"The cat sat on the mat.", "A feline was resting on a rug.",
81
"Stars twinkle brightly in the night sky."
82
]
85
- embeddings = await embedding_client.embeddings.create(
83
+ embeddings = await client.embeddings.create(
84
model=model_name,
85
input=input_texts,
86
encoding_format="float",
@@ -90,11 +88,14 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
88
assert embeddings.id is not None
89
assert len(embeddings.data) == 3
90
assert len(embeddings.data[0].embedding) == 4096
91
+ assert embeddings.usage.completion_tokens == 0
92
+ assert embeddings.usage.prompt_tokens == 32
93
+ assert embeddings.usage.total_tokens == 32
94
95
# test List[List[int]]
96
input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
97
[25, 32, 64, 77]]
97
- embeddings = await embedding_client.embeddings.create(
98
+ embeddings = await client.embeddings.create(
99
model=model_name,
100
input=input_tokens,
101
encoding_format="float",
@@ -108,22 +109,70 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
109
110
111
@pytest.mark.asyncio
111
-@pytest.mark.parametrize(
112
- "model_name",
113
- [EMBEDDING_MODEL_NAME],
114
-)
115
-async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
112
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
113
+async def test_conversation_embedding(server: RemoteOpenAIServer,
114
+ client: openai.AsyncOpenAI,
115
+ model_name: str):
116
+ messages = [{
117
+ "role": "user",
118
+ "content": "The cat sat on the mat.",
119
+ }, {
120
+ "role": "assistant",
121
+ "content": "A feline was resting on a rug.",
122
+ }, {
123
+ "role": "user",
124
+ "content": "Stars twinkle brightly in the night sky.",
125
+ }]
126
+
127
+ chat_response = requests.post(server.url_for("v1/embeddings"),
128
+ json={
129
+ "model": model_name,
130
+ "messages": messages,
131
+ "encoding_format": "float",
132
+ })
133
+ chat_response.raise_for_status()
134
+ chat_embeddings = chat_response.json()
135
+
136
+ tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
137
+ prompt = tokenizer.apply_chat_template(
138
+ messages,
139
+ chat_template=DUMMY_CHAT_TEMPLATE,
140
+ add_generation_prompt=True,
141
+ continue_final_message=False,
142
+ tokenize=False,
143
+ )
144
+ completion_response = await client.embeddings.create(
145
+ model=model_name,
146
+ input=prompt,
147
+ encoding_format="float",
148
+ # To be consistent with chat
149
+ extra_body={"add_special_tokens": False},
150
+ )
151
+ completion_embeddings = completion_response.model_dump(mode="json")
152
+
153
+ assert chat_embeddings.pop("id") is not None
154
+ assert completion_embeddings.pop("id") is not None
155
+ assert chat_embeddings.pop("created") <= completion_embeddings.pop(
156
+ "created")
157
+ assert chat_embeddings == completion_embeddings
158
+
159
+
160
+@pytest.mark.asyncio
161
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
162
+async def test_batch_base64_embedding(client: openai.AsyncOpenAI,
163
model_name: str):
164
input_texts = [
165
"Hello my name is",
166
"The best thing about vLLM is that it supports many different models"
167
]
168
122
- responses_float = await embedding_client.embeddings.create(
123
- input=input_texts, model=model_name, encoding_format="float")
169
+ responses_float = await client.embeddings.create(input=input_texts,
170
+ model=model_name,
171
+ encoding_format="float")
172
125
- responses_base64 = await embedding_client.embeddings.create(
126
- input=input_texts, model=model_name, encoding_format="base64")
173
+ responses_base64 = await client.embeddings.create(input=input_texts,
174
+ model=model_name,
175
+ encoding_format="base64")
176
177
decoded_responses_base64_data = []
178
for data in responses_base64.data:
@@ -137,8 +186,8 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
186
1]
187
188
# Default response is float32 decoded from base64 by OpenAI Client
140
- responses_default = await embedding_client.embeddings.create(
141
- input=input_texts, model=model_name)
189
+ responses_default = await client.embeddings.create(input=input_texts,
190
+ model=model_name)
191
192
assert responses_float.data[0].embedding == responses_default.data[
193
0].embedding
@@ -147,18 +196,15 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
196
197
198
@pytest.mark.asyncio
150
-@pytest.mark.parametrize(
151
- "model_name",
152
- [EMBEDDING_MODEL_NAME],
153
-)
154
-async def test_single_embedding_truncation(
155
- embedding_client: openai.AsyncOpenAI, model_name: str):
199
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
200
+async def test_single_embedding_truncation(client: openai.AsyncOpenAI,
201
+ model_name: str):
202
input_texts = [
203
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
204
]
205
206
# test single embedding
161
- embeddings = await embedding_client.embeddings.create(
207
+ embeddings = await client.embeddings.create(
208
model=model_name,
209
input=input_texts,
210
extra_body={"truncate_prompt_tokens": 10})
@@ -173,7 +219,7 @@ async def test_single_embedding_truncation(
219
1, 24428, 289, 18341, 26165, 285, 19323, 283, 289, 26789, 3871, 28728,
220
9901, 340, 2229, 385, 340, 315, 28741, 28804, 2
221
]
176
- embeddings = await embedding_client.embeddings.create(
222
+ embeddings = await client.embeddings.create(
223
model=model_name,
224
input=input_tokens,
225
extra_body={"truncate_prompt_tokens": 10})
@@ -187,18 +233,15 @@ async def test_single_embedding_truncation(
233
234
235
@pytest.mark.asyncio
190
-@pytest.mark.parametrize(
191
- "model_name",
192
- [EMBEDDING_MODEL_NAME],
193
-)
194
-async def test_single_embedding_truncation_invalid(
195
- embedding_client: openai.AsyncOpenAI, model_name: str):
236
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
237
+async def test_single_embedding_truncation_invalid(client: openai.AsyncOpenAI,
238
+ model_name: str):
239
input_texts = [
240
"Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
241
]
242
243
with pytest.raises(openai.BadRequestError):
201
- embeddings = await embedding_client.embeddings.create(
244
+ embeddings = await client.embeddings.create(
245
model=model_name,
246
input=input_texts,
247
extra_body={"truncate_prompt_tokens": 8193})
tests/entrypoints/openai/test_metrics.py
+6
-8
@@ -79,9 +79,8 @@ EXPECTED_VALUES = {
79
80
81
@pytest.mark.asyncio
82
-async def test_metrics_counts(client: openai.AsyncOpenAI):
83
- base_url = str(client.base_url)[:-3].strip("/")
84
-
82
+async def test_metrics_counts(server: RemoteOpenAIServer,
83
+ client: openai.AsyncClient):
84
for _ in range(_NUM_REQUESTS):
85
# sending a request triggers the metrics to be logged.
86
await client.completions.create(
@@ -89,7 +88,7 @@ async def test_metrics_counts(client: openai.AsyncOpenAI):
88
prompt=_TOKENIZED_PROMPT,
89
max_tokens=_NUM_GENERATION_TOKENS_PER_REQUEST)
90
92
- response = requests.get(base_url + "/metrics")
91
+ response = requests.get(server.url_for("metrics"))
92
print(response.text)
93
assert response.status_code == HTTPStatus.OK
94
@@ -170,16 +169,15 @@ EXPECTED_METRICS = [
169
170
171
@pytest.mark.asyncio
173
-async def test_metrics_exist(client: openai.AsyncOpenAI):
174
- base_url = str(client.base_url)[:-3].strip("/")
175
-
172
+async def test_metrics_exist(server: RemoteOpenAIServer,
173
+ client: openai.AsyncClient):
174
# sending a request triggers the metrics to be logged.
175
await client.completions.create(model=MODEL_NAME,
176
prompt="Hello, my name is",
177
max_tokens=5,
178
temperature=0.0)
179
182
- response = requests.get(base_url + "/metrics")
180
+ response = requests.get(server.url_for("metrics"))
181
assert response.status_code == HTTPStatus.OK
182
183
for metric in EXPECTED_METRICS:
tests/entrypoints/openai/test_tokenization.py
+18
-14
@@ -1,4 +1,3 @@
1
-import openai # use the official client for correctness check
1
import pytest
2
import pytest_asyncio
3
import requests
@@ -55,9 +54,11 @@ async def client(server):
54
[(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
55
indirect=["tokenizer_name"],
56
)
58
-async def test_tokenize_completions(client: openai.AsyncOpenAI,
59
- model_name: str, tokenizer_name: str):
60
- base_url = str(client.base_url)[:-3].strip("/")
57
+async def test_tokenize_completions(
58
+ server: RemoteOpenAIServer,
59
+ model_name: str,
60
+ tokenizer_name: str,
61
+):
62
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
63
tokenizer_mode="fast")
64
@@ -65,7 +66,7 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
66
prompt = "vllm1 This is a test prompt."
67
tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
68
68
- response = requests.post(base_url + "/tokenize",
69
+ response = requests.post(server.url_for("tokenize"),
70
json={
71
"add_special_tokens": add_special,
72
"model": model_name,
@@ -86,9 +87,11 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
87
[(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
88
indirect=["tokenizer_name"],
89
)
89
-async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
90
- tokenizer_name: str):
91
- base_url = str(client.base_url)[:-3].strip("/")
90
+async def test_tokenize_chat(
91
+ server: RemoteOpenAIServer,
92
+ model_name: str,
93
+ tokenizer_name: str,
94
+):
95
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
96
tokenizer_mode="fast")
97
@@ -121,7 +124,7 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
124
tokens = tokenizer.encode(prompt,
125
add_special_tokens=add_special)
126
124
- response = requests.post(base_url + "/tokenize",
127
+ response = requests.post(server.url_for("tokenize"),
128
json={
129
"add_generation_prompt":
130
add_generation,
@@ -146,17 +149,18 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
149
[(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
150
indirect=["tokenizer_name"],
151
)
149
-async def test_detokenize(client: openai.AsyncOpenAI, model_name: str,
150
- tokenizer_name: str):
151
- base_url = str(client.base_url)[:-3].strip("/")
152
+async def test_detokenize(
153
+ server: RemoteOpenAIServer,
154
+ model_name: str,
155
+ tokenizer_name: str,
156
+):
157
tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
158
tokenizer_mode="fast")
159
160
prompt = "This is a test prompt. vllm1"
161
tokens = tokenizer.encode(prompt, add_special_tokens=False)
162
158
- print(f"CALLING {base_url} FOR {model_name}")
159
- response = requests.post(base_url + "/detokenize",
163
+ response = requests.post(server.url_for("detokenize"),
164
json={
165
"model": model_name,
166
"tokens": tokens
tests/entrypoints/openai/test_vision_embedding.py
new
+94
@@ -0,0 +1,94 @@
1
+from typing import Dict
2
+
3
+import pytest
4
+import pytest_asyncio
5
+import requests
6
+
7
+from vllm.multimodal.utils import encode_image_base64, fetch_image
8
+
9
+from ...utils import RemoteOpenAIServer
10
+
11
+MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
12
+MAXIMUM_IMAGES = 2
13
+
14
+# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15
+TEST_IMAGE_URLS = [
16
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
17
+ "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png",
18
+ "https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Venn_diagram_rgb.svg/1280px-Venn_diagram_rgb.svg.png",
19
+ "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png",
20
+]
21
+
22
+
23
+@pytest.fixture(scope="module")
24
+def server():
25
+ args = [
26
+ "--task",
27
+ "embedding",
28
+ "--dtype",
29
+ "bfloat16",
30
+ "--max-model-len",
31
+ "2048",
32
+ "--max-num-seqs",
33
+ "5",
34
+ "--enforce-eager",
35
+ "--trust-remote-code",
36
+ "--limit-mm-per-prompt",
37
+ f"image={MAXIMUM_IMAGES}",
38
+ ]
39
+
40
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
41
+ yield remote_server
42
+
43
+
44
+@pytest_asyncio.fixture
45
+async def client(server):
46
+ async with server.get_async_client() as async_client:
47
+ yield async_client
48
+
49
+
50
+@pytest.fixture(scope="session")
51
+def base64_encoded_image() -> Dict[str, str]:
52
+ return {
53
+ image_url: encode_image_base64(fetch_image(image_url))
54
+ for image_url in TEST_IMAGE_URLS
55
+ }
56
+
57
+
58
+@pytest.mark.asyncio
59
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
60
+@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
61
+async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
62
+ image_url: str):
63
+ messages = [{
64
+ "role":
65
+ "user",
66
+ "content": [
67
+ {
68
+ "type": "image_url",
69
+ "image_url": {
70
+ "url": image_url
71
+ }
72
+ },
73
+ {
74
+ "type": "text",
75
+ "text": "Represent the given image."
76
+ },
77
+ ],
78
+ }]
79
+
80
+ response = requests.post(server.url_for("v1/embeddings"),
81
+ json={
82
+ "model": model_name,
83
+ "messages": messages,
84
+ "encoding_format": "float"
85
+ })
86
+ response.raise_for_status()
87
+
88
+ embeddings = response.json()
89
+ assert embeddings["id"] is not None
90
+ assert len(embeddings["data"]) == 1
91
+ assert len(embeddings["data"][0]["embedding"]) == 3072
92
+ assert embeddings["usage"]["completion_tokens"] == 0
93
+ assert embeddings["usage"]["prompt_tokens"] == 771
94
+ assert embeddings["usage"]["total_tokens"] == 771
vllm/entrypoints/openai/api_server.py
+58
-38
@@ -11,7 +11,7 @@ from argparse import Namespace
11
from contextlib import asynccontextmanager
12
from functools import partial
13
from http import HTTPStatus
14
-from typing import AsyncIterator, Set
14
+from typing import AsyncIterator, Optional, Set
15
16
import uvloop
17
from fastapi import APIRouter, FastAPI, Request
@@ -51,7 +51,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
51
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
52
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
53
from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
54
-from vllm.entrypoints.openai.serving_engine import BaseModelPath
54
+from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
55
from vllm.entrypoints.openai.serving_tokenization import (
56
OpenAIServingTokenization)
57
from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -248,20 +248,25 @@ def mount_metrics(app: FastAPI):
248
app.routes.append(metrics_route)
249
250
251
-def chat(request: Request) -> OpenAIServingChat:
251
+def base(request: Request) -> OpenAIServing:
252
+ # Reuse the existing instance
253
+ return tokenization(request)
254
+
255
+
256
+def chat(request: Request) -> Optional[OpenAIServingChat]:
257
return request.app.state.openai_serving_chat
258
259
255
-def completion(request: Request) -> OpenAIServingCompletion:
260
+def completion(request: Request) -> Optional[OpenAIServingCompletion]:
261
return request.app.state.openai_serving_completion
262
263
259
-def tokenization(request: Request) -> OpenAIServingTokenization:
260
- return request.app.state.openai_serving_tokenization
264
+def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
265
+ return request.app.state.openai_serving_embedding
266
267
263
-def embedding(request: Request) -> OpenAIServingEmbedding:
264
- return request.app.state.openai_serving_embedding
268
+def tokenization(request: Request) -> OpenAIServingTokenization:
269
+ return request.app.state.openai_serving_tokenization
270
271
272
def engine_client(request: Request) -> EngineClient:
@@ -277,7 +282,9 @@ async def health(raw_request: Request) -> Response:
282
283
@router.post("/tokenize")
284
async def tokenize(request: TokenizeRequest, raw_request: Request):
280
- generator = await tokenization(raw_request).create_tokenize(request)
285
+ handler = tokenization(raw_request)
286
+
287
+ generator = await handler.create_tokenize(request)
288
if isinstance(generator, ErrorResponse):
289
return JSONResponse(content=generator.model_dump(),
290
status_code=generator.code)
@@ -289,7 +296,9 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
296
297
@router.post("/detokenize")
298
async def detokenize(request: DetokenizeRequest, raw_request: Request):
292
- generator = await tokenization(raw_request).create_detokenize(request)
299
+ handler = tokenization(raw_request)
300
+
301
+ generator = await handler.create_detokenize(request)
302
if isinstance(generator, ErrorResponse):
303
return JSONResponse(content=generator.model_dump(),
304
status_code=generator.code)
@@ -301,7 +310,9 @@ async def detokenize(request: DetokenizeRequest, raw_request: Request):
310
311
@router.get("/v1/models")
312
async def show_available_models(raw_request: Request):
304
- models = await completion(raw_request).show_available_models()
313
+ handler = base(raw_request)
314
+
315
+ models = await handler.show_available_models()
316
return JSONResponse(content=models.model_dump())
317
318
@@ -314,9 +325,12 @@ async def show_version():
325
@router.post("/v1/chat/completions")
326
async def create_chat_completion(request: ChatCompletionRequest,
327
raw_request: Request):
328
+ handler = chat(raw_request)
329
+ if handler is None:
330
+ return base(raw_request).create_error_response(
331
+ message="The model does not support Chat Completions API")
332
318
- generator = await chat(raw_request).create_chat_completion(
319
- request, raw_request)
333
+ generator = await handler.create_chat_completion(request, raw_request)
334
335
if isinstance(generator, ErrorResponse):
336
return JSONResponse(content=generator.model_dump(),
@@ -330,8 +344,12 @@ async def create_chat_completion(request: ChatCompletionRequest,
344
345
@router.post("/v1/completions")
346
async def create_completion(request: CompletionRequest, raw_request: Request):
333
- generator = await completion(raw_request).create_completion(
334
- request, raw_request)
347
+ handler = completion(raw_request)
348
+ if handler is None:
349
+ return base(raw_request).create_error_response(
350
+ message="The model does not support Completions API")
351
+
352
+ generator = await handler.create_completion(request, raw_request)
353
if isinstance(generator, ErrorResponse):
354
return JSONResponse(content=generator.model_dump(),
355
status_code=generator.code)
@@ -343,8 +361,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
361
362
@router.post("/v1/embeddings")
363
async def create_embedding(request: EmbeddingRequest, raw_request: Request):
346
- generator = await embedding(raw_request).create_embedding(
347
- request, raw_request)
364
+ handler = embedding(raw_request)
365
+ if handler is None:
366
+ return base(raw_request).create_error_response(
367
+ message="The model does not support Embeddings API")
368
+
369
+ generator = await handler.create_embedding(request, raw_request)
370
if isinstance(generator, ErrorResponse):
371
return JSONResponse(content=generator.model_dump(),
372
status_code=generator.code)
@@ -382,30 +404,26 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
404
@router.post("/v1/load_lora_adapter")
405
async def load_lora_adapter(request: LoadLoraAdapterRequest,
406
raw_request: Request):
385
- response = await chat(raw_request).load_lora_adapter(request)
386
- if isinstance(response, ErrorResponse):
387
- return JSONResponse(content=response.model_dump(),
388
- status_code=response.code)
389
-
390
- response = await completion(raw_request).load_lora_adapter(request)
391
- if isinstance(response, ErrorResponse):
392
- return JSONResponse(content=response.model_dump(),
393
- status_code=response.code)
407
+ for route in [chat, completion, embedding]:
408
+ handler = route(raw_request)
409
+ if handler is not None:
410
+ response = await handler.load_lora_adapter(request)
411
+ if isinstance(response, ErrorResponse):
412
+ return JSONResponse(content=response.model_dump(),
413
+ status_code=response.code)
414
415
return Response(status_code=200, content=response)
416
417
@router.post("/v1/unload_lora_adapter")
418
async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
419
raw_request: Request):
400
- response = await chat(raw_request).unload_lora_adapter(request)
401
- if isinstance(response, ErrorResponse):
402
- return JSONResponse(content=response.model_dump(),
403
- status_code=response.code)
404
-
405
- response = await completion(raw_request).unload_lora_adapter(request)
406
- if isinstance(response, ErrorResponse):
407
- return JSONResponse(content=response.model_dump(),
408
- status_code=response.code)
420
+ for route in [chat, completion, embedding]:
421
+ handler = route(raw_request)
422
+ if handler is not None:
423
+ response = await handler.unload_lora_adapter(request)
424
+ if isinstance(response, ErrorResponse):
425
+ return JSONResponse(content=response.model_dump(),
426
+ status_code=response.code)
427
428
return Response(status_code=200, content=response)
429
@@ -501,7 +519,8 @@ def init_app_state(
519
chat_template=args.chat_template,
520
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
521
enable_auto_tools=args.enable_auto_tool_choice,
504
- tool_parser=args.tool_call_parser)
522
+ tool_parser=args.tool_call_parser,
523
+ ) if model_config.task == "generate" else None
524
state.openai_serving_completion = OpenAIServingCompletion(
525
engine_client,
526
model_config,
@@ -510,13 +529,14 @@ def init_app_state(
529
prompt_adapters=args.prompt_adapters,
530
request_logger=request_logger,
531
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
513
- )
532
+ ) if model_config.task == "generate" else None
533
state.openai_serving_embedding = OpenAIServingEmbedding(
534
engine_client,
535
model_config,
536
base_model_paths,
537
request_logger=request_logger,
519
- )
538
+ chat_template=args.chat_template,
539
+ ) if model_config.task == "embedding" else None
540
state.openai_serving_tokenization = OpenAIServingTokenization(
541
engine_client,
542
model_config,
vllm/entrypoints/openai/protocol.py
+84
-3
@@ -708,7 +708,7 @@ class CompletionRequest(OpenAIBaseModel):
708
return data
709
710
711
-class EmbeddingRequest(OpenAIBaseModel):
711
+class EmbeddingCompletionRequest(OpenAIBaseModel):
712
# Ordered by official OpenAI API documentation
713
# https://platform.openai.com/docs/api-reference/embeddings
714
model: str
@@ -720,10 +720,15 @@ class EmbeddingRequest(OpenAIBaseModel):
720
721
# doc: begin-embedding-pooling-params
722
additional_data: Optional[Any] = None
723
-
723
# doc: end-embedding-pooling-params
724
725
# doc: begin-embedding-extra-params
726
+ add_special_tokens: bool = Field(
727
+ default=True,
728
+ description=(
729
+ "If true (the default), special tokens (e.g. BOS) will be added to "
730
+ "the prompt."),
731
+ )
732
priority: int = Field(
733
default=0,
734
description=(
@@ -737,6 +742,82 @@ class EmbeddingRequest(OpenAIBaseModel):
742
return PoolingParams(additional_data=self.additional_data)
743
744
745
+class EmbeddingChatRequest(OpenAIBaseModel):
746
+ model: str
747
+ messages: List[ChatCompletionMessageParam]
748
+
749
+ encoding_format: Literal["float", "base64"] = "float"
750
+ dimensions: Optional[int] = None
751
+ user: Optional[str] = None
752
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
753
+
754
+ # doc: begin-chat-embedding-pooling-params
755
+ additional_data: Optional[Any] = None
756
+ # doc: end-chat-embedding-pooling-params
757
+
758
+ # doc: begin-chat-embedding-extra-params
759
+ add_generation_prompt: bool = Field(
760
+ default=True,
761
+ description=
762
+ ("If true, the generation prompt will be added to the chat template. "
763
+ "This is a parameter used by chat template in tokenizer config of the "
764
+ "model."),
765
+ )
766
+ continue_final_message: bool = Field(
767
+ default=False,
768
+ description=
769
+ ("If this is set, the chat will be formatted so that the final "
770
+ "message in the chat is open-ended, without any EOS tokens. The "
771
+ "model will continue this message rather than starting a new one. "
772
+ "This allows you to \"prefill\" part of the model's response for it. "
773
+ "Cannot be used at the same time as `add_generation_prompt`."),
774
+ )
775
+ add_special_tokens: bool = Field(
776
+ default=False,
777
+ description=(
778
+ "If true, special tokens (e.g. BOS) will be added to the prompt "
779
+ "on top of what is added by the chat template. "
780
+ "For most models, the chat template takes care of adding the "
781
+ "special tokens so this should be set to false (as is the "
782
+ "default)."),
783
+ )
784
+ chat_template: Optional[str] = Field(
785
+ default=None,
786
+ description=(
787
+ "A Jinja template to use for this conversion. "
788
+ "As of transformers v4.44, default chat template is no longer "
789
+ "allowed, so you must provide a chat template if the tokenizer "
790
+ "does not define one."),
791
+ )
792
+ chat_template_kwargs: Optional[Dict[str, Any]] = Field(
793
+ default=None,
794
+ description=("Additional kwargs to pass to the template renderer. "
795
+ "Will be accessible by the chat template."),
796
+ )
797
+ priority: int = Field(
798
+ default=0,
799
+ description=(
800
+ "The priority of the request (lower means earlier handling; "
801
+ "default: 0). Any priority other than 0 will raise an error "
802
+ "if the served model does not use priority scheduling."))
803
+ # doc: end-chat-embedding-extra-params
804
+
805
+ @model_validator(mode="before")
806
+ @classmethod
807
+ def check_generation_prompt(cls, data):
808
+ if data.get("continue_final_message") and data.get(
809
+ "add_generation_prompt"):
810
+ raise ValueError("Cannot set both `continue_final_message` and "
811
+ "`add_generation_prompt` to True.")
812
+ return data
813
+
814
+ def to_pooling_params(self):
815
+ return PoolingParams(additional_data=self.additional_data)
816
+
817
+
818
+EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]
819
+
820
+
821
class CompletionLogProbs(OpenAIBaseModel):
822
text_offset: List[int] = Field(default_factory=list)
823
token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -799,7 +880,7 @@ class EmbeddingResponseData(OpenAIBaseModel):
880
881
882
class EmbeddingResponse(OpenAIBaseModel):
802
- id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
883
+ id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
884
object: str = "list"
885
created: int = Field(default_factory=lambda: int(time.time()))
886
model: str
vllm/entrypoints/openai/run_batch.py
+26
-8
@@ -217,13 +217,14 @@ async def main(args):
217
prompt_adapters=None,
218
request_logger=request_logger,
219
chat_template=None,
220
- )
220
+ ) if model_config.task == "generate" else None
221
openai_serving_embedding = OpenAIServingEmbedding(
222
engine,
223
model_config,
224
base_model_paths,
225
request_logger=request_logger,
226
- )
226
+ chat_template=None,
227
+ ) if model_config.task == "embedding" else None
228
229
tracker = BatchProgressTracker()
230
logger.info("Reading batch from %s...", args.input_file)
@@ -240,14 +241,31 @@ async def main(args):
241
242
# Determine the type of request and run it.
243
if request.url == "/v1/chat/completions":
243
- response_futures.append(
244
- run_request(openai_serving_chat.create_chat_completion,
245
- request, tracker))
244
+ handler_fn = (None if openai_serving_chat is None else
245
+ openai_serving_chat.create_chat_completion)
246
+ if handler_fn is None:
247
+ response_futures.append(
248
+ make_async_error_request_output(
249
+ request,
250
+ error_msg=
251
+ "The model does not support Chat Completions API",
252
+ ))
253
+ continue
254
+
255
+ response_futures.append(run_request(handler_fn, request, tracker))
256
tracker.submitted()
257
elif request.url == "/v1/embeddings":
248
- response_futures.append(
249
- run_request(openai_serving_embedding.create_embedding, request,
250
- tracker))
258
+ handler_fn = (None if openai_serving_embedding is None else
259
+ openai_serving_embedding.create_embedding)
260
+ if handler_fn is None:
261
+ response_futures.append(
262
+ make_async_error_request_output(
263
+ request,
264
+ error_msg="The model does not support Embeddings API",
265
+ ))
266
+ continue
267
+
268
+ response_futures.append(run_request(handler_fn, request, tracker))
269
tracker.submitted()
270
else:
271
response_futures.append(
vllm/entrypoints/openai/serving_chat.py
+90
-132
@@ -10,11 +10,7 @@ from fastapi import Request
10
11
from vllm.config import ModelConfig
12
from vllm.engine.protocol import EngineClient
13
-from vllm.entrypoints.chat_utils import (ConversationMessage,
14
- apply_hf_chat_template,
15
- apply_mistral_chat_template,
16
- load_chat_template,
17
- parse_chat_messages_futures)
13
+from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
14
from vllm.entrypoints.logger import RequestLogger
15
from vllm.entrypoints.openai.protocol import (
16
ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -27,16 +23,12 @@ from vllm.entrypoints.openai.protocol import (
23
from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24
LoRAModulePath,
25
OpenAIServing,
30
- PromptAdapterPath,
31
- TextTokensPrompt)
26
+ PromptAdapterPath)
27
from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
33
-from vllm.inputs import TokensPrompt
28
from vllm.logger import init_logger
29
from vllm.outputs import CompletionOutput, RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
38
-from vllm.tracing import (contains_trace_headers, extract_trace_headers,
39
- log_tracing_disabled_warning)
32
from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33
from vllm.utils import iterate_with_cancellation
34
@@ -94,12 +86,12 @@ class OpenAIServingChat(OpenAIServing):
86
raw_request: Optional[Request] = None,
87
) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
88
ErrorResponse]:
97
- """Completion API similar to OpenAI's API.
89
+ """
90
+ Chat Completion API similar to OpenAI's API.
91
92
See https://platform.openai.com/docs/api-reference/chat/create
93
for the API specification. This API mimics the OpenAI
101
- ChatCompletion API.
102
-
94
+ Chat Completion API.
95
"""
96
error_check_ret = await self._check_model(request)
97
if error_check_ret is not None:
@@ -118,143 +110,106 @@ class OpenAIServingChat(OpenAIServing):
110
prompt_adapter_request,
111
) = self._maybe_get_adapters(request)
112
121
- model_config = self.model_config
113
tokenizer = await self.engine_client.get_tokenizer(lora_request)
123
-
124
- conversation, mm_data_future = parse_chat_messages_futures(
125
- request.messages, model_config, tokenizer)
114
+ tool_parser = self.tool_parser
115
+
116
+ # validation for OpenAI tools
117
+ # tool_choice = "required" is not supported
118
+ if request.tool_choice == "required":
119
+ return self.create_error_response(
120
+ "tool_choice = \"required\" is not supported!")
121
+
122
+ if (request.tool_choice == "auto" and
123
+ not (self.enable_auto_tools and tool_parser is not None)
124
+ and not isinstance(tokenizer, MistralTokenizer)):
125
+ # for hf tokenizers, "auto" tools requires
126
+ # --enable-auto-tool-choice and --tool-call-parser
127
+ return self.create_error_response(
128
+ "\"auto\" tool choice requires "
129
+ "--enable-auto-tool-choice and --tool-call-parser to be set"
130
+ )
131
132
tool_dicts = None if request.tools is None else [
133
tool.model_dump() for tool in request.tools
134
]
135
131
- prompt: Union[str, List[int]]
132
- is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
133
- if is_mistral_tokenizer:
134
- prompt = apply_mistral_chat_template(
135
- tokenizer,
136
- messages=request.messages,
137
- chat_template=request.chat_template or self.chat_template,
138
- add_generation_prompt=request.add_generation_prompt,
139
- continue_final_message=request.continue_final_message,
140
- tools=tool_dicts,
141
- documents=request.documents,
142
- **(request.chat_template_kwargs or {}),
143
- )
144
- else:
145
- prompt = apply_hf_chat_template(
146
- tokenizer,
147
- conversation=conversation,
148
- chat_template=request.chat_template or self.chat_template,
149
- add_generation_prompt=request.add_generation_prompt,
150
- continue_final_message=request.continue_final_message,
151
- tools=tool_dicts,
152
- documents=request.documents,
153
- **(request.chat_template_kwargs or {}),
154
- )
155
- except Exception as e:
156
- logger.exception("Error in applying chat template from request")
157
- return self.create_error_response(str(e))
158
-
159
- try:
160
- mm_data = await mm_data_future
161
- except Exception as e:
162
- logger.exception("Error in loading multi-modal data")
136
+ (
137
+ conversation,
138
+ request_prompts,
139
+ engine_prompts,
140
+ ) = await self._preprocess_chat(
141
+ request,
142
+ tokenizer,
143
+ request.messages,
144
+ chat_template=request.chat_template or self.chat_template,
145
+ add_generation_prompt=request.add_generation_prompt,
146
+ continue_final_message=request.continue_final_message,
147
+ tool_dicts=tool_dicts,
148
+ documents=request.documents,
149
+ chat_template_kwargs=request.chat_template_kwargs,
150
+ tool_parser=tool_parser,
151
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
152
+ add_special_tokens=request.add_special_tokens,
153
+ )
154
+ except ValueError as e:
155
+ logger.exception("Error in preprocessing prompt inputs")
156
return self.create_error_response(str(e))
157
165
- # validation for OpenAI tools
166
- # tool_choice = "required" is not supported
167
- if request.tool_choice == "required":
168
- return self.create_error_response(
169
- "tool_choice = \"required\" is not supported!")
170
-
171
- if not is_mistral_tokenizer and request.tool_choice == "auto" and not (
172
- self.enable_auto_tools and self.tool_parser is not None):
173
- # for hf tokenizers, "auto" tools requires
174
- # --enable-auto-tool-choice and --tool-call-parser
175
- return self.create_error_response(
176
- "\"auto\" tool choice requires "
177
- "--enable-auto-tool-choice and --tool-call-parser to be set")
178
-
179
- request_id = f"chat-{request.request_id}"
158
+ request_id = f"chatcmpl-{request.request_id}"
159
160
request_metadata = RequestResponseMetadata(request_id=request_id)
161
if raw_request:
162
raw_request.state.request_metadata = request_metadata
163
164
+ # Schedule the request and get the result generator.
165
+ generators: List[AsyncGenerator[RequestOutput, None]] = []
166
try:
186
- if self.enable_auto_tools and self.tool_parser:
187
- request = self.tool_parser(tokenizer).adjust_request(
188
- request=request)
189
-
190
- if isinstance(prompt, str):
191
- prompt_inputs = self._tokenize_prompt_input(
192
- request,
193
- tokenizer,
194
- prompt,
195
- truncate_prompt_tokens=request.truncate_prompt_tokens,
196
- add_special_tokens=request.add_special_tokens,
197
- )
198
- else:
199
- assert isinstance(prompt, list) and isinstance(
200
- prompt[0], int
201
- ), "Prompt has to be either a string or a list of token ids"
202
- prompt_inputs = TextTokensPrompt(
203
- prompt=tokenizer.decode(prompt), prompt_token_ids=prompt)
204
-
205
- assert prompt_inputs is not None
206
-
207
- sampling_params: Union[SamplingParams, BeamSearchParams]
208
- default_max_tokens = self.max_model_len - len(
209
- prompt_inputs["prompt_token_ids"])
210
- if request.use_beam_search:
211
- sampling_params = request.to_beam_search_params(
212
- default_max_tokens)
213
- else:
214
- sampling_params = request.to_sampling_params(
215
- default_max_tokens)
216
-
217
- self._log_inputs(request_id,
218
- prompt_inputs,
219
- params=sampling_params,
220
- lora_request=lora_request,
221
- prompt_adapter_request=prompt_adapter_request)
222
-
223
- engine_inputs = TokensPrompt(
224
- prompt_token_ids=prompt_inputs["prompt_token_ids"])
225
- if mm_data is not None:
226
- engine_inputs["multi_modal_data"] = mm_data
227
-
228
- is_tracing_enabled = (await
229
- self.engine_client.is_tracing_enabled())
230
- trace_headers = None
231
- if is_tracing_enabled and raw_request:
232
- trace_headers = extract_trace_headers(raw_request.headers)
233
- if (not is_tracing_enabled and raw_request
234
- and contains_trace_headers(raw_request.headers)):
235
- log_tracing_disabled_warning()
236
-
237
- if isinstance(sampling_params, BeamSearchParams):
238
- result_generator = self.engine_client.beam_search(
239
- prompt=engine_inputs,
240
- model_config=self.model_config,
241
- request_id=request_id,
242
- params=sampling_params,
243
- )
244
- else:
245
- result_generator = self.engine_client.generate(
246
- engine_inputs,
247
- sampling_params,
248
- request_id,
249
- lora_request=lora_request,
250
- trace_headers=trace_headers,
251
- prompt_adapter_request=prompt_adapter_request,
252
- priority=request.priority,
253
- )
167
+ for i, engine_prompt in enumerate(engine_prompts):
168
+ sampling_params: Union[SamplingParams, BeamSearchParams]
169
+ default_max_tokens = self.max_model_len - len(
170
+ engine_prompt["prompt_token_ids"])
171
+ if request.use_beam_search:
172
+ sampling_params = request.to_beam_search_params(
173
+ default_max_tokens)
174
+ else:
175
+ sampling_params = request.to_sampling_params(
176
+ default_max_tokens)
177
+
178
+ self._log_inputs(request_id,
179
+ request_prompts[i],
180
+ params=sampling_params,
181
+ lora_request=lora_request,
182
+ prompt_adapter_request=prompt_adapter_request)
183
+
184
+ trace_headers = (None if raw_request is None else await
185
+ self._get_trace_headers(raw_request.headers))
186
+
187
+ if isinstance(sampling_params, BeamSearchParams):
188
+ generator = self.engine_client.beam_search(
189
+ prompt=engine_prompt,
190
+ model_config=self.model_config,
191
+ request_id=request_id,
192
+ params=sampling_params,
193
+ )
194
+ else:
195
+ generator = self.engine_client.generate(
196
+ engine_prompt,
197
+ sampling_params,
198
+ request_id,
199
+ lora_request=lora_request,
200
+ trace_headers=trace_headers,
201
+ prompt_adapter_request=prompt_adapter_request,
202
+ priority=request.priority,
203
+ )
204
+
205
+ generators.append(generator)
206
except ValueError as e:
207
# TODO: Use a vllm-specific Validation Error
208
return self.create_error_response(str(e))
209
210
+ assert len(generators) == 1
211
+ result_generator, = generators
212
+
213
if raw_request:
214
result_generator = iterate_with_cancellation(
215
result_generator, raw_request.is_disconnected)
@@ -626,6 +581,9 @@ class OpenAIServingChat(OpenAIServing):
581
final_res = res
582
except asyncio.CancelledError:
583
return self.create_error_response("Client disconnected")
584
+ except ValueError as e:
585
+ # TODO: Use a vllm-specific Validation Error
586
+ return self.create_error_response(str(e))
587
588
assert final_res is not None
589
vllm/entrypoints/openai/serving_completion.py
+32
-43
@@ -1,7 +1,6 @@
1
import asyncio
2
import time
3
-from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4
- Optional)
3
+from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Tuple, Union, cast
6
@@ -30,18 +29,11 @@ from vllm.logger import init_logger
29
from vllm.outputs import RequestOutput
30
from vllm.sampling_params import BeamSearchParams, SamplingParams
31
from vllm.sequence import Logprob
33
-from vllm.tracing import (contains_trace_headers, extract_trace_headers,
34
- log_tracing_disabled_warning)
32
from vllm.transformers_utils.tokenizer import AnyTokenizer
33
from vllm.utils import merge_async_iterators, random_uuid
34
35
logger = init_logger(__name__)
36
40
-TypeTokenIDs = List[int]
41
-TypeTopLogProbs = List[Optional[Dict[int, float]]]
42
-TypeCreateLogProbsFn = Callable[
43
- [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
44
-
37
38
class OpenAIServingCompletion(OpenAIServing):
39
@@ -101,8 +93,6 @@ class OpenAIServingCompletion(OpenAIServing):
93
if raw_request:
94
raw_request.state.request_metadata = request_metadata
95
104
- # Schedule the request and get the result generator.
105
- generators: List[AsyncGenerator[RequestOutput, None]] = []
96
try:
97
(
98
lora_request,
@@ -111,19 +101,24 @@ class OpenAIServingCompletion(OpenAIServing):
101
102
tokenizer = await self.engine_client.get_tokenizer(lora_request)
103
114
- prompts = list(
115
- self._tokenize_prompt_input_or_inputs(
116
- request,
117
- tokenizer,
118
- request.prompt,
119
- truncate_prompt_tokens=request.truncate_prompt_tokens,
120
- add_special_tokens=request.add_special_tokens,
121
- ))
104
+ request_prompts, engine_prompts = self._preprocess_completion(
105
+ request,
106
+ tokenizer,
107
+ request.prompt,
108
+ truncate_prompt_tokens=request.truncate_prompt_tokens,
109
+ add_special_tokens=request.add_special_tokens,
110
+ )
111
+ except ValueError as e:
112
+ logger.exception("Error in preprocessing prompt inputs")
113
+ return self.create_error_response(str(e))
114
123
- for i, prompt_inputs in enumerate(prompts):
115
+ # Schedule the request and get the result generator.
116
+ generators: List[AsyncGenerator[RequestOutput, None]] = []
117
+ try:
118
+ for i, engine_prompt in enumerate(engine_prompts):
119
sampling_params: Union[SamplingParams, BeamSearchParams]
120
default_max_tokens = self.max_model_len - len(
126
- prompt_inputs["prompt_token_ids"])
121
+ engine_prompt["prompt_token_ids"])
122
if request.use_beam_search:
123
sampling_params = request.to_beam_search_params(
124
default_max_tokens)
@@ -134,36 +129,24 @@ class OpenAIServingCompletion(OpenAIServing):
129
request_id_item = f"{request_id}-{i}"
130
131
self._log_inputs(request_id_item,
137
- prompt_inputs,
132
+ request_prompts[i],
133
params=sampling_params,
134
lora_request=lora_request,
135
prompt_adapter_request=prompt_adapter_request)
136
142
- is_tracing_enabled = (await
143
- self.engine_client.is_tracing_enabled())
144
- trace_headers = None
145
- if is_tracing_enabled:
146
- trace_headers = extract_trace_headers(raw_request.headers)
147
- if not is_tracing_enabled and contains_trace_headers(
148
- raw_request.headers):
149
- log_tracing_disabled_warning()
137
+ trace_headers = (await
138
+ self._get_trace_headers(raw_request.headers))
139
140
if isinstance(sampling_params, BeamSearchParams):
141
generator = self.engine_client.beam_search(
153
- prompt={
154
- "prompt_token_ids":
155
- prompt_inputs["prompt_token_ids"]
156
- },
142
+ prompt=engine_prompt,
143
model_config=self.model_config,
144
request_id=request_id,
145
params=sampling_params,
146
)
147
else:
148
generator = self.engine_client.generate(
163
- {
164
- "prompt_token_ids":
165
- prompt_inputs["prompt_token_ids"]
166
- },
149
+ engine_prompt,
150
sampling_params,
151
request_id_item,
152
lora_request=lora_request,
@@ -180,6 +163,8 @@ class OpenAIServingCompletion(OpenAIServing):
163
result_generator = merge_async_iterators(
164
*generators, is_cancelled=raw_request.is_disconnected)
165
166
+ num_prompts = len(engine_prompts)
167
+
168
# Similar to the OpenAI API, when n != best_of, we do not stream the
169
# results. In addition, we do not stream the results when use
170
# beam search.
@@ -195,16 +180,22 @@ class OpenAIServingCompletion(OpenAIServing):
180
request_id,
181
created_time,
182
model_name,
198
- num_prompts=len(prompts),
183
+ num_prompts=num_prompts,
184
tokenizer=tokenizer,
185
request_metadata=request_metadata)
186
187
# Non-streaming response
203
- final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
188
+ final_res_batch: List[Optional[RequestOutput]] = [None] * num_prompts
189
try:
190
async for i, res in result_generator:
191
final_res_batch[i] = res
192
+ except asyncio.CancelledError:
193
+ return self.create_error_response("Client disconnected")
194
+ except ValueError as e:
195
+ # TODO: Use a vllm-specific Validation Error
196
+ return self.create_error_response(str(e))
197
198
+ try:
199
for i, final_res in enumerate(final_res_batch):
200
assert final_res is not None
201
@@ -212,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
203
# We did not pass it into vLLM engine to avoid being redundant
204
# with the inputs token IDs
205
if final_res.prompt is None:
215
- final_res.prompt = prompts[i]["prompt"]
206
+ final_res.prompt = request_prompts[i]["prompt"]
207
208
final_res_batch_checked = cast(List[RequestOutput],
209
final_res_batch)
@@ -226,8 +217,6 @@ class OpenAIServingCompletion(OpenAIServing):
217
tokenizer,
218
request_metadata,
219
)
229
- except asyncio.CancelledError:
230
- return self.create_error_response("Client disconnected")
220
except ValueError as e:
221
# TODO: Use a vllm-specific Validation Error
222
return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+54
-33
@@ -9,8 +9,10 @@ from typing_extensions import assert_never
9
10
from vllm.config import ModelConfig
11
from vllm.engine.protocol import EngineClient
12
+from vllm.entrypoints.chat_utils import load_chat_template
13
from vllm.entrypoints.logger import RequestLogger
13
-from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14
+from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
15
+ EmbeddingRequest,
16
EmbeddingResponse,
17
EmbeddingResponseData,
18
ErrorResponse, UsageInfo)
@@ -21,8 +23,6 @@ from vllm.utils import merge_async_iterators, random_uuid
23
24
logger = init_logger(__name__)
25
24
-TypeTokenIDs = List[int]
25
-
26
27
def _get_embedding(
28
output: EmbeddingOutput,
@@ -76,6 +76,7 @@ class OpenAIServingEmbedding(OpenAIServing):
76
base_model_paths: List[BaseModelPath],
77
*,
78
request_logger: Optional[RequestLogger],
79
+ chat_template: Optional[str],
80
):
81
super().__init__(engine_client=engine_client,
82
model_config=model_config,
@@ -83,21 +84,20 @@ class OpenAIServingEmbedding(OpenAIServing):
84
lora_modules=None,
85
prompt_adapters=None,
86
request_logger=request_logger)
86
- self._enabled = self._check_embedding_mode(
87
- model_config.task == "embedding")
87
+
88
+ self.chat_template = load_chat_template(chat_template)
89
90
async def create_embedding(
91
self,
92
request: EmbeddingRequest,
93
raw_request: Optional[Request] = None,
94
) -> Union[EmbeddingResponse, ErrorResponse]:
94
- """Completion API similar to OpenAI's API.
95
+ """
96
+ Embedding API similar to OpenAI's API.
97
98
See https://platform.openai.com/docs/api-reference/embeddings/create
99
for the API specification. This API mimics the OpenAI Embedding API.
100
"""
99
- if not self._enabled:
100
- return self.create_error_response("Embedding API disabled")
101
error_check_ret = await self._check_model(request)
102
if error_check_ret is not None:
103
return error_check_ret
@@ -122,8 +122,6 @@ class OpenAIServingEmbedding(OpenAIServing):
122
"greater than max_model_len."
123
" Please, select a smaller truncation size.")
124
125
- # Schedule the request and get the result generator.
126
- generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
125
try:
126
(
127
lora_request,
@@ -132,32 +130,60 @@ class OpenAIServingEmbedding(OpenAIServing):
130
131
tokenizer = await self.engine_client.get_tokenizer(lora_request)
132
135
- pooling_params = request.to_pooling_params()
133
+ if prompt_adapter_request is not None:
134
+ raise NotImplementedError("Prompt adapter is not supported "
135
+ "for embedding models")
136
+
137
+ if isinstance(request, EmbeddingChatRequest):
138
+ (
139
+ _,
140
+ request_prompts,
141
+ engine_prompts,
142
+ ) = await self._preprocess_chat(
143
+ request,
144
+ tokenizer,
145
+ request.messages,
146
+ chat_template=request.chat_template or self.chat_template,
147
+ add_generation_prompt=request.add_generation_prompt,
148
+ continue_final_message=request.continue_final_message,
149
+ truncate_prompt_tokens=truncate_prompt_tokens,
150
+ add_special_tokens=request.add_special_tokens,
151
+ )
152
+ else:
153
+ request_prompts, engine_prompts = self._preprocess_completion(
154
+ request,
155
+ tokenizer,
156
+ request.input,
157
+ truncate_prompt_tokens=truncate_prompt_tokens,
158
+ add_special_tokens=request.add_special_tokens,
159
+ )
160
+ except ValueError as e:
161
+ logger.exception("Error in preprocessing prompt inputs")
162
+ return self.create_error_response(str(e))
163
137
- prompts = list(
138
- self._tokenize_prompt_input_or_inputs(request, tokenizer,
139
- request.input,
140
- truncate_prompt_tokens))
164
+ # Schedule the request and get the result generator.
165
+ generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
166
+ try:
167
+ pooling_params = request.to_pooling_params()
168
142
- for i, prompt_inputs in enumerate(prompts):
169
+ for i, engine_prompt in enumerate(engine_prompts):
170
request_id_item = f"{request_id}-{i}"
171
172
self._log_inputs(request_id_item,
146
- prompt_inputs,
173
+ request_prompts[i],
174
params=pooling_params,
175
lora_request=lora_request,
176
prompt_adapter_request=prompt_adapter_request)
177
151
- if prompt_adapter_request is not None:
152
- raise NotImplementedError(
153
- "Prompt adapter is not supported "
154
- "for embedding models")
178
+ trace_headers = (None if raw_request is None else await
179
+ self._get_trace_headers(raw_request.headers))
180
181
generator = self.engine_client.encode(
157
- {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
182
+ engine_prompt,
183
pooling_params,
184
request_id_item,
185
lora_request=lora_request,
186
+ trace_headers=trace_headers,
187
priority=request.priority,
188
)
189
@@ -171,13 +197,18 @@ class OpenAIServingEmbedding(OpenAIServing):
197
is_cancelled=raw_request.is_disconnected if raw_request else None,
198
)
199
200
+ num_prompts = len(engine_prompts)
201
+
202
# Non-streaming response
203
final_res_batch: List[Optional[EmbeddingRequestOutput]]
176
- final_res_batch = [None] * len(prompts)
204
+ final_res_batch = [None] * num_prompts
205
try:
206
async for i, res in result_generator:
207
final_res_batch[i] = res
208
+ except asyncio.CancelledError:
209
+ return self.create_error_response("Client disconnected")
210
211
+ try:
212
for final_res in final_res_batch:
213
assert final_res is not None
214
@@ -187,18 +218,8 @@ class OpenAIServingEmbedding(OpenAIServing):
218
response = request_output_to_embedding_response(
219
final_res_batch_checked, request_id, created_time, model_name,
220
encoding_format)
190
- except asyncio.CancelledError:
191
- return self.create_error_response("Client disconnected")
221
except ValueError as e:
222
# TODO: Use a vllm-specific Validation Error
223
return self.create_error_response(str(e))
224
225
return response
197
-
198
- def _check_embedding_mode(self, embedding_mode: bool) -> bool:
199
- if not embedding_mode:
200
- logger.warning(
201
- "embedding_mode is False. Embedding API will not work.")
202
- else:
203
- logger.info("Activating the server engine with embedding enabled.")
204
- return embedding_mode
vllm/entrypoints/openai/serving_engine.py
+150
-9
@@ -2,28 +2,38 @@ import json
2
import pathlib
3
from dataclasses import dataclass
4
from http import HTTPStatus
5
-from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
5
+from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
6
+ Optional, Sequence, Tuple, TypedDict, Union)
7
8
from pydantic import Field
9
+from starlette.datastructures import Headers
10
from typing_extensions import Annotated
11
12
from vllm.config import ModelConfig
13
from vllm.engine.protocol import EngineClient
14
+from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
15
+ ConversationMessage,
16
+ apply_hf_chat_template,
17
+ apply_mistral_chat_template,
18
+ parse_chat_messages_futures)
19
from vllm.entrypoints.logger import RequestLogger
20
# yapf conflicts with isort for this block
21
# yapf: disable
22
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
23
CompletionRequest,
24
DetokenizeRequest,
18
- EmbeddingRequest, ErrorResponse,
25
+ EmbeddingChatRequest,
26
+ EmbeddingCompletionRequest,
27
+ ErrorResponse,
28
LoadLoraAdapterRequest,
29
ModelCard, ModelList,
30
ModelPermission,
31
TokenizeChatRequest,
32
TokenizeCompletionRequest,
24
- TokenizeRequest,
33
UnloadLoraAdapterRequest)
34
+from vllm.entrypoints.openai.tool_parsers import ToolParser
35
# yapf: enable
36
+from vllm.inputs import TokensPrompt
37
from vllm.inputs.parse import parse_and_batch_prompt
38
from vllm.logger import init_logger
39
from vllm.lora.request import LoRARequest
@@ -31,8 +41,10 @@ from vllm.pooling_params import PoolingParams
41
from vllm.prompt_adapter.request import PromptAdapterRequest
42
from vllm.sampling_params import BeamSearchParams, SamplingParams
43
from vllm.sequence import Logprob
34
-from vllm.transformers_utils.tokenizer import AnyTokenizer
35
-from vllm.utils import AtomicCounter
44
+from vllm.tracing import (contains_trace_headers, extract_trace_headers,
45
+ log_tracing_disabled_warning)
46
+from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
47
+from vllm.utils import AtomicCounter, is_list_of
48
49
logger = init_logger(__name__)
50
@@ -56,8 +68,14 @@ class LoRAModulePath:
68
base_model_name: Optional[str] = None
69
70
59
-AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
60
- EmbeddingRequest, TokenizeRequest]
71
+CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
72
+ EmbeddingCompletionRequest,
73
+ TokenizeCompletionRequest]
74
+
75
+ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
76
+ TokenizeChatRequest]
77
+
78
+AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest]
79
80
81
class TextTokensPrompt(TypedDict):
@@ -65,6 +83,9 @@ class TextTokensPrompt(TypedDict):
83
prompt_token_ids: List[int]
84
85
86
+RequestPrompt = Union[List[int], str, TextTokensPrompt]
87
+
88
+
89
class OpenAIServing:
90
91
def __init__(
@@ -246,7 +267,8 @@ class OpenAIServing:
267
token_num = len(input_ids)
268
269
# Note: EmbeddingRequest doesn't have max_tokens
249
- if isinstance(request, EmbeddingRequest):
270
+ if isinstance(request,
271
+ (EmbeddingChatRequest, EmbeddingCompletionRequest)):
272
if token_num > self.max_model_len:
273
raise ValueError(
274
f"This model's maximum context length is "
@@ -373,10 +395,115 @@ class OpenAIServing:
395
truncate_prompt_tokens=truncate_prompt_tokens,
396
)
397
398
+ def _preprocess_completion(
399
+ self,
400
+ request: CompletionLikeRequest,
401
+ tokenizer: AnyTokenizer,
402
+ input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
403
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
404
+ add_special_tokens: bool = True,
405
+ ) -> Tuple[Sequence[TextTokensPrompt], List[TokensPrompt]]:
406
+ request_prompts = [
407
+ request_prompt
408
+ for request_prompt in self._tokenize_prompt_input_or_inputs(
409
+ request,
410
+ tokenizer,
411
+ input_or_inputs,
412
+ truncate_prompt_tokens=truncate_prompt_tokens,
413
+ add_special_tokens=add_special_tokens,
414
+ )
415
+ ]
416
+
417
+ engine_prompts = [
418
+ TokensPrompt(prompt_token_ids=request_prompt["prompt_token_ids"])
419
+ for request_prompt in request_prompts
420
+ ]
421
+
422
+ return request_prompts, engine_prompts
423
+
424
+ async def _preprocess_chat(
425
+ self,
426
+ request: ChatLikeRequest,
427
+ tokenizer: AnyTokenizer,
428
+ messages: List[ChatCompletionMessageParam],
429
+ chat_template: Optional[str] = None,
430
+ add_generation_prompt: bool = True,
431
+ continue_final_message: bool = False,
432
+ tool_dicts: Optional[List[Dict[str, Any]]] = None,
433
+ documents: Optional[List[Dict[str, str]]] = None,
434
+ chat_template_kwargs: Optional[Dict[str, Any]] = None,
435
+ tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
436
+ truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
437
+ add_special_tokens: bool = False,
438
+ ) -> Tuple[List[ConversationMessage], Sequence[RequestPrompt],
439
+ List[TokensPrompt]]:
440
+ conversation, mm_data_future = parse_chat_messages_futures(
441
+ messages,
442
+ self.model_config,
443
+ tokenizer,
444
+ )
445
+
446
+ request_prompt: Union[str, List[int]]
447
+ is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
448
+ if is_mistral_tokenizer:
449
+ request_prompt = apply_mistral_chat_template(
450
+ tokenizer,
451
+ messages=messages,
452
+ chat_template=chat_template,
453
+ add_generation_prompt=add_generation_prompt,
454
+ continue_final_message=continue_final_message,
455
+ tools=tool_dicts,
456
+ documents=documents,
457
+ **(chat_template_kwargs or {}),
458
+ )
459
+ else:
460
+ request_prompt = apply_hf_chat_template(
461
+ tokenizer,
462
+ conversation=conversation,
463
+ chat_template=chat_template,
464
+ add_generation_prompt=add_generation_prompt,
465
+ continue_final_message=continue_final_message,
466
+ tools=tool_dicts,
467
+ documents=documents,
468
+ **(chat_template_kwargs or {}),
469
+ )
470
+
471
+ mm_data = await mm_data_future
472
+
473
+ if tool_parser is not None:
474
+ if not isinstance(request, ChatCompletionRequest):
475
+ msg = "Tool usage is only supported for Chat Completions API"
476
+ raise NotImplementedError(msg)
477
+
478
+ request = tool_parser(tokenizer).adjust_request(request=request)
479
+
480
+ if isinstance(request_prompt, str):
481
+ prompt_inputs = self._tokenize_prompt_input(
482
+ request,
483
+ tokenizer,
484
+ request_prompt,
485
+ truncate_prompt_tokens=truncate_prompt_tokens,
486
+ add_special_tokens=add_special_tokens,
487
+ )
488
+ else:
489
+ # For MistralTokenizer
490
+ assert is_list_of(request_prompt, int), (
491
+ "Prompt has to be either a string or a list of token ids")
492
+ prompt_inputs = TextTokensPrompt(
493
+ prompt=tokenizer.decode(request_prompt),
494
+ prompt_token_ids=request_prompt)
495
+
496
+ engine_prompt = TokensPrompt(
497
+ prompt_token_ids=prompt_inputs["prompt_token_ids"])
498
+ if mm_data is not None:
499
+ engine_prompt["multi_modal_data"] = mm_data
500
+
501
+ return conversation, [request_prompt], [engine_prompt]
502
+
503
def _log_inputs(
504
self,
505
request_id: str,
379
- inputs: Union[str, List[int], TextTokensPrompt],
506
+ inputs: RequestPrompt,
507
params: Optional[Union[SamplingParams, PoolingParams,
508
BeamSearchParams]],
509
lora_request: Optional[LoRARequest],
@@ -404,6 +531,20 @@ class OpenAIServing:
531
prompt_adapter_request=prompt_adapter_request,
532
)
533
534
+ async def _get_trace_headers(
535
+ self,
536
+ headers: Headers,
537
+ ) -> Optional[Mapping[str, str]]:
538
+ is_tracing_enabled = await self.engine_client.is_tracing_enabled()
539
+
540
+ if is_tracing_enabled:
541
+ return extract_trace_headers(headers)
542
+
543
+ if contains_trace_headers(headers):
544
+ log_tracing_disabled_warning()
545
+
546
+ return None
547
+
548
@staticmethod
549
def _get_decoded_token(logprob: Logprob,
550
token_id: int,
vllm/entrypoints/openai/serving_tokenization.py
+37
-50
@@ -2,10 +2,7 @@ from typing import List, Optional, Union
2
3
from vllm.config import ModelConfig
4
from vllm.engine.protocol import EngineClient
5
-from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6
- apply_mistral_chat_template,
7
- load_chat_template,
8
- parse_chat_messages_futures)
5
+from vllm.entrypoints.chat_utils import load_chat_template
6
from vllm.entrypoints.logger import RequestLogger
7
# yapf conflicts with isort for this block
8
# yapf: disable
@@ -20,7 +17,6 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
17
LoRAModulePath,
18
OpenAIServing)
19
from vllm.logger import init_logger
23
-from vllm.transformers_utils.tokenizer import MistralTokenizer
20
from vllm.utils import random_uuid
21
22
logger = init_logger(__name__)
@@ -62,59 +58,51 @@ class OpenAIServingTokenization(OpenAIServing):
58
59
request_id = f"tokn-{random_uuid()}"
60
65
- (
66
- lora_request,
67
- prompt_adapter_request,
68
- ) = self._maybe_get_adapters(request)
69
-
70
- tokenizer = await self.engine_client.get_tokenizer(lora_request)
71
-
72
- prompt: Union[str, List[int]]
73
- if isinstance(request, TokenizeChatRequest):
74
- model_config = self.model_config
75
-
76
- conversation, mm_data_future = parse_chat_messages_futures(
77
- request.messages, model_config, tokenizer)
78
-
79
- mm_data = await mm_data_future
80
- if mm_data:
81
- logger.warning(
82
- "Multi-modal inputs are ignored during tokenization")
83
-
84
- if isinstance(tokenizer, MistralTokenizer):
85
- prompt = apply_mistral_chat_template(
61
+ try:
62
+ (
63
+ lora_request,
64
+ prompt_adapter_request,
65
+ ) = self._maybe_get_adapters(request)
66
+
67
+ tokenizer = await self.engine_client.get_tokenizer(lora_request)
68
+
69
+ if isinstance(request, TokenizeChatRequest):
70
+ (
71
+ _,
72
+ request_prompts,
73
+ engine_prompts,
74
+ ) = await self._preprocess_chat(
75
+ request,
76
tokenizer,
87
- messages=request.messages,
77
+ request.messages,
78
chat_template=self.chat_template,
79
add_generation_prompt=request.add_generation_prompt,
80
continue_final_message=request.continue_final_message,
81
+ add_special_tokens=request.add_special_tokens,
82
)
83
else:
93
- prompt = apply_hf_chat_template(
84
+ request_prompts, engine_prompts = self._preprocess_completion(
85
+ request,
86
tokenizer,
95
- conversation=conversation,
96
- chat_template=self.chat_template,
97
- add_generation_prompt=request.add_generation_prompt,
98
- continue_final_message=request.continue_final_message,
87
+ request.prompt,
88
+ add_special_tokens=request.add_special_tokens,
89
)
100
- else:
101
- prompt = request.prompt
90
+ except ValueError as e:
91
+ logger.exception("Error in preprocessing prompt inputs")
92
+ return self.create_error_response(str(e))
93
103
- self._log_inputs(request_id,
104
- prompt,
105
- params=None,
106
- lora_request=lora_request,
107
- prompt_adapter_request=prompt_adapter_request)
94
+ input_ids: List[int] = []
95
+ for i, engine_prompt in enumerate(engine_prompts):
96
+ self._log_inputs(request_id,
97
+ request_prompts[i],
98
+ params=None,
99
+ lora_request=lora_request,
100
+ prompt_adapter_request=prompt_adapter_request)
101
109
- # Silently ignore prompt adapter since it does not affect tokenization
102
+ # Silently ignore prompt adapter since it does not affect
103
+ # tokenization (Unlike in Embeddings API where an error is raised)
104
111
- prompt_input = self._tokenize_prompt_input(
112
- request,
113
- tokenizer,
114
- prompt,
115
- add_special_tokens=request.add_special_tokens,
116
- )
117
- input_ids = prompt_input["prompt_token_ids"]
105
+ input_ids.extend(engine_prompt["prompt_token_ids"])
106
107
return TokenizeResponse(tokens=input_ids,
108
count=len(input_ids),
@@ -143,9 +131,8 @@ class OpenAIServingTokenization(OpenAIServing):
131
lora_request=lora_request,
132
prompt_adapter_request=prompt_adapter_request)
133
146
- if prompt_adapter_request is not None:
147
- raise NotImplementedError("Prompt adapter is not supported "
148
- "for tokenization")
134
+ # Silently ignore prompt adapter since it does not affect tokenization
135
+ # (Unlike in Embeddings API where an error is raised)
136
137
prompt_input = self._tokenize_prompt_input(
138
request,
vllm/pooling_params.py
+2
-2
@@ -7,7 +7,7 @@ class PoolingParams(
7
msgspec.Struct,
8
omit_defaults=True, # type: ignore[call-arg]
9
array_like=True): # type: ignore[call-arg]
10
- """Pooling parameters for pooling.
10
+ """Pooling parameters for embeddings API.
11
12
Attributes:
13
additional_data: Any additional data needed for pooling.
@@ -16,7 +16,7 @@ class PoolingParams(
16
17
def clone(self) -> "PoolingParams":
18
"""Returns a deep copy of the PoolingParams instance."""
19
- return PoolingParams(additional_data=self.additional_data, )
19
+ return PoolingParams(additional_data=self.additional_data)
20
21
def __repr__(self) -> str:
22
return (f"PoolingParams("