@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 06386a64dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Frontend] Chat-based Embeddings API (#9759)

Cyrus Leung committed Nov 1, 2024 at 16:13 UTC 06386a64dd706cf3fdab82510124ca2c2f9eee9d
21 files changed +853 -415
docs/requirements-docs.txt
+2
@@ -13,5 +13,7 @@ torch
13 py-cpuinfo
14 transformers
15 mistral_common >= 1.3.4
16 +aiohttp
17 +starlette
18 openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
19 partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
\ No newline at end of file
docs/source/conf.py
+1 -1
@@ -96,7 +96,6 @@ def setup(app):
96
97 # Mock out external dependencies here, otherwise the autodoc pages may be blank.
98 autodoc_mock_imports = [
99 - "aiohttp",
99 "compressed_tensors",
100 "cpuinfo",
101 "cv2",
@@ -143,6 +142,7 @@ intersphinx_mapping = {
142 "python": ("https://docs.python.org/3", None),
143 "typing_extensions":
144 ("https://typing-extensions.readthedocs.io/en/latest", None),
145 + "aiohttp": ("https://docs.aiohttp.org/en/stable", None),
146 "pillow": ("https://pillow.readthedocs.io/en/stable", None),
147 "numpy": ("https://numpy.org/doc/stable", None),
148 "torch": ("https://pytorch.org/docs/stable", None),
docs/source/dev/pooling_params.rst new
+5
@@ -0,0 +1,5 @@
1 +Pooling Parameters
2 +==================
3 +
4 +.. autoclass:: vllm.PoolingParams
5 + :members:
docs/source/getting_started/quickstart.rst
+4 -4
@@ -138,10 +138,10 @@ Since this server is compatible with OpenAI API, you can use it as a drop-in rep
138
139 A more detailed client example can be found `here <https://github.com/vllm-project/vllm/blob/main/examples/openai_completion_client.py>`__.
140
141 -OpenAI Chat API with vLLM
142 -~~~~~~~~~~~~~~~~~~~~~~~~~~
141 +OpenAI Chat Completions API with vLLM
142 +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
143
144 -vLLM is designed to also support the OpenAI Chat API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
144 +vLLM is designed to also support the OpenAI Chat Completions API. The chat interface is a more dynamic, interactive way to communicate with the model, allowing back-and-forth exchanges that can be stored in the chat history. This is useful for tasks that require context or more detailed explanations.
145
146 You can use the `create chat completion <https://platform.openai.com/docs/api-reference/chat/completions/create>`_ endpoint to interact with the model:
147
@@ -157,7 +157,7 @@ You can use the `create chat completion <https://platform.openai.com/docs/api-re
157 $ ]
158 $ }'
159
160 -Alternatively, you can use the `openai` python package:
160 +Alternatively, you can use the ``openai`` python package:
161
162 .. code-block:: python
163
docs/source/index.rst
+1
@@ -134,6 +134,7 @@ Documentation
134 :caption: Developer Documentation
135
136 dev/sampling_params
137 + dev/pooling_params
138 dev/offline_inference/offline_index
139 dev/engine/engine_index
140 dev/kernel/paged_attention
docs/source/models/vlm.rst
+51 -3
@@ -185,7 +185,7 @@ Below is an example on how to launch the same ``microsoft/Phi-3.5-vision-instruc
185 --trust-remote-code --max-model-len 4096 --limit-mm-per-prompt image=2
186
187 .. important::
188 - Since OpenAI Vision API is based on `Chat Completions <https://platform.openai.com/docs/api-reference/chat>`_ API,
188 + Since OpenAI Vision API is based on `Chat Completions API <https://platform.openai.com/docs/api-reference/chat>`_,
189 a chat template is **required** to launch the API server.
190
191 Although Phi-3.5-Vision comes with a chat template, for other models you may have to provide one if the model's tokenizer does not come with it.
@@ -243,6 +243,10 @@ To consume the server, you can use the OpenAI client like in the example below:
243
244 A full code example can be found in `examples/openai_api_client_for_multimodal.py <https://github.com/vllm-project/vllm/blob/main/examples/openai_api_client_for_multimodal.py>`_.
245
246 +.. tip::
247 + There is no need to place image placeholders in the text content of the API request - they are already represented by the image content.
248 + In fact, you can place image placeholders in the middle of the text by interleaving text and image content.
249 +
250 .. note::
251
252 By default, the timeout for fetching images through http url is ``5`` seconds. You can override this by setting the environment variable:
@@ -251,5 +255,49 @@ A full code example can be found in `examples/openai_api_client_for_multimodal.p
255
256 $ export VLLM_IMAGE_FETCH_TIMEOUT=<timeout>
257
254 -.. note::
255 - There is no need to format the prompt in the API request since it will be handled by the server.
258 +Chat Embeddings API
259 +^^^^^^^^^^^^^^^^^^^
260 +
261 +vLLM's Chat Embeddings API is a superset of OpenAI's `Embeddings API <https://platform.openai.com/docs/api-reference/embeddings>`_,
262 +where a list of ``messages`` can be passed instead of batched ``inputs``. This enables multi-modal inputs to be passed to embedding models.
263 +
264 +.. tip::
265 + The schema of ``messages`` is exactly the same as in Chat Completions API.
266 +
267 +In this example, we will serve the ``TIGER-Lab/VLM2Vec-Full`` model.
268 +
269 +.. code-block:: bash
270 +
271 + vllm serve TIGER-Lab/VLM2Vec-Full --task embedding \
272 + --trust-remote-code --max-model-len 4096
273 +
274 +.. important::
275 +
276 + Since VLM2Vec has the same model architecture as Phi-3.5-Vision, we have to explicitly pass ``--task embedding``
277 + to run this model in embedding mode instead of text generation mode.
278 +
279 +Since this schema is not defined by OpenAI client, we post a request to the server using the lower-level ``requests`` library:
280 +
281 +.. code-block:: python
282 +
283 + import requests
284 +
285 + image_url = "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
286 +
287 + response = requests.post(
288 + "http://localhost:8000/v1/embeddings",
289 + json={
290 + "model": "TIGER-Lab/VLM2Vec-Full",
291 + "messages": [{
292 + "role": "user",
293 + "content": [
294 + {"type": "image_url", "image_url": {"url": image_url}},
295 + {"type": "text", "text": "Represent the given image."},
296 + ],
297 + }],
298 + "encoding_format": "float",
299 + },
300 + )
301 + response.raise_for_status()
302 + response_json = response.json()
303 + print("Embedding output:", response_json["data"][0]["embedding"])
docs/source/serving/openai_compatible_server.md
+44 -11
@@ -26,13 +26,26 @@ print(completion.choices[0].message)
26 ```
27
28 ## API Reference
29 -Please see the [OpenAI API Reference](https://platform.openai.com/docs/api-reference) for more information on the API. We support all parameters except:
30 -- Chat: `tools`, and `tool_choice`.
31 -- Completions: `suffix`.
29
33 -vLLM also provides experimental support for OpenAI Vision API compatible inference. See more details in [Using VLMs](../models/vlm.rst).
30 +We currently support the following OpenAI APIs:
31 +
32 +- [Completions API](https://platform.openai.com/docs/api-reference/completions)
33 + - *Note: `suffix` parameter is not supported.*
34 +- [Chat Completions API](https://platform.openai.com/docs/api-reference/chat)
35 + - [Vision](https://platform.openai.com/docs/guides/vision)-related parameters are supported; see [Using VLMs](../models/vlm.rst).
36 + - *Note: `image_url.detail` parameter is not supported.*
37 + - We also support `audio_url` content type for audio files.
38 + - Refer to [vllm.entrypoints.chat_utils](https://github.com/vllm-project/vllm/tree/main/vllm/entrypoints/chat_utils.py) for the exact schema.
39 + - *TODO: Support `input_audio` content type as defined [here](https://github.com/openai/openai-python/blob/v1.52.2/src/openai/types/chat/chat_completion_content_part_input_audio_param.py).*
40 + - *Note: `parallel_tool_calls` and `user` parameters are ignored.*
41 +- [Embeddings API](https://platform.openai.com/docs/api-reference/embeddings)
42 + - Instead of `inputs`, you can pass in a list of `messages` (same schema as Chat Completions API),
43 + which will be treated as a single prompt to the model according to its chat template.
44 + - This enables multi-modal inputs to be passed to embedding models, see [Using VLMs](../models/vlm.rst).
45 + - *Note: You should run `vllm serve` with `--task embedding` to ensure that the model is being run in embedding mode.*
46
47 ## Extra Parameters
48 +
49 vLLM supports a set of parameters that are not part of the OpenAI API.
50 In order to use them, you can pass them as extra parameters in the OpenAI client.
51 Or directly merge them into the JSON payload if you are using HTTP call directly.
@@ -49,7 +62,26 @@ completion = client.chat.completions.create(
62 )
63 ```
64
52 -### Extra Parameters for Chat API
65 +### Extra Parameters for Completions API
66 +
67 +The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
68 +
69 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
70 +:language: python
71 +:start-after: begin-completion-sampling-params
72 +:end-before: end-completion-sampling-params
73 +```
74 +
75 +The following extra parameters are supported:
76 +
77 +```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
78 +:language: python
79 +:start-after: begin-completion-extra-params
80 +:end-before: end-completion-extra-params
81 +```
82 +
83 +### Extra Parameters for Chat Completions API
84 +
85 The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
86
87 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
@@ -66,21 +98,22 @@ The following extra parameters are supported:
98 :end-before: end-chat-completion-extra-params
99 ```
100
69 -### Extra Parameters for Completions API
70 -The following [sampling parameters (click through to see documentation)](../dev/sampling_params.rst) are supported.
101 +### Extra Parameters for Embeddings API
102 +
103 +The following [pooling parameters (click through to see documentation)](../dev/pooling_params.rst) are supported.
104
105 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
106 :language: python
74 -:start-after: begin-completion-sampling-params
75 -:end-before: end-completion-sampling-params
107 +:start-after: begin-embedding-pooling-params
108 +:end-before: end-embedding-pooling-params
109 ```
110
111 The following extra parameters are supported:
112
113 ```{literalinclude} ../../../vllm/entrypoints/openai/protocol.py
114 :language: python
82 -:start-after: begin-completion-extra-params
83 -:end-before: end-completion-extra-params
115 +:start-after: begin-embedding-extra-params
116 +:end-before: end-embedding-extra-params
117 ```
118
119 ## Chat Template
tests/entrypoints/openai/test_basic.py
+4 -9
@@ -1,7 +1,6 @@
1 from http import HTTPStatus
2 from typing import List
3
4 -import openai
4 import pytest
5 import pytest_asyncio
6 import requests
@@ -83,10 +82,8 @@ async def client(server):
82 indirect=True,
83 )
84 @pytest.mark.asyncio
86 -async def test_show_version(client: openai.AsyncOpenAI):
87 - base_url = str(client.base_url)[:-3].strip("/")
88 -
89 - response = requests.get(base_url + "/version")
85 +async def test_show_version(server: RemoteOpenAIServer):
86 + response = requests.get(server.url_for("version"))
87 response.raise_for_status()
88
89 assert response.json() == {"version": VLLM_VERSION}
@@ -102,9 +99,7 @@ async def test_show_version(client: openai.AsyncOpenAI):
99 indirect=True,
100 )
101 @pytest.mark.asyncio
105 -async def test_check_health(client: openai.AsyncOpenAI):
106 - base_url = str(client.base_url)[:-3].strip("/")
107 -
108 - response = requests.get(base_url + "/health")
102 +async def test_check_health(server: RemoteOpenAIServer):
103 + response = requests.get(server.url_for("health"))
104
105 assert response.status_code == HTTPStatus.OK
tests/entrypoints/openai/test_embedding.py
+90 -47
@@ -4,14 +4,18 @@ import numpy as np
4 import openai
5 import pytest
6 import pytest_asyncio
7 +import requests
8 +
9 +from vllm.transformers_utils.tokenizer import get_tokenizer
10
11 from ...utils import RemoteOpenAIServer
12
10 -EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
13 +MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
14 +DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
15
16
17 @pytest.fixture(scope="module")
14 -def embedding_server():
18 +def server():
19 args = [
20 # use half precision for speed and memory savings in CI environment
21 "--dtype",
@@ -19,31 +23,29 @@ def embedding_server():
23 "--enforce-eager",
24 "--max-model-len",
25 "8192",
26 + "--chat-template",
27 + DUMMY_CHAT_TEMPLATE,
28 ]
29
24 - with RemoteOpenAIServer(EMBEDDING_MODEL_NAME, args) as remote_server:
30 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
31 yield remote_server
32
33
34 @pytest_asyncio.fixture
29 -async def embedding_client(embedding_server):
30 - async with embedding_server.get_async_client() as async_client:
35 +async def client(server):
36 + async with server.get_async_client() as async_client:
37 yield async_client
38
39
40 @pytest.mark.asyncio
35 -@pytest.mark.parametrize(
36 - "model_name",
37 - [EMBEDDING_MODEL_NAME],
38 -)
39 -async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
40 - model_name: str):
41 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
42 +async def test_single_embedding(client: openai.AsyncOpenAI, model_name: str):
43 input_texts = [
44 "The chef prepared a delicious meal.",
45 ]
46
47 # test single embedding
46 - embeddings = await embedding_client.embeddings.create(
48 + embeddings = await client.embeddings.create(
49 model=model_name,
50 input=input_texts,
51 encoding_format="float",
@@ -57,7 +59,7 @@ async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
59
60 # test using token IDs
61 input_tokens = [1, 1, 1, 1, 1]
60 - embeddings = await embedding_client.embeddings.create(
62 + embeddings = await client.embeddings.create(
63 model=model_name,
64 input=input_tokens,
65 encoding_format="float",
@@ -71,18 +73,14 @@ async def test_single_embedding(embedding_client: openai.AsyncOpenAI,
73
74
75 @pytest.mark.asyncio
74 -@pytest.mark.parametrize(
75 - "model_name",
76 - [EMBEDDING_MODEL_NAME],
77 -)
78 -async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
79 - model_name: str):
76 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
77 +async def test_batch_embedding(client: openai.AsyncOpenAI, model_name: str):
78 # test List[str]
79 input_texts = [
80 "The cat sat on the mat.", "A feline was resting on a rug.",
81 "Stars twinkle brightly in the night sky."
82 ]
85 - embeddings = await embedding_client.embeddings.create(
83 + embeddings = await client.embeddings.create(
84 model=model_name,
85 input=input_texts,
86 encoding_format="float",
@@ -90,11 +88,14 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
88 assert embeddings.id is not None
89 assert len(embeddings.data) == 3
90 assert len(embeddings.data[0].embedding) == 4096
91 + assert embeddings.usage.completion_tokens == 0
92 + assert embeddings.usage.prompt_tokens == 32
93 + assert embeddings.usage.total_tokens == 32
94
95 # test List[List[int]]
96 input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
97 [25, 32, 64, 77]]
97 - embeddings = await embedding_client.embeddings.create(
98 + embeddings = await client.embeddings.create(
99 model=model_name,
100 input=input_tokens,
101 encoding_format="float",
@@ -108,22 +109,70 @@ async def test_batch_embedding(embedding_client: openai.AsyncOpenAI,
109
110
111 @pytest.mark.asyncio
111 -@pytest.mark.parametrize(
112 - "model_name",
113 - [EMBEDDING_MODEL_NAME],
114 -)
115 -async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
112 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
113 +async def test_conversation_embedding(server: RemoteOpenAIServer,
114 + client: openai.AsyncOpenAI,
115 + model_name: str):
116 + messages = [{
117 + "role": "user",
118 + "content": "The cat sat on the mat.",
119 + }, {
120 + "role": "assistant",
121 + "content": "A feline was resting on a rug.",
122 + }, {
123 + "role": "user",
124 + "content": "Stars twinkle brightly in the night sky.",
125 + }]
126 +
127 + chat_response = requests.post(server.url_for("v1/embeddings"),
128 + json={
129 + "model": model_name,
130 + "messages": messages,
131 + "encoding_format": "float",
132 + })
133 + chat_response.raise_for_status()
134 + chat_embeddings = chat_response.json()
135 +
136 + tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
137 + prompt = tokenizer.apply_chat_template(
138 + messages,
139 + chat_template=DUMMY_CHAT_TEMPLATE,
140 + add_generation_prompt=True,
141 + continue_final_message=False,
142 + tokenize=False,
143 + )
144 + completion_response = await client.embeddings.create(
145 + model=model_name,
146 + input=prompt,
147 + encoding_format="float",
148 + # To be consistent with chat
149 + extra_body={"add_special_tokens": False},
150 + )
151 + completion_embeddings = completion_response.model_dump(mode="json")
152 +
153 + assert chat_embeddings.pop("id") is not None
154 + assert completion_embeddings.pop("id") is not None
155 + assert chat_embeddings.pop("created") <= completion_embeddings.pop(
156 + "created")
157 + assert chat_embeddings == completion_embeddings
158 +
159 +
160 +@pytest.mark.asyncio
161 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
162 +async def test_batch_base64_embedding(client: openai.AsyncOpenAI,
163 model_name: str):
164 input_texts = [
165 "Hello my name is",
166 "The best thing about vLLM is that it supports many different models"
167 ]
168
122 - responses_float = await embedding_client.embeddings.create(
123 - input=input_texts, model=model_name, encoding_format="float")
169 + responses_float = await client.embeddings.create(input=input_texts,
170 + model=model_name,
171 + encoding_format="float")
172
125 - responses_base64 = await embedding_client.embeddings.create(
126 - input=input_texts, model=model_name, encoding_format="base64")
173 + responses_base64 = await client.embeddings.create(input=input_texts,
174 + model=model_name,
175 + encoding_format="base64")
176
177 decoded_responses_base64_data = []
178 for data in responses_base64.data:
@@ -137,8 +186,8 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
186 1]
187
188 # Default response is float32 decoded from base64 by OpenAI Client
140 - responses_default = await embedding_client.embeddings.create(
141 - input=input_texts, model=model_name)
189 + responses_default = await client.embeddings.create(input=input_texts,
190 + model=model_name)
191
192 assert responses_float.data[0].embedding == responses_default.data[
193 0].embedding
@@ -147,18 +196,15 @@ async def test_batch_base64_embedding(embedding_client: openai.AsyncOpenAI,
196
197
198 @pytest.mark.asyncio
150 -@pytest.mark.parametrize(
151 - "model_name",
152 - [EMBEDDING_MODEL_NAME],
153 -)
154 -async def test_single_embedding_truncation(
155 - embedding_client: openai.AsyncOpenAI, model_name: str):
199 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
200 +async def test_single_embedding_truncation(client: openai.AsyncOpenAI,
201 + model_name: str):
202 input_texts = [
203 "Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
204 ]
205
206 # test single embedding
161 - embeddings = await embedding_client.embeddings.create(
207 + embeddings = await client.embeddings.create(
208 model=model_name,
209 input=input_texts,
210 extra_body={"truncate_prompt_tokens": 10})
@@ -173,7 +219,7 @@ async def test_single_embedding_truncation(
219 1, 24428, 289, 18341, 26165, 285, 19323, 283, 289, 26789, 3871, 28728,
220 9901, 340, 2229, 385, 340, 315, 28741, 28804, 2
221 ]
176 - embeddings = await embedding_client.embeddings.create(
222 + embeddings = await client.embeddings.create(
223 model=model_name,
224 input=input_tokens,
225 extra_body={"truncate_prompt_tokens": 10})
@@ -187,18 +233,15 @@ async def test_single_embedding_truncation(
233
234
235 @pytest.mark.asyncio
190 -@pytest.mark.parametrize(
191 - "model_name",
192 - [EMBEDDING_MODEL_NAME],
193 -)
194 -async def test_single_embedding_truncation_invalid(
195 - embedding_client: openai.AsyncOpenAI, model_name: str):
236 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
237 +async def test_single_embedding_truncation_invalid(client: openai.AsyncOpenAI,
238 + model_name: str):
239 input_texts = [
240 "Como o Brasil pode fomentar o desenvolvimento de modelos de IA?",
241 ]
242
243 with pytest.raises(openai.BadRequestError):
201 - embeddings = await embedding_client.embeddings.create(
244 + embeddings = await client.embeddings.create(
245 model=model_name,
246 input=input_texts,
247 extra_body={"truncate_prompt_tokens": 8193})
tests/entrypoints/openai/test_metrics.py
+6 -8
@@ -79,9 +79,8 @@ EXPECTED_VALUES = {
79
80
81 @pytest.mark.asyncio
82 -async def test_metrics_counts(client: openai.AsyncOpenAI):
83 - base_url = str(client.base_url)[:-3].strip("/")
84 -
82 +async def test_metrics_counts(server: RemoteOpenAIServer,
83 + client: openai.AsyncClient):
84 for _ in range(_NUM_REQUESTS):
85 # sending a request triggers the metrics to be logged.
86 await client.completions.create(
@@ -89,7 +88,7 @@ async def test_metrics_counts(client: openai.AsyncOpenAI):
88 prompt=_TOKENIZED_PROMPT,
89 max_tokens=_NUM_GENERATION_TOKENS_PER_REQUEST)
90
92 - response = requests.get(base_url + "/metrics")
91 + response = requests.get(server.url_for("metrics"))
92 print(response.text)
93 assert response.status_code == HTTPStatus.OK
94
@@ -170,16 +169,15 @@ EXPECTED_METRICS = [
169
170
171 @pytest.mark.asyncio
173 -async def test_metrics_exist(client: openai.AsyncOpenAI):
174 - base_url = str(client.base_url)[:-3].strip("/")
175 -
172 +async def test_metrics_exist(server: RemoteOpenAIServer,
173 + client: openai.AsyncClient):
174 # sending a request triggers the metrics to be logged.
175 await client.completions.create(model=MODEL_NAME,
176 prompt="Hello, my name is",
177 max_tokens=5,
178 temperature=0.0)
179
182 - response = requests.get(base_url + "/metrics")
180 + response = requests.get(server.url_for("metrics"))
181 assert response.status_code == HTTPStatus.OK
182
183 for metric in EXPECTED_METRICS:
tests/entrypoints/openai/test_tokenization.py
+18 -14
@@ -1,4 +1,3 @@
1 -import openai # use the official client for correctness check
1 import pytest
2 import pytest_asyncio
3 import requests
@@ -55,9 +54,11 @@ async def client(server):
54 [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
55 indirect=["tokenizer_name"],
56 )
58 -async def test_tokenize_completions(client: openai.AsyncOpenAI,
59 - model_name: str, tokenizer_name: str):
60 - base_url = str(client.base_url)[:-3].strip("/")
57 +async def test_tokenize_completions(
58 + server: RemoteOpenAIServer,
59 + model_name: str,
60 + tokenizer_name: str,
61 +):
62 tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
63 tokenizer_mode="fast")
64
@@ -65,7 +66,7 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
66 prompt = "vllm1 This is a test prompt."
67 tokens = tokenizer.encode(prompt, add_special_tokens=add_special)
68
68 - response = requests.post(base_url + "/tokenize",
69 + response = requests.post(server.url_for("tokenize"),
70 json={
71 "add_special_tokens": add_special,
72 "model": model_name,
@@ -86,9 +87,11 @@ async def test_tokenize_completions(client: openai.AsyncOpenAI,
87 [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
88 indirect=["tokenizer_name"],
89 )
89 -async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
90 - tokenizer_name: str):
91 - base_url = str(client.base_url)[:-3].strip("/")
90 +async def test_tokenize_chat(
91 + server: RemoteOpenAIServer,
92 + model_name: str,
93 + tokenizer_name: str,
94 +):
95 tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
96 tokenizer_mode="fast")
97
@@ -121,7 +124,7 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
124 tokens = tokenizer.encode(prompt,
125 add_special_tokens=add_special)
126
124 - response = requests.post(base_url + "/tokenize",
127 + response = requests.post(server.url_for("tokenize"),
128 json={
129 "add_generation_prompt":
130 add_generation,
@@ -146,17 +149,18 @@ async def test_tokenize_chat(client: openai.AsyncOpenAI, model_name: str,
149 [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
150 indirect=["tokenizer_name"],
151 )
149 -async def test_detokenize(client: openai.AsyncOpenAI, model_name: str,
150 - tokenizer_name: str):
151 - base_url = str(client.base_url)[:-3].strip("/")
152 +async def test_detokenize(
153 + server: RemoteOpenAIServer,
154 + model_name: str,
155 + tokenizer_name: str,
156 +):
157 tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
158 tokenizer_mode="fast")
159
160 prompt = "This is a test prompt. vllm1"
161 tokens = tokenizer.encode(prompt, add_special_tokens=False)
162
158 - print(f"CALLING {base_url} FOR {model_name}")
159 - response = requests.post(base_url + "/detokenize",
163 + response = requests.post(server.url_for("detokenize"),
164 json={
165 "model": model_name,
166 "tokens": tokens
tests/entrypoints/openai/test_vision_embedding.py new
+94
@@ -0,0 +1,94 @@
1 +from typing import Dict
2 +
3 +import pytest
4 +import pytest_asyncio
5 +import requests
6 +
7 +from vllm.multimodal.utils import encode_image_base64, fetch_image
8 +
9 +from ...utils import RemoteOpenAIServer
10 +
11 +MODEL_NAME = "TIGER-Lab/VLM2Vec-Full"
12 +MAXIMUM_IMAGES = 2
13 +
14 +# Test different image extensions (JPG/PNG) and formats (gray/RGB/RGBA)
15 +TEST_IMAGE_URLS = [
16 + "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
17 + "https://upload.wikimedia.org/wikipedia/commons/f/fa/Grayscale_8bits_palette_sample_image.png",
18 + "https://upload.wikimedia.org/wikipedia/commons/thumb/9/91/Venn_diagram_rgb.svg/1280px-Venn_diagram_rgb.svg.png",
19 + "https://upload.wikimedia.org/wikipedia/commons/0/0b/RGBA_comp.png",
20 +]
21 +
22 +
23 +@pytest.fixture(scope="module")
24 +def server():
25 + args = [
26 + "--task",
27 + "embedding",
28 + "--dtype",
29 + "bfloat16",
30 + "--max-model-len",
31 + "2048",
32 + "--max-num-seqs",
33 + "5",
34 + "--enforce-eager",
35 + "--trust-remote-code",
36 + "--limit-mm-per-prompt",
37 + f"image={MAXIMUM_IMAGES}",
38 + ]
39 +
40 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
41 + yield remote_server
42 +
43 +
44 +@pytest_asyncio.fixture
45 +async def client(server):
46 + async with server.get_async_client() as async_client:
47 + yield async_client
48 +
49 +
50 +@pytest.fixture(scope="session")
51 +def base64_encoded_image() -> Dict[str, str]:
52 + return {
53 + image_url: encode_image_base64(fetch_image(image_url))
54 + for image_url in TEST_IMAGE_URLS
55 + }
56 +
57 +
58 +@pytest.mark.asyncio
59 +@pytest.mark.parametrize("model_name", [MODEL_NAME])
60 +@pytest.mark.parametrize("image_url", TEST_IMAGE_URLS)
61 +async def test_image_embedding(server: RemoteOpenAIServer, model_name: str,
62 + image_url: str):
63 + messages = [{
64 + "role":
65 + "user",
66 + "content": [
67 + {
68 + "type": "image_url",
69 + "image_url": {
70 + "url": image_url
71 + }
72 + },
73 + {
74 + "type": "text",
75 + "text": "Represent the given image."
76 + },
77 + ],
78 + }]
79 +
80 + response = requests.post(server.url_for("v1/embeddings"),
81 + json={
82 + "model": model_name,
83 + "messages": messages,
84 + "encoding_format": "float"
85 + })
86 + response.raise_for_status()
87 +
88 + embeddings = response.json()
89 + assert embeddings["id"] is not None
90 + assert len(embeddings["data"]) == 1
91 + assert len(embeddings["data"][0]["embedding"]) == 3072
92 + assert embeddings["usage"]["completion_tokens"] == 0
93 + assert embeddings["usage"]["prompt_tokens"] == 771
94 + assert embeddings["usage"]["total_tokens"] == 771
vllm/entrypoints/openai/api_server.py
+58 -38
@@ -11,7 +11,7 @@ from argparse import Namespace
11 from contextlib import asynccontextmanager
12 from functools import partial
13 from http import HTTPStatus
14 -from typing import AsyncIterator, Set
14 +from typing import AsyncIterator, Optional, Set
15
16 import uvloop
17 from fastapi import APIRouter, FastAPI, Request
@@ -51,7 +51,7 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
51 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
52 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
53 from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
54 -from vllm.entrypoints.openai.serving_engine import BaseModelPath
54 +from vllm.entrypoints.openai.serving_engine import BaseModelPath, OpenAIServing
55 from vllm.entrypoints.openai.serving_tokenization import (
56 OpenAIServingTokenization)
57 from vllm.entrypoints.openai.tool_parsers import ToolParserManager
@@ -248,20 +248,25 @@ def mount_metrics(app: FastAPI):
248 app.routes.append(metrics_route)
249
250
251 -def chat(request: Request) -> OpenAIServingChat:
251 +def base(request: Request) -> OpenAIServing:
252 + # Reuse the existing instance
253 + return tokenization(request)
254 +
255 +
256 +def chat(request: Request) -> Optional[OpenAIServingChat]:
257 return request.app.state.openai_serving_chat
258
259
255 -def completion(request: Request) -> OpenAIServingCompletion:
260 +def completion(request: Request) -> Optional[OpenAIServingCompletion]:
261 return request.app.state.openai_serving_completion
262
263
259 -def tokenization(request: Request) -> OpenAIServingTokenization:
260 - return request.app.state.openai_serving_tokenization
264 +def embedding(request: Request) -> Optional[OpenAIServingEmbedding]:
265 + return request.app.state.openai_serving_embedding
266
267
263 -def embedding(request: Request) -> OpenAIServingEmbedding:
264 - return request.app.state.openai_serving_embedding
268 +def tokenization(request: Request) -> OpenAIServingTokenization:
269 + return request.app.state.openai_serving_tokenization
270
271
272 def engine_client(request: Request) -> EngineClient:
@@ -277,7 +282,9 @@ async def health(raw_request: Request) -> Response:
282
283 @router.post("/tokenize")
284 async def tokenize(request: TokenizeRequest, raw_request: Request):
280 - generator = await tokenization(raw_request).create_tokenize(request)
285 + handler = tokenization(raw_request)
286 +
287 + generator = await handler.create_tokenize(request)
288 if isinstance(generator, ErrorResponse):
289 return JSONResponse(content=generator.model_dump(),
290 status_code=generator.code)
@@ -289,7 +296,9 @@ async def tokenize(request: TokenizeRequest, raw_request: Request):
296
297 @router.post("/detokenize")
298 async def detokenize(request: DetokenizeRequest, raw_request: Request):
292 - generator = await tokenization(raw_request).create_detokenize(request)
299 + handler = tokenization(raw_request)
300 +
301 + generator = await handler.create_detokenize(request)
302 if isinstance(generator, ErrorResponse):
303 return JSONResponse(content=generator.model_dump(),
304 status_code=generator.code)
@@ -301,7 +310,9 @@ async def detokenize(request: DetokenizeRequest, raw_request: Request):
310
311 @router.get("/v1/models")
312 async def show_available_models(raw_request: Request):
304 - models = await completion(raw_request).show_available_models()
313 + handler = base(raw_request)
314 +
315 + models = await handler.show_available_models()
316 return JSONResponse(content=models.model_dump())
317
318
@@ -314,9 +325,12 @@ async def show_version():
325 @router.post("/v1/chat/completions")
326 async def create_chat_completion(request: ChatCompletionRequest,
327 raw_request: Request):
328 + handler = chat(raw_request)
329 + if handler is None:
330 + return base(raw_request).create_error_response(
331 + message="The model does not support Chat Completions API")
332
318 - generator = await chat(raw_request).create_chat_completion(
319 - request, raw_request)
333 + generator = await handler.create_chat_completion(request, raw_request)
334
335 if isinstance(generator, ErrorResponse):
336 return JSONResponse(content=generator.model_dump(),
@@ -330,8 +344,12 @@ async def create_chat_completion(request: ChatCompletionRequest,
344
345 @router.post("/v1/completions")
346 async def create_completion(request: CompletionRequest, raw_request: Request):
333 - generator = await completion(raw_request).create_completion(
334 - request, raw_request)
347 + handler = completion(raw_request)
348 + if handler is None:
349 + return base(raw_request).create_error_response(
350 + message="The model does not support Completions API")
351 +
352 + generator = await handler.create_completion(request, raw_request)
353 if isinstance(generator, ErrorResponse):
354 return JSONResponse(content=generator.model_dump(),
355 status_code=generator.code)
@@ -343,8 +361,12 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
361
362 @router.post("/v1/embeddings")
363 async def create_embedding(request: EmbeddingRequest, raw_request: Request):
346 - generator = await embedding(raw_request).create_embedding(
347 - request, raw_request)
364 + handler = embedding(raw_request)
365 + if handler is None:
366 + return base(raw_request).create_error_response(
367 + message="The model does not support Embeddings API")
368 +
369 + generator = await handler.create_embedding(request, raw_request)
370 if isinstance(generator, ErrorResponse):
371 return JSONResponse(content=generator.model_dump(),
372 status_code=generator.code)
@@ -382,30 +404,26 @@ if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
404 @router.post("/v1/load_lora_adapter")
405 async def load_lora_adapter(request: LoadLoraAdapterRequest,
406 raw_request: Request):
385 - response = await chat(raw_request).load_lora_adapter(request)
386 - if isinstance(response, ErrorResponse):
387 - return JSONResponse(content=response.model_dump(),
388 - status_code=response.code)
389 -
390 - response = await completion(raw_request).load_lora_adapter(request)
391 - if isinstance(response, ErrorResponse):
392 - return JSONResponse(content=response.model_dump(),
393 - status_code=response.code)
407 + for route in [chat, completion, embedding]:
408 + handler = route(raw_request)
409 + if handler is not None:
410 + response = await handler.load_lora_adapter(request)
411 + if isinstance(response, ErrorResponse):
412 + return JSONResponse(content=response.model_dump(),
413 + status_code=response.code)
414
415 return Response(status_code=200, content=response)
416
417 @router.post("/v1/unload_lora_adapter")
418 async def unload_lora_adapter(request: UnloadLoraAdapterRequest,
419 raw_request: Request):
400 - response = await chat(raw_request).unload_lora_adapter(request)
401 - if isinstance(response, ErrorResponse):
402 - return JSONResponse(content=response.model_dump(),
403 - status_code=response.code)
404 -
405 - response = await completion(raw_request).unload_lora_adapter(request)
406 - if isinstance(response, ErrorResponse):
407 - return JSONResponse(content=response.model_dump(),
408 - status_code=response.code)
420 + for route in [chat, completion, embedding]:
421 + handler = route(raw_request)
422 + if handler is not None:
423 + response = await handler.unload_lora_adapter(request)
424 + if isinstance(response, ErrorResponse):
425 + return JSONResponse(content=response.model_dump(),
426 + status_code=response.code)
427
428 return Response(status_code=200, content=response)
429
@@ -501,7 +519,8 @@ def init_app_state(
519 chat_template=args.chat_template,
520 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
521 enable_auto_tools=args.enable_auto_tool_choice,
504 - tool_parser=args.tool_call_parser)
522 + tool_parser=args.tool_call_parser,
523 + ) if model_config.task == "generate" else None
524 state.openai_serving_completion = OpenAIServingCompletion(
525 engine_client,
526 model_config,
@@ -510,13 +529,14 @@ def init_app_state(
529 prompt_adapters=args.prompt_adapters,
530 request_logger=request_logger,
531 return_tokens_as_token_ids=args.return_tokens_as_token_ids,
513 - )
532 + ) if model_config.task == "generate" else None
533 state.openai_serving_embedding = OpenAIServingEmbedding(
534 engine_client,
535 model_config,
536 base_model_paths,
537 request_logger=request_logger,
519 - )
538 + chat_template=args.chat_template,
539 + ) if model_config.task == "embedding" else None
540 state.openai_serving_tokenization = OpenAIServingTokenization(
541 engine_client,
542 model_config,
vllm/entrypoints/openai/protocol.py
+84 -3
@@ -708,7 +708,7 @@ class CompletionRequest(OpenAIBaseModel):
708 return data
709
710
711 -class EmbeddingRequest(OpenAIBaseModel):
711 +class EmbeddingCompletionRequest(OpenAIBaseModel):
712 # Ordered by official OpenAI API documentation
713 # https://platform.openai.com/docs/api-reference/embeddings
714 model: str
@@ -720,10 +720,15 @@ class EmbeddingRequest(OpenAIBaseModel):
720
721 # doc: begin-embedding-pooling-params
722 additional_data: Optional[Any] = None
723 -
723 # doc: end-embedding-pooling-params
724
725 # doc: begin-embedding-extra-params
726 + add_special_tokens: bool = Field(
727 + default=True,
728 + description=(
729 + "If true (the default), special tokens (e.g. BOS) will be added to "
730 + "the prompt."),
731 + )
732 priority: int = Field(
733 default=0,
734 description=(
@@ -737,6 +742,82 @@ class EmbeddingRequest(OpenAIBaseModel):
742 return PoolingParams(additional_data=self.additional_data)
743
744
745 +class EmbeddingChatRequest(OpenAIBaseModel):
746 + model: str
747 + messages: List[ChatCompletionMessageParam]
748 +
749 + encoding_format: Literal["float", "base64"] = "float"
750 + dimensions: Optional[int] = None
751 + user: Optional[str] = None
752 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
753 +
754 + # doc: begin-chat-embedding-pooling-params
755 + additional_data: Optional[Any] = None
756 + # doc: end-chat-embedding-pooling-params
757 +
758 + # doc: begin-chat-embedding-extra-params
759 + add_generation_prompt: bool = Field(
760 + default=True,
761 + description=
762 + ("If true, the generation prompt will be added to the chat template. "
763 + "This is a parameter used by chat template in tokenizer config of the "
764 + "model."),
765 + )
766 + continue_final_message: bool = Field(
767 + default=False,
768 + description=
769 + ("If this is set, the chat will be formatted so that the final "
770 + "message in the chat is open-ended, without any EOS tokens. The "
771 + "model will continue this message rather than starting a new one. "
772 + "This allows you to \"prefill\" part of the model's response for it. "
773 + "Cannot be used at the same time as `add_generation_prompt`."),
774 + )
775 + add_special_tokens: bool = Field(
776 + default=False,
777 + description=(
778 + "If true, special tokens (e.g. BOS) will be added to the prompt "
779 + "on top of what is added by the chat template. "
780 + "For most models, the chat template takes care of adding the "
781 + "special tokens so this should be set to false (as is the "
782 + "default)."),
783 + )
784 + chat_template: Optional[str] = Field(
785 + default=None,
786 + description=(
787 + "A Jinja template to use for this conversion. "
788 + "As of transformers v4.44, default chat template is no longer "
789 + "allowed, so you must provide a chat template if the tokenizer "
790 + "does not define one."),
791 + )
792 + chat_template_kwargs: Optional[Dict[str, Any]] = Field(
793 + default=None,
794 + description=("Additional kwargs to pass to the template renderer. "
795 + "Will be accessible by the chat template."),
796 + )
797 + priority: int = Field(
798 + default=0,
799 + description=(
800 + "The priority of the request (lower means earlier handling; "
801 + "default: 0). Any priority other than 0 will raise an error "
802 + "if the served model does not use priority scheduling."))
803 + # doc: end-chat-embedding-extra-params
804 +
805 + @model_validator(mode="before")
806 + @classmethod
807 + def check_generation_prompt(cls, data):
808 + if data.get("continue_final_message") and data.get(
809 + "add_generation_prompt"):
810 + raise ValueError("Cannot set both `continue_final_message` and "
811 + "`add_generation_prompt` to True.")
812 + return data
813 +
814 + def to_pooling_params(self):
815 + return PoolingParams(additional_data=self.additional_data)
816 +
817 +
818 +EmbeddingRequest = Union[EmbeddingCompletionRequest, EmbeddingChatRequest]
819 +
820 +
821 class CompletionLogProbs(OpenAIBaseModel):
822 text_offset: List[int] = Field(default_factory=list)
823 token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -799,7 +880,7 @@ class EmbeddingResponseData(OpenAIBaseModel):
880
881
882 class EmbeddingResponse(OpenAIBaseModel):
802 - id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
883 + id: str = Field(default_factory=lambda: f"embd-{random_uuid()}")
884 object: str = "list"
885 created: int = Field(default_factory=lambda: int(time.time()))
886 model: str
vllm/entrypoints/openai/run_batch.py
+26 -8
@@ -217,13 +217,14 @@ async def main(args):
217 prompt_adapters=None,
218 request_logger=request_logger,
219 chat_template=None,
220 - )
220 + ) if model_config.task == "generate" else None
221 openai_serving_embedding = OpenAIServingEmbedding(
222 engine,
223 model_config,
224 base_model_paths,
225 request_logger=request_logger,
226 - )
226 + chat_template=None,
227 + ) if model_config.task == "embedding" else None
228
229 tracker = BatchProgressTracker()
230 logger.info("Reading batch from %s...", args.input_file)
@@ -240,14 +241,31 @@ async def main(args):
241
242 # Determine the type of request and run it.
243 if request.url == "/v1/chat/completions":
243 - response_futures.append(
244 - run_request(openai_serving_chat.create_chat_completion,
245 - request, tracker))
244 + handler_fn = (None if openai_serving_chat is None else
245 + openai_serving_chat.create_chat_completion)
246 + if handler_fn is None:
247 + response_futures.append(
248 + make_async_error_request_output(
249 + request,
250 + error_msg=
251 + "The model does not support Chat Completions API",
252 + ))
253 + continue
254 +
255 + response_futures.append(run_request(handler_fn, request, tracker))
256 tracker.submitted()
257 elif request.url == "/v1/embeddings":
248 - response_futures.append(
249 - run_request(openai_serving_embedding.create_embedding, request,
250 - tracker))
258 + handler_fn = (None if openai_serving_embedding is None else
259 + openai_serving_embedding.create_embedding)
260 + if handler_fn is None:
261 + response_futures.append(
262 + make_async_error_request_output(
263 + request,
264 + error_msg="The model does not support Embeddings API",
265 + ))
266 + continue
267 +
268 + response_futures.append(run_request(handler_fn, request, tracker))
269 tracker.submitted()
270 else:
271 response_futures.append(
vllm/entrypoints/openai/serving_chat.py
+90 -132
@@ -10,11 +10,7 @@ from fastapi import Request
10
11 from vllm.config import ModelConfig
12 from vllm.engine.protocol import EngineClient
13 -from vllm.entrypoints.chat_utils import (ConversationMessage,
14 - apply_hf_chat_template,
15 - apply_mistral_chat_template,
16 - load_chat_template,
17 - parse_chat_messages_futures)
13 +from vllm.entrypoints.chat_utils import ConversationMessage, load_chat_template
14 from vllm.entrypoints.logger import RequestLogger
15 from vllm.entrypoints.openai.protocol import (
16 ChatCompletionLogProb, ChatCompletionLogProbs,
@@ -27,16 +23,12 @@ from vllm.entrypoints.openai.protocol import (
23 from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
24 LoRAModulePath,
25 OpenAIServing,
30 - PromptAdapterPath,
31 - TextTokensPrompt)
26 + PromptAdapterPath)
27 from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager
33 -from vllm.inputs import TokensPrompt
28 from vllm.logger import init_logger
29 from vllm.outputs import CompletionOutput, RequestOutput
30 from vllm.sampling_params import BeamSearchParams, SamplingParams
31 from vllm.sequence import Logprob
38 -from vllm.tracing import (contains_trace_headers, extract_trace_headers,
39 - log_tracing_disabled_warning)
32 from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
33 from vllm.utils import iterate_with_cancellation
34
@@ -94,12 +86,12 @@ class OpenAIServingChat(OpenAIServing):
86 raw_request: Optional[Request] = None,
87 ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse,
88 ErrorResponse]:
97 - """Completion API similar to OpenAI's API.
89 + """
90 + Chat Completion API similar to OpenAI's API.
91
92 See https://platform.openai.com/docs/api-reference/chat/create
93 for the API specification. This API mimics the OpenAI
101 - ChatCompletion API.
102 -
94 + Chat Completion API.
95 """
96 error_check_ret = await self._check_model(request)
97 if error_check_ret is not None:
@@ -118,143 +110,106 @@ class OpenAIServingChat(OpenAIServing):
110 prompt_adapter_request,
111 ) = self._maybe_get_adapters(request)
112
121 - model_config = self.model_config
113 tokenizer = await self.engine_client.get_tokenizer(lora_request)
123 -
124 - conversation, mm_data_future = parse_chat_messages_futures(
125 - request.messages, model_config, tokenizer)
114 + tool_parser = self.tool_parser
115 +
116 + # validation for OpenAI tools
117 + # tool_choice = "required" is not supported
118 + if request.tool_choice == "required":
119 + return self.create_error_response(
120 + "tool_choice = \"required\" is not supported!")
121 +
122 + if (request.tool_choice == "auto" and
123 + not (self.enable_auto_tools and tool_parser is not None)
124 + and not isinstance(tokenizer, MistralTokenizer)):
125 + # for hf tokenizers, "auto" tools requires
126 + # --enable-auto-tool-choice and --tool-call-parser
127 + return self.create_error_response(
128 + "\"auto\" tool choice requires "
129 + "--enable-auto-tool-choice and --tool-call-parser to be set"
130 + )
131
132 tool_dicts = None if request.tools is None else [
133 tool.model_dump() for tool in request.tools
134 ]
135
131 - prompt: Union[str, List[int]]
132 - is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
133 - if is_mistral_tokenizer:
134 - prompt = apply_mistral_chat_template(
135 - tokenizer,
136 - messages=request.messages,
137 - chat_template=request.chat_template or self.chat_template,
138 - add_generation_prompt=request.add_generation_prompt,
139 - continue_final_message=request.continue_final_message,
140 - tools=tool_dicts,
141 - documents=request.documents,
142 - **(request.chat_template_kwargs or {}),
143 - )
144 - else:
145 - prompt = apply_hf_chat_template(
146 - tokenizer,
147 - conversation=conversation,
148 - chat_template=request.chat_template or self.chat_template,
149 - add_generation_prompt=request.add_generation_prompt,
150 - continue_final_message=request.continue_final_message,
151 - tools=tool_dicts,
152 - documents=request.documents,
153 - **(request.chat_template_kwargs or {}),
154 - )
155 - except Exception as e:
156 - logger.exception("Error in applying chat template from request")
157 - return self.create_error_response(str(e))
158 -
159 - try:
160 - mm_data = await mm_data_future
161 - except Exception as e:
162 - logger.exception("Error in loading multi-modal data")
136 + (
137 + conversation,
138 + request_prompts,
139 + engine_prompts,
140 + ) = await self._preprocess_chat(
141 + request,
142 + tokenizer,
143 + request.messages,
144 + chat_template=request.chat_template or self.chat_template,
145 + add_generation_prompt=request.add_generation_prompt,
146 + continue_final_message=request.continue_final_message,
147 + tool_dicts=tool_dicts,
148 + documents=request.documents,
149 + chat_template_kwargs=request.chat_template_kwargs,
150 + tool_parser=tool_parser,
151 + truncate_prompt_tokens=request.truncate_prompt_tokens,
152 + add_special_tokens=request.add_special_tokens,
153 + )
154 + except ValueError as e:
155 + logger.exception("Error in preprocessing prompt inputs")
156 return self.create_error_response(str(e))
157
165 - # validation for OpenAI tools
166 - # tool_choice = "required" is not supported
167 - if request.tool_choice == "required":
168 - return self.create_error_response(
169 - "tool_choice = \"required\" is not supported!")
170 -
171 - if not is_mistral_tokenizer and request.tool_choice == "auto" and not (
172 - self.enable_auto_tools and self.tool_parser is not None):
173 - # for hf tokenizers, "auto" tools requires
174 - # --enable-auto-tool-choice and --tool-call-parser
175 - return self.create_error_response(
176 - "\"auto\" tool choice requires "
177 - "--enable-auto-tool-choice and --tool-call-parser to be set")
178 -
179 - request_id = f"chat-{request.request_id}"
158 + request_id = f"chatcmpl-{request.request_id}"
159
160 request_metadata = RequestResponseMetadata(request_id=request_id)
161 if raw_request:
162 raw_request.state.request_metadata = request_metadata
163
164 + # Schedule the request and get the result generator.
165 + generators: List[AsyncGenerator[RequestOutput, None]] = []
166 try:
186 - if self.enable_auto_tools and self.tool_parser:
187 - request = self.tool_parser(tokenizer).adjust_request(
188 - request=request)
189 -
190 - if isinstance(prompt, str):
191 - prompt_inputs = self._tokenize_prompt_input(
192 - request,
193 - tokenizer,
194 - prompt,
195 - truncate_prompt_tokens=request.truncate_prompt_tokens,
196 - add_special_tokens=request.add_special_tokens,
197 - )
198 - else:
199 - assert isinstance(prompt, list) and isinstance(
200 - prompt[0], int
201 - ), "Prompt has to be either a string or a list of token ids"
202 - prompt_inputs = TextTokensPrompt(
203 - prompt=tokenizer.decode(prompt), prompt_token_ids=prompt)
204 -
205 - assert prompt_inputs is not None
206 -
207 - sampling_params: Union[SamplingParams, BeamSearchParams]
208 - default_max_tokens = self.max_model_len - len(
209 - prompt_inputs["prompt_token_ids"])
210 - if request.use_beam_search:
211 - sampling_params = request.to_beam_search_params(
212 - default_max_tokens)
213 - else:
214 - sampling_params = request.to_sampling_params(
215 - default_max_tokens)
216 -
217 - self._log_inputs(request_id,
218 - prompt_inputs,
219 - params=sampling_params,
220 - lora_request=lora_request,
221 - prompt_adapter_request=prompt_adapter_request)
222 -
223 - engine_inputs = TokensPrompt(
224 - prompt_token_ids=prompt_inputs["prompt_token_ids"])
225 - if mm_data is not None:
226 - engine_inputs["multi_modal_data"] = mm_data
227 -
228 - is_tracing_enabled = (await
229 - self.engine_client.is_tracing_enabled())
230 - trace_headers = None
231 - if is_tracing_enabled and raw_request:
232 - trace_headers = extract_trace_headers(raw_request.headers)
233 - if (not is_tracing_enabled and raw_request
234 - and contains_trace_headers(raw_request.headers)):
235 - log_tracing_disabled_warning()
236 -
237 - if isinstance(sampling_params, BeamSearchParams):
238 - result_generator = self.engine_client.beam_search(
239 - prompt=engine_inputs,
240 - model_config=self.model_config,
241 - request_id=request_id,
242 - params=sampling_params,
243 - )
244 - else:
245 - result_generator = self.engine_client.generate(
246 - engine_inputs,
247 - sampling_params,
248 - request_id,
249 - lora_request=lora_request,
250 - trace_headers=trace_headers,
251 - prompt_adapter_request=prompt_adapter_request,
252 - priority=request.priority,
253 - )
167 + for i, engine_prompt in enumerate(engine_prompts):
168 + sampling_params: Union[SamplingParams, BeamSearchParams]
169 + default_max_tokens = self.max_model_len - len(
170 + engine_prompt["prompt_token_ids"])
171 + if request.use_beam_search:
172 + sampling_params = request.to_beam_search_params(
173 + default_max_tokens)
174 + else:
175 + sampling_params = request.to_sampling_params(
176 + default_max_tokens)
177 +
178 + self._log_inputs(request_id,
179 + request_prompts[i],
180 + params=sampling_params,
181 + lora_request=lora_request,
182 + prompt_adapter_request=prompt_adapter_request)
183 +
184 + trace_headers = (None if raw_request is None else await
185 + self._get_trace_headers(raw_request.headers))
186 +
187 + if isinstance(sampling_params, BeamSearchParams):
188 + generator = self.engine_client.beam_search(
189 + prompt=engine_prompt,
190 + model_config=self.model_config,
191 + request_id=request_id,
192 + params=sampling_params,
193 + )
194 + else:
195 + generator = self.engine_client.generate(
196 + engine_prompt,
197 + sampling_params,
198 + request_id,
199 + lora_request=lora_request,
200 + trace_headers=trace_headers,
201 + prompt_adapter_request=prompt_adapter_request,
202 + priority=request.priority,
203 + )
204 +
205 + generators.append(generator)
206 except ValueError as e:
207 # TODO: Use a vllm-specific Validation Error
208 return self.create_error_response(str(e))
209
210 + assert len(generators) == 1
211 + result_generator, = generators
212 +
213 if raw_request:
214 result_generator = iterate_with_cancellation(
215 result_generator, raw_request.is_disconnected)
@@ -626,6 +581,9 @@ class OpenAIServingChat(OpenAIServing):
581 final_res = res
582 except asyncio.CancelledError:
583 return self.create_error_response("Client disconnected")
584 + except ValueError as e:
585 + # TODO: Use a vllm-specific Validation Error
586 + return self.create_error_response(str(e))
587
588 assert final_res is not None
589
vllm/entrypoints/openai/serving_completion.py
+32 -43
@@ -1,7 +1,6 @@
1 import asyncio
2 import time
3 -from typing import (AsyncGenerator, AsyncIterator, Callable, Dict, List,
4 - Optional)
3 +from typing import AsyncGenerator, AsyncIterator, Dict, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Tuple, Union, cast
6
@@ -30,18 +29,11 @@ from vllm.logger import init_logger
29 from vllm.outputs import RequestOutput
30 from vllm.sampling_params import BeamSearchParams, SamplingParams
31 from vllm.sequence import Logprob
33 -from vllm.tracing import (contains_trace_headers, extract_trace_headers,
34 - log_tracing_disabled_warning)
32 from vllm.transformers_utils.tokenizer import AnyTokenizer
33 from vllm.utils import merge_async_iterators, random_uuid
34
35 logger = init_logger(__name__)
36
40 -TypeTokenIDs = List[int]
41 -TypeTopLogProbs = List[Optional[Dict[int, float]]]
42 -TypeCreateLogProbsFn = Callable[
43 - [TypeTokenIDs, TypeTopLogProbs, Optional[int], int], CompletionLogProbs]
44 -
37
38 class OpenAIServingCompletion(OpenAIServing):
39
@@ -101,8 +93,6 @@ class OpenAIServingCompletion(OpenAIServing):
93 if raw_request:
94 raw_request.state.request_metadata = request_metadata
95
104 - # Schedule the request and get the result generator.
105 - generators: List[AsyncGenerator[RequestOutput, None]] = []
96 try:
97 (
98 lora_request,
@@ -111,19 +101,24 @@ class OpenAIServingCompletion(OpenAIServing):
101
102 tokenizer = await self.engine_client.get_tokenizer(lora_request)
103
114 - prompts = list(
115 - self._tokenize_prompt_input_or_inputs(
116 - request,
117 - tokenizer,
118 - request.prompt,
119 - truncate_prompt_tokens=request.truncate_prompt_tokens,
120 - add_special_tokens=request.add_special_tokens,
121 - ))
104 + request_prompts, engine_prompts = self._preprocess_completion(
105 + request,
106 + tokenizer,
107 + request.prompt,
108 + truncate_prompt_tokens=request.truncate_prompt_tokens,
109 + add_special_tokens=request.add_special_tokens,
110 + )
111 + except ValueError as e:
112 + logger.exception("Error in preprocessing prompt inputs")
113 + return self.create_error_response(str(e))
114
123 - for i, prompt_inputs in enumerate(prompts):
115 + # Schedule the request and get the result generator.
116 + generators: List[AsyncGenerator[RequestOutput, None]] = []
117 + try:
118 + for i, engine_prompt in enumerate(engine_prompts):
119 sampling_params: Union[SamplingParams, BeamSearchParams]
120 default_max_tokens = self.max_model_len - len(
126 - prompt_inputs["prompt_token_ids"])
121 + engine_prompt["prompt_token_ids"])
122 if request.use_beam_search:
123 sampling_params = request.to_beam_search_params(
124 default_max_tokens)
@@ -134,36 +129,24 @@ class OpenAIServingCompletion(OpenAIServing):
129 request_id_item = f"{request_id}-{i}"
130
131 self._log_inputs(request_id_item,
137 - prompt_inputs,
132 + request_prompts[i],
133 params=sampling_params,
134 lora_request=lora_request,
135 prompt_adapter_request=prompt_adapter_request)
136
142 - is_tracing_enabled = (await
143 - self.engine_client.is_tracing_enabled())
144 - trace_headers = None
145 - if is_tracing_enabled:
146 - trace_headers = extract_trace_headers(raw_request.headers)
147 - if not is_tracing_enabled and contains_trace_headers(
148 - raw_request.headers):
149 - log_tracing_disabled_warning()
137 + trace_headers = (await
138 + self._get_trace_headers(raw_request.headers))
139
140 if isinstance(sampling_params, BeamSearchParams):
141 generator = self.engine_client.beam_search(
153 - prompt={
154 - "prompt_token_ids":
155 - prompt_inputs["prompt_token_ids"]
156 - },
142 + prompt=engine_prompt,
143 model_config=self.model_config,
144 request_id=request_id,
145 params=sampling_params,
146 )
147 else:
148 generator = self.engine_client.generate(
163 - {
164 - "prompt_token_ids":
165 - prompt_inputs["prompt_token_ids"]
166 - },
149 + engine_prompt,
150 sampling_params,
151 request_id_item,
152 lora_request=lora_request,
@@ -180,6 +163,8 @@ class OpenAIServingCompletion(OpenAIServing):
163 result_generator = merge_async_iterators(
164 *generators, is_cancelled=raw_request.is_disconnected)
165
166 + num_prompts = len(engine_prompts)
167 +
168 # Similar to the OpenAI API, when n != best_of, we do not stream the
169 # results. In addition, we do not stream the results when use
170 # beam search.
@@ -195,16 +180,22 @@ class OpenAIServingCompletion(OpenAIServing):
180 request_id,
181 created_time,
182 model_name,
198 - num_prompts=len(prompts),
183 + num_prompts=num_prompts,
184 tokenizer=tokenizer,
185 request_metadata=request_metadata)
186
187 # Non-streaming response
203 - final_res_batch: List[Optional[RequestOutput]] = [None] * len(prompts)
188 + final_res_batch: List[Optional[RequestOutput]] = [None] * num_prompts
189 try:
190 async for i, res in result_generator:
191 final_res_batch[i] = res
192 + except asyncio.CancelledError:
193 + return self.create_error_response("Client disconnected")
194 + except ValueError as e:
195 + # TODO: Use a vllm-specific Validation Error
196 + return self.create_error_response(str(e))
197
198 + try:
199 for i, final_res in enumerate(final_res_batch):
200 assert final_res is not None
201
@@ -212,7 +203,7 @@ class OpenAIServingCompletion(OpenAIServing):
203 # We did not pass it into vLLM engine to avoid being redundant
204 # with the inputs token IDs
205 if final_res.prompt is None:
215 - final_res.prompt = prompts[i]["prompt"]
206 + final_res.prompt = request_prompts[i]["prompt"]
207
208 final_res_batch_checked = cast(List[RequestOutput],
209 final_res_batch)
@@ -226,8 +217,6 @@ class OpenAIServingCompletion(OpenAIServing):
217 tokenizer,
218 request_metadata,
219 )
229 - except asyncio.CancelledError:
230 - return self.create_error_response("Client disconnected")
220 except ValueError as e:
221 # TODO: Use a vllm-specific Validation Error
222 return self.create_error_response(str(e))
vllm/entrypoints/openai/serving_embedding.py
+54 -33
@@ -9,8 +9,10 @@ from typing_extensions import assert_never
9
10 from vllm.config import ModelConfig
11 from vllm.engine.protocol import EngineClient
12 +from vllm.entrypoints.chat_utils import load_chat_template
13 from vllm.entrypoints.logger import RequestLogger
13 -from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
14 +from vllm.entrypoints.openai.protocol import (EmbeddingChatRequest,
15 + EmbeddingRequest,
16 EmbeddingResponse,
17 EmbeddingResponseData,
18 ErrorResponse, UsageInfo)
@@ -21,8 +23,6 @@ from vllm.utils import merge_async_iterators, random_uuid
23
24 logger = init_logger(__name__)
25
24 -TypeTokenIDs = List[int]
25 -
26
27 def _get_embedding(
28 output: EmbeddingOutput,
@@ -76,6 +76,7 @@ class OpenAIServingEmbedding(OpenAIServing):
76 base_model_paths: List[BaseModelPath],
77 *,
78 request_logger: Optional[RequestLogger],
79 + chat_template: Optional[str],
80 ):
81 super().__init__(engine_client=engine_client,
82 model_config=model_config,
@@ -83,21 +84,20 @@ class OpenAIServingEmbedding(OpenAIServing):
84 lora_modules=None,
85 prompt_adapters=None,
86 request_logger=request_logger)
86 - self._enabled = self._check_embedding_mode(
87 - model_config.task == "embedding")
87 +
88 + self.chat_template = load_chat_template(chat_template)
89
90 async def create_embedding(
91 self,
92 request: EmbeddingRequest,
93 raw_request: Optional[Request] = None,
94 ) -> Union[EmbeddingResponse, ErrorResponse]:
94 - """Completion API similar to OpenAI's API.
95 + """
96 + Embedding API similar to OpenAI's API.
97
98 See https://platform.openai.com/docs/api-reference/embeddings/create
99 for the API specification. This API mimics the OpenAI Embedding API.
100 """
99 - if not self._enabled:
100 - return self.create_error_response("Embedding API disabled")
101 error_check_ret = await self._check_model(request)
102 if error_check_ret is not None:
103 return error_check_ret
@@ -122,8 +122,6 @@ class OpenAIServingEmbedding(OpenAIServing):
122 "greater than max_model_len."
123 " Please, select a smaller truncation size.")
124
125 - # Schedule the request and get the result generator.
126 - generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
125 try:
126 (
127 lora_request,
@@ -132,32 +130,60 @@ class OpenAIServingEmbedding(OpenAIServing):
130
131 tokenizer = await self.engine_client.get_tokenizer(lora_request)
132
135 - pooling_params = request.to_pooling_params()
133 + if prompt_adapter_request is not None:
134 + raise NotImplementedError("Prompt adapter is not supported "
135 + "for embedding models")
136 +
137 + if isinstance(request, EmbeddingChatRequest):
138 + (
139 + _,
140 + request_prompts,
141 + engine_prompts,
142 + ) = await self._preprocess_chat(
143 + request,
144 + tokenizer,
145 + request.messages,
146 + chat_template=request.chat_template or self.chat_template,
147 + add_generation_prompt=request.add_generation_prompt,
148 + continue_final_message=request.continue_final_message,
149 + truncate_prompt_tokens=truncate_prompt_tokens,
150 + add_special_tokens=request.add_special_tokens,
151 + )
152 + else:
153 + request_prompts, engine_prompts = self._preprocess_completion(
154 + request,
155 + tokenizer,
156 + request.input,
157 + truncate_prompt_tokens=truncate_prompt_tokens,
158 + add_special_tokens=request.add_special_tokens,
159 + )
160 + except ValueError as e:
161 + logger.exception("Error in preprocessing prompt inputs")
162 + return self.create_error_response(str(e))
163
137 - prompts = list(
138 - self._tokenize_prompt_input_or_inputs(request, tokenizer,
139 - request.input,
140 - truncate_prompt_tokens))
164 + # Schedule the request and get the result generator.
165 + generators: List[AsyncGenerator[EmbeddingRequestOutput, None]] = []
166 + try:
167 + pooling_params = request.to_pooling_params()
168
142 - for i, prompt_inputs in enumerate(prompts):
169 + for i, engine_prompt in enumerate(engine_prompts):
170 request_id_item = f"{request_id}-{i}"
171
172 self._log_inputs(request_id_item,
146 - prompt_inputs,
173 + request_prompts[i],
174 params=pooling_params,
175 lora_request=lora_request,
176 prompt_adapter_request=prompt_adapter_request)
177
151 - if prompt_adapter_request is not None:
152 - raise NotImplementedError(
153 - "Prompt adapter is not supported "
154 - "for embedding models")
178 + trace_headers = (None if raw_request is None else await
179 + self._get_trace_headers(raw_request.headers))
180
181 generator = self.engine_client.encode(
157 - {"prompt_token_ids": prompt_inputs["prompt_token_ids"]},
182 + engine_prompt,
183 pooling_params,
184 request_id_item,
185 lora_request=lora_request,
186 + trace_headers=trace_headers,
187 priority=request.priority,
188 )
189
@@ -171,13 +197,18 @@ class OpenAIServingEmbedding(OpenAIServing):
197 is_cancelled=raw_request.is_disconnected if raw_request else None,
198 )
199
200 + num_prompts = len(engine_prompts)
201 +
202 # Non-streaming response
203 final_res_batch: List[Optional[EmbeddingRequestOutput]]
176 - final_res_batch = [None] * len(prompts)
204 + final_res_batch = [None] * num_prompts
205 try:
206 async for i, res in result_generator:
207 final_res_batch[i] = res
208 + except asyncio.CancelledError:
209 + return self.create_error_response("Client disconnected")
210
211 + try:
212 for final_res in final_res_batch:
213 assert final_res is not None
214
@@ -187,18 +218,8 @@ class OpenAIServingEmbedding(OpenAIServing):
218 response = request_output_to_embedding_response(
219 final_res_batch_checked, request_id, created_time, model_name,
220 encoding_format)
190 - except asyncio.CancelledError:
191 - return self.create_error_response("Client disconnected")
221 except ValueError as e:
222 # TODO: Use a vllm-specific Validation Error
223 return self.create_error_response(str(e))
224
225 return response
197 -
198 - def _check_embedding_mode(self, embedding_mode: bool) -> bool:
199 - if not embedding_mode:
200 - logger.warning(
201 - "embedding_mode is False. Embedding API will not work.")
202 - else:
203 - logger.info("Activating the server engine with embedding enabled.")
204 - return embedding_mode
vllm/entrypoints/openai/serving_engine.py
+150 -9
@@ -2,28 +2,38 @@ import json
2 import pathlib
3 from dataclasses import dataclass
4 from http import HTTPStatus
5 -from typing import Iterable, Iterator, List, Optional, Tuple, TypedDict, Union
5 +from typing import (Any, Callable, Dict, Iterable, Iterator, List, Mapping,
6 + Optional, Sequence, Tuple, TypedDict, Union)
7
8 from pydantic import Field
9 +from starlette.datastructures import Headers
10 from typing_extensions import Annotated
11
12 from vllm.config import ModelConfig
13 from vllm.engine.protocol import EngineClient
14 +from vllm.entrypoints.chat_utils import (ChatCompletionMessageParam,
15 + ConversationMessage,
16 + apply_hf_chat_template,
17 + apply_mistral_chat_template,
18 + parse_chat_messages_futures)
19 from vllm.entrypoints.logger import RequestLogger
20 # yapf conflicts with isort for this block
21 # yapf: disable
22 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
23 CompletionRequest,
24 DetokenizeRequest,
18 - EmbeddingRequest, ErrorResponse,
25 + EmbeddingChatRequest,
26 + EmbeddingCompletionRequest,
27 + ErrorResponse,
28 LoadLoraAdapterRequest,
29 ModelCard, ModelList,
30 ModelPermission,
31 TokenizeChatRequest,
32 TokenizeCompletionRequest,
24 - TokenizeRequest,
33 UnloadLoraAdapterRequest)
34 +from vllm.entrypoints.openai.tool_parsers import ToolParser
35 # yapf: enable
36 +from vllm.inputs import TokensPrompt
37 from vllm.inputs.parse import parse_and_batch_prompt
38 from vllm.logger import init_logger
39 from vllm.lora.request import LoRARequest
@@ -31,8 +41,10 @@ from vllm.pooling_params import PoolingParams
41 from vllm.prompt_adapter.request import PromptAdapterRequest
42 from vllm.sampling_params import BeamSearchParams, SamplingParams
43 from vllm.sequence import Logprob
34 -from vllm.transformers_utils.tokenizer import AnyTokenizer
35 -from vllm.utils import AtomicCounter
44 +from vllm.tracing import (contains_trace_headers, extract_trace_headers,
45 + log_tracing_disabled_warning)
46 +from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
47 +from vllm.utils import AtomicCounter, is_list_of
48
49 logger = init_logger(__name__)
50
@@ -56,8 +68,14 @@ class LoRAModulePath:
68 base_model_name: Optional[str] = None
69
70
59 -AnyRequest = Union[ChatCompletionRequest, CompletionRequest, DetokenizeRequest,
60 - EmbeddingRequest, TokenizeRequest]
71 +CompletionLikeRequest = Union[CompletionRequest, DetokenizeRequest,
72 + EmbeddingCompletionRequest,
73 + TokenizeCompletionRequest]
74 +
75 +ChatLikeRequest = Union[ChatCompletionRequest, EmbeddingChatRequest,
76 + TokenizeChatRequest]
77 +
78 +AnyRequest = Union[CompletionLikeRequest, ChatLikeRequest]
79
80
81 class TextTokensPrompt(TypedDict):
@@ -65,6 +83,9 @@ class TextTokensPrompt(TypedDict):
83 prompt_token_ids: List[int]
84
85
86 +RequestPrompt = Union[List[int], str, TextTokensPrompt]
87 +
88 +
89 class OpenAIServing:
90
91 def __init__(
@@ -246,7 +267,8 @@ class OpenAIServing:
267 token_num = len(input_ids)
268
269 # Note: EmbeddingRequest doesn't have max_tokens
249 - if isinstance(request, EmbeddingRequest):
270 + if isinstance(request,
271 + (EmbeddingChatRequest, EmbeddingCompletionRequest)):
272 if token_num > self.max_model_len:
273 raise ValueError(
274 f"This model's maximum context length is "
@@ -373,10 +395,115 @@ class OpenAIServing:
395 truncate_prompt_tokens=truncate_prompt_tokens,
396 )
397
398 + def _preprocess_completion(
399 + self,
400 + request: CompletionLikeRequest,
401 + tokenizer: AnyTokenizer,
402 + input_or_inputs: Union[str, List[str], List[int], List[List[int]]],
403 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
404 + add_special_tokens: bool = True,
405 + ) -> Tuple[Sequence[TextTokensPrompt], List[TokensPrompt]]:
406 + request_prompts = [
407 + request_prompt
408 + for request_prompt in self._tokenize_prompt_input_or_inputs(
409 + request,
410 + tokenizer,
411 + input_or_inputs,
412 + truncate_prompt_tokens=truncate_prompt_tokens,
413 + add_special_tokens=add_special_tokens,
414 + )
415 + ]
416 +
417 + engine_prompts = [
418 + TokensPrompt(prompt_token_ids=request_prompt["prompt_token_ids"])
419 + for request_prompt in request_prompts
420 + ]
421 +
422 + return request_prompts, engine_prompts
423 +
424 + async def _preprocess_chat(
425 + self,
426 + request: ChatLikeRequest,
427 + tokenizer: AnyTokenizer,
428 + messages: List[ChatCompletionMessageParam],
429 + chat_template: Optional[str] = None,
430 + add_generation_prompt: bool = True,
431 + continue_final_message: bool = False,
432 + tool_dicts: Optional[List[Dict[str, Any]]] = None,
433 + documents: Optional[List[Dict[str, str]]] = None,
434 + chat_template_kwargs: Optional[Dict[str, Any]] = None,
435 + tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
436 + truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None,
437 + add_special_tokens: bool = False,
438 + ) -> Tuple[List[ConversationMessage], Sequence[RequestPrompt],
439 + List[TokensPrompt]]:
440 + conversation, mm_data_future = parse_chat_messages_futures(
441 + messages,
442 + self.model_config,
443 + tokenizer,
444 + )
445 +
446 + request_prompt: Union[str, List[int]]
447 + is_mistral_tokenizer = isinstance(tokenizer, MistralTokenizer)
448 + if is_mistral_tokenizer:
449 + request_prompt = apply_mistral_chat_template(
450 + tokenizer,
451 + messages=messages,
452 + chat_template=chat_template,
453 + add_generation_prompt=add_generation_prompt,
454 + continue_final_message=continue_final_message,
455 + tools=tool_dicts,
456 + documents=documents,
457 + **(chat_template_kwargs or {}),
458 + )
459 + else:
460 + request_prompt = apply_hf_chat_template(
461 + tokenizer,
462 + conversation=conversation,
463 + chat_template=chat_template,
464 + add_generation_prompt=add_generation_prompt,
465 + continue_final_message=continue_final_message,
466 + tools=tool_dicts,
467 + documents=documents,
468 + **(chat_template_kwargs or {}),
469 + )
470 +
471 + mm_data = await mm_data_future
472 +
473 + if tool_parser is not None:
474 + if not isinstance(request, ChatCompletionRequest):
475 + msg = "Tool usage is only supported for Chat Completions API"
476 + raise NotImplementedError(msg)
477 +
478 + request = tool_parser(tokenizer).adjust_request(request=request)
479 +
480 + if isinstance(request_prompt, str):
481 + prompt_inputs = self._tokenize_prompt_input(
482 + request,
483 + tokenizer,
484 + request_prompt,
485 + truncate_prompt_tokens=truncate_prompt_tokens,
486 + add_special_tokens=add_special_tokens,
487 + )
488 + else:
489 + # For MistralTokenizer
490 + assert is_list_of(request_prompt, int), (
491 + "Prompt has to be either a string or a list of token ids")
492 + prompt_inputs = TextTokensPrompt(
493 + prompt=tokenizer.decode(request_prompt),
494 + prompt_token_ids=request_prompt)
495 +
496 + engine_prompt = TokensPrompt(
497 + prompt_token_ids=prompt_inputs["prompt_token_ids"])
498 + if mm_data is not None:
499 + engine_prompt["multi_modal_data"] = mm_data
500 +
501 + return conversation, [request_prompt], [engine_prompt]
502 +
503 def _log_inputs(
504 self,
505 request_id: str,
379 - inputs: Union[str, List[int], TextTokensPrompt],
506 + inputs: RequestPrompt,
507 params: Optional[Union[SamplingParams, PoolingParams,
508 BeamSearchParams]],
509 lora_request: Optional[LoRARequest],
@@ -404,6 +531,20 @@ class OpenAIServing:
531 prompt_adapter_request=prompt_adapter_request,
532 )
533
534 + async def _get_trace_headers(
535 + self,
536 + headers: Headers,
537 + ) -> Optional[Mapping[str, str]]:
538 + is_tracing_enabled = await self.engine_client.is_tracing_enabled()
539 +
540 + if is_tracing_enabled:
541 + return extract_trace_headers(headers)
542 +
543 + if contains_trace_headers(headers):
544 + log_tracing_disabled_warning()
545 +
546 + return None
547 +
548 @staticmethod
549 def _get_decoded_token(logprob: Logprob,
550 token_id: int,
vllm/entrypoints/openai/serving_tokenization.py
+37 -50
@@ -2,10 +2,7 @@ from typing import List, Optional, Union
2
3 from vllm.config import ModelConfig
4 from vllm.engine.protocol import EngineClient
5 -from vllm.entrypoints.chat_utils import (apply_hf_chat_template,
6 - apply_mistral_chat_template,
7 - load_chat_template,
8 - parse_chat_messages_futures)
5 +from vllm.entrypoints.chat_utils import load_chat_template
6 from vllm.entrypoints.logger import RequestLogger
7 # yapf conflicts with isort for this block
8 # yapf: disable
@@ -20,7 +17,6 @@ from vllm.entrypoints.openai.serving_engine import (BaseModelPath,
17 LoRAModulePath,
18 OpenAIServing)
19 from vllm.logger import init_logger
23 -from vllm.transformers_utils.tokenizer import MistralTokenizer
20 from vllm.utils import random_uuid
21
22 logger = init_logger(__name__)
@@ -62,59 +58,51 @@ class OpenAIServingTokenization(OpenAIServing):
58
59 request_id = f"tokn-{random_uuid()}"
60
65 - (
66 - lora_request,
67 - prompt_adapter_request,
68 - ) = self._maybe_get_adapters(request)
69 -
70 - tokenizer = await self.engine_client.get_tokenizer(lora_request)
71 -
72 - prompt: Union[str, List[int]]
73 - if isinstance(request, TokenizeChatRequest):
74 - model_config = self.model_config
75 -
76 - conversation, mm_data_future = parse_chat_messages_futures(
77 - request.messages, model_config, tokenizer)
78 -
79 - mm_data = await mm_data_future
80 - if mm_data:
81 - logger.warning(
82 - "Multi-modal inputs are ignored during tokenization")
83 -
84 - if isinstance(tokenizer, MistralTokenizer):
85 - prompt = apply_mistral_chat_template(
61 + try:
62 + (
63 + lora_request,
64 + prompt_adapter_request,
65 + ) = self._maybe_get_adapters(request)
66 +
67 + tokenizer = await self.engine_client.get_tokenizer(lora_request)
68 +
69 + if isinstance(request, TokenizeChatRequest):
70 + (
71 + _,
72 + request_prompts,
73 + engine_prompts,
74 + ) = await self._preprocess_chat(
75 + request,
76 tokenizer,
87 - messages=request.messages,
77 + request.messages,
78 chat_template=self.chat_template,
79 add_generation_prompt=request.add_generation_prompt,
80 continue_final_message=request.continue_final_message,
81 + add_special_tokens=request.add_special_tokens,
82 )
83 else:
93 - prompt = apply_hf_chat_template(
84 + request_prompts, engine_prompts = self._preprocess_completion(
85 + request,
86 tokenizer,
95 - conversation=conversation,
96 - chat_template=self.chat_template,
97 - add_generation_prompt=request.add_generation_prompt,
98 - continue_final_message=request.continue_final_message,
87 + request.prompt,
88 + add_special_tokens=request.add_special_tokens,
89 )
100 - else:
101 - prompt = request.prompt
90 + except ValueError as e:
91 + logger.exception("Error in preprocessing prompt inputs")
92 + return self.create_error_response(str(e))
93
103 - self._log_inputs(request_id,
104 - prompt,
105 - params=None,
106 - lora_request=lora_request,
107 - prompt_adapter_request=prompt_adapter_request)
94 + input_ids: List[int] = []
95 + for i, engine_prompt in enumerate(engine_prompts):
96 + self._log_inputs(request_id,
97 + request_prompts[i],
98 + params=None,
99 + lora_request=lora_request,
100 + prompt_adapter_request=prompt_adapter_request)
101
109 - # Silently ignore prompt adapter since it does not affect tokenization
102 + # Silently ignore prompt adapter since it does not affect
103 + # tokenization (Unlike in Embeddings API where an error is raised)
104
111 - prompt_input = self._tokenize_prompt_input(
112 - request,
113 - tokenizer,
114 - prompt,
115 - add_special_tokens=request.add_special_tokens,
116 - )
117 - input_ids = prompt_input["prompt_token_ids"]
105 + input_ids.extend(engine_prompt["prompt_token_ids"])
106
107 return TokenizeResponse(tokens=input_ids,
108 count=len(input_ids),
@@ -143,9 +131,8 @@ class OpenAIServingTokenization(OpenAIServing):
131 lora_request=lora_request,
132 prompt_adapter_request=prompt_adapter_request)
133
146 - if prompt_adapter_request is not None:
147 - raise NotImplementedError("Prompt adapter is not supported "
148 - "for tokenization")
134 + # Silently ignore prompt adapter since it does not affect tokenization
135 + # (Unlike in Embeddings API where an error is raised)
136
137 prompt_input = self._tokenize_prompt_input(
138 request,
vllm/pooling_params.py
+2 -2
@@ -7,7 +7,7 @@ class PoolingParams(
7 msgspec.Struct,
8 omit_defaults=True, # type: ignore[call-arg]
9 array_like=True): # type: ignore[call-arg]
10 - """Pooling parameters for pooling.
10 + """Pooling parameters for embeddings API.
11
12 Attributes:
13 additional_data: Any additional data needed for pooling.
@@ -16,7 +16,7 @@ class PoolingParams(
16
17 def clone(self) -> "PoolingParams":
18 """Returns a deep copy of the PoolingParams instance."""
19 - return PoolingParams(additional_data=self.additional_data, )
19 + return PoolingParams(additional_data=self.additional_data)
20
21 def __repr__(self) -> str:
22 return (f"PoolingParams("