A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Bugfix] Refactor `/invocations` to be task-agnostic (#20764)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jul 11, 2025 at 18:20 UTC
cbd14ed5613c6c20b3225e81f32a9bfe3a0d32ac
9 files changed
+352
-75
tests/entrypoints/openai/test_chat.py
+33
-4
@@ -1113,10 +1113,7 @@ async def test_http_chat_no_model_name_with_curl(server: RemoteOpenAIServer):
1113
1114
1115
@pytest.mark.asyncio
1116
-@pytest.mark.parametrize("model_name", [MODEL_NAME, ""])
1117
-async def test_http_chat_no_model_name_with_openai(server: RemoteOpenAIServer,
1118
- model_name: str):
1119
-
1116
+async def test_http_chat_no_model_name_with_openai(server: RemoteOpenAIServer):
1117
openai_api_key = "EMPTY"
1118
openai_api_base = f"http://localhost:{server.port}/v1"
1119
@@ -1135,3 +1132,35 @@ async def test_http_chat_no_model_name_with_openai(server: RemoteOpenAIServer,
1132
messages=messages,
1133
)
1134
assert response.model == MODEL_NAME
1135
+
1136
+
1137
+@pytest.mark.asyncio
1138
+async def test_invocations(server: RemoteOpenAIServer,
1139
+ client: openai.AsyncOpenAI):
1140
+ messages = [{
1141
+ "role": "system",
1142
+ "content": "you are a helpful assistant"
1143
+ }, {
1144
+ "role": "user",
1145
+ "content": "what is 1+1?"
1146
+ }]
1147
+
1148
+ request_args = {
1149
+ "model": MODEL_NAME,
1150
+ "messages": messages,
1151
+ "max_completion_tokens": 5,
1152
+ "temperature": 0.0,
1153
+ "logprobs": False,
1154
+ }
1155
+
1156
+ chat_completion = await client.chat.completions.create(**request_args)
1157
+
1158
+ invocation_response = requests.post(server.url_for("invocations"),
1159
+ json=request_args)
1160
+ invocation_response.raise_for_status()
1161
+
1162
+ chat_output = chat_completion.model_dump()
1163
+ invocation_output = invocation_response.json()
1164
+
1165
+ assert chat_output.keys() == invocation_output.keys()
1166
+ assert chat_output["choices"] == invocation_output["choices"]
tests/entrypoints/openai/test_classification.py
+22
@@ -155,3 +155,25 @@ def test_batch_classification_empty_list(server: RemoteOpenAIServer,
155
assert output.object == "list"
156
assert isinstance(output.data, list)
157
assert len(output.data) == 0
158
+
159
+
160
+@pytest.mark.asyncio
161
+async def test_invocations(server: RemoteOpenAIServer):
162
+ request_args = {
163
+ "model": MODEL_NAME,
164
+ "input": "This product was excellent and exceeded my expectations"
165
+ }
166
+
167
+ classification_response = requests.post(server.url_for("classify"),
168
+ json=request_args)
169
+ classification_response.raise_for_status()
170
+
171
+ invocation_response = requests.post(server.url_for("invocations"),
172
+ json=request_args)
173
+ invocation_response.raise_for_status()
174
+
175
+ classification_output = classification_response.json()
176
+ invocation_output = invocation_response.json()
177
+
178
+ assert classification_output.keys() == invocation_output.keys()
179
+ assert classification_output["data"] == invocation_output["data"]
tests/entrypoints/openai/test_completion.py
+25
@@ -11,6 +11,7 @@ import openai # use the official client for correctness check
11
import pytest
12
import pytest_asyncio
13
import regex as re
14
+import requests
15
# downloading lora to test lora requests
16
from huggingface_hub import snapshot_download
17
from openai import BadRequestError
@@ -833,3 +834,27 @@ async def test_echo_stream_completion(client: openai.AsyncOpenAI,
834
assert content is not None and saying in content
835
else:
836
assert content is not None and saying not in content
837
+
838
+
839
+@pytest.mark.asyncio
840
+async def test_invocations(server: RemoteOpenAIServer,
841
+ client: openai.AsyncOpenAI):
842
+ request_args = {
843
+ "model": MODEL_NAME,
844
+ "prompt": "Hello, my name is",
845
+ "max_tokens": 5,
846
+ "temperature": 0.0,
847
+ "logprobs": None,
848
+ }
849
+
850
+ completion = await client.completions.create(**request_args)
851
+
852
+ invocation_response = requests.post(server.url_for("invocations"),
853
+ json=request_args)
854
+ invocation_response.raise_for_status()
855
+
856
+ completion_output = completion.model_dump()
857
+ invocation_output = invocation_response.json()
858
+
859
+ assert completion_output.keys() == invocation_output.keys()
860
+ assert completion_output["choices"] == invocation_output["choices"]
tests/entrypoints/openai/test_embedding.py
+60
@@ -296,3 +296,63 @@ async def test_single_embedding_truncation_invalid(client: openai.AsyncOpenAI,
296
assert "error" in response.object
297
assert "truncate_prompt_tokens value is greater than max_model_len. "\
298
"Please, select a smaller truncation size." in response.message
299
+
300
+
301
+@pytest.mark.asyncio
302
+async def test_invocations(server: RemoteOpenAIServer,
303
+ client: openai.AsyncOpenAI):
304
+ input_texts = [
305
+ "The chef prepared a delicious meal.",
306
+ ]
307
+
308
+ request_args = {
309
+ "model": MODEL_NAME,
310
+ "input": input_texts,
311
+ "encoding_format": "float",
312
+ }
313
+
314
+ completion_response = await client.embeddings.create(**request_args)
315
+
316
+ invocation_response = requests.post(server.url_for("invocations"),
317
+ json=request_args)
318
+ invocation_response.raise_for_status()
319
+
320
+ completion_output = completion_response.model_dump()
321
+ invocation_output = invocation_response.json()
322
+
323
+ assert completion_output.keys() == invocation_output.keys()
324
+ assert completion_output["data"] == invocation_output["data"]
325
+
326
+
327
+@pytest.mark.asyncio
328
+async def test_invocations_conversation(server: RemoteOpenAIServer):
329
+ messages = [{
330
+ "role": "user",
331
+ "content": "The cat sat on the mat.",
332
+ }, {
333
+ "role": "assistant",
334
+ "content": "A feline was resting on a rug.",
335
+ }, {
336
+ "role": "user",
337
+ "content": "Stars twinkle brightly in the night sky.",
338
+ }]
339
+
340
+ request_args = {
341
+ "model": MODEL_NAME,
342
+ "messages": messages,
343
+ "encoding_format": "float",
344
+ }
345
+
346
+ chat_response = requests.post(server.url_for("v1/embeddings"),
347
+ json=request_args)
348
+ chat_response.raise_for_status()
349
+
350
+ invocation_response = requests.post(server.url_for("invocations"),
351
+ json=request_args)
352
+ invocation_response.raise_for_status()
353
+
354
+ chat_output = chat_response.json()
355
+ invocation_output = invocation_response.json()
356
+
357
+ assert chat_output.keys() == invocation_output.keys()
358
+ assert chat_output["data"] == invocation_output["data"]
tests/entrypoints/openai/test_pooling.py
+91
-22
@@ -13,7 +13,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
13
14
from ...utils import RemoteOpenAIServer
15
16
-MODEL_NAME = "jason9693/Qwen2.5-1.5B-apeach"
16
+MODEL_NAME = "internlm/internlm2-1_8b-reward"
17
DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' + message['content'] + '\\n'}}{% endfor %}""" # noqa: E501
18
19
@@ -21,15 +21,16 @@ DUMMY_CHAT_TEMPLATE = """{% for message in messages %}{{message['role'] + ': ' +
21
def server():
22
args = [
23
"--task",
24
- "classify",
24
+ "reward",
25
# use half precision for speed and memory savings in CI environment
26
"--dtype",
27
"bfloat16",
28
"--enforce-eager",
29
"--max-model-len",
30
- "8192",
30
+ "512",
31
"--chat-template",
32
DUMMY_CHAT_TEMPLATE,
33
+ "--trust-remote-code",
34
]
35
36
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
@@ -57,10 +58,10 @@ async def test_single_pooling(server: RemoteOpenAIServer, model_name: str):
58
59
assert poolings.id is not None
60
assert len(poolings.data) == 1
60
- assert len(poolings.data[0].data) == 2
61
+ assert len(poolings.data[0].data) == 8
62
assert poolings.usage.completion_tokens == 0
62
- assert poolings.usage.prompt_tokens == 7
63
- assert poolings.usage.total_tokens == 7
63
+ assert poolings.usage.prompt_tokens == 8
64
+ assert poolings.usage.total_tokens == 8
65
66
# test using token IDs
67
input_tokens = [1, 1, 1, 1, 1]
@@ -77,7 +78,7 @@ async def test_single_pooling(server: RemoteOpenAIServer, model_name: str):
78
79
assert poolings.id is not None
80
assert len(poolings.data) == 1
80
- assert len(poolings.data[0].data) == 2
81
+ assert len(poolings.data[0].data) == 5
82
assert poolings.usage.completion_tokens == 0
83
assert poolings.usage.prompt_tokens == 5
84
assert poolings.usage.total_tokens == 5
@@ -104,10 +105,10 @@ async def test_batch_pooling(server: RemoteOpenAIServer, model_name: str):
105
106
assert poolings.id is not None
107
assert len(poolings.data) == 3
107
- assert len(poolings.data[0].data) == 2
108
+ assert len(poolings.data[0].data) == 8
109
assert poolings.usage.completion_tokens == 0
109
- assert poolings.usage.prompt_tokens == 25
110
- assert poolings.usage.total_tokens == 25
110
+ assert poolings.usage.prompt_tokens == 29
111
+ assert poolings.usage.total_tokens == 29
112
113
# test list[list[int]]
114
input_tokens = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
@@ -125,7 +126,7 @@ async def test_batch_pooling(server: RemoteOpenAIServer, model_name: str):
126
127
assert poolings.id is not None
128
assert len(poolings.data) == 4
128
- assert len(poolings.data[0].data) == 2
129
+ assert len(poolings.data[0].data) == 5
130
assert poolings.usage.completion_tokens == 0
131
assert poolings.usage.prompt_tokens == 17
132
assert poolings.usage.total_tokens == 17
@@ -157,7 +158,11 @@ async def test_conversation_pooling(server: RemoteOpenAIServer,
158
chat_response.raise_for_status()
159
chat_poolings = PoolingResponse.model_validate(chat_response.json())
160
160
- tokenizer = get_tokenizer(tokenizer_name=model_name, tokenizer_mode="fast")
161
+ tokenizer = get_tokenizer(
162
+ tokenizer_name=model_name,
163
+ tokenizer_mode="fast",
164
+ trust_remote_code=True,
165
+ )
166
prompt = tokenizer.apply_chat_template(
167
messages,
168
chat_template=DUMMY_CHAT_TEMPLATE,
@@ -206,6 +211,9 @@ async def test_batch_base64_pooling(server: RemoteOpenAIServer,
211
)
212
float_response.raise_for_status()
213
responses_float = PoolingResponse.model_validate(float_response.json())
214
+ float_data = [
215
+ np.array(d.data).squeeze(-1).tolist() for d in responses_float.data
216
+ ]
217
218
base64_response = requests.post(
219
server.url_for("pooling"),
@@ -224,11 +232,10 @@ async def test_batch_base64_pooling(server: RemoteOpenAIServer,
232
np.frombuffer(base64.b64decode(data.data),
233
dtype="float32").tolist())
234
227
- check_embeddings_close(
228
- embeddings_0_lst=[d.data for d in responses_float.data],
229
- embeddings_1_lst=decoded_responses_base64_data,
230
- name_0="float32",
231
- name_1="base64")
235
+ check_embeddings_close(embeddings_0_lst=float_data,
236
+ embeddings_1_lst=decoded_responses_base64_data,
237
+ name_0="float32",
238
+ name_1="base64")
239
240
# Default response is float32 decoded from base64 by OpenAI Client
241
default_response = requests.post(
@@ -240,9 +247,71 @@ async def test_batch_base64_pooling(server: RemoteOpenAIServer,
247
)
248
default_response.raise_for_status()
249
responses_default = PoolingResponse.model_validate(default_response.json())
250
+ default_data = [
251
+ np.array(d.data).squeeze(-1).tolist() for d in responses_default.data
252
+ ]
253
+
254
+ check_embeddings_close(embeddings_0_lst=float_data,
255
+ embeddings_1_lst=default_data,
256
+ name_0="float32",
257
+ name_1="default")
258
+
259
+
260
+@pytest.mark.asyncio
261
+async def test_invocations(server: RemoteOpenAIServer):
262
+ input_texts = [
263
+ "The chef prepared a delicious meal.",
264
+ ]
265
+
266
+ request_args = {
267
+ "model": MODEL_NAME,
268
+ "input": input_texts,
269
+ "encoding_format": "float",
270
+ }
271
+
272
+ completion_response = requests.post(server.url_for("pooling"),
273
+ json=request_args)
274
+ completion_response.raise_for_status()
275
+
276
+ invocation_response = requests.post(server.url_for("invocations"),
277
+ json=request_args)
278
+ invocation_response.raise_for_status()
279
+
280
+ completion_output = completion_response.json()
281
+ invocation_output = invocation_response.json()
282
+
283
+ assert completion_output.keys() == invocation_output.keys()
284
+ assert completion_output["data"] == invocation_output["data"]
285
+
286
+
287
+@pytest.mark.asyncio
288
+async def test_invocations_conversation(server: RemoteOpenAIServer):
289
+ messages = [{
290
+ "role": "user",
291
+ "content": "The cat sat on the mat.",
292
+ }, {
293
+ "role": "assistant",
294
+ "content": "A feline was resting on a rug.",
295
+ }, {
296
+ "role": "user",
297
+ "content": "Stars twinkle brightly in the night sky.",
298
+ }]
299
+
300
+ request_args = {
301
+ "model": MODEL_NAME,
302
+ "messages": messages,
303
+ "encoding_format": "float",
304
+ }
305
+
306
+ chat_response = requests.post(server.url_for("pooling"), json=request_args)
307
+ chat_response.raise_for_status()
308
+
309
+ invocation_response = requests.post(server.url_for("invocations"),
310
+ json=request_args)
311
+ invocation_response.raise_for_status()
312
+
313
+ chat_output = chat_response.json()
314
+ invocation_output = invocation_response.json()
315
244
- check_embeddings_close(
245
- embeddings_0_lst=[d.data for d in responses_default.data],
246
- embeddings_1_lst=[d.data for d in responses_default.data],
247
- name_0="float32",
248
- name_1="base64")
316
+ assert chat_output.keys() == invocation_output.keys()
317
+ assert chat_output["data"] == invocation_output["data"]
tests/entrypoints/openai/test_rerank.py
+27
@@ -94,3 +94,30 @@ def test_rerank_max_model_len(server: RemoteOpenAIServer, model_name: str):
94
# Assert just a small fragments of the response
95
assert "Please reduce the length of the input." in \
96
rerank_response.text
97
+
98
+
99
+def test_invocations(server: RemoteOpenAIServer):
100
+ query = "What is the capital of France?"
101
+ documents = [
102
+ "The capital of Brazil is Brasilia.", "The capital of France is Paris."
103
+ ]
104
+
105
+ request_args = {
106
+ "model": MODEL_NAME,
107
+ "query": query,
108
+ "documents": documents,
109
+ }
110
+
111
+ rerank_response = requests.post(server.url_for("rerank"),
112
+ json=request_args)
113
+ rerank_response.raise_for_status()
114
+
115
+ invocation_response = requests.post(server.url_for("invocations"),
116
+ json=request_args)
117
+ invocation_response.raise_for_status()
118
+
119
+ rerank_output = rerank_response.json()
120
+ invocation_output = invocation_response.json()
121
+
122
+ assert rerank_output.keys() == invocation_output.keys()
123
+ assert rerank_output["results"] == invocation_output["results"]
tests/entrypoints/openai/test_score.py
+25
@@ -191,3 +191,28 @@ class TestModel:
191
assert score_response.status_code == 400
192
assert "Please, select a smaller truncation size." in \
193
score_response.text
194
+
195
+ def test_invocations(self, server: RemoteOpenAIServer, model: dict[str,
196
+ Any]):
197
+ text_1 = "What is the capital of France?"
198
+ text_2 = "The capital of France is Paris."
199
+
200
+ request_args = {
201
+ "model": model["name"],
202
+ "text_1": text_1,
203
+ "text_2": text_2,
204
+ }
205
+
206
+ score_response = requests.post(server.url_for("score"),
207
+ json=request_args)
208
+ score_response.raise_for_status()
209
+
210
+ invocation_response = requests.post(server.url_for("invocations"),
211
+ json=request_args)
212
+ invocation_response.raise_for_status()
213
+
214
+ score_output = score_response.json()
215
+ invocation_output = invocation_response.json()
216
+
217
+ assert score_output.keys() == invocation_output.keys()
218
+ assert score_output["data"] == invocation_output["data"]
vllm/entrypoints/openai/api_server.py
+45
-47
@@ -18,7 +18,7 @@ from collections.abc import AsyncIterator, Awaitable
18
from contextlib import asynccontextmanager
19
from functools import partial
20
from http import HTTPStatus
21
-from typing import Annotated, Any, Optional
21
+from typing import Annotated, Any, Callable, Optional
22
23
import prometheus_client
24
import pydantic
@@ -61,13 +61,9 @@ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
61
CompletionResponse,
62
DetokenizeRequest,
63
DetokenizeResponse,
64
- EmbeddingChatRequest,
65
- EmbeddingCompletionRequest,
64
EmbeddingRequest,
65
EmbeddingResponse, ErrorResponse,
66
LoadLoRAAdapterRequest,
69
- PoolingChatRequest,
70
- PoolingCompletionRequest,
67
PoolingRequest, PoolingResponse,
68
RerankRequest, RerankResponse,
69
ResponsesRequest,
@@ -434,6 +430,7 @@ async def get_server_load_metrics(request: Request):
430
# - /v1/chat/completions
431
# - /v1/completions
432
# - /v1/audio/transcriptions
433
+ # - /v1/audio/translations
434
# - /v1/embeddings
435
# - /pooling
436
# - /classify
@@ -957,31 +954,6 @@ async def do_rerank_v2(request: RerankRequest, raw_request: Request):
954
return await do_rerank(request, raw_request)
955
956
960
-TASK_HANDLERS: dict[str, dict[str, tuple]] = {
961
- "generate": {
962
- "messages": (ChatCompletionRequest, create_chat_completion),
963
- "default": (CompletionRequest, create_completion),
964
- },
965
- "embed": {
966
- "messages": (EmbeddingChatRequest, create_embedding),
967
- "default": (EmbeddingCompletionRequest, create_embedding),
968
- },
969
- "score": {
970
- "default": (RerankRequest, do_rerank)
971
- },
972
- "rerank": {
973
- "default": (RerankRequest, do_rerank)
974
- },
975
- "reward": {
976
- "messages": (PoolingChatRequest, create_pooling),
977
- "default": (PoolingCompletionRequest, create_pooling),
978
- },
979
- "classify": {
980
- "messages": (PoolingChatRequest, create_pooling),
981
- "default": (PoolingCompletionRequest, create_pooling),
982
- },
983
-}
984
-
957
if envs.VLLM_SERVER_DEV_MODE:
958
logger.warning("SECURITY WARNING: Development endpoints are enabled! "
959
"This should NOT be used in production!")
@@ -1033,6 +1005,30 @@ if envs.VLLM_SERVER_DEV_MODE:
1005
return JSONResponse(content={"is_sleeping": is_sleeping})
1006
1007
1008
+# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
1009
+# (requires typing_extensions >= 4.13)
1010
+RequestType = Any
1011
+GetHandlerFn = Callable[[Request], Optional[OpenAIServing]]
1012
+EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
1013
+
1014
+# NOTE: Items defined earlier take higher priority
1015
+INVOCATION_TYPES: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = [
1016
+ (ChatCompletionRequest, (chat, create_chat_completion)),
1017
+ (CompletionRequest, (completion, create_completion)),
1018
+ (EmbeddingRequest, (embedding, create_embedding)),
1019
+ (ClassificationRequest, (classify, create_classify)),
1020
+ (ScoreRequest, (score, create_score)),
1021
+ (RerankRequest, (rerank, do_rerank)),
1022
+ (PoolingRequest, (pooling, create_pooling)),
1023
+]
1024
+
1025
+# NOTE: Construct the TypeAdapters only once
1026
+INVOCATION_VALIDATORS = [
1027
+ (pydantic.TypeAdapter(request_type), (get_handler, endpoint))
1028
+ for request_type, (get_handler, endpoint) in INVOCATION_TYPES
1029
+]
1030
+
1031
+
1032
@router.post("/invocations",
1033
dependencies=[Depends(validate_json_request)],
1034
responses={
@@ -1047,32 +1043,34 @@ if envs.VLLM_SERVER_DEV_MODE:
1043
},
1044
})
1045
async def invocations(raw_request: Request):
1050
- """
1051
- For SageMaker, routes requests to other handlers based on model `task`.
1052
- """
1046
+ """For SageMaker, routes requests based on the request type."""
1047
try:
1048
body = await raw_request.json()
1049
except json.JSONDecodeError as e:
1050
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST.value,
1051
detail=f"JSON decode error: {e}") from e
1052
1059
- task = raw_request.app.state.task
1053
+ valid_endpoints = [(validator, endpoint)
1054
+ for validator, (get_handler,
1055
+ endpoint) in INVOCATION_VALIDATORS
1056
+ if get_handler(raw_request) is not None]
1057
1061
- if task not in TASK_HANDLERS:
1062
- raise HTTPException(
1063
- status_code=400,
1064
- detail=f"Unsupported task: '{task}' for '/invocations'. "
1065
- f"Expected one of {set(TASK_HANDLERS.keys())}")
1058
+ for request_validator, endpoint in valid_endpoints:
1059
+ try:
1060
+ request = request_validator.validate_python(body)
1061
+ except pydantic.ValidationError:
1062
+ continue
1063
1067
- handler_config = TASK_HANDLERS[task]
1068
- if "messages" in body:
1069
- request_model, handler = handler_config["messages"]
1070
- else:
1071
- request_model, handler = handler_config["default"]
1064
+ return await endpoint(request, raw_request)
1065
1073
- # this is required since we lose the FastAPI automatic casting
1074
- request = request_model.model_validate(body)
1075
- return await handler(request, raw_request)
1066
+ type_names = [
1067
+ t.__name__ if isinstance(t := validator._type, type) else str(t)
1068
+ for validator, _ in valid_endpoints
1069
+ ]
1070
+ msg = ("Cannot find suitable handler for request. "
1071
+ f"Expected one of: {type_names}")
1072
+ res = base(raw_request).create_error_response(message=msg)
1073
+ return JSONResponse(content=res.model_dump(), status_code=res.code)
1074
1075
1076
if envs.VLLM_TORCH_PROFILER_DIR:
vllm/entrypoints/openai/protocol.py
+24
-2
@@ -11,6 +11,12 @@ from typing import Annotated, Any, ClassVar, Literal, Optional, Union
11
import regex as re
12
import torch
13
from fastapi import HTTPException, UploadFile
14
+# yapf: disable
15
+from openai.types.chat.chat_completion_audio import (
16
+ ChatCompletionAudio as OpenAIChatCompletionAudio)
17
+from openai.types.chat.chat_completion_message import (
18
+ Annotation as OpenAIAnnotation)
19
+# yapf: enable
20
from openai.types.responses import (ResponseInputParam, ResponseOutputItem,
21
ResponseOutputMessage, ResponsePrompt,
22
ResponseStatus, ResponseTextConfig)
@@ -1393,11 +1399,16 @@ class CompletionResponseChoice(OpenAIBaseModel):
1399
1400
class CompletionResponse(OpenAIBaseModel):
1401
id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
1396
- object: str = "text_completion"
1402
+ object: Literal["text_completion"] = "text_completion"
1403
created: int = Field(default_factory=lambda: int(time.time()))
1404
model: str
1405
choices: list[CompletionResponseChoice]
1406
+ service_tier: Optional[Literal["auto", "default", "flex", "scale",
1407
+ "priority"]] = None
1408
+ system_fingerprint: Optional[str] = None
1409
usage: UsageInfo
1410
+
1411
+ # vLLM-specific fields that are not in OpenAI spec
1412
kv_transfer_params: Optional[dict[str, Any]] = Field(
1413
default=None, description="KVTransfer parameters.")
1414
@@ -1549,10 +1560,16 @@ class ExtractedToolCallInformation(BaseModel):
1560
1561
class ChatMessage(OpenAIBaseModel):
1562
role: str
1552
- reasoning_content: Optional[str] = None
1563
content: Optional[str] = None
1564
+ refusal: Optional[str] = None
1565
+ annotations: Optional[OpenAIAnnotation] = None
1566
+ audio: Optional[OpenAIChatCompletionAudio] = None
1567
+ function_call: Optional[FunctionCall] = None
1568
tool_calls: list[ToolCall] = Field(default_factory=list)
1569
1570
+ # vLLM-specific fields that are not in OpenAI spec
1571
+ reasoning_content: Optional[str] = None
1572
+
1573
1574
class ChatCompletionLogProb(OpenAIBaseModel):
1575
token: str
@@ -1587,7 +1604,12 @@ class ChatCompletionResponse(OpenAIBaseModel):
1604
created: int = Field(default_factory=lambda: int(time.time()))
1605
model: str
1606
choices: list[ChatCompletionResponseChoice]
1607
+ service_tier: Optional[Literal["auto", "default", "flex", "scale",
1608
+ "priority"]] = None
1609
+ system_fingerprint: Optional[str] = None
1610
usage: UsageInfo
1611
+
1612
+ # vLLM-specific fields that are not in OpenAI spec
1613
prompt_logprobs: Optional[list[Optional[dict[int, Logprob]]]] = None
1614
kv_transfer_params: Optional[dict[str, Any]] = Field(
1615
default=None, description="KVTransfer parameters.")