A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Enable request body OpenAPI spec for OpenAI endpoints (#865)
lplcor committed
Aug 29, 2023 at 21:54 UTC
becd7a56f115f425e8ca99c785b12d545ac30042
1 file changed
+3
-4
vllm/entrypoints/openai/api_server.py
+3
-4
@@ -178,7 +178,8 @@ def create_logprobs(token_ids: List[int],
178
179
180
@app.post("/v1/chat/completions")
181
-async def create_chat_completion(raw_request: Request):
181
+async def create_chat_completion(request: ChatCompletionRequest,
182
+ raw_request: Request):
183
"""Completion API similar to OpenAI's API.
184
185
See https://platform.openai.com/docs/api-reference/chat/create
@@ -188,7 +189,6 @@ async def create_chat_completion(raw_request: Request):
189
- function_call (Users should implement this by themselves)
190
- logit_bias (to be supported by vLLM engine)
191
"""
191
- request = ChatCompletionRequest(**await raw_request.json())
192
logger.info(f"Received chat completion request: {request}")
193
194
error_check_ret = await check_model(request)
@@ -348,7 +348,7 @@ async def create_chat_completion(raw_request: Request):
348
349
350
@app.post("/v1/completions")
351
-async def create_completion(raw_request: Request):
351
+async def create_completion(request: CompletionRequest, raw_request: Request):
352
"""Completion API similar to OpenAI's API.
353
354
See https://platform.openai.com/docs/api-reference/completions/create
@@ -361,7 +361,6 @@ async def create_completion(raw_request: Request):
361
suffix)
362
- logit_bias (to be supported by vLLM engine)
363
"""
364
- request = CompletionRequest(**await raw_request.json())
364
logger.info(f"Received completion request: {request}")
365
366
error_check_ret = await check_model(request)