@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / becd7a56f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Enable request body OpenAPI spec for OpenAI endpoints (#865)

lplcor committed Aug 29, 2023 at 21:54 UTC becd7a56f115f425e8ca99c785b12d545ac30042
1 file changed +3 -4
vllm/entrypoints/openai/api_server.py
+3 -4
@@ -178,7 +178,8 @@ def create_logprobs(token_ids: List[int],
178
179
180 @app.post("/v1/chat/completions")
181 -async def create_chat_completion(raw_request: Request):
181 +async def create_chat_completion(request: ChatCompletionRequest,
182 + raw_request: Request):
183 """Completion API similar to OpenAI's API.
184
185 See https://platform.openai.com/docs/api-reference/chat/create
@@ -188,7 +189,6 @@ async def create_chat_completion(raw_request: Request):
189 - function_call (Users should implement this by themselves)
190 - logit_bias (to be supported by vLLM engine)
191 """
191 - request = ChatCompletionRequest(**await raw_request.json())
192 logger.info(f"Received chat completion request: {request}")
193
194 error_check_ret = await check_model(request)
@@ -348,7 +348,7 @@ async def create_chat_completion(raw_request: Request):
348
349
350 @app.post("/v1/completions")
351 -async def create_completion(raw_request: Request):
351 +async def create_completion(request: CompletionRequest, raw_request: Request):
352 """Completion API similar to OpenAI's API.
353
354 See https://platform.openai.com/docs/api-reference/completions/create
@@ -361,7 +361,6 @@ async def create_completion(raw_request: Request):
361 suffix)
362 - logit_bias (to be supported by vLLM engine)
363 """
364 - request = CompletionRequest(**await raw_request.json())
364 logger.info(f"Received completion request: {request}")
365
366 error_check_ret = await check_model(request)