@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ce2c050dd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Fix] Correct OpenAI batch response format (#5554)

zifeitong committed Jun 15, 2024 at 16:57 UTC 3ce2c050dd919542ef5355635edf71349ea597f2
2 files changed +25 -5
vllm/entrypoints/openai/protocol.py
+12 -1
@@ -672,6 +672,17 @@ class BatchRequestInput(OpenAIBaseModel):
672 body: Union[ChatCompletionRequest, ]
673
674
675 +class BatchResponseData(OpenAIBaseModel):
676 + # HTTP status code of the response.
677 + status_code: int = 200
678 +
679 + # An unique identifier for the API request.
680 + request_id: str
681 +
682 + # The body of the response.
683 + body: Union[ChatCompletionResponse, ]
684 +
685 +
686 class BatchRequestOutput(OpenAIBaseModel):
687 """
688 The per-line object of the batch output and error files
@@ -683,7 +694,7 @@ class BatchRequestOutput(OpenAIBaseModel):
694 # inputs.
695 custom_id: str
696
686 - response: Optional[ChatCompletionResponse]
697 + response: Optional[BatchResponseData]
698
699 # For requests that failed with a non-HTTP error, this will contain more
700 # information on the cause of the failure.
vllm/entrypoints/openai/run_batch.py
+13 -4
@@ -10,7 +10,9 @@ from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.entrypoints.openai.protocol import (BatchRequestInput,
12 BatchRequestOutput,
13 - ChatCompletionResponse)
13 + BatchResponseData,
14 + ChatCompletionResponse,
15 + ErrorResponse)
16 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
17 from vllm.logger import init_logger
18 from vllm.usage.usage_lib import UsageContext
@@ -77,20 +79,27 @@ async def run_request(chat_serving: OpenAIServingChat,
79 request: BatchRequestInput) -> BatchRequestOutput:
80 chat_request = request.body
81 chat_response = await chat_serving.create_chat_completion(chat_request)
82 +
83 if isinstance(chat_response, ChatCompletionResponse):
84 batch_output = BatchRequestOutput(
85 id=f"vllm-{random_uuid()}",
86 custom_id=request.custom_id,
84 - response=chat_response,
87 + response=BatchResponseData(
88 + body=chat_response, request_id=f"vllm-batch-{random_uuid()}"),
89 error=None,
90 )
87 - else:
91 + elif isinstance(chat_response, ErrorResponse):
92 batch_output = BatchRequestOutput(
93 id=f"vllm-{random_uuid()}",
94 custom_id=request.custom_id,
91 - response=None,
95 + response=BatchResponseData(
96 + status_code=chat_response.code,
97 + request_id=f"vllm-batch-{random_uuid()}"),
98 error=chat_response,
99 )
100 + else:
101 + raise ValueError("Request must not be sent in stream mode")
102 +
103 return batch_output
104
105