A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Fix] Correct OpenAI batch response format (#5554)
zifeitong committed
Jun 15, 2024 at 16:57 UTC
3ce2c050dd919542ef5355635edf71349ea597f2
2 files changed
+25
-5
vllm/entrypoints/openai/protocol.py
+12
-1
@@ -672,6 +672,17 @@ class BatchRequestInput(OpenAIBaseModel):
672
body: Union[ChatCompletionRequest, ]
673
674
675
+class BatchResponseData(OpenAIBaseModel):
676
+ # HTTP status code of the response.
677
+ status_code: int = 200
678
+
679
+ # An unique identifier for the API request.
680
+ request_id: str
681
+
682
+ # The body of the response.
683
+ body: Union[ChatCompletionResponse, ]
684
+
685
+
686
class BatchRequestOutput(OpenAIBaseModel):
687
"""
688
The per-line object of the batch output and error files
@@ -683,7 +694,7 @@ class BatchRequestOutput(OpenAIBaseModel):
694
# inputs.
695
custom_id: str
696
686
- response: Optional[ChatCompletionResponse]
697
+ response: Optional[BatchResponseData]
698
699
# For requests that failed with a non-HTTP error, this will contain more
700
# information on the cause of the failure.
vllm/entrypoints/openai/run_batch.py
+13
-4
@@ -10,7 +10,9 @@ from vllm.engine.arg_utils import AsyncEngineArgs, nullable_str
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (BatchRequestInput,
12
BatchRequestOutput,
13
- ChatCompletionResponse)
13
+ BatchResponseData,
14
+ ChatCompletionResponse,
15
+ ErrorResponse)
16
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
17
from vllm.logger import init_logger
18
from vllm.usage.usage_lib import UsageContext
@@ -77,20 +79,27 @@ async def run_request(chat_serving: OpenAIServingChat,
79
request: BatchRequestInput) -> BatchRequestOutput:
80
chat_request = request.body
81
chat_response = await chat_serving.create_chat_completion(chat_request)
82
+
83
if isinstance(chat_response, ChatCompletionResponse):
84
batch_output = BatchRequestOutput(
85
id=f"vllm-{random_uuid()}",
86
custom_id=request.custom_id,
84
- response=chat_response,
87
+ response=BatchResponseData(
88
+ body=chat_response, request_id=f"vllm-batch-{random_uuid()}"),
89
error=None,
90
)
87
- else:
91
+ elif isinstance(chat_response, ErrorResponse):
92
batch_output = BatchRequestOutput(
93
id=f"vllm-{random_uuid()}",
94
custom_id=request.custom_id,
91
- response=None,
95
+ response=BatchResponseData(
96
+ status_code=chat_response.code,
97
+ request_id=f"vllm-batch-{random_uuid()}"),
98
error=chat_response,
99
)
100
+ else:
101
+ raise ValueError("Request must not be sent in stream mode")
102
+
103
return batch_output
104
105