A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Frontend] Create ErrorResponse instead of raising exceptions in run_batch (#8347)
Pooya Davoodi committed
Sep 10, 2024 at 22:30 UTC
cea95dfb941878b3370a7c40ca7ab2d549524445
2 files changed
+31
-4
tests/entrypoints/openai/test_run_batch.py
+3
-1
@@ -8,7 +8,9 @@ from vllm.entrypoints.openai.protocol import BatchRequestOutput
8
INPUT_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
9
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
10
11
-{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NonExistModel", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
11
+{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NonExistModel", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
12
+{"custom_id": "request-4", "method": "POST", "url": "/bad_url", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
13
+{"custom_id": "request-5", "method": "POST", "url": "/v1/chat/completions", "body": {"stream": "True", "model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
14
15
INVALID_INPUT_BATCH = """{"invalid_field": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
16
{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
vllm/entrypoints/openai/run_batch.py
+28
-3
@@ -1,4 +1,5 @@
1
import asyncio
2
+from http import HTTPStatus
3
from io import StringIO
4
from typing import Awaitable, Callable, List, Optional
5
@@ -135,6 +136,25 @@ async def write_file(path_or_url: str, data: str) -> None:
136
f.write(data)
137
138
139
+def make_error_request_output(request: BatchRequestInput,
140
+ error_msg: str) -> BatchRequestOutput:
141
+ batch_output = BatchRequestOutput(
142
+ id=f"vllm-{random_uuid()}",
143
+ custom_id=request.custom_id,
144
+ response=BatchResponseData(
145
+ status_code=HTTPStatus.BAD_REQUEST,
146
+ request_id=f"vllm-batch-{random_uuid()}",
147
+ ),
148
+ error=error_msg,
149
+ )
150
+ return batch_output
151
+
152
+
153
+async def make_async_error_request_output(
154
+ request: BatchRequestInput, error_msg: str) -> BatchRequestOutput:
155
+ return make_error_request_output(request, error_msg)
156
+
157
+
158
async def run_request(serving_engine_func: Callable,
159
request: BatchRequestInput,
160
tracker: BatchProgressTracker) -> BatchRequestOutput:
@@ -158,7 +178,8 @@ async def run_request(serving_engine_func: Callable,
178
error=response,
179
)
180
else:
161
- raise ValueError("Request must not be sent in stream mode")
181
+ batch_output = make_error_request_output(
182
+ request, error_msg="Request must not be sent in stream mode")
183
184
tracker.completed()
185
return batch_output
@@ -225,8 +246,12 @@ async def main(args):
246
tracker))
247
tracker.submitted()
248
else:
228
- raise ValueError("Only /v1/chat/completions and /v1/embeddings are"
229
- "supported in the batch endpoint.")
249
+ response_futures.append(
250
+ make_async_error_request_output(
251
+ request,
252
+ error_msg="Only /v1/chat/completions and "
253
+ "/v1/embeddings are supported in the batch endpoint.",
254
+ ))
255
256
with tracker.pbar():
257
responses = await asyncio.gather(*response_futures)