@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cea95dfb94
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Frontend] Create ErrorResponse instead of raising exceptions in run_batch (#8347)

Pooya Davoodi committed Sep 10, 2024 at 22:30 UTC cea95dfb941878b3370a7c40ca7ab2d549524445
2 files changed +31 -4
tests/entrypoints/openai/test_run_batch.py
+3 -1
@@ -8,7 +8,9 @@ from vllm.entrypoints.openai.protocol import BatchRequestOutput
8 INPUT_BATCH = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
9 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
10
11 -{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NonExistModel", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
11 +{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NonExistModel", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
12 +{"custom_id": "request-4", "method": "POST", "url": "/bad_url", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
13 +{"custom_id": "request-5", "method": "POST", "url": "/v1/chat/completions", "body": {"stream": "True", "model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
14
15 INVALID_INPUT_BATCH = """{"invalid_field": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are a helpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}
16 {"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "NousResearch/Meta-Llama-3-8B-Instruct", "messages": [{"role": "system", "content": "You are an unhelpful assistant."},{"role": "user", "content": "Hello world!"}],"max_tokens": 1000}}"""
vllm/entrypoints/openai/run_batch.py
+28 -3
@@ -1,4 +1,5 @@
1 import asyncio
2 +from http import HTTPStatus
3 from io import StringIO
4 from typing import Awaitable, Callable, List, Optional
5
@@ -135,6 +136,25 @@ async def write_file(path_or_url: str, data: str) -> None:
136 f.write(data)
137
138
139 +def make_error_request_output(request: BatchRequestInput,
140 + error_msg: str) -> BatchRequestOutput:
141 + batch_output = BatchRequestOutput(
142 + id=f"vllm-{random_uuid()}",
143 + custom_id=request.custom_id,
144 + response=BatchResponseData(
145 + status_code=HTTPStatus.BAD_REQUEST,
146 + request_id=f"vllm-batch-{random_uuid()}",
147 + ),
148 + error=error_msg,
149 + )
150 + return batch_output
151 +
152 +
153 +async def make_async_error_request_output(
154 + request: BatchRequestInput, error_msg: str) -> BatchRequestOutput:
155 + return make_error_request_output(request, error_msg)
156 +
157 +
158 async def run_request(serving_engine_func: Callable,
159 request: BatchRequestInput,
160 tracker: BatchProgressTracker) -> BatchRequestOutput:
@@ -158,7 +178,8 @@ async def run_request(serving_engine_func: Callable,
178 error=response,
179 )
180 else:
161 - raise ValueError("Request must not be sent in stream mode")
181 + batch_output = make_error_request_output(
182 + request, error_msg="Request must not be sent in stream mode")
183
184 tracker.completed()
185 return batch_output
@@ -225,8 +246,12 @@ async def main(args):
246 tracker))
247 tracker.submitted()
248 else:
228 - raise ValueError("Only /v1/chat/completions and /v1/embeddings are"
229 - "supported in the batch endpoint.")
249 + response_futures.append(
250 + make_async_error_request_output(
251 + request,
252 + error_msg="Only /v1/chat/completions and "
253 + "/v1/embeddings are supported in the batch endpoint.",
254 + ))
255
256 with tracker.pbar():
257 responses = await asyncio.gather(*response_futures)