A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
clean api code, remove redundant background task. (#1102)
Roy committed
Sep 22, 2023 at 04:25 UTC
2d1e86f1b15396119321cfb3a77acde72b0c08ee
2 files changed
+7
-27
vllm/entrypoints/api_server.py
+2
-8
@@ -2,7 +2,7 @@ import argparse
2
import json
3
from typing import AsyncGenerator
4
5
-from fastapi import BackgroundTasks, FastAPI, Request
5
+from fastapi import FastAPI, Request
6
from fastapi.responses import JSONResponse, Response, StreamingResponse
7
import uvicorn
8
@@ -44,14 +44,8 @@ async def generate(request: Request) -> Response:
44
ret = {"text": text_outputs}
45
yield (json.dumps(ret) + "\0").encode("utf-8")
46
47
- async def abort_request() -> None:
48
- await engine.abort(request_id)
49
-
47
if stream:
51
- background_tasks = BackgroundTasks()
52
- # Abort the request if the client disconnects.
53
- background_tasks.add_task(abort_request)
54
- return StreamingResponse(stream_results(), background=background_tasks)
48
+ return StreamingResponse(stream_results())
49
50
# Non-streaming case
51
final_output = None
vllm/entrypoints/openai/api_server.py
+5
-19
@@ -10,7 +10,7 @@ from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
10
11
import fastapi
12
import uvicorn
13
-from fastapi import BackgroundTasks, Request
13
+from fastapi import Request
14
from fastapi.exceptions import RequestValidationError
15
from fastapi.middleware.cors import CORSMiddleware
16
from fastapi.responses import JSONResponse, StreamingResponse
@@ -229,9 +229,6 @@ async def create_chat_completion(request: ChatCompletionRequest,
229
result_generator = engine.generate(prompt, sampling_params, request_id,
230
token_ids)
231
232
- async def abort_request() -> None:
233
- await engine.abort(request_id)
234
-
232
def create_stream_response_json(
233
index: int,
234
text: str,
@@ -291,19 +288,15 @@ async def create_chat_completion(request: ChatCompletionRequest,
288
289
# Streaming response
290
if request.stream:
294
- background_tasks = BackgroundTasks()
295
- # Abort the request if the client disconnects.
296
- background_tasks.add_task(abort_request)
291
return StreamingResponse(completion_stream_generator(),
298
- media_type="text/event-stream",
299
- background=background_tasks)
292
+ media_type="text/event-stream")
293
294
# Non-streaming response
295
final_res: RequestOutput = None
296
async for res in result_generator:
297
if await raw_request.is_disconnected():
298
# Abort the request if the client disconnects.
306
- await abort_request()
299
+ await engine.abort(request_id)
300
return create_error_response(HTTPStatus.BAD_REQUEST,
301
"Client disconnected")
302
final_res = res
@@ -448,9 +441,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
441
and (request.best_of is None or request.n == request.best_of)
442
and not request.use_beam_search)
443
451
- async def abort_request() -> None:
452
- await engine.abort(request_id)
453
-
444
def create_stream_response_json(
445
index: int,
446
text: str,
@@ -510,19 +500,15 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
500
501
# Streaming response
502
if stream:
513
- background_tasks = BackgroundTasks()
514
- # Abort the request if the client disconnects.
515
- background_tasks.add_task(abort_request)
503
return StreamingResponse(completion_stream_generator(),
517
- media_type="text/event-stream",
518
- background=background_tasks)
504
+ media_type="text/event-stream")
505
506
# Non-streaming response
507
final_res: RequestOutput = None
508
async for res in result_generator:
509
if await raw_request.is_disconnected():
510
# Abort the request if the client disconnects.
525
- await abort_request()
511
+ await engine.abort(request_id)
512
return create_error_response(HTTPStatus.BAD_REQUEST,
513
"Client disconnected")
514
final_res = res