@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2d1e86f1b1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

clean api code, remove redundant background task. (#1102)

Roy committed Sep 22, 2023 at 04:25 UTC 2d1e86f1b15396119321cfb3a77acde72b0c08ee
2 files changed +7 -27
vllm/entrypoints/api_server.py
+2 -8
@@ -2,7 +2,7 @@ import argparse
2 import json
3 from typing import AsyncGenerator
4
5 -from fastapi import BackgroundTasks, FastAPI, Request
5 +from fastapi import FastAPI, Request
6 from fastapi.responses import JSONResponse, Response, StreamingResponse
7 import uvicorn
8
@@ -44,14 +44,8 @@ async def generate(request: Request) -> Response:
44 ret = {"text": text_outputs}
45 yield (json.dumps(ret) + "\0").encode("utf-8")
46
47 - async def abort_request() -> None:
48 - await engine.abort(request_id)
49 -
47 if stream:
51 - background_tasks = BackgroundTasks()
52 - # Abort the request if the client disconnects.
53 - background_tasks.add_task(abort_request)
54 - return StreamingResponse(stream_results(), background=background_tasks)
48 + return StreamingResponse(stream_results())
49
50 # Non-streaming case
51 final_output = None
vllm/entrypoints/openai/api_server.py
+5 -19
@@ -10,7 +10,7 @@ from typing import AsyncGenerator, Dict, List, Optional, Tuple, Union
10
11 import fastapi
12 import uvicorn
13 -from fastapi import BackgroundTasks, Request
13 +from fastapi import Request
14 from fastapi.exceptions import RequestValidationError
15 from fastapi.middleware.cors import CORSMiddleware
16 from fastapi.responses import JSONResponse, StreamingResponse
@@ -229,9 +229,6 @@ async def create_chat_completion(request: ChatCompletionRequest,
229 result_generator = engine.generate(prompt, sampling_params, request_id,
230 token_ids)
231
232 - async def abort_request() -> None:
233 - await engine.abort(request_id)
234 -
232 def create_stream_response_json(
233 index: int,
234 text: str,
@@ -291,19 +288,15 @@ async def create_chat_completion(request: ChatCompletionRequest,
288
289 # Streaming response
290 if request.stream:
294 - background_tasks = BackgroundTasks()
295 - # Abort the request if the client disconnects.
296 - background_tasks.add_task(abort_request)
291 return StreamingResponse(completion_stream_generator(),
298 - media_type="text/event-stream",
299 - background=background_tasks)
292 + media_type="text/event-stream")
293
294 # Non-streaming response
295 final_res: RequestOutput = None
296 async for res in result_generator:
297 if await raw_request.is_disconnected():
298 # Abort the request if the client disconnects.
306 - await abort_request()
299 + await engine.abort(request_id)
300 return create_error_response(HTTPStatus.BAD_REQUEST,
301 "Client disconnected")
302 final_res = res
@@ -448,9 +441,6 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
441 and (request.best_of is None or request.n == request.best_of)
442 and not request.use_beam_search)
443
451 - async def abort_request() -> None:
452 - await engine.abort(request_id)
453 -
444 def create_stream_response_json(
445 index: int,
446 text: str,
@@ -510,19 +500,15 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
500
501 # Streaming response
502 if stream:
513 - background_tasks = BackgroundTasks()
514 - # Abort the request if the client disconnects.
515 - background_tasks.add_task(abort_request)
503 return StreamingResponse(completion_stream_generator(),
517 - media_type="text/event-stream",
518 - background=background_tasks)
504 + media_type="text/event-stream")
505
506 # Non-streaming response
507 final_res: RequestOutput = None
508 async for res in result_generator:
509 if await raw_request.is_disconnected():
510 # Abort the request if the client disconnects.
525 - await abort_request()
511 + await engine.abort(request_id)
512 return create_error_response(HTTPStatus.BAD_REQUEST,
513 "Client disconnected")
514 final_res = res