A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Non-streaming simple fastapi server (#144)
Zhuohan Li committed
Jun 11, 2023 at 01:43 UTC
5020e1e80c33e57ec9d4f7e17f35bc87b1a7cb01
3 files changed
+61
-20
cacheflow/entrypoints/openai/openai_frontend.py
+1
-1
@@ -233,7 +233,7 @@ async def create_completion(raw_request: Request):
233
async for res in result_generator:
234
if await raw_request.is_disconnected():
235
# Abort the request if the client disconnects.
236
- await server.abort(request_id)
236
+ await abort_request()
237
return create_error_response(HTTPStatus.BAD_REQUEST,
238
"Client disconnected")
239
final_res = res
cacheflow/entrypoints/simple_fastapi_frontend.py
+29
-10
@@ -3,7 +3,7 @@ import json
3
from typing import AsyncGenerator
4
5
from fastapi import BackgroundTasks, FastAPI, Request
6
-from fastapi.responses import StreamingResponse
6
+from fastapi.responses import Response, StreamingResponse
7
import uvicorn
8
9
from cacheflow.sampling_params import SamplingParams
@@ -17,19 +17,22 @@ app = FastAPI()
17
18
19
@app.post("/generate")
20
-async def generate_stream(request: Request) -> StreamingResponse:
20
+async def generate(request: Request) -> Response:
21
""" Stream the results of the generation request.
22
23
The request should be a JSON object with the following fields:
24
- prompt: the prompt to use for the generation.
25
+ - stream: whether to stream the results or not.
26
- other fields: the sampling parameters (See `SamplingParams` for details).
27
"""
28
request_dict = await request.json()
29
prompt = request_dict.pop("prompt")
30
+ stream = request_dict.pop("stream", False)
31
sampling_params = SamplingParams(**request_dict)
32
request_id = random_uuid()
33
results_generator = server.generate(prompt, sampling_params, request_id)
34
35
+ # Streaming case
36
async def stream_results() -> AsyncGenerator[bytes, None]:
37
async for request_output in results_generator:
38
prompt = request_output.prompt
@@ -37,19 +40,35 @@ async def generate_stream(request: Request) -> StreamingResponse:
40
prompt + output.text
41
for output in request_output.outputs
42
]
40
- ret = {
41
- "text": text_outputs,
42
- "error": 0,
43
- }
43
+ ret = {"text": text_outputs}
44
yield (json.dumps(ret) + "\0").encode("utf-8")
45
46
async def abort_request() -> None:
47
await server.abort(request_id)
48
49
- background_tasks = BackgroundTasks()
50
- # Abort the request if the client disconnects.
51
- background_tasks.add_task(abort_request)
52
- return StreamingResponse(stream_results(), background=background_tasks)
49
+ if stream:
50
+ background_tasks = BackgroundTasks()
51
+ # Abort the request if the client disconnects.
52
+ background_tasks.add_task(abort_request)
53
+ return StreamingResponse(stream_results(), background=background_tasks)
54
+
55
+ # Non-streaming case
56
+ final_output = None
57
+ async for request_output in results_generator:
58
+ if await request.is_disconnected():
59
+ # Abort the request if the client disconnects.
60
+ await server.abort(request_id)
61
+ return Response(status_code=499)
62
+ final_output = request_output
63
+
64
+ assert final_output is not None
65
+ prompt = final_output.prompt
66
+ text_outputs = [
67
+ prompt + output.text
68
+ for output in final_output.outputs
69
+ ]
70
+ ret = {"text": text_outputs}
71
+ return Response(content=json.dumps(ret))
72
73
74
if __name__ == "__main__":
examples/simple_fastapi_client.py
+31
-9
@@ -1,15 +1,17 @@
1
import argparse
2
-import requests
2
import json
3
+import requests
4
+from typing import Iterable, List
5
5
-def clear_line(n=1):
6
+def clear_line(n:int = 1) -> None:
7
LINE_UP = '\033[1A'
8
LINE_CLEAR = '\x1b[2K'
9
for i in range(n):
10
print(LINE_UP, end=LINE_CLEAR, flush=True)
11
12
12
-def http_request(prompt: str, api_url: str, n: int = 1):
13
+def post_http_request(prompt: str, api_url: str, n: int = 1,
14
+ stream: bool = False) -> requests.Response:
15
headers = {"User-Agent": "Test Client"}
16
pload = {
17
"prompt": prompt,
@@ -17,32 +19,52 @@ def http_request(prompt: str, api_url: str, n: int = 1):
19
"use_beam_search": True,
20
"temperature": 0.0,
21
"max_tokens": 16,
22
+ "stream": stream,
23
}
24
response = requests.post(api_url, headers=headers, json=pload, stream=True)
25
+ return response
26
+
27
23
- for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
28
+def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
29
+ for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False,
30
+ delimiter=b"\0"):
31
if chunk:
32
data = json.loads(chunk.decode("utf-8"))
33
output = data["text"]
34
yield output
35
36
37
+def get_response(response: requests.Response) -> List[str]:
38
+ data = json.loads(response.content)
39
+ output = data["text"]
40
+ return output
41
+
42
+
43
if __name__ == "__main__":
44
parser = argparse.ArgumentParser()
45
parser.add_argument("--host", type=str, default="localhost")
46
parser.add_argument("--port", type=int, default=8001)
47
parser.add_argument("--n", type=int, default=4)
48
parser.add_argument("--prompt", type=str, default="San Francisco is a")
49
+ parser.add_argument("--stream", action="store_true")
50
args = parser.parse_args()
51
prompt = args.prompt
52
api_url = f"http://{args.host}:{args.port}/generate"
53
n = args.n
54
+ stream = args.stream
55
56
print(f"Prompt: {prompt}\n", flush=True)
42
- num_printed_lines = 0
43
- for h in http_request(prompt, api_url, n):
44
- clear_line(num_printed_lines)
57
+ response = post_http_request(prompt, api_url, n, stream)
58
+
59
+ if stream:
60
num_printed_lines = 0
46
- for i, line in enumerate(h):
47
- num_printed_lines += 1
61
+ for h in get_streaming_response(response):
62
+ clear_line(num_printed_lines)
63
+ num_printed_lines = 0
64
+ for i, line in enumerate(h):
65
+ num_printed_lines += 1
66
+ print(f"Beam candidate {i}: {line}", flush=True)
67
+ else:
68
+ output = get_response(response)
69
+ for i, line in enumerate(output):
70
print(f"Beam candidate {i}: {line}", flush=True)