@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5020e1e80c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Non-streaming simple fastapi server (#144)

Zhuohan Li committed Jun 11, 2023 at 01:43 UTC 5020e1e80c33e57ec9d4f7e17f35bc87b1a7cb01
3 files changed +61 -20
cacheflow/entrypoints/openai/openai_frontend.py
+1 -1
@@ -233,7 +233,7 @@ async def create_completion(raw_request: Request):
233 async for res in result_generator:
234 if await raw_request.is_disconnected():
235 # Abort the request if the client disconnects.
236 - await server.abort(request_id)
236 + await abort_request()
237 return create_error_response(HTTPStatus.BAD_REQUEST,
238 "Client disconnected")
239 final_res = res
cacheflow/entrypoints/simple_fastapi_frontend.py
+29 -10
@@ -3,7 +3,7 @@ import json
3 from typing import AsyncGenerator
4
5 from fastapi import BackgroundTasks, FastAPI, Request
6 -from fastapi.responses import StreamingResponse
6 +from fastapi.responses import Response, StreamingResponse
7 import uvicorn
8
9 from cacheflow.sampling_params import SamplingParams
@@ -17,19 +17,22 @@ app = FastAPI()
17
18
19 @app.post("/generate")
20 -async def generate_stream(request: Request) -> StreamingResponse:
20 +async def generate(request: Request) -> Response:
21 """ Stream the results of the generation request.
22
23 The request should be a JSON object with the following fields:
24 - prompt: the prompt to use for the generation.
25 + - stream: whether to stream the results or not.
26 - other fields: the sampling parameters (See `SamplingParams` for details).
27 """
28 request_dict = await request.json()
29 prompt = request_dict.pop("prompt")
30 + stream = request_dict.pop("stream", False)
31 sampling_params = SamplingParams(**request_dict)
32 request_id = random_uuid()
33 results_generator = server.generate(prompt, sampling_params, request_id)
34
35 + # Streaming case
36 async def stream_results() -> AsyncGenerator[bytes, None]:
37 async for request_output in results_generator:
38 prompt = request_output.prompt
@@ -37,19 +40,35 @@ async def generate_stream(request: Request) -> StreamingResponse:
40 prompt + output.text
41 for output in request_output.outputs
42 ]
40 - ret = {
41 - "text": text_outputs,
42 - "error": 0,
43 - }
43 + ret = {"text": text_outputs}
44 yield (json.dumps(ret) + "\0").encode("utf-8")
45
46 async def abort_request() -> None:
47 await server.abort(request_id)
48
49 - background_tasks = BackgroundTasks()
50 - # Abort the request if the client disconnects.
51 - background_tasks.add_task(abort_request)
52 - return StreamingResponse(stream_results(), background=background_tasks)
49 + if stream:
50 + background_tasks = BackgroundTasks()
51 + # Abort the request if the client disconnects.
52 + background_tasks.add_task(abort_request)
53 + return StreamingResponse(stream_results(), background=background_tasks)
54 +
55 + # Non-streaming case
56 + final_output = None
57 + async for request_output in results_generator:
58 + if await request.is_disconnected():
59 + # Abort the request if the client disconnects.
60 + await server.abort(request_id)
61 + return Response(status_code=499)
62 + final_output = request_output
63 +
64 + assert final_output is not None
65 + prompt = final_output.prompt
66 + text_outputs = [
67 + prompt + output.text
68 + for output in final_output.outputs
69 + ]
70 + ret = {"text": text_outputs}
71 + return Response(content=json.dumps(ret))
72
73
74 if __name__ == "__main__":
examples/simple_fastapi_client.py
+31 -9
@@ -1,15 +1,17 @@
1 import argparse
2 -import requests
2 import json
3 +import requests
4 +from typing import Iterable, List
5
5 -def clear_line(n=1):
6 +def clear_line(n:int = 1) -> None:
7 LINE_UP = '\033[1A'
8 LINE_CLEAR = '\x1b[2K'
9 for i in range(n):
10 print(LINE_UP, end=LINE_CLEAR, flush=True)
11
12
12 -def http_request(prompt: str, api_url: str, n: int = 1):
13 +def post_http_request(prompt: str, api_url: str, n: int = 1,
14 + stream: bool = False) -> requests.Response:
15 headers = {"User-Agent": "Test Client"}
16 pload = {
17 "prompt": prompt,
@@ -17,32 +19,52 @@ def http_request(prompt: str, api_url: str, n: int = 1):
19 "use_beam_search": True,
20 "temperature": 0.0,
21 "max_tokens": 16,
22 + "stream": stream,
23 }
24 response = requests.post(api_url, headers=headers, json=pload, stream=True)
25 + return response
26 +
27
23 - for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"):
28 +def get_streaming_response(response: requests.Response) -> Iterable[List[str]]:
29 + for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False,
30 + delimiter=b"\0"):
31 if chunk:
32 data = json.loads(chunk.decode("utf-8"))
33 output = data["text"]
34 yield output
35
36
37 +def get_response(response: requests.Response) -> List[str]:
38 + data = json.loads(response.content)
39 + output = data["text"]
40 + return output
41 +
42 +
43 if __name__ == "__main__":
44 parser = argparse.ArgumentParser()
45 parser.add_argument("--host", type=str, default="localhost")
46 parser.add_argument("--port", type=int, default=8001)
47 parser.add_argument("--n", type=int, default=4)
48 parser.add_argument("--prompt", type=str, default="San Francisco is a")
49 + parser.add_argument("--stream", action="store_true")
50 args = parser.parse_args()
51 prompt = args.prompt
52 api_url = f"http://{args.host}:{args.port}/generate"
53 n = args.n
54 + stream = args.stream
55
56 print(f"Prompt: {prompt}\n", flush=True)
42 - num_printed_lines = 0
43 - for h in http_request(prompt, api_url, n):
44 - clear_line(num_printed_lines)
57 + response = post_http_request(prompt, api_url, n, stream)
58 +
59 + if stream:
60 num_printed_lines = 0
46 - for i, line in enumerate(h):
47 - num_printed_lines += 1
61 + for h in get_streaming_response(response):
62 + clear_line(num_printed_lines)
63 + num_printed_lines = 0
64 + for i, line in enumerate(h):
65 + num_printed_lines += 1
66 + print(f"Beam candidate {i}: {line}", flush=True)
67 + else:
68 + output = get_response(response)
69 + for i, line in enumerate(output):
70 print(f"Beam candidate {i}: {line}", flush=True)