@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0370afa2e5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Remove benchmark_async_llm_server.py (#155)

Zhuohan Li committed Jun 19, 2023 at 11:12 UTC 0370afa2e55c4934e8d38fa1dfe22e7a8e64345a
1 file changed -60
benchmarks/benchmark_async_llm_server.py deleted
-60
@@ -1,60 +0,0 @@
1 -import argparse
2 -import json
3 -import threading
4 -import time
5 -
6 -import requests
7 -
8 -
9 -def main(args: argparse.Namespace):
10 - prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11 - for i in range(args.n_threads)]
12 -
13 - api_url = f"http://{args.host}:{args.port}/generate"
14 - headers = {"User-Agent": "vLLM Benchmark Client"}
15 - ploads = [{
16 - "prompt": p,
17 - "max_tokens": args.max_tokens,
18 - "temperature": 0.0,
19 - "ignore_eos": True,
20 - } for p in prompts]
21 -
22 - def send_request(results, i):
23 - response = requests.post(api_url, headers=headers, json=ploads[i],
24 - stream=True)
25 - results[i] = response
26 -
27 - # use args.n_threads to prompt the backend
28 - tik = time.time()
29 - threads = []
30 - results = [None] * args.n_threads
31 - for i in range(args.n_threads):
32 - t = threading.Thread(target=send_request, args=(results, i))
33 - t.start()
34 - threads.append(t)
35 -
36 - for t in threads:
37 - t.join()
38 -
39 - print(f"Time (POST): {time.time() - tik} s")
40 - n_words = 0
41 -
42 - for i, response in enumerate(results):
43 - k = list(response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"))
44 - response_new_words = json.loads(k[-2].decode("utf-8"))["text"][0]
45 - n_words += len(response_new_words.split(" ")) - len(prompts[i].split(" "))
46 -
47 - time_seconds = time.time() - tik
48 - print(f"Time (total): {time_seconds:.3f}s to finish, n_threads: {args.n_threads}, "
49 - f"throughput: {n_words / time_seconds} words/s.")
50 -
51 -
52 -if __name__ == "__main__":
53 - parser = argparse.ArgumentParser()
54 - parser.add_argument("--host", type=str, default="localhost")
55 - parser.add_argument("--port", type=int, default=8000)
56 - parser.add_argument("--max-tokens", type=int, default=128)
57 - parser.add_argument("--n-threads", type=int, default=128)
58 - args = parser.parse_args()
59 -
60 - main(args)