A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Remove benchmark_async_llm_server.py (#155)
Zhuohan Li committed
Jun 19, 2023 at 11:12 UTC
0370afa2e55c4934e8d38fa1dfe22e7a8e64345a
1 file changed
-60
benchmarks/benchmark_async_llm_server.py
deleted
-60
@@ -1,60 +0,0 @@
1
-import argparse
2
-import json
3
-import threading
4
-import time
5
-
6
-import requests
7
-
8
-
9
-def main(args: argparse.Namespace):
10
- prompts = [f"Tell me a story with more than {''.join([str(i+1)] * 5)} words"
11
- for i in range(args.n_threads)]
12
-
13
- api_url = f"http://{args.host}:{args.port}/generate"
14
- headers = {"User-Agent": "vLLM Benchmark Client"}
15
- ploads = [{
16
- "prompt": p,
17
- "max_tokens": args.max_tokens,
18
- "temperature": 0.0,
19
- "ignore_eos": True,
20
- } for p in prompts]
21
-
22
- def send_request(results, i):
23
- response = requests.post(api_url, headers=headers, json=ploads[i],
24
- stream=True)
25
- results[i] = response
26
-
27
- # use args.n_threads to prompt the backend
28
- tik = time.time()
29
- threads = []
30
- results = [None] * args.n_threads
31
- for i in range(args.n_threads):
32
- t = threading.Thread(target=send_request, args=(results, i))
33
- t.start()
34
- threads.append(t)
35
-
36
- for t in threads:
37
- t.join()
38
-
39
- print(f"Time (POST): {time.time() - tik} s")
40
- n_words = 0
41
-
42
- for i, response in enumerate(results):
43
- k = list(response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"))
44
- response_new_words = json.loads(k[-2].decode("utf-8"))["text"][0]
45
- n_words += len(response_new_words.split(" ")) - len(prompts[i].split(" "))
46
-
47
- time_seconds = time.time() - tik
48
- print(f"Time (total): {time_seconds:.3f}s to finish, n_threads: {args.n_threads}, "
49
- f"throughput: {n_words / time_seconds} words/s.")
50
-
51
-
52
-if __name__ == "__main__":
53
- parser = argparse.ArgumentParser()
54
- parser.add_argument("--host", type=str, default="localhost")
55
- parser.add_argument("--port", type=int, default=8000)
56
- parser.add_argument("--max-tokens", type=int, default=128)
57
- parser.add_argument("--n-threads", type=int, default=128)
58
- args = parser.parse_args()
59
-
60
- main(args)