A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Serving Benchmark Refactoring (#2433)
Roger Wang committed
Feb 12, 2024 at 22:53 UTC
a4211a4dc3a83d9e58eb7ee2f015aa033159c267
4 files changed
+553
-125
.buildkite/run-benchmarks.sh
+10
-4
@@ -6,15 +6,16 @@ set -o pipefail
6
# cd into parent directory of this file
7
cd "$(dirname "${BASH_SOURCE[0]}")/.."
8
9
-(wget && curl) || (apt-get update && apt-get install -y wget curl)
9
+(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
10
11
-# run benchmarks and upload the result to buildkite
11
+# run python-based benchmarks and upload the result to buildkite
12
python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
13
bench_latency_exit_code=$?
14
15
python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
16
bench_throughput_exit_code=$?
17
18
+# run server-based benchmarks and upload the result to buildkite
19
python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &
20
server_pid=$!
21
wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
@@ -22,11 +23,14 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/r
23
# wait for server to start, timeout after 600 seconds
24
timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
25
python3 benchmarks/benchmark_serving.py \
26
+ --backend openai \
27
--dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
28
--model meta-llama/Llama-2-7b-chat-hf \
29
--num-prompts 20 \
30
--endpoint /v1/completions \
29
- --tokenizer meta-llama/Llama-2-7b-chat-hf 2>&1 | tee benchmark_serving.txt
31
+ --tokenizer meta-llama/Llama-2-7b-chat-hf \
32
+ --save-result \
33
+ 2>&1 | tee benchmark_serving.txt
34
bench_serving_exit_code=$?
35
kill $server_pid
36
@@ -44,7 +48,7 @@ sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line
48
echo "### Serving Benchmarks" >> benchmark_results.md
49
sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
50
echo "" >> benchmark_results.md
47
-tail -n 5 benchmark_serving.txt >> benchmark_results.md # last 5 lines
51
+tail -n 13 benchmark_serving.txt >> benchmark_results.md # last 13 lines
52
53
# upload the results to buildkite
54
/workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
@@ -61,3 +65,5 @@ fi
65
if [ $bench_serving_exit_code -ne 0 ]; then
66
exit $bench_serving_exit_code
67
fi
68
+
69
+/workspace/buildkite-agent artifact upload openai-*.json
benchmarks/backend_request_func.py
new
+284
@@ -0,0 +1,284 @@
1
+import json
2
+import os
3
+import time
4
+from dataclasses import dataclass
5
+from typing import Optional
6
+
7
+import aiohttp
8
+from tqdm.asyncio import tqdm
9
+
10
+AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
11
+
12
+
13
+@dataclass
14
+class RequestFuncInput:
15
+ prompt: str
16
+ api_url: str
17
+ prompt_len: int
18
+ output_len: int
19
+ model: str
20
+ best_of: int = 1
21
+ use_beam_search: bool = False
22
+
23
+
24
+@dataclass
25
+class RequestFuncOutput:
26
+ generated_text: str = ""
27
+ success: bool = False
28
+ latency: float = 0
29
+ ttft: float = 0
30
+ prompt_len: int = 0
31
+
32
+
33
+async def async_request_tgi(
34
+ request_func_input: RequestFuncInput,
35
+ pbar: Optional[tqdm] = None,
36
+) -> RequestFuncOutput:
37
+ api_url = request_func_input.api_url
38
+ assert api_url.endswith("generate_stream")
39
+
40
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
41
+ assert not request_func_input.use_beam_search
42
+ params = {
43
+ "best_of": request_func_input.best_of,
44
+ "max_new_tokens": request_func_input.output_len,
45
+ "do_sample": True,
46
+ "temperature": 0.01, # TGI does not accept 0.0 temperature.
47
+ "top_p": 0.99, # TGI does not accept 1.0 top_p.
48
+ }
49
+ payload = {
50
+ "inputs": request_func_input.prompt,
51
+ "parameters": params,
52
+ }
53
+ output = RequestFuncOutput()
54
+ output.prompt_len = request_func_input.prompt_len
55
+
56
+ ttft = 0
57
+ st = time.perf_counter()
58
+ try:
59
+ async with session.post(url=api_url, json=payload) as response:
60
+ if response.status == 200:
61
+ async for data in response.content.iter_any():
62
+ if ttft == 0:
63
+ ttft = time.perf_counter() - st
64
+ output.ttft = ttft
65
+ output.latency = time.perf_counter() - st
66
+
67
+ body = data.decode("utf-8").lstrip("data:")
68
+ output.generated_text = json.loads(body)["generated_text"]
69
+ output.success = True
70
+ else:
71
+ output.success = False
72
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
73
+ output.success = False
74
+
75
+ if pbar:
76
+ pbar.update(1)
77
+ return output
78
+
79
+
80
+async def async_request_vllm(
81
+ request_func_input: RequestFuncInput,
82
+ pbar: Optional[tqdm] = None,
83
+) -> RequestFuncOutput:
84
+ api_url = request_func_input.api_url
85
+ assert api_url.endswith("generate")
86
+
87
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
88
+ payload = {
89
+ "prompt": request_func_input.prompt,
90
+ "n": 1,
91
+ "best_of": request_func_input.best_of,
92
+ "use_beam_search": request_func_input.use_beam_search,
93
+ "temperature": 0.0 if request_func_input.use_beam_search else 1.0,
94
+ "top_p": 1.0,
95
+ "max_tokens": request_func_input.output_len,
96
+ "ignore_eos": True,
97
+ "stream": True,
98
+ }
99
+ output = RequestFuncOutput()
100
+ output.prompt_len = request_func_input.prompt_len
101
+
102
+ ttft = 0
103
+ st = time.perf_counter()
104
+ try:
105
+ async with session.post(url=api_url, json=payload) as response:
106
+ if response.status == 200:
107
+ async for data in response.content.iter_any():
108
+ if ttft == 0:
109
+ ttft = time.perf_counter() - st
110
+ output.ttft = ttft
111
+ output.latency = time.perf_counter() - st
112
+
113
+ # When streaming, '\0' is appended to the end of the response.
114
+ body = data.decode("utf-8").strip("\0")
115
+ output.generated_text = json.loads(
116
+ body)["text"][0][len(request_func_input.prompt):]
117
+ output.success = True
118
+
119
+ else:
120
+ output.success = False
121
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
122
+ output.success = False
123
+
124
+ if pbar:
125
+ pbar.update(1)
126
+ return output
127
+
128
+
129
+async def async_request_trt_llm(
130
+ request_func_input: RequestFuncInput,
131
+ pbar: Optional[tqdm] = None,
132
+) -> RequestFuncOutput:
133
+ api_url = request_func_input.api_url
134
+ assert api_url.endswith("generate_stream")
135
+
136
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
137
+ assert not request_func_input.use_beam_search
138
+ assert request_func_input.best_of == 1
139
+ payload = {
140
+ "accumulate_tokens": True,
141
+ "text_input": request_func_input.prompt,
142
+ "temperature": 0.0,
143
+ "top_p": 1.0,
144
+ "max_tokens": request_func_input.output_len,
145
+ "stream": True,
146
+ }
147
+ output = RequestFuncOutput()
148
+ output.prompt_len = request_func_input.prompt_len
149
+ ttft = 0
150
+
151
+ st = time.perf_counter()
152
+ try:
153
+ async with session.post(url=api_url, json=payload) as resp:
154
+ if resp.status == 200:
155
+ async for data in resp.content.iter_any():
156
+ if ttft == 0:
157
+ ttft = time.perf_counter() - st
158
+ output.ttft = ttft
159
+ output.latency = time.perf_counter() - st
160
+
161
+ body = data.decode("utf-8").lstrip("data:")
162
+ output.generated_text = json.loads(body)["text_output"]
163
+ output.success = True
164
+
165
+ else:
166
+ output.success = False
167
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
168
+ output.success = False
169
+
170
+ if pbar:
171
+ pbar.update(1)
172
+ return output
173
+
174
+
175
+async def async_request_deepspeed_mii(
176
+ request_func_input: RequestFuncInput,
177
+ pbar: Optional[tqdm] = None,
178
+) -> RequestFuncOutput:
179
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
180
+ assert request_func_input.best_of == 1
181
+ assert not request_func_input.use_beam_search
182
+
183
+ payload = {
184
+ "prompts": request_func_input.prompt,
185
+ "max_new_tokens": request_func_input.output_len,
186
+ "ignore_eos": True,
187
+ "do_sample": True,
188
+ "temperature":
189
+ 0.01, # deepspeed-mii does not accept 0.0 temperature.
190
+ "top_p": 1.0,
191
+ }
192
+ output = RequestFuncOutput()
193
+ output.prompt_len = request_func_input.prompt_len
194
+
195
+ # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.
196
+ # https://github.com/microsoft/DeepSpeed-MII/pull/311
197
+ output.ttft = 0
198
+
199
+ st = time.perf_counter()
200
+ try:
201
+ async with session.post(url=request_func_input.api_url,
202
+ json=payload) as resp:
203
+ if resp.status == 200:
204
+ parsed_resp = await resp.json()
205
+ output.latency = time.perf_counter() - st
206
+ output.generated_text = parsed_resp[0]["generated_text"]
207
+ output.success = True
208
+ else:
209
+ output.success = False
210
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
211
+ output.success = False
212
+
213
+ if pbar:
214
+ pbar.update(1)
215
+ return output
216
+
217
+
218
+async def async_request_openai_completions(
219
+ request_func_input: RequestFuncInput,
220
+ pbar: Optional[tqdm] = None,
221
+) -> RequestFuncOutput:
222
+ api_url = request_func_input.api_url
223
+ assert api_url.endswith("v1/completions")
224
+
225
+ async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
226
+ assert not request_func_input.use_beam_search
227
+ payload = {
228
+ "model": request_func_input.model,
229
+ "prompt": request_func_input.prompt,
230
+ "temperature": 0.0,
231
+ "best_of": request_func_input.best_of,
232
+ "max_tokens": request_func_input.output_len,
233
+ "stream": True,
234
+ }
235
+ headers = {
236
+ "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
237
+ }
238
+
239
+ output = RequestFuncOutput()
240
+ output.prompt_len = request_func_input.prompt_len
241
+
242
+ generated_text = ""
243
+ ttft = 0
244
+ st = time.perf_counter()
245
+ try:
246
+ async with session.post(url=api_url, json=payload,
247
+ headers=headers) as response:
248
+ if response.status == 200:
249
+ async for chunk in response.content:
250
+ if ttft == 0:
251
+ ttft = time.perf_counter() - st
252
+ output.ttft = ttft
253
+
254
+ chunk = chunk.strip()
255
+ if not chunk:
256
+ continue
257
+
258
+ chunk = chunk.decode("utf-8").lstrip("data: ")
259
+ if chunk == "[DONE]":
260
+ latency = time.perf_counter() - st
261
+ else:
262
+ body = json.loads(chunk)
263
+ generated_text += body["choices"][0]["text"]
264
+
265
+ output.generated_text = generated_text
266
+ output.success = True
267
+ output.latency = latency
268
+ else:
269
+ output.success = False
270
+ except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
271
+ output.success = False
272
+
273
+ if pbar:
274
+ pbar.update(1)
275
+ return output
276
+
277
+
278
+ASYNC_REQUEST_FUNCS = {
279
+ "tgi": async_request_tgi,
280
+ "vllm": async_request_vllm,
281
+ "deepspeed-mii": async_request_deepspeed_mii,
282
+ "openai": async_request_openai_completions,
283
+ "tensorrt-llm": async_request_trt_llm,
284
+}
benchmarks/benchmark_serving.py
+258
-120
@@ -20,16 +20,36 @@ import asyncio
20
import json
21
import random
22
import time
23
+from dataclasses import dataclass
24
+from datetime import datetime
25
from typing import AsyncGenerator, List, Tuple
26
25
-import aiohttp
27
import numpy as np
28
from tqdm.asyncio import tqdm
29
from transformers import PreTrainedTokenizerBase
30
from vllm.transformers_utils.tokenizer import get_tokenizer
31
31
-# (prompt len, output len, latency)
32
-REQUEST_LATENCY: List[Tuple[int, int, float]] = []
32
+from backend_request_func import (
33
+ ASYNC_REQUEST_FUNCS,
34
+ RequestFuncInput,
35
+ RequestFuncOutput,
36
+)
37
+
38
+
39
+@dataclass
40
+class BenchmarkMetrics:
41
+ completed: int
42
+ total_input: int
43
+ total_output: int
44
+ request_throughput: float
45
+ input_throughput: float
46
+ output_throughput: float
47
+ mean_ttft_ms: float
48
+ median_ttft_ms: float
49
+ p99_ttft_ms: float
50
+ mean_tpot_ms: float
51
+ median_tpot_ms: float
52
+ p99_tpot_ms: float
53
54
55
def sample_requests(
@@ -46,6 +66,11 @@ def sample_requests(
66
dataset = [(data["conversations"][0]["value"],
67
data["conversations"][1]["value"]) for data in dataset]
68
69
+ # some of these will be filtered out, so sample more than we need
70
+ sampled_indices = random.sample(range(len(dataset)),
71
+ int(num_requests * 1.2))
72
+ dataset = [dataset[i] for i in sampled_indices]
73
+
74
# Tokenize the prompts and completions.
75
prompts = [prompt for prompt, _ in dataset]
76
prompt_token_ids = tokenizer(prompts).input_ids
@@ -92,80 +117,125 @@ async def get_request(
117
await asyncio.sleep(interval)
118
119
95
-async def send_request(backend: str, model: str, api_url: str, prompt: str,
96
- prompt_len: int, output_len: int, best_of: int,
97
- use_beam_search: bool, pbar: tqdm) -> None:
98
- request_start_time = time.perf_counter()
99
-
100
- headers = {"User-Agent": "Benchmark Client"}
101
- if backend == "vllm":
102
- pload = {
103
- "prompt": prompt,
104
- "n": 1,
105
- "best_of": best_of,
106
- "use_beam_search": use_beam_search,
107
- "temperature": 0.0 if use_beam_search else 1.0,
108
- "top_p": 1.0,
109
- "max_tokens": output_len,
110
- "ignore_eos": True,
111
- "stream": False,
112
- }
113
- if model is not None:
114
- pload["model"] = model
115
- elif backend == "tgi":
116
- assert not use_beam_search
117
- params = {
118
- "best_of": best_of,
119
- "max_new_tokens": output_len,
120
- "do_sample": True,
121
- }
122
- pload = {
123
- "inputs": prompt,
124
- "parameters": params,
125
- }
126
- else:
127
- raise ValueError(f"Unknown backend: {backend}")
128
-
129
- timeout = aiohttp.ClientTimeout(total=3 * 3600)
130
- async with aiohttp.ClientSession(timeout=timeout) as session:
131
- while True:
132
- async with session.post(api_url, headers=headers,
133
- json=pload) as response:
134
- chunks = []
135
- async for chunk, _ in response.content.iter_chunks():
136
- chunks.append(chunk)
137
- output = b"".join(chunks).decode("utf-8")
138
- output = json.loads(output)
120
+def calculate_metrics(
121
+ input_requests: List[Tuple[str, int, int]],
122
+ outputs: List[RequestFuncOutput],
123
+ dur_s: float,
124
+ tokenizer: PreTrainedTokenizerBase,
125
+) -> BenchmarkMetrics:
126
+ total_output = 0
127
+ total_input = 0
128
+ completed = 0
129
+ per_token_latencies = []
130
+ ttfts = []
131
+ for i in range(len(outputs)):
132
+ if outputs[i].success:
133
+ output_len = len(tokenizer.encode(outputs[i].generated_text))
134
+ total_output += output_len
135
+ total_input += input_requests[i][1]
136
+ per_token_latencies.append(outputs[i].latency / output_len)
137
+ ttfts.append(outputs[i].ttft)
138
+ completed += 1
139
140
- # Re-send the request if it failed.
141
- if "error" not in output:
142
- break
140
+ metrics = BenchmarkMetrics(
141
+ completed=completed,
142
+ total_input=total_input,
143
+ total_output=total_output,
144
+ request_throughput=completed / dur_s,
145
+ input_throughput=total_input / dur_s,
146
+ output_throughput=total_output / dur_s,
147
+ mean_ttft_ms=np.mean(ttfts) * 1000,
148
+ median_ttft_ms=np.median(ttfts) * 1000,
149
+ p99_ttft_ms=np.percentile(ttfts, 99) * 1000,
150
+ mean_tpot_ms=np.mean(per_token_latencies) * 1000,
151
+ median_tpot_ms=np.median(per_token_latencies) * 1000,
152
+ p99_tpot_ms=np.percentile(per_token_latencies, 99) * 1000,
153
+ )
154
144
- request_end_time = time.perf_counter()
145
- request_latency = request_end_time - request_start_time
146
- REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
147
- pbar.update(1)
155
+ return metrics
156
157
158
async def benchmark(
159
backend: str,
152
- model: str,
160
api_url: str,
161
+ model_id: str,
162
+ tokenizer: PreTrainedTokenizerBase,
163
input_requests: List[Tuple[str, int, int]],
164
best_of: int,
165
use_beam_search: bool,
166
request_rate: float,
158
-) -> None:
159
- tasks: List[asyncio.Task] = []
160
- pbar = tqdm(total=len(input_requests))
167
+ disable_tqdm: bool,
168
+):
169
+ if backend in ASYNC_REQUEST_FUNCS:
170
+ request_func = ASYNC_REQUEST_FUNCS.get(backend)
171
+ else:
172
+ raise ValueError(f"Unknown backend: {backend}")
173
+
174
+ pbar = None if disable_tqdm else tqdm(total=len(input_requests))
175
+
176
+ print(f"Traffic request rate: {request_rate}")
177
+
178
+ benchmark_start_time = time.perf_counter()
179
+ tasks = []
180
async for request in get_request(input_requests, request_rate):
181
prompt, prompt_len, output_len = request
163
- task = asyncio.create_task(
164
- send_request(backend, model, api_url, prompt, prompt_len,
165
- output_len, best_of, use_beam_search, pbar))
166
- tasks.append(task)
167
- await asyncio.gather(*tasks)
168
- pbar.close()
182
+ request_func_input = RequestFuncInput(
183
+ model=model_id,
184
+ prompt=prompt,
185
+ api_url=api_url,
186
+ prompt_len=prompt_len,
187
+ output_len=output_len,
188
+ best_of=best_of,
189
+ use_beam_search=use_beam_search,
190
+ )
191
+ tasks.append(
192
+ asyncio.create_task(
193
+ request_func(request_func_input=request_func_input,
194
+ pbar=pbar)))
195
+ outputs = await asyncio.gather(*tasks)
196
+
197
+ if not disable_tqdm:
198
+ pbar.close()
199
+
200
+ benchmark_duration = time.perf_counter() - benchmark_start_time
201
+
202
+ metrics = calculate_metrics(
203
+ input_requests=input_requests,
204
+ outputs=outputs,
205
+ dur_s=benchmark_duration,
206
+ tokenizer=tokenizer,
207
+ )
208
+
209
+ print(f"Successful requests: {metrics.completed}")
210
+ print(f"Benchmark duration: {benchmark_duration:2f} s")
211
+ print(f"Total input tokens: {metrics.total_input}")
212
+ print(f"Total generated tokens: {metrics.total_output}")
213
+ print(f"Request throughput: {metrics.request_throughput:.2f} requests/s")
214
+ print(f"Input token throughput: {metrics.input_throughput:.2f} tokens/s")
215
+ print(f"Output token throughput: {metrics.output_throughput:.2f} tokens/s")
216
+ print(f"Mean TTFT: {metrics.mean_ttft_ms:.2f} ms")
217
+ print(f"Median TTFT: {metrics.median_ttft_ms:.2f} ms")
218
+ print(f"P99 TTFT: {metrics.p99_ttft_ms:.2f} ms")
219
+ print(f"Mean TPOT: {metrics.mean_tpot_ms:.2f} ms")
220
+ print(f"Median TPOT: {metrics.median_tpot_ms:.2f} ms")
221
+ print(f"P99 TPOT: {metrics.p99_tpot_ms:.2f} ms")
222
+
223
+ result = {
224
+ "duration": benchmark_duration,
225
+ "completed": metrics.completed,
226
+ "total_input_tokens": metrics.total_input,
227
+ "total_output_tokens": metrics.total_output,
228
+ "request_inthroughput": metrics.request_throughput,
229
+ "input_throughput": metrics.input_throughput,
230
+ "output_throughput": metrics.output_throughput,
231
+ "mean_ttft_ms": metrics.mean_ttft_ms,
232
+ "median_ttft_ms": metrics.median_ttft_ms,
233
+ "p99_ttft_ms": metrics.p99_ttft_ms,
234
+ "mean_tpot_ms": metrics.mean_tpot_ms,
235
+ "median_tpot_ms": metrics.median_tpot_ms,
236
+ "p99_tpot_ms": metrics.p99_tpot_ms
237
+ }
238
+ return result
239
240
241
def main(args: argparse.Namespace):
@@ -173,77 +243,145 @@ def main(args: argparse.Namespace):
243
random.seed(args.seed)
244
np.random.seed(args.seed)
245
176
- api_url = f"{args.protocol}://{args.host}:{args.port}{args.endpoint}"
177
- tokenizer = get_tokenizer(args.tokenizer,
246
+ backend = args.backend
247
+ model_id = args.model
248
+ tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
249
+
250
+ if args.base_url is not None:
251
+ api_url = f"{args.base_url}{args.endpoint}"
252
+ else:
253
+ api_url = f"http://{args.host}:{args.port}{args.endpoint}"
254
+
255
+ tokenizer = get_tokenizer(tokenizer_id,
256
trust_remote_code=args.trust_remote_code)
257
input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
258
181
- benchmark_start_time = time.perf_counter()
182
- asyncio.run(
183
- benchmark(args.backend, args.model, api_url, input_requests,
184
- args.best_of, args.use_beam_search, args.request_rate))
185
- benchmark_end_time = time.perf_counter()
186
- benchmark_time = benchmark_end_time - benchmark_start_time
187
- print(f"Total time: {benchmark_time:.2f} s")
188
- print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
189
-
190
- # Compute the latency statistics.
191
- avg_latency = np.mean([latency for _, _, latency in REQUEST_LATENCY])
192
- print(f"Average latency: {avg_latency:.2f} s")
193
- avg_per_token_latency = np.mean([
194
- latency / (prompt_len + output_len)
195
- for prompt_len, output_len, latency in REQUEST_LATENCY
196
- ])
197
- print(f"Average latency per token: {avg_per_token_latency:.2f} s")
198
- avg_per_output_token_latency = np.mean(
199
- [latency / output_len for _, output_len, latency in REQUEST_LATENCY])
200
- print("Average latency per output token: "
201
- f"{avg_per_output_token_latency:.2f} s")
259
+ benchmark_result = asyncio.run(
260
+ benchmark(
261
+ backend=backend,
262
+ api_url=api_url,
263
+ model_id=model_id,
264
+ tokenizer=tokenizer,
265
+ input_requests=input_requests,
266
+ best_of=args.best_of,
267
+ use_beam_search=args.use_beam_search,
268
+ request_rate=args.request_rate,
269
+ disable_tqdm=args.disable_tqdm,
270
+ ))
271
+
272
+ # Save config and results to json
273
+ if args.save_result:
274
+ result_json = {}
275
+
276
+ # Setup
277
+ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
278
+ result_json["date"] = current_dt
279
+ result_json["backend"] = backend
280
+ result_json["version"] = args.version
281
+ result_json["model_id"] = model_id
282
+ result_json["tokenizer_id"] = tokenizer_id
283
+ result_json["best_of"] = args.best_of
284
+ result_json["use_beam_search"] = args.use_beam_search
285
+ result_json["num_prompts"] = args.num_prompts
286
+
287
+ # Traffic
288
+ result_json["request_rate"] = (
289
+ args.request_rate if args.request_rate < float("inf") else "inf")
290
+
291
+ # Merge with benchmark result
292
+ result_json = {**result_json, **benchmark_result}
293
+
294
+ # Save to file
295
+ base_model_id = model_id.split("/")[-1]
296
+ file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
297
+ with open(file_name, "w") as outfile:
298
+ json.dump(result_json, outfile)
299
300
301
if __name__ == "__main__":
302
parser = argparse.ArgumentParser(
303
description="Benchmark the online serving throughput.")
207
- parser.add_argument("--backend",
208
- type=str,
209
- default="vllm",
210
- choices=["vllm", "tgi"])
211
- parser.add_argument("--protocol",
212
- type=str,
213
- default="http",
214
- choices=["http", "https"])
304
+ parser.add_argument(
305
+ "--backend",
306
+ type=str,
307
+ default="vllm",
308
+ choices=list(ASYNC_REQUEST_FUNCS.keys()),
309
+ )
310
+ parser.add_argument(
311
+ "--version",
312
+ type=str,
313
+ default="N/A",
314
+ help="Version of the serving backend/engine.",
315
+ )
316
+ parser.add_argument(
317
+ "--base-url",
318
+ type=str,
319
+ default=None,
320
+ help="Server or API base url if not using http host and port.",
321
+ )
322
parser.add_argument("--host", type=str, default="localhost")
323
parser.add_argument("--port", type=int, default=8000)
217
- parser.add_argument("--endpoint", type=str, default="/generate")
218
- parser.add_argument("--model", type=str, default=None)
324
+ parser.add_argument(
325
+ "--endpoint",
326
+ type=str,
327
+ default="/generate",
328
+ help="API endpoint.",
329
+ )
330
parser.add_argument("--dataset",
331
type=str,
332
required=True,
333
help="Path to the dataset.")
223
- parser.add_argument("--tokenizer",
224
- type=str,
225
- required=True,
226
- help="Name or path of the tokenizer.")
227
- parser.add_argument("--best-of",
228
- type=int,
229
- default=1,
230
- help="Generates `best_of` sequences per prompt and "
231
- "returns the best one.")
334
+ parser.add_argument(
335
+ "--model",
336
+ type=str,
337
+ required=True,
338
+ help="Name of the model.",
339
+ )
340
+ parser.add_argument(
341
+ "--tokenizer",
342
+ type=str,
343
+ help=
344
+ "Name or path of the tokenizer, if not using the default model tokenizer.",
345
+ )
346
+ parser.add_argument(
347
+ "--best-of",
348
+ type=int,
349
+ default=1,
350
+ help="Generates `best_of` sequences per prompt and "
351
+ "returns the best one.",
352
+ )
353
parser.add_argument("--use-beam-search", action="store_true")
233
- parser.add_argument("--num-prompts",
234
- type=int,
235
- default=1000,
236
- help="Number of prompts to process.")
237
- parser.add_argument("--request-rate",
238
- type=float,
239
- default=float("inf"),
240
- help="Number of requests per second. If this is inf, "
241
- "then all the requests are sent at time 0. "
242
- "Otherwise, we use Poisson process to synthesize "
243
- "the request arrival times.")
354
+ parser.add_argument(
355
+ "--num-prompts",
356
+ type=int,
357
+ default=1000,
358
+ help="Number of prompts to process.",
359
+ )
360
+ parser.add_argument(
361
+ "--request-rate",
362
+ type=float,
363
+ default=float("inf"),
364
+ help="Number of requests per second. If this is inf, "
365
+ "then all the requests are sent at time 0. "
366
+ "Otherwise, we use Poisson process to synthesize "
367
+ "the request arrival times.",
368
+ )
369
parser.add_argument("--seed", type=int, default=0)
245
- parser.add_argument('--trust-remote-code',
246
- action='store_true',
247
- help='trust remote code from huggingface')
370
+ parser.add_argument(
371
+ "--trust-remote-code",
372
+ action="store_true",
373
+ help="Trust remote code from huggingface",
374
+ )
375
+ parser.add_argument(
376
+ "--disable-tqdm",
377
+ action="store_true",
378
+ help="Specify to disbale tqdm progress bar.",
379
+ )
380
+ parser.add_argument(
381
+ "--save-result",
382
+ action="store_true",
383
+ help="Specify to save benchmark results to a json file",
384
+ )
385
+
386
args = parser.parse_args()
387
main(args)
benchmarks/launch_tgi_server.sh
+1
-1
@@ -6,7 +6,7 @@ TOKENS=$2
6
7
docker run --gpus all --shm-size 1g -p $PORT:80 \
8
-v $PWD/data:/data \
9
- ghcr.io/huggingface/text-generation-inference:0.8 \
9
+ ghcr.io/huggingface/text-generation-inference:1.4.0 \
10
--model-id $MODEL \
11
--sharded false \
12
--max-input-length 1024 \