@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a4211a4dc3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Serving Benchmark Refactoring (#2433)

Roger Wang committed Feb 12, 2024 at 22:53 UTC a4211a4dc3a83d9e58eb7ee2f015aa033159c267
4 files changed +553 -125
.buildkite/run-benchmarks.sh
+10 -4
@@ -6,15 +6,16 @@ set -o pipefail
6 # cd into parent directory of this file
7 cd "$(dirname "${BASH_SOURCE[0]}")/.."
8
9 -(wget && curl) || (apt-get update && apt-get install -y wget curl)
9 +(which wget && which curl) || (apt-get update && apt-get install -y wget curl)
10
11 -# run benchmarks and upload the result to buildkite
11 +# run python-based benchmarks and upload the result to buildkite
12 python3 benchmarks/benchmark_latency.py 2>&1 | tee benchmark_latency.txt
13 bench_latency_exit_code=$?
14
15 python3 benchmarks/benchmark_throughput.py --input-len 256 --output-len 256 2>&1 | tee benchmark_throughput.txt
16 bench_throughput_exit_code=$?
17
18 +# run server-based benchmarks and upload the result to buildkite
19 python3 -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf &
20 server_pid=$!
21 wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
@@ -22,11 +23,14 @@ wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/r
23 # wait for server to start, timeout after 600 seconds
24 timeout 600 bash -c 'until curl localhost:8000/v1/models; do sleep 1; done' || exit 1
25 python3 benchmarks/benchmark_serving.py \
26 + --backend openai \
27 --dataset ./ShareGPT_V3_unfiltered_cleaned_split.json \
28 --model meta-llama/Llama-2-7b-chat-hf \
29 --num-prompts 20 \
30 --endpoint /v1/completions \
29 - --tokenizer meta-llama/Llama-2-7b-chat-hf 2>&1 | tee benchmark_serving.txt
31 + --tokenizer meta-llama/Llama-2-7b-chat-hf \
32 + --save-result \
33 + 2>&1 | tee benchmark_serving.txt
34 bench_serving_exit_code=$?
35 kill $server_pid
36
@@ -44,7 +48,7 @@ sed -n '$p' benchmark_throughput.txt >> benchmark_results.md # last line
48 echo "### Serving Benchmarks" >> benchmark_results.md
49 sed -n '1p' benchmark_serving.txt >> benchmark_results.md # first line
50 echo "" >> benchmark_results.md
47 -tail -n 5 benchmark_serving.txt >> benchmark_results.md # last 5 lines
51 +tail -n 13 benchmark_serving.txt >> benchmark_results.md # last 13 lines
52
53 # upload the results to buildkite
54 /workspace/buildkite-agent annotate --style "info" --context "benchmark-results" < benchmark_results.md
@@ -61,3 +65,5 @@ fi
65 if [ $bench_serving_exit_code -ne 0 ]; then
66 exit $bench_serving_exit_code
67 fi
68 +
69 +/workspace/buildkite-agent artifact upload openai-*.json
benchmarks/backend_request_func.py new
+284
@@ -0,0 +1,284 @@
1 +import json
2 +import os
3 +import time
4 +from dataclasses import dataclass
5 +from typing import Optional
6 +
7 +import aiohttp
8 +from tqdm.asyncio import tqdm
9 +
10 +AIOHTTP_TIMEOUT = aiohttp.ClientTimeout(total=6 * 60 * 60)
11 +
12 +
13 +@dataclass
14 +class RequestFuncInput:
15 + prompt: str
16 + api_url: str
17 + prompt_len: int
18 + output_len: int
19 + model: str
20 + best_of: int = 1
21 + use_beam_search: bool = False
22 +
23 +
24 +@dataclass
25 +class RequestFuncOutput:
26 + generated_text: str = ""
27 + success: bool = False
28 + latency: float = 0
29 + ttft: float = 0
30 + prompt_len: int = 0
31 +
32 +
33 +async def async_request_tgi(
34 + request_func_input: RequestFuncInput,
35 + pbar: Optional[tqdm] = None,
36 +) -> RequestFuncOutput:
37 + api_url = request_func_input.api_url
38 + assert api_url.endswith("generate_stream")
39 +
40 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
41 + assert not request_func_input.use_beam_search
42 + params = {
43 + "best_of": request_func_input.best_of,
44 + "max_new_tokens": request_func_input.output_len,
45 + "do_sample": True,
46 + "temperature": 0.01, # TGI does not accept 0.0 temperature.
47 + "top_p": 0.99, # TGI does not accept 1.0 top_p.
48 + }
49 + payload = {
50 + "inputs": request_func_input.prompt,
51 + "parameters": params,
52 + }
53 + output = RequestFuncOutput()
54 + output.prompt_len = request_func_input.prompt_len
55 +
56 + ttft = 0
57 + st = time.perf_counter()
58 + try:
59 + async with session.post(url=api_url, json=payload) as response:
60 + if response.status == 200:
61 + async for data in response.content.iter_any():
62 + if ttft == 0:
63 + ttft = time.perf_counter() - st
64 + output.ttft = ttft
65 + output.latency = time.perf_counter() - st
66 +
67 + body = data.decode("utf-8").lstrip("data:")
68 + output.generated_text = json.loads(body)["generated_text"]
69 + output.success = True
70 + else:
71 + output.success = False
72 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
73 + output.success = False
74 +
75 + if pbar:
76 + pbar.update(1)
77 + return output
78 +
79 +
80 +async def async_request_vllm(
81 + request_func_input: RequestFuncInput,
82 + pbar: Optional[tqdm] = None,
83 +) -> RequestFuncOutput:
84 + api_url = request_func_input.api_url
85 + assert api_url.endswith("generate")
86 +
87 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
88 + payload = {
89 + "prompt": request_func_input.prompt,
90 + "n": 1,
91 + "best_of": request_func_input.best_of,
92 + "use_beam_search": request_func_input.use_beam_search,
93 + "temperature": 0.0 if request_func_input.use_beam_search else 1.0,
94 + "top_p": 1.0,
95 + "max_tokens": request_func_input.output_len,
96 + "ignore_eos": True,
97 + "stream": True,
98 + }
99 + output = RequestFuncOutput()
100 + output.prompt_len = request_func_input.prompt_len
101 +
102 + ttft = 0
103 + st = time.perf_counter()
104 + try:
105 + async with session.post(url=api_url, json=payload) as response:
106 + if response.status == 200:
107 + async for data in response.content.iter_any():
108 + if ttft == 0:
109 + ttft = time.perf_counter() - st
110 + output.ttft = ttft
111 + output.latency = time.perf_counter() - st
112 +
113 + # When streaming, '\0' is appended to the end of the response.
114 + body = data.decode("utf-8").strip("\0")
115 + output.generated_text = json.loads(
116 + body)["text"][0][len(request_func_input.prompt):]
117 + output.success = True
118 +
119 + else:
120 + output.success = False
121 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
122 + output.success = False
123 +
124 + if pbar:
125 + pbar.update(1)
126 + return output
127 +
128 +
129 +async def async_request_trt_llm(
130 + request_func_input: RequestFuncInput,
131 + pbar: Optional[tqdm] = None,
132 +) -> RequestFuncOutput:
133 + api_url = request_func_input.api_url
134 + assert api_url.endswith("generate_stream")
135 +
136 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
137 + assert not request_func_input.use_beam_search
138 + assert request_func_input.best_of == 1
139 + payload = {
140 + "accumulate_tokens": True,
141 + "text_input": request_func_input.prompt,
142 + "temperature": 0.0,
143 + "top_p": 1.0,
144 + "max_tokens": request_func_input.output_len,
145 + "stream": True,
146 + }
147 + output = RequestFuncOutput()
148 + output.prompt_len = request_func_input.prompt_len
149 + ttft = 0
150 +
151 + st = time.perf_counter()
152 + try:
153 + async with session.post(url=api_url, json=payload) as resp:
154 + if resp.status == 200:
155 + async for data in resp.content.iter_any():
156 + if ttft == 0:
157 + ttft = time.perf_counter() - st
158 + output.ttft = ttft
159 + output.latency = time.perf_counter() - st
160 +
161 + body = data.decode("utf-8").lstrip("data:")
162 + output.generated_text = json.loads(body)["text_output"]
163 + output.success = True
164 +
165 + else:
166 + output.success = False
167 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
168 + output.success = False
169 +
170 + if pbar:
171 + pbar.update(1)
172 + return output
173 +
174 +
175 +async def async_request_deepspeed_mii(
176 + request_func_input: RequestFuncInput,
177 + pbar: Optional[tqdm] = None,
178 +) -> RequestFuncOutput:
179 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
180 + assert request_func_input.best_of == 1
181 + assert not request_func_input.use_beam_search
182 +
183 + payload = {
184 + "prompts": request_func_input.prompt,
185 + "max_new_tokens": request_func_input.output_len,
186 + "ignore_eos": True,
187 + "do_sample": True,
188 + "temperature":
189 + 0.01, # deepspeed-mii does not accept 0.0 temperature.
190 + "top_p": 1.0,
191 + }
192 + output = RequestFuncOutput()
193 + output.prompt_len = request_func_input.prompt_len
194 +
195 + # DeepSpeed-MII doesn't support streaming as of Jan 28 2024, will use 0 as placeholder.
196 + # https://github.com/microsoft/DeepSpeed-MII/pull/311
197 + output.ttft = 0
198 +
199 + st = time.perf_counter()
200 + try:
201 + async with session.post(url=request_func_input.api_url,
202 + json=payload) as resp:
203 + if resp.status == 200:
204 + parsed_resp = await resp.json()
205 + output.latency = time.perf_counter() - st
206 + output.generated_text = parsed_resp[0]["generated_text"]
207 + output.success = True
208 + else:
209 + output.success = False
210 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
211 + output.success = False
212 +
213 + if pbar:
214 + pbar.update(1)
215 + return output
216 +
217 +
218 +async def async_request_openai_completions(
219 + request_func_input: RequestFuncInput,
220 + pbar: Optional[tqdm] = None,
221 +) -> RequestFuncOutput:
222 + api_url = request_func_input.api_url
223 + assert api_url.endswith("v1/completions")
224 +
225 + async with aiohttp.ClientSession(timeout=AIOHTTP_TIMEOUT) as session:
226 + assert not request_func_input.use_beam_search
227 + payload = {
228 + "model": request_func_input.model,
229 + "prompt": request_func_input.prompt,
230 + "temperature": 0.0,
231 + "best_of": request_func_input.best_of,
232 + "max_tokens": request_func_input.output_len,
233 + "stream": True,
234 + }
235 + headers = {
236 + "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
237 + }
238 +
239 + output = RequestFuncOutput()
240 + output.prompt_len = request_func_input.prompt_len
241 +
242 + generated_text = ""
243 + ttft = 0
244 + st = time.perf_counter()
245 + try:
246 + async with session.post(url=api_url, json=payload,
247 + headers=headers) as response:
248 + if response.status == 200:
249 + async for chunk in response.content:
250 + if ttft == 0:
251 + ttft = time.perf_counter() - st
252 + output.ttft = ttft
253 +
254 + chunk = chunk.strip()
255 + if not chunk:
256 + continue
257 +
258 + chunk = chunk.decode("utf-8").lstrip("data: ")
259 + if chunk == "[DONE]":
260 + latency = time.perf_counter() - st
261 + else:
262 + body = json.loads(chunk)
263 + generated_text += body["choices"][0]["text"]
264 +
265 + output.generated_text = generated_text
266 + output.success = True
267 + output.latency = latency
268 + else:
269 + output.success = False
270 + except (aiohttp.ClientOSError, aiohttp.ServerDisconnectedError):
271 + output.success = False
272 +
273 + if pbar:
274 + pbar.update(1)
275 + return output
276 +
277 +
278 +ASYNC_REQUEST_FUNCS = {
279 + "tgi": async_request_tgi,
280 + "vllm": async_request_vllm,
281 + "deepspeed-mii": async_request_deepspeed_mii,
282 + "openai": async_request_openai_completions,
283 + "tensorrt-llm": async_request_trt_llm,
284 +}
benchmarks/benchmark_serving.py
+258 -120
@@ -20,16 +20,36 @@ import asyncio
20 import json
21 import random
22 import time
23 +from dataclasses import dataclass
24 +from datetime import datetime
25 from typing import AsyncGenerator, List, Tuple
26
25 -import aiohttp
27 import numpy as np
28 from tqdm.asyncio import tqdm
29 from transformers import PreTrainedTokenizerBase
30 from vllm.transformers_utils.tokenizer import get_tokenizer
31
31 -# (prompt len, output len, latency)
32 -REQUEST_LATENCY: List[Tuple[int, int, float]] = []
32 +from backend_request_func import (
33 + ASYNC_REQUEST_FUNCS,
34 + RequestFuncInput,
35 + RequestFuncOutput,
36 +)
37 +
38 +
39 +@dataclass
40 +class BenchmarkMetrics:
41 + completed: int
42 + total_input: int
43 + total_output: int
44 + request_throughput: float
45 + input_throughput: float
46 + output_throughput: float
47 + mean_ttft_ms: float
48 + median_ttft_ms: float
49 + p99_ttft_ms: float
50 + mean_tpot_ms: float
51 + median_tpot_ms: float
52 + p99_tpot_ms: float
53
54
55 def sample_requests(
@@ -46,6 +66,11 @@ def sample_requests(
66 dataset = [(data["conversations"][0]["value"],
67 data["conversations"][1]["value"]) for data in dataset]
68
69 + # some of these will be filtered out, so sample more than we need
70 + sampled_indices = random.sample(range(len(dataset)),
71 + int(num_requests * 1.2))
72 + dataset = [dataset[i] for i in sampled_indices]
73 +
74 # Tokenize the prompts and completions.
75 prompts = [prompt for prompt, _ in dataset]
76 prompt_token_ids = tokenizer(prompts).input_ids
@@ -92,80 +117,125 @@ async def get_request(
117 await asyncio.sleep(interval)
118
119
95 -async def send_request(backend: str, model: str, api_url: str, prompt: str,
96 - prompt_len: int, output_len: int, best_of: int,
97 - use_beam_search: bool, pbar: tqdm) -> None:
98 - request_start_time = time.perf_counter()
99 -
100 - headers = {"User-Agent": "Benchmark Client"}
101 - if backend == "vllm":
102 - pload = {
103 - "prompt": prompt,
104 - "n": 1,
105 - "best_of": best_of,
106 - "use_beam_search": use_beam_search,
107 - "temperature": 0.0 if use_beam_search else 1.0,
108 - "top_p": 1.0,
109 - "max_tokens": output_len,
110 - "ignore_eos": True,
111 - "stream": False,
112 - }
113 - if model is not None:
114 - pload["model"] = model
115 - elif backend == "tgi":
116 - assert not use_beam_search
117 - params = {
118 - "best_of": best_of,
119 - "max_new_tokens": output_len,
120 - "do_sample": True,
121 - }
122 - pload = {
123 - "inputs": prompt,
124 - "parameters": params,
125 - }
126 - else:
127 - raise ValueError(f"Unknown backend: {backend}")
128 -
129 - timeout = aiohttp.ClientTimeout(total=3 * 3600)
130 - async with aiohttp.ClientSession(timeout=timeout) as session:
131 - while True:
132 - async with session.post(api_url, headers=headers,
133 - json=pload) as response:
134 - chunks = []
135 - async for chunk, _ in response.content.iter_chunks():
136 - chunks.append(chunk)
137 - output = b"".join(chunks).decode("utf-8")
138 - output = json.loads(output)
120 +def calculate_metrics(
121 + input_requests: List[Tuple[str, int, int]],
122 + outputs: List[RequestFuncOutput],
123 + dur_s: float,
124 + tokenizer: PreTrainedTokenizerBase,
125 +) -> BenchmarkMetrics:
126 + total_output = 0
127 + total_input = 0
128 + completed = 0
129 + per_token_latencies = []
130 + ttfts = []
131 + for i in range(len(outputs)):
132 + if outputs[i].success:
133 + output_len = len(tokenizer.encode(outputs[i].generated_text))
134 + total_output += output_len
135 + total_input += input_requests[i][1]
136 + per_token_latencies.append(outputs[i].latency / output_len)
137 + ttfts.append(outputs[i].ttft)
138 + completed += 1
139
140 - # Re-send the request if it failed.
141 - if "error" not in output:
142 - break
140 + metrics = BenchmarkMetrics(
141 + completed=completed,
142 + total_input=total_input,
143 + total_output=total_output,
144 + request_throughput=completed / dur_s,
145 + input_throughput=total_input / dur_s,
146 + output_throughput=total_output / dur_s,
147 + mean_ttft_ms=np.mean(ttfts) * 1000,
148 + median_ttft_ms=np.median(ttfts) * 1000,
149 + p99_ttft_ms=np.percentile(ttfts, 99) * 1000,
150 + mean_tpot_ms=np.mean(per_token_latencies) * 1000,
151 + median_tpot_ms=np.median(per_token_latencies) * 1000,
152 + p99_tpot_ms=np.percentile(per_token_latencies, 99) * 1000,
153 + )
154
144 - request_end_time = time.perf_counter()
145 - request_latency = request_end_time - request_start_time
146 - REQUEST_LATENCY.append((prompt_len, output_len, request_latency))
147 - pbar.update(1)
155 + return metrics
156
157
158 async def benchmark(
159 backend: str,
152 - model: str,
160 api_url: str,
161 + model_id: str,
162 + tokenizer: PreTrainedTokenizerBase,
163 input_requests: List[Tuple[str, int, int]],
164 best_of: int,
165 use_beam_search: bool,
166 request_rate: float,
158 -) -> None:
159 - tasks: List[asyncio.Task] = []
160 - pbar = tqdm(total=len(input_requests))
167 + disable_tqdm: bool,
168 +):
169 + if backend in ASYNC_REQUEST_FUNCS:
170 + request_func = ASYNC_REQUEST_FUNCS.get(backend)
171 + else:
172 + raise ValueError(f"Unknown backend: {backend}")
173 +
174 + pbar = None if disable_tqdm else tqdm(total=len(input_requests))
175 +
176 + print(f"Traffic request rate: {request_rate}")
177 +
178 + benchmark_start_time = time.perf_counter()
179 + tasks = []
180 async for request in get_request(input_requests, request_rate):
181 prompt, prompt_len, output_len = request
163 - task = asyncio.create_task(
164 - send_request(backend, model, api_url, prompt, prompt_len,
165 - output_len, best_of, use_beam_search, pbar))
166 - tasks.append(task)
167 - await asyncio.gather(*tasks)
168 - pbar.close()
182 + request_func_input = RequestFuncInput(
183 + model=model_id,
184 + prompt=prompt,
185 + api_url=api_url,
186 + prompt_len=prompt_len,
187 + output_len=output_len,
188 + best_of=best_of,
189 + use_beam_search=use_beam_search,
190 + )
191 + tasks.append(
192 + asyncio.create_task(
193 + request_func(request_func_input=request_func_input,
194 + pbar=pbar)))
195 + outputs = await asyncio.gather(*tasks)
196 +
197 + if not disable_tqdm:
198 + pbar.close()
199 +
200 + benchmark_duration = time.perf_counter() - benchmark_start_time
201 +
202 + metrics = calculate_metrics(
203 + input_requests=input_requests,
204 + outputs=outputs,
205 + dur_s=benchmark_duration,
206 + tokenizer=tokenizer,
207 + )
208 +
209 + print(f"Successful requests: {metrics.completed}")
210 + print(f"Benchmark duration: {benchmark_duration:2f} s")
211 + print(f"Total input tokens: {metrics.total_input}")
212 + print(f"Total generated tokens: {metrics.total_output}")
213 + print(f"Request throughput: {metrics.request_throughput:.2f} requests/s")
214 + print(f"Input token throughput: {metrics.input_throughput:.2f} tokens/s")
215 + print(f"Output token throughput: {metrics.output_throughput:.2f} tokens/s")
216 + print(f"Mean TTFT: {metrics.mean_ttft_ms:.2f} ms")
217 + print(f"Median TTFT: {metrics.median_ttft_ms:.2f} ms")
218 + print(f"P99 TTFT: {metrics.p99_ttft_ms:.2f} ms")
219 + print(f"Mean TPOT: {metrics.mean_tpot_ms:.2f} ms")
220 + print(f"Median TPOT: {metrics.median_tpot_ms:.2f} ms")
221 + print(f"P99 TPOT: {metrics.p99_tpot_ms:.2f} ms")
222 +
223 + result = {
224 + "duration": benchmark_duration,
225 + "completed": metrics.completed,
226 + "total_input_tokens": metrics.total_input,
227 + "total_output_tokens": metrics.total_output,
228 + "request_inthroughput": metrics.request_throughput,
229 + "input_throughput": metrics.input_throughput,
230 + "output_throughput": metrics.output_throughput,
231 + "mean_ttft_ms": metrics.mean_ttft_ms,
232 + "median_ttft_ms": metrics.median_ttft_ms,
233 + "p99_ttft_ms": metrics.p99_ttft_ms,
234 + "mean_tpot_ms": metrics.mean_tpot_ms,
235 + "median_tpot_ms": metrics.median_tpot_ms,
236 + "p99_tpot_ms": metrics.p99_tpot_ms
237 + }
238 + return result
239
240
241 def main(args: argparse.Namespace):
@@ -173,77 +243,145 @@ def main(args: argparse.Namespace):
243 random.seed(args.seed)
244 np.random.seed(args.seed)
245
176 - api_url = f"{args.protocol}://{args.host}:{args.port}{args.endpoint}"
177 - tokenizer = get_tokenizer(args.tokenizer,
246 + backend = args.backend
247 + model_id = args.model
248 + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
249 +
250 + if args.base_url is not None:
251 + api_url = f"{args.base_url}{args.endpoint}"
252 + else:
253 + api_url = f"http://{args.host}:{args.port}{args.endpoint}"
254 +
255 + tokenizer = get_tokenizer(tokenizer_id,
256 trust_remote_code=args.trust_remote_code)
257 input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
258
181 - benchmark_start_time = time.perf_counter()
182 - asyncio.run(
183 - benchmark(args.backend, args.model, api_url, input_requests,
184 - args.best_of, args.use_beam_search, args.request_rate))
185 - benchmark_end_time = time.perf_counter()
186 - benchmark_time = benchmark_end_time - benchmark_start_time
187 - print(f"Total time: {benchmark_time:.2f} s")
188 - print(f"Throughput: {args.num_prompts / benchmark_time:.2f} requests/s")
189 -
190 - # Compute the latency statistics.
191 - avg_latency = np.mean([latency for _, _, latency in REQUEST_LATENCY])
192 - print(f"Average latency: {avg_latency:.2f} s")
193 - avg_per_token_latency = np.mean([
194 - latency / (prompt_len + output_len)
195 - for prompt_len, output_len, latency in REQUEST_LATENCY
196 - ])
197 - print(f"Average latency per token: {avg_per_token_latency:.2f} s")
198 - avg_per_output_token_latency = np.mean(
199 - [latency / output_len for _, output_len, latency in REQUEST_LATENCY])
200 - print("Average latency per output token: "
201 - f"{avg_per_output_token_latency:.2f} s")
259 + benchmark_result = asyncio.run(
260 + benchmark(
261 + backend=backend,
262 + api_url=api_url,
263 + model_id=model_id,
264 + tokenizer=tokenizer,
265 + input_requests=input_requests,
266 + best_of=args.best_of,
267 + use_beam_search=args.use_beam_search,
268 + request_rate=args.request_rate,
269 + disable_tqdm=args.disable_tqdm,
270 + ))
271 +
272 + # Save config and results to json
273 + if args.save_result:
274 + result_json = {}
275 +
276 + # Setup
277 + current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
278 + result_json["date"] = current_dt
279 + result_json["backend"] = backend
280 + result_json["version"] = args.version
281 + result_json["model_id"] = model_id
282 + result_json["tokenizer_id"] = tokenizer_id
283 + result_json["best_of"] = args.best_of
284 + result_json["use_beam_search"] = args.use_beam_search
285 + result_json["num_prompts"] = args.num_prompts
286 +
287 + # Traffic
288 + result_json["request_rate"] = (
289 + args.request_rate if args.request_rate < float("inf") else "inf")
290 +
291 + # Merge with benchmark result
292 + result_json = {**result_json, **benchmark_result}
293 +
294 + # Save to file
295 + base_model_id = model_id.split("/")[-1]
296 + file_name = f"{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
297 + with open(file_name, "w") as outfile:
298 + json.dump(result_json, outfile)
299
300
301 if __name__ == "__main__":
302 parser = argparse.ArgumentParser(
303 description="Benchmark the online serving throughput.")
207 - parser.add_argument("--backend",
208 - type=str,
209 - default="vllm",
210 - choices=["vllm", "tgi"])
211 - parser.add_argument("--protocol",
212 - type=str,
213 - default="http",
214 - choices=["http", "https"])
304 + parser.add_argument(
305 + "--backend",
306 + type=str,
307 + default="vllm",
308 + choices=list(ASYNC_REQUEST_FUNCS.keys()),
309 + )
310 + parser.add_argument(
311 + "--version",
312 + type=str,
313 + default="N/A",
314 + help="Version of the serving backend/engine.",
315 + )
316 + parser.add_argument(
317 + "--base-url",
318 + type=str,
319 + default=None,
320 + help="Server or API base url if not using http host and port.",
321 + )
322 parser.add_argument("--host", type=str, default="localhost")
323 parser.add_argument("--port", type=int, default=8000)
217 - parser.add_argument("--endpoint", type=str, default="/generate")
218 - parser.add_argument("--model", type=str, default=None)
324 + parser.add_argument(
325 + "--endpoint",
326 + type=str,
327 + default="/generate",
328 + help="API endpoint.",
329 + )
330 parser.add_argument("--dataset",
331 type=str,
332 required=True,
333 help="Path to the dataset.")
223 - parser.add_argument("--tokenizer",
224 - type=str,
225 - required=True,
226 - help="Name or path of the tokenizer.")
227 - parser.add_argument("--best-of",
228 - type=int,
229 - default=1,
230 - help="Generates `best_of` sequences per prompt and "
231 - "returns the best one.")
334 + parser.add_argument(
335 + "--model",
336 + type=str,
337 + required=True,
338 + help="Name of the model.",
339 + )
340 + parser.add_argument(
341 + "--tokenizer",
342 + type=str,
343 + help=
344 + "Name or path of the tokenizer, if not using the default model tokenizer.",
345 + )
346 + parser.add_argument(
347 + "--best-of",
348 + type=int,
349 + default=1,
350 + help="Generates `best_of` sequences per prompt and "
351 + "returns the best one.",
352 + )
353 parser.add_argument("--use-beam-search", action="store_true")
233 - parser.add_argument("--num-prompts",
234 - type=int,
235 - default=1000,
236 - help="Number of prompts to process.")
237 - parser.add_argument("--request-rate",
238 - type=float,
239 - default=float("inf"),
240 - help="Number of requests per second. If this is inf, "
241 - "then all the requests are sent at time 0. "
242 - "Otherwise, we use Poisson process to synthesize "
243 - "the request arrival times.")
354 + parser.add_argument(
355 + "--num-prompts",
356 + type=int,
357 + default=1000,
358 + help="Number of prompts to process.",
359 + )
360 + parser.add_argument(
361 + "--request-rate",
362 + type=float,
363 + default=float("inf"),
364 + help="Number of requests per second. If this is inf, "
365 + "then all the requests are sent at time 0. "
366 + "Otherwise, we use Poisson process to synthesize "
367 + "the request arrival times.",
368 + )
369 parser.add_argument("--seed", type=int, default=0)
245 - parser.add_argument('--trust-remote-code',
246 - action='store_true',
247 - help='trust remote code from huggingface')
370 + parser.add_argument(
371 + "--trust-remote-code",
372 + action="store_true",
373 + help="Trust remote code from huggingface",
374 + )
375 + parser.add_argument(
376 + "--disable-tqdm",
377 + action="store_true",
378 + help="Specify to disbale tqdm progress bar.",
379 + )
380 + parser.add_argument(
381 + "--save-result",
382 + action="store_true",
383 + help="Specify to save benchmark results to a json file",
384 + )
385 +
386 args = parser.parse_args()
387 main(args)
benchmarks/launch_tgi_server.sh
+1 -1
@@ -6,7 +6,7 @@ TOKENS=$2
6
7 docker run --gpus all --shm-size 1g -p $PORT:80 \
8 -v $PWD/data:/data \
9 - ghcr.io/huggingface/text-generation-inference:0.8 \
9 + ghcr.io/huggingface/text-generation-inference:1.4.0 \
10 --model-id $MODEL \
11 --sharded false \
12 --max-input-length 1024 \