A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Support OpenAI API server in `benchmark_serving.py` (#2172)
Harry Mellor committed
Jan 19, 2024 at 04:34 UTC
2709c0009aa434fbf2ef0fe48ca3094a2268b190
2 files changed
+51
-32
.gitignore
+3
@@ -181,3 +181,6 @@ _build/
181
# hip files generated by PyTorch
182
*.hip
183
*_hip*
184
+
185
+# Benchmark dataset
186
+*.json
benchmarks/benchmark_serving.py
+48
-32
@@ -24,6 +24,7 @@ from typing import AsyncGenerator, List, Tuple
24
25
import aiohttp
26
import numpy as np
27
+from tqdm.asyncio import tqdm
28
from transformers import PreTrainedTokenizerBase
29
from vllm.transformers_utils.tokenizer import get_tokenizer
30
@@ -40,15 +41,10 @@ def sample_requests(
41
with open(dataset_path) as f:
42
dataset = json.load(f)
43
# Filter out the conversations with less than 2 turns.
43
- dataset = [
44
- data for data in dataset
45
- if len(data["conversations"]) >= 2
46
- ]
44
+ dataset = [data for data in dataset if len(data["conversations"]) >= 2]
45
# Only keep the first two turns of each conversation.
48
- dataset = [
49
- (data["conversations"][0]["value"], data["conversations"][1]["value"])
50
- for data in dataset
51
- ]
46
+ dataset = [(data["conversations"][0]["value"],
47
+ data["conversations"][1]["value"]) for data in dataset]
48
49
# Tokenize the prompts and completions.
50
prompts = [prompt for prompt, _ in dataset]
@@ -98,6 +94,7 @@ async def get_request(
94
95
async def send_request(
96
backend: str,
97
+ model: str,
98
api_url: str,
99
prompt: str,
100
prompt_len: int,
@@ -120,6 +117,8 @@ async def send_request(
117
"ignore_eos": True,
118
"stream": False,
119
}
120
+ if model is not None:
121
+ pload["model"] = model
122
elif backend == "tgi":
123
assert not use_beam_search
124
params = {
@@ -137,7 +136,8 @@ async def send_request(
136
timeout = aiohttp.ClientTimeout(total=3 * 3600)
137
async with aiohttp.ClientSession(timeout=timeout) as session:
138
while True:
140
- async with session.post(api_url, headers=headers, json=pload) as response:
139
+ async with session.post(api_url, headers=headers,
140
+ json=pload) as response:
141
chunks = []
142
async for chunk, _ in response.content.iter_chunks():
143
chunks.append(chunk)
@@ -155,6 +155,7 @@ async def send_request(
155
156
async def benchmark(
157
backend: str,
158
+ model: str,
159
api_url: str,
160
input_requests: List[Tuple[str, int, int]],
161
best_of: int,
@@ -164,11 +165,11 @@ async def benchmark(
165
tasks: List[asyncio.Task] = []
166
async for request in get_request(input_requests, request_rate):
167
prompt, prompt_len, output_len = request
167
- task = asyncio.create_task(send_request(backend, api_url, prompt,
168
- prompt_len, output_len,
169
- best_of, use_beam_search))
168
+ task = asyncio.create_task(
169
+ send_request(backend, model, api_url, prompt, prompt_len,
170
+ output_len, best_of, use_beam_search))
171
tasks.append(task)
171
- await asyncio.gather(*tasks)
172
+ await tqdm.gather(*tasks)
173
174
175
def main(args: argparse.Namespace):
@@ -176,13 +177,15 @@ def main(args: argparse.Namespace):
177
random.seed(args.seed)
178
np.random.seed(args.seed)
179
179
- api_url = f"http://{args.host}:{args.port}/generate"
180
- tokenizer = get_tokenizer(args.tokenizer, trust_remote_code=args.trust_remote_code)
180
+ api_url = f"http://{args.host}:{args.port}{args.endpoint}"
181
+ tokenizer = get_tokenizer(args.tokenizer,
182
+ trust_remote_code=args.trust_remote_code)
183
input_requests = sample_requests(args.dataset, args.num_prompts, tokenizer)
184
185
benchmark_start_time = time.perf_counter()
184
- asyncio.run(benchmark(args.backend, api_url, input_requests, args.best_of,
185
- args.use_beam_search, args.request_rate))
186
+ asyncio.run(
187
+ benchmark(args.backend, args.model, api_url, input_requests,
188
+ args.best_of, args.use_beam_search, args.request_rate))
189
benchmark_end_time = time.perf_counter()
190
benchmark_time = benchmark_end_time - benchmark_start_time
191
print(f"Total time: {benchmark_time:.2f} s")
@@ -196,10 +199,8 @@ def main(args: argparse.Namespace):
199
for prompt_len, output_len, latency in REQUEST_LATENCY
200
])
201
print(f"Average latency per token: {avg_per_token_latency:.2f} s")
199
- avg_per_output_token_latency = np.mean([
200
- latency / output_len
201
- for _, output_len, latency in REQUEST_LATENCY
202
- ])
202
+ avg_per_output_token_latency = np.mean(
203
+ [latency / output_len for _, output_len, latency in REQUEST_LATENCY])
204
print("Average latency per output token: "
205
f"{avg_per_output_token_latency:.2f} s")
206
@@ -207,27 +208,42 @@ def main(args: argparse.Namespace):
208
if __name__ == "__main__":
209
parser = argparse.ArgumentParser(
210
description="Benchmark the online serving throughput.")
210
- parser.add_argument("--backend", type=str, default="vllm",
211
+ parser.add_argument("--backend",
212
+ type=str,
213
+ default="vllm",
214
choices=["vllm", "tgi"])
215
parser.add_argument("--host", type=str, default="localhost")
216
parser.add_argument("--port", type=int, default=8000)
214
- parser.add_argument("--dataset", type=str, required=True,
217
+ parser.add_argument("--endpoint", type=str, default="/generate")
218
+ parser.add_argument("--model", type=str, default=None)
219
+ parser.add_argument("--dataset",
220
+ type=str,
221
+ required=True,
222
help="Path to the dataset.")
216
- parser.add_argument("--tokenizer", type=str, required=True,
223
+ parser.add_argument("--tokenizer",
224
+ type=str,
225
+ required=True,
226
help="Name or path of the tokenizer.")
218
- parser.add_argument("--best-of", type=int, default=1,
227
+ parser.add_argument("--best-of",
228
+ type=int,
229
+ default=1,
230
help="Generates `best_of` sequences per prompt and "
220
- "returns the best one.")
231
+ "returns the best one.")
232
parser.add_argument("--use-beam-search", action="store_true")
222
- parser.add_argument("--num-prompts", type=int, default=1000,
233
+ parser.add_argument("--num-prompts",
234
+ type=int,
235
+ default=1000,
236
help="Number of prompts to process.")
224
- parser.add_argument("--request-rate", type=float, default=float("inf"),
237
+ parser.add_argument("--request-rate",
238
+ type=float,
239
+ default=float("inf"),
240
help="Number of requests per second. If this is inf, "
226
- "then all the requests are sent at time 0. "
227
- "Otherwise, we use Poisson process to synthesize "
228
- "the request arrival times.")
241
+ "then all the requests are sent at time 0. "
242
+ "Otherwise, we use Poisson process to synthesize "
243
+ "the request arrival times.")
244
parser.add_argument("--seed", type=int, default=0)
230
- parser.add_argument('--trust-remote-code', action='store_true',
245
+ parser.add_argument('--trust-remote-code',
246
+ action='store_true',
247
help='trust remote code from huggingface')
248
args = parser.parse_args()
249
main(args)