A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
Benchmark serving structured output (#10880)
Signed-off-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
Chendi.Xue committed
Dec 4, 2024 at 15:28 UTC
82eb5ea8f3bd3aabbe5c2fd43e37d263768603c5
2 files changed
+887
benchmarks/backend_request_func.py
+6
@@ -24,6 +24,7 @@ class RequestFuncInput:
24
model: str
25
best_of: int = 1
26
logprobs: Optional[int] = None
27
+ extra_body: Optional[dict] = None
28
multi_modal_content: Optional[dict] = None
29
ignore_eos: bool = False
30
@@ -36,6 +37,7 @@ class RequestFuncOutput:
37
ttft: float = 0.0 # Time to first token
38
itl: List[float] = field(
39
default_factory=list) # List of inter-token latencies
40
+ tpot: float = 0.0 # avg next-token latencies
41
prompt_len: int = 0
42
error: str = ""
43
@@ -242,6 +244,8 @@ async def async_request_openai_completions(
244
"stream": True,
245
"ignore_eos": request_func_input.ignore_eos,
246
}
247
+ if request_func_input.extra_body:
248
+ payload.update(request_func_input.extra_body)
249
headers = {
250
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
251
}
@@ -336,6 +340,8 @@ async def async_request_openai_chat_completions(
340
"stream": True,
341
"ignore_eos": request_func_input.ignore_eos,
342
}
343
+ if request_func_input.extra_body:
344
+ payload.update(request_func_input.extra_body)
345
headers = {
346
"Content-Type": "application/json",
347
"Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
benchmarks/benchmark_serving_guided.py
new
+881
@@ -0,0 +1,881 @@
1
+r"""Benchmark online serving throughput with guided decoding.
2
+
3
+On the server side, run one of the following commands:
4
+ (vLLM OpenAI API server)
5
+ vllm serve <your_model> --disable-log-requests
6
+
7
+ (TGI backend)
8
+ ./launch_tgi_server.sh <your_model> <max_batch_total_tokens>
9
+
10
+On the client side, run:
11
+ python benchmarks/benchmark_serving.py \
12
+ --backend <backend> \
13
+ --model <your_model> \
14
+ --dataset json \
15
+ --guided-decoding-ratio 1.0 \
16
+ --guided-decoding-backend xgrammar \
17
+ --request-rate 10 \
18
+ --num-prompts 1000
19
+
20
+ when using tgi backend, add
21
+ --endpoint /generate_stream
22
+ to the end of the command above.
23
+"""
24
+import argparse
25
+import asyncio
26
+import dataclasses
27
+import json
28
+import os
29
+import random
30
+import time
31
+import warnings
32
+from dataclasses import dataclass
33
+from typing import AsyncGenerator, List, Optional, Tuple
34
+
35
+import datasets
36
+import numpy as np
37
+import pandas as pd
38
+from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
39
+ RequestFuncOutput)
40
+from tqdm.asyncio import tqdm
41
+from transformers import PreTrainedTokenizerBase
42
+
43
+try:
44
+ from vllm.transformers_utils.tokenizer import get_tokenizer
45
+except ImportError:
46
+ from backend_request_func import get_tokenizer
47
+
48
+try:
49
+ from vllm.utils import FlexibleArgumentParser
50
+except ImportError:
51
+ from argparse import ArgumentParser as FlexibleArgumentParser
52
+
53
+MILLISECONDS_TO_SECONDS_CONVERSION = 1000
54
+
55
+
56
+@dataclass
57
+class BenchmarkMetrics:
58
+ completed: int
59
+ total_input: int
60
+ total_output: int
61
+ request_throughput: float
62
+ request_goodput: float
63
+ output_throughput: float
64
+ total_token_throughput: float
65
+ mean_ttft_ms: float
66
+ median_ttft_ms: float
67
+ std_ttft_ms: float
68
+ percentiles_ttft_ms: List[Tuple[float, float]]
69
+ mean_tpot_ms: float
70
+ median_tpot_ms: float
71
+ std_tpot_ms: float
72
+ percentiles_tpot_ms: List[Tuple[float, float]]
73
+ mean_itl_ms: float
74
+ median_itl_ms: float
75
+ std_itl_ms: float
76
+ percentiles_itl_ms: List[Tuple[float, float]]
77
+ # E2EL stands for end-to-end latency per request.
78
+ # It is the time taken on the client side from sending
79
+ # a request to receiving a complete response.
80
+ mean_e2el_ms: float
81
+ median_e2el_ms: float
82
+ std_e2el_ms: float
83
+ percentiles_e2el_ms: List[Tuple[float, float]]
84
+
85
+
86
+@dataclasses.dataclass
87
+class SampleRequest:
88
+ """A class representing a single inference request for benchmarking.
89
+
90
+ Attributes:
91
+ prompt: The input text prompt for the model.
92
+ multi_modal_data: Optional dictionary containing multi-modal data (e.g.
93
+ images).
94
+ prompt_len: The length of the prompt in tokens.
95
+ expected_output_len: The expected length of the output in tokens.
96
+ """
97
+ prompt: str
98
+ prompt_len: int
99
+ expected_output_len: int
100
+ schema: dict
101
+ structure_type: str
102
+ completion: str = None
103
+
104
+
105
+def sample_requests(tokenizer: PreTrainedTokenizerBase,
106
+ args: argparse.Namespace) -> List[SampleRequest]:
107
+ if args.dataset == 'json':
108
+ if args.json_schema_path is None:
109
+ dir_path = os.path.dirname(os.path.realpath(__file__))
110
+ args.json_schema_path = os.path.join(dir_path,
111
+ "structured_schemas",
112
+ "structured_schema_1.json")
113
+ with open(args.json_schema_path) as f:
114
+ schema = json.load(f)
115
+ prompt = f"Generate an example of a user profile given the following schema: {json.dumps(schema)}" # noqa: E501
116
+ input_len = len(tokenizer(prompt).input_ids)
117
+ print(f"Input length of the prompt: {input_len} tokens")
118
+ requests = [
119
+ SampleRequest(prompt=prompt,
120
+ prompt_len=input_len,
121
+ expected_output_len=args.output_len,
122
+ schema=schema,
123
+ structure_type=args.structure_type)
124
+ for _ in range(args.num_prompts)
125
+ ]
126
+
127
+ elif args.dataset == "grammar":
128
+ schema = """
129
+ ?start: select_statement
130
+
131
+ ?select_statement: "SELECT " column_list " FROM " table_name
132
+
133
+ ?column_list: column_name ("," column_name)*
134
+
135
+ ?table_name: identifier
136
+
137
+ ?column_name: identifier
138
+
139
+ ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
140
+ """
141
+ prompt = "Generate an SQL query to show the 'username' \
142
+ and 'email' from the 'users' table."
143
+
144
+ input_len = len(tokenizer(prompt).input_ids)
145
+ print(f"Input length of the prompt: {input_len} tokens")
146
+ requests = [
147
+ SampleRequest(prompt=prompt,
148
+ prompt_len=input_len,
149
+ expected_output_len=args.output_len,
150
+ schema=schema,
151
+ structure_type=args.structure_type)
152
+ for _ in range(args.num_prompts)
153
+ ]
154
+
155
+ elif args.dataset == "regex":
156
+ regex = r"\w+@\w+\.com\n"
157
+ args.regex = regex
158
+ prompt = "Generate an email address for Alan Turing, \
159
+ who works in Enigma. End in .com and new line. \
160
+ Example result: alan.turing@enigma.com\n"
161
+
162
+ input_len = len(tokenizer(prompt).input_ids)
163
+ print(f"Input length of the prompt: {input_len} tokens")
164
+ requests = [
165
+ SampleRequest(prompt=prompt,
166
+ prompt_len=input_len,
167
+ expected_output_len=args.output_len,
168
+ schema=regex,
169
+ structure_type=args.structure_type)
170
+ for _ in range(args.num_prompts)
171
+ ]
172
+
173
+ elif args.dataset == "choice":
174
+ choice = ["Positive", "Negative"]
175
+ args.choice = choice
176
+ prompt = "Classify this sentiment: vLLM is wonderful!"
177
+ input_len = len(tokenizer(prompt).input_ids)
178
+ print(f"Input length of the prompt: {input_len} tokens")
179
+ requests = [
180
+ SampleRequest(prompt=prompt,
181
+ prompt_len=input_len,
182
+ expected_output_len=args.output_len,
183
+ schema=choice,
184
+ structure_type=args.structure_type)
185
+ for _ in range(args.num_prompts)
186
+ ]
187
+
188
+ elif args.dataset == "xgrammar_bench":
189
+ requests: List[SampleRequest] = []
190
+ dataset = datasets.load_dataset("NousResearch/json-mode-eval",
191
+ split="train")
192
+ print(f"dataset has {len(dataset)} entries")
193
+ len_dataset = len(dataset)
194
+ for data_point_idx in range(args.num_prompts):
195
+ idx = data_point_idx
196
+ while idx >= len_dataset:
197
+ idx -= len_dataset
198
+ schema = dataset["schema"][idx]
199
+ prompt = tokenizer.apply_chat_template(dataset["prompt"][idx],
200
+ tokenize=False)
201
+ input_len = len(tokenizer(prompt).input_ids)
202
+ completion = dataset["completion"][idx]
203
+
204
+ requests.append(
205
+ SampleRequest(prompt=prompt,
206
+ prompt_len=input_len,
207
+ expected_output_len=args.output_len,
208
+ schema=schema,
209
+ structure_type=args.structure_type,
210
+ completion=completion))
211
+
212
+ return requests
213
+
214
+
215
+async def get_request(
216
+ input_requests: List[SampleRequest],
217
+ request_rate: float,
218
+ burstiness: float = 1.0,
219
+) -> AsyncGenerator[Tuple[int, SampleRequest], None]:
220
+ """
221
+ Asynchronously generates requests at a specified rate
222
+ with OPTIONAL burstiness.
223
+
224
+ Args:
225
+ input_requests:
226
+ A list of input requests, each represented as a tuple.
227
+ request_rate:
228
+ The rate at which requests are generated (requests/s).
229
+ burstiness (optional):
230
+ The burstiness factor of the request generation.
231
+ Only takes effect when request_rate is not inf.
232
+ Default value is 1, which follows a Poisson process.
233
+ Otherwise, the request intervals follow a gamma distribution.
234
+ A lower burstiness value (0 < burstiness < 1) results
235
+ in more bursty requests, while a higher burstiness value
236
+ (burstiness > 1) results in a more uniform arrival of requests.
237
+ """
238
+ input_requests = iter(input_requests)
239
+
240
+ # Calculate scale parameter theta to maintain the desired request_rate.
241
+ assert burstiness > 0, (
242
+ f"A positive burstiness factor is expected, but given {burstiness}.")
243
+ theta = 1.0 / (request_rate * burstiness)
244
+
245
+ for i, request in enumerate(input_requests):
246
+ yield i, request
247
+
248
+ if request_rate == float("inf"):
249
+ # If the request rate is infinity, then we don't need to wait.
250
+ continue
251
+
252
+ # Sample the request interval from the gamma distribution.
253
+ # If burstiness is 1, it follows exponential distribution.
254
+ interval = np.random.gamma(shape=burstiness, scale=theta)
255
+ # The next request will be sent after the interval.
256
+ await asyncio.sleep(interval)
257
+
258
+
259
+def calculate_metrics(
260
+ input_requests: List[Tuple[str, int, int]],
261
+ outputs: List[RequestFuncOutput],
262
+ dur_s: float,
263
+ tokenizer: PreTrainedTokenizerBase,
264
+ selected_percentile_metrics: List[str],
265
+ selected_percentiles: List[float],
266
+) -> Tuple[BenchmarkMetrics, List[int]]:
267
+ actual_output_lens: List[int] = []
268
+ total_input = 0
269
+ completed = 0
270
+ good_completed = 0
271
+ itls: List[float] = []
272
+ tpots: List[float] = []
273
+ all_tpots: List[float] = []
274
+ ttfts: List[float] = []
275
+ e2els: List[float] = []
276
+ for i in range(len(outputs)):
277
+ if outputs[i].success:
278
+ # We use the tokenizer to count the number of output tokens for all
279
+ # serving backends instead of looking at len(outputs[i].itl) since
280
+ # multiple output tokens may be bundled together
281
+ # Note : this may inflate the output token count slightly
282
+ output_len = len(
283
+ tokenizer(outputs[i].generated_text,
284
+ add_special_tokens=False).input_ids)
285
+ actual_output_lens.append(output_len)
286
+ total_input += input_requests[i].prompt_len
287
+ tpot = 0
288
+ if output_len > 1:
289
+ tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
290
+ 1)
291
+ tpots.append(tpot)
292
+ outputs[i].tpot = sum(tpots) / len(tpots) if len(tpots) else 0
293
+ # Note: if output_len <= 1, we regard tpot as 0 for goodput
294
+ all_tpots.append(tpot)
295
+ itls += outputs[i].itl
296
+ ttfts.append(outputs[i].ttft)
297
+ e2els.append(outputs[i].latency)
298
+ completed += 1
299
+ else:
300
+ actual_output_lens.append(0)
301
+
302
+ if completed == 0:
303
+ warnings.warn(
304
+ "All requests failed. This is likely due to a misconfiguration "
305
+ "on the benchmark arguments.",
306
+ stacklevel=2)
307
+ metrics = BenchmarkMetrics(
308
+ completed=completed,
309
+ total_input=total_input,
310
+ total_output=sum(actual_output_lens),
311
+ request_throughput=completed / dur_s,
312
+ request_goodput=good_completed / dur_s,
313
+ output_throughput=sum(actual_output_lens) / dur_s,
314
+ total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
315
+ mean_ttft_ms=np.mean(ttfts or 0) *
316
+ 1000, # ttfts is empty if streaming is not supported by backend
317
+ std_ttft_ms=np.std(ttfts or 0) * 1000,
318
+ median_ttft_ms=np.median(ttfts or 0) * 1000,
319
+ percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
320
+ for p in selected_percentiles],
321
+ mean_tpot_ms=np.mean(tpots or 0) * 1000,
322
+ std_tpot_ms=np.std(tpots or 0) * 1000,
323
+ median_tpot_ms=np.median(tpots or 0) * 1000,
324
+ percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
325
+ for p in selected_percentiles],
326
+ mean_itl_ms=np.mean(itls or 0) * 1000,
327
+ std_itl_ms=np.std(itls or 0) * 1000,
328
+ median_itl_ms=np.median(itls or 0) * 1000,
329
+ percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
330
+ for p in selected_percentiles],
331
+ mean_e2el_ms=np.mean(e2els or 0) * 1000,
332
+ std_e2el_ms=np.std(e2els or 0) * 1000,
333
+ median_e2el_ms=np.median(e2els or 0) * 1000,
334
+ percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
335
+ for p in selected_percentiles],
336
+ )
337
+
338
+ return metrics, actual_output_lens
339
+
340
+
341
+async def benchmark(
342
+ backend: str,
343
+ api_url: str,
344
+ base_url: str,
345
+ model_id: str,
346
+ tokenizer: PreTrainedTokenizerBase,
347
+ input_requests: List[SampleRequest],
348
+ request_rate: float,
349
+ burstiness: float,
350
+ disable_tqdm: bool,
351
+ profile: bool,
352
+ selected_percentile_metrics: List[str],
353
+ selected_percentiles: List[str],
354
+ ignore_eos: bool,
355
+ max_concurrency: Optional[int],
356
+ guided_decoding_ratio: float,
357
+ guided_decoding_backend: str,
358
+):
359
+ if backend in ASYNC_REQUEST_FUNCS:
360
+ request_func = ASYNC_REQUEST_FUNCS[backend]
361
+ else:
362
+ raise ValueError(f"Unknown backend: {backend}")
363
+
364
+ def prepare_extra_body(request) -> dict:
365
+ extra_body = {}
366
+ # Add the schema to the extra_body
367
+ extra_body[request.structure_type] = request.schema
368
+ # Add the specific guided_decoding_backend
369
+ extra_body["guided_decoding_backend"] = guided_decoding_backend
370
+ return extra_body
371
+
372
+ print("Starting initial single prompt test run...")
373
+ guided_decoding_req_idx = random.sample(
374
+ range(len(input_requests)),
375
+ int(len(input_requests) * guided_decoding_ratio))
376
+
377
+ test_request = input_requests[0]
378
+ test_input = RequestFuncInput(
379
+ model=model_id,
380
+ prompt=test_request.prompt,
381
+ api_url=api_url,
382
+ prompt_len=test_request.prompt_len,
383
+ output_len=test_request.expected_output_len,
384
+ ignore_eos=ignore_eos,
385
+ extra_body=prepare_extra_body(test_request),
386
+ )
387
+ test_output = await request_func(request_func_input=test_input)
388
+ if not test_output.success:
389
+ raise ValueError(
390
+ "Initial test run failed - Please make sure benchmark arguments "
391
+ f"are correctly specified. Error: {test_output.error}")
392
+ else:
393
+ print("Initial test run completed. Starting main benchmark run...")
394
+
395
+ if profile:
396
+ print("Starting profiler...")
397
+ profile_input = RequestFuncInput(
398
+ model=model_id,
399
+ prompt=test_request.prompt,
400
+ api_url=base_url + "/start_profile",
401
+ prompt_len=test_request.prompt_len,
402
+ output_len=test_request.expected_output_len,
403
+ ignore_eos=ignore_eos,
404
+ extra_body=prepare_extra_body(test_request),
405
+ )
406
+ profile_output = await request_func(request_func_input=profile_input)
407
+ if profile_output.success:
408
+ print("Profiler started")
409
+
410
+ if burstiness == 1.0:
411
+ distribution = "Poisson process"
412
+ else:
413
+ distribution = "Gamma distribution"
414
+
415
+ print(f"Traffic request rate: {request_rate}")
416
+ print(f"Burstiness factor: {burstiness} ({distribution})")
417
+ print(f"Maximum request concurrency: {max_concurrency}")
418
+
419
+ pbar = None if disable_tqdm else tqdm(total=len(input_requests))
420
+
421
+ # This can be used once the minimum Python version is 3.10 or higher,
422
+ # and it will simplify the code in limited_request_func.
423
+ # semaphore = (asyncio.Semaphore(max_concurrency)
424
+ # if max_concurrency else contextlib.nullcontext())
425
+ semaphore = (asyncio.Semaphore(max_concurrency)
426
+ if max_concurrency else None)
427
+
428
+ async def limited_request_func(request_func_input, pbar):
429
+ if semaphore is None:
430
+ return await request_func(request_func_input=request_func_input,
431
+ pbar=pbar)
432
+ async with semaphore:
433
+ return await request_func(request_func_input=request_func_input,
434
+ pbar=pbar)
435
+
436
+ benchmark_start_time = time.perf_counter()
437
+ tasks: List[asyncio.Task] = []
438
+ expected: List[str] = []
439
+ async for i, request in get_request(input_requests, request_rate,
440
+ burstiness):
441
+ extra_body = prepare_extra_body(
442
+ request) if i in guided_decoding_req_idx else None
443
+ request_func_input = RequestFuncInput(
444
+ model=model_id,
445
+ prompt=request.prompt,
446
+ api_url=api_url,
447
+ prompt_len=request.prompt_len,
448
+ output_len=request.expected_output_len,
449
+ ignore_eos=ignore_eos,
450
+ extra_body=extra_body,
451
+ )
452
+ expected.append(request.completion)
453
+ tasks.append(
454
+ asyncio.create_task(
455
+ limited_request_func(request_func_input=request_func_input,
456
+ pbar=pbar)))
457
+ outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
458
+
459
+ if profile:
460
+ print("Stopping profiler...")
461
+ profile_input = RequestFuncInput(
462
+ model=model_id,
463
+ prompt=test_request.prompt,
464
+ api_url=base_url + "/stop_profile",
465
+ prompt_len=test_request.prompt_len,
466
+ output_len=test_request.expected_output_len,
467
+ extra_body={test_request.structure_type: test_request.schema},
468
+ )
469
+ profile_output = await request_func(request_func_input=profile_input)
470
+ if profile_output.success:
471
+ print("Profiler stopped")
472
+
473
+ if pbar is not None:
474
+ pbar.close()
475
+
476
+ benchmark_duration = time.perf_counter() - benchmark_start_time
477
+
478
+ metrics, actual_output_lens = calculate_metrics(
479
+ input_requests=input_requests,
480
+ outputs=outputs,
481
+ dur_s=benchmark_duration,
482
+ tokenizer=tokenizer,
483
+ selected_percentile_metrics=selected_percentile_metrics,
484
+ selected_percentiles=selected_percentiles,
485
+ )
486
+
487
+ print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
488
+ print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
489
+ print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
490
+ benchmark_duration))
491
+ print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
492
+ print("{:<40} {:<10}".format("Total generated tokens:",
493
+ metrics.total_output))
494
+ print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
495
+ metrics.request_throughput))
496
+ print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
497
+ metrics.output_throughput))
498
+ print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
499
+ metrics.total_token_throughput))
500
+
501
+ result = {
502
+ "duration":
503
+ benchmark_duration,
504
+ "completed":
505
+ metrics.completed,
506
+ "total_input_tokens":
507
+ metrics.total_input,
508
+ "total_output_tokens":
509
+ metrics.total_output,
510
+ "request_throughput":
511
+ metrics.request_throughput,
512
+ "output_throughput":
513
+ metrics.output_throughput,
514
+ "total_token_throughput":
515
+ metrics.total_token_throughput,
516
+ "ttft_description":
517
+ pd.Series([output.ttft for output in outputs]).describe().to_dict(),
518
+ "tpot_description":
519
+ pd.Series([output.tpot for output in outputs]).describe().to_dict(),
520
+ "input_lens": [output.prompt_len for output in outputs],
521
+ "output_lens":
522
+ actual_output_lens,
523
+ "ttfts": [output.ttft for output in outputs],
524
+ "itls": [output.itl for output in outputs],
525
+ "errors": [output.error for output in outputs],
526
+ }
527
+
528
+ ret = [{
529
+ 'generated': output.generated_text,
530
+ 'expected': gt
531
+ } for output, gt in zip(outputs, expected)]
532
+
533
+ def process_one_metric(
534
+ # E.g., "ttft"
535
+ metric_attribute_name: str,
536
+ # E.g., "TTFT"
537
+ metric_name: str,
538
+ # E.g., "Time to First Token"
539
+ metric_header: str,
540
+ ):
541
+ # This function prints and adds statistics of the specified
542
+ # metric.
543
+ if metric_attribute_name not in selected_percentile_metrics:
544
+ return
545
+ print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
546
+ print("{:<40} {:<10.2f}".format(
547
+ f"Mean {metric_name} (ms):",
548
+ getattr(metrics, f"mean_{metric_attribute_name}_ms")))
549
+ print("{:<40} {:<10.2f}".format(
550
+ f"Median {metric_name} (ms):",
551
+ getattr(metrics, f"median_{metric_attribute_name}_ms")))
552
+ result[f"mean_{metric_attribute_name}_ms"] = getattr(
553
+ metrics, f"mean_{metric_attribute_name}_ms")
554
+ result[f"median_{metric_attribute_name}_ms"] = getattr(
555
+ metrics, f"median_{metric_attribute_name}_ms")
556
+ result[f"std_{metric_attribute_name}_ms"] = getattr(
557
+ metrics, f"std_{metric_attribute_name}_ms")
558
+ for p, value in getattr(metrics,
559
+ f"percentiles_{metric_attribute_name}_ms"):
560
+ p_word = str(int(p)) if int(p) == p else str(p)
561
+ print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
562
+ value))
563
+ result[f"p{p_word}_{metric_attribute_name}_ms"] = value
564
+
565
+ process_one_metric("ttft", "TTFT", "Time to First Token")
566
+ process_one_metric("tpot", "TPOT",
567
+ "Time per Output Token (excl. 1st token)")
568
+ process_one_metric("itl", "ITL", "Inter-token Latency")
569
+ process_one_metric("e2el", "E2EL", "End-to-end Latency")
570
+
571
+ print("=" * 50)
572
+
573
+ return result, ret
574
+
575
+
576
+def evaluate(ret, args):
577
+
578
+ def _eval_correctness_json(expected, actual):
579
+ # extract json string from string using regex
580
+ import re
581
+ actual = actual.replace('\n', '').replace(' ', '').strip()
582
+ try:
583
+ actual = re.search(r'\{.*\}', actual).group()
584
+ actual = json.loads(actual)
585
+ except Exception:
586
+ return False
587
+
588
+ return True
589
+
590
+ def _eval_correctness_choice(expected, actual):
591
+ return actual in args.choice
592
+
593
+ def _eval_correctness_regex(expected, actual):
594
+ import re
595
+ return re.match(args.regex, actual) is not None
596
+
597
+ def _eval_correctness(expected, actual):
598
+ if args.structure_type == 'guided_json':
599
+ return _eval_correctness_json(expected, actual)
600
+ elif args.structure_type == 'guided_regex':
601
+ return _eval_correctness_regex(expected, actual)
602
+ elif args.structure_type == 'guided_choice':
603
+ return _eval_correctness_choice(expected, actual)
604
+ else:
605
+ return None
606
+
607
+ scores = []
608
+ for res in ret:
609
+ score = _eval_correctness(res['expected'], res['generated'])
610
+ res['correctness'] = score
611
+ scores.append(score)
612
+
613
+ not_none_scores = [score for score in scores if score is not None]
614
+
615
+ return (sum(not_none_scores) / len(not_none_scores) *
616
+ 100) if len(not_none_scores) > 0 else None
617
+
618
+
619
+def main(args: argparse.Namespace):
620
+ print(args)
621
+ random.seed(args.seed)
622
+ np.random.seed(args.seed)
623
+
624
+ backend = args.backend
625
+ model_id = args.model
626
+ tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
627
+
628
+ if args.base_url is not None:
629
+ api_url = f"{args.base_url}{args.endpoint}"
630
+ base_url = f"{args.base_url}"
631
+ else:
632
+ api_url = f"http://{args.host}:{args.port}{args.endpoint}"
633
+ base_url = f"http://{args.host}:{args.port}"
634
+
635
+ tokenizer = get_tokenizer(tokenizer_id,
636
+ trust_remote_code=args.trust_remote_code)
637
+
638
+ if args.dataset == 'grammar':
639
+ args.structure_type = 'guided_grammar'
640
+ elif args.dataset == 'regex':
641
+ args.structure_type = 'guided_regex'
642
+ elif args.dataset == 'choice':
643
+ args.structure_type = 'guided_choice'
644
+ else:
645
+ args.structure_type = 'guided_json'
646
+
647
+ if args.no_guided_decoding:
648
+ args.guided_decoding_ratio = 0
649
+ if args.save_results:
650
+ result_file_name = f'{args.guided_decoding_ratio}guided'
651
+ result_file_name += f"_{backend}"
652
+ result_file_name += f"_{args.request_rate}qps"
653
+ result_file_name += f"_{args.model.split('/')[-1]}"
654
+ result_file_name += f"_{args.dataset}"
655
+ result_file_name += f"_{args.num_prompts}"
656
+ result_file_name += f"_out{args.output_len}"
657
+ result_file_name += ".txt"
658
+ else:
659
+ result_file_name = None
660
+
661
+ input_requests = sample_requests(tokenizer, args)
662
+
663
+ benchmark_result, ret = asyncio.run(
664
+ benchmark(
665
+ backend=backend,
666
+ api_url=api_url,
667
+ base_url=base_url,
668
+ model_id=model_id,
669
+ tokenizer=tokenizer,
670
+ input_requests=input_requests,
671
+ request_rate=args.request_rate,
672
+ burstiness=args.burstiness,
673
+ disable_tqdm=args.disable_tqdm,
674
+ profile=args.profile,
675
+ selected_percentile_metrics=args.percentile_metrics.split(","),
676
+ selected_percentiles=[
677
+ float(p) for p in args.metric_percentiles.split(",")
678
+ ],
679
+ ignore_eos=args.ignore_eos,
680
+ max_concurrency=args.max_concurrency,
681
+ guided_decoding_ratio=args.guided_decoding_ratio,
682
+ guided_decoding_backend=args.guided_decoding_backend,
683
+ ))
684
+
685
+ # Save config and results to json
686
+ score = evaluate(ret, args)
687
+ print("correct_rate(%)", score, '\n')
688
+ if args.save_results:
689
+ results = {
690
+ "backend":
691
+ backend,
692
+ "model_id":
693
+ model_id,
694
+ "tokenizer_id":
695
+ tokenizer_id,
696
+ "num_prompts":
697
+ args.num_prompts,
698
+ "request_rate":
699
+ args.request_rate if args.request_rate < float("inf") else "inf",
700
+ "burstiness":
701
+ args.burstiness,
702
+ "max_concurrency":
703
+ args.max_concurrency,
704
+ "correct_rate(%)":
705
+ score
706
+ }
707
+ results = {"outputs": ret, **results, **benchmark_result}
708
+
709
+ # Save to file
710
+ if args.result_filename:
711
+ result_file_name = args.result_filename
712
+ if args.result_dir:
713
+ result_file_name = os.path.join(args.result_dir, result_file_name)
714
+ with open(result_file_name, "w", encoding='utf-8') as outfile:
715
+ json.dump(results, outfile, indent=4)
716
+
717
+
718
+if __name__ == "__main__":
719
+ parser = FlexibleArgumentParser(
720
+ description="Benchmark the online serving throughput.")
721
+ parser.add_argument(
722
+ "--backend",
723
+ type=str,
724
+ default="vllm",
725
+ choices=list(ASYNC_REQUEST_FUNCS.keys()),
726
+ )
727
+ parser.add_argument(
728
+ "--base-url",
729
+ type=str,
730
+ default=None,
731
+ help="Server or API base url if not using http host and port.",
732
+ )
733
+ parser.add_argument("--host", type=str, default="localhost")
734
+ parser.add_argument("--port", type=int, default=8000)
735
+ parser.add_argument(
736
+ "--endpoint",
737
+ type=str,
738
+ default="/v1/completions",
739
+ help="API endpoint.",
740
+ )
741
+ parser.add_argument(
742
+ "--dataset",
743
+ default='json',
744
+ choices=['json', 'grammar', 'regex', 'choice', 'xgrammar_bench'])
745
+ parser.add_argument("--json_schema_path",
746
+ type=str,
747
+ default=None,
748
+ help="Path to json schema.")
749
+ parser.add_argument(
750
+ "--max-concurrency",
751
+ type=int,
752
+ default=None,
753
+ help="Maximum number of concurrent requests. This can be used "
754
+ "to help simulate an environment where a higher level component "
755
+ "is enforcing a maximum number of concurrent requests. While the "
756
+ "--request-rate argument controls the rate at which requests are "
757
+ "initiated, this argument will control how many are actually allowed "
758
+ "to execute at a time. This means that when used in combination, the "
759
+ "actual request rate may be lower than specified with --request-rate, "
760
+ "if the server is not processing requests fast enough to keep up.")
761
+ parser.add_argument(
762
+ "--model",
763
+ type=str,
764
+ required=True,
765
+ help="Name of the model.",
766
+ )
767
+ parser.add_argument(
768
+ "--tokenizer",
769
+ type=str,
770
+ help=
771
+ "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
772
+ )
773
+ parser.add_argument(
774
+ "--num-prompts",
775
+ type=int,
776
+ default=1000,
777
+ help="Number of prompts to process.",
778
+ )
779
+ parser.add_argument(
780
+ "--output-len",
781
+ type=int,
782
+ default=128,
783
+ help="Number of output tokens.",
784
+ )
785
+ parser.add_argument(
786
+ "--request-rate",
787
+ type=float,
788
+ default=float("inf"),
789
+ help="Number of requests per second. If this is inf, "
790
+ "then all the requests are sent at time 0. "
791
+ "Otherwise, we use Poisson process or gamma distribution "
792
+ "to synthesize the request arrival times.",
793
+ )
794
+ parser.add_argument(
795
+ "--burstiness",
796
+ type=float,
797
+ default=1.0,
798
+ help="Burstiness factor of the request generation. "
799
+ "Only take effect when request_rate is not inf. "
800
+ "Default value is 1, which follows Poisson process. "
801
+ "Otherwise, the request intervals follow a gamma distribution. "
802
+ "A lower burstiness value (0 < burstiness < 1) results in more "
803
+ "bursty requests. A higher burstiness value (burstiness > 1) "
804
+ "results in a more uniform arrival of requests.",
805
+ )
806
+ parser.add_argument("--seed", type=int, default=0)
807
+ parser.add_argument(
808
+ "--trust-remote-code",
809
+ action="store_true",
810
+ help="Trust remote code from huggingface",
811
+ )
812
+ parser.add_argument(
813
+ "--disable-tqdm",
814
+ action="store_true",
815
+ help="Specify to disable tqdm progress bar.",
816
+ )
817
+ parser.add_argument(
818
+ "--save-results",
819
+ action="store_true",
820
+ help="Specify to save benchmark results to a json file",
821
+ )
822
+ parser.add_argument(
823
+ "--profile",
824
+ action="store_true",
825
+ help="Use Torch Profiler. The endpoint must be launched with "
826
+ "VLLM_TORCH_PROFILER_DIR to enable profiler.",
827
+ )
828
+ parser.add_argument(
829
+ "--result-dir",
830
+ type=str,
831
+ default=None,
832
+ help="Specify directory to save benchmark json results."
833
+ "If not specified, results are saved in the current directory.",
834
+ )
835
+ parser.add_argument(
836
+ "--result-filename",
837
+ type=str,
838
+ default=None,
839
+ help="Specify the filename to save benchmark json results."
840
+ "If not specified, results will be saved in "
841
+ "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
842
+ " format.",
843
+ )
844
+ parser.add_argument(
845
+ "--ignore-eos",
846
+ action="store_true",
847
+ help="Set ignore_eos flag when sending the benchmark request."
848
+ "Warning: ignore_eos is not supported in deepspeed_mii and tgi.")
849
+ parser.add_argument(
850
+ "--percentile-metrics",
851
+ type=str,
852
+ default="ttft,tpot,itl",
853
+ help="Comma-seperated list of selected metrics to report percentils. "
854
+ "This argument specifies the metrics to report percentiles. "
855
+ "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
856
+ "Default value is \"ttft,tpot,itl\".")
857
+ parser.add_argument(
858
+ "--metric-percentiles",
859
+ type=str,
860
+ default="99",
861
+ help="Comma-seperated list of percentiles for selected metrics. "
862
+ "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
863
+ "Default value is \"99\". "
864
+ "Use \"--percentile-metrics\" to select metrics.",
865
+ )
866
+ parser.add_argument("--no-guided-decoding",
867
+ action='store_true',
868
+ default=False,
869
+ help="Whether to disable JSON decoding or not.")
870
+ parser.add_argument("--guided-decoding-ratio",
871
+ type=float,
872
+ default=1.0,
873
+ help="Ratio of Guided Decoding requests")
874
+ parser.add_argument("--guided-decoding-backend",
875
+ type=str,
876
+ choices=["outlines", "lm-format-enforcer", "xgrammar"],
877
+ default="xgrammar",
878
+ help="Backend to use for guided decoding")
879
+
880
+ args = parser.parse_args()
881
+ main(args)