A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Benchmark] Benchmark structured output with datasets (#10557)
Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Aaron Pham <contact@aarnphm.xyz>
Chendi.Xue committed
Dec 3, 2024 at 18:21 UTC
381ac93bb5a41347a025367bc58119cb45357095
2 files changed
+607
benchmarks/benchmark_guided.py
new
+494
@@ -0,0 +1,494 @@
1
+"""Benchmark guided decoding throughput."""
2
+import argparse
3
+import dataclasses
4
+import json
5
+import os
6
+import random
7
+import time
8
+from typing import List
9
+
10
+import datasets
11
+import pandas as pd
12
+import uvloop
13
+from transformers import AutoTokenizer, PreTrainedTokenizerBase
14
+
15
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16
+from vllm.entrypoints.openai.api_server import (
17
+ build_async_engine_client_from_engine_args)
18
+from vllm.sampling_params import GuidedDecodingParams
19
+from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20
+
21
+
22
+@dataclasses.dataclass
23
+class SampleRequest:
24
+ """A class representing a single inference request for benchmarking.
25
+
26
+ Attributes:
27
+ prompt: The input text prompt for the model.
28
+ multi_modal_data: Optional dictionary containing multi-modal data (e.g.
29
+ images).
30
+ prompt_len: The length of the prompt in tokens.
31
+ expected_output_len: The expected length of the output in tokens.
32
+ """
33
+ prompt: str
34
+ prompt_len: int
35
+ expected_output_len: int
36
+ schema: dict
37
+ structure_type: str = 'json'
38
+ completion: str = None
39
+
40
+
41
+def run_vllm(requests: List[SampleRequest],
42
+ engine_args: EngineArgs,
43
+ n: int,
44
+ guided_decoding_rate: float = 1.0,
45
+ warmup: bool = False) -> float:
46
+ from vllm import LLM, SamplingParams
47
+ llm = LLM(**vars(engine_args))
48
+
49
+ # Add the requests to the engine.
50
+ prompts: List[str] = []
51
+ sampling_params: List[SamplingParams] = []
52
+ # create a list containing random selected true or false
53
+ guided_decoding_req_idx = random.sample(
54
+ range(len(requests)), int(len(requests) * guided_decoding_rate))
55
+
56
+ if warmup:
57
+ print(">>>>> Running warmup prompt, for the first 5")
58
+ # We setup the first 5 requests to warmup FSM
59
+ # if using xgrammar dataset, we will skip warmup
60
+ warmup_requests = requests[:5]
61
+ for i, request in enumerate(warmup_requests):
62
+ prompts.append(request.prompt)
63
+ sampling_params.append(
64
+ SamplingParams(
65
+ n=n,
66
+ temperature=1.0,
67
+ top_p=1.0,
68
+ ignore_eos=True,
69
+ max_tokens=request.expected_output_len,
70
+ guided_decoding=GuidedDecodingParams(json=request.schema)
71
+ if guided_decoding_rate > 0 else None,
72
+ ))
73
+ llm.generate(prompts, sampling_params, use_tqdm=False)
74
+
75
+ print(">>>>> Benchmark started...")
76
+ prompts = []
77
+ sampling_params = []
78
+ for i, request in enumerate(requests):
79
+ prompts.append(request.prompt)
80
+ sampling_params.append(
81
+ SamplingParams(
82
+ n=n,
83
+ temperature=1.0,
84
+ top_p=1.0,
85
+ ignore_eos=True,
86
+ max_tokens=request.expected_output_len,
87
+ guided_decoding=GuidedDecodingParams(
88
+ **{request.structure_type: request.schema})
89
+ if i in guided_decoding_req_idx else None,
90
+ ))
91
+
92
+ start = time.perf_counter()
93
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=False)
94
+ ret = []
95
+ for output, request in zip(outputs, requests):
96
+ generated_text = output.outputs[0].text
97
+ ret.append({
98
+ "generated": generated_text,
99
+ "expected": request.completion
100
+ })
101
+ end = time.perf_counter()
102
+ return end - start, ret
103
+
104
+
105
+async def run_vllm_async(
106
+ requests: List[SampleRequest],
107
+ engine_args: AsyncEngineArgs,
108
+ n: int,
109
+ guided_decoding_rate: float = 1.0,
110
+ warmup: bool = False,
111
+ disable_frontend_multiprocessing: bool = False) -> float:
112
+ from vllm import SamplingParams
113
+
114
+ async with build_async_engine_client_from_engine_args(
115
+ engine_args, disable_frontend_multiprocessing) as llm:
116
+
117
+ # Add the requests to the engine.
118
+ prompts: List[str] = []
119
+ sampling_params: List[SamplingParams] = []
120
+ guided_decoding_req_idx = random.sample(
121
+ range(len(requests)), int(len(requests) * guided_decoding_rate))
122
+
123
+ if warmup:
124
+ print(">>>>>> Running warmup prompt, for the first 5")
125
+ # We setup the first 5 requests to warmup FSM
126
+ # if using xgrammar dataset, we will skip warmup
127
+ warmup_requests = requests[:5]
128
+ for i, request in enumerate(warmup_requests):
129
+ prompts.append(request.prompt)
130
+ sampling_params.append(
131
+ SamplingParams(
132
+ n=n,
133
+ temperature=1.0,
134
+ top_p=1.0,
135
+ ignore_eos=True,
136
+ max_tokens=request.expected_output_len,
137
+ guided_decoding=GuidedDecodingParams(
138
+ json=request.schema)
139
+ if guided_decoding_rate > 0 else None,
140
+ ))
141
+ generators = []
142
+ for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
143
+ generator = llm.generate(prompt, sp, request_id=f"test{i}")
144
+ generators.append(generator)
145
+ all_gens = merge_async_iterators(*generators)
146
+ async for i, res in all_gens:
147
+ pass
148
+
149
+ print(">>>>> Benchmark started...")
150
+ prompts = []
151
+ sampling_params = []
152
+ for i, request in enumerate(requests):
153
+ prompts.append(request.prompt)
154
+ sampling_params.append(
155
+ SamplingParams(
156
+ n=n,
157
+ temperature=1.0,
158
+ top_p=1.0,
159
+ ignore_eos=True,
160
+ max_tokens=request.expected_output_len,
161
+ guided_decoding=GuidedDecodingParams(json=request.schema)
162
+ if i in guided_decoding_req_idx else None,
163
+ ))
164
+
165
+ generators = []
166
+ start_time = []
167
+ latencies = []
168
+ start = time.perf_counter()
169
+ for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
170
+ generator = llm.generate(prompt, sp, request_id=f"test{i}")
171
+ generators.append(generator)
172
+ start_time.append(time.perf_counter())
173
+ latencies.append([])
174
+ all_gens = merge_async_iterators(*generators)
175
+ generated_texts = [''] * len(requests)
176
+ async for i, res in all_gens:
177
+ generated_texts[i] = res.outputs[0].text
178
+ lat = time.perf_counter() - start_time[i]
179
+ latencies[i].append(lat)
180
+ ret = [{
181
+ 'generated': gt,
182
+ 'expected': req.completion
183
+ } for gt, req in zip(generated_texts, requests)]
184
+ end = time.perf_counter()
185
+ first_latency = pd.Series([lat[0] * 1000 for lat in latencies])
186
+ next_latency = pd.Series([(lat[-1] - lat[0]) / len(lat[1:]) * 1000
187
+ for lat in latencies])
188
+ return end - start, ret, (first_latency, next_latency)
189
+
190
+
191
+def sample_requests(tokenizer: PreTrainedTokenizerBase,
192
+ args: argparse.Namespace) -> List[SampleRequest]:
193
+ if args.dataset == 'json':
194
+ if args.json_schema_path is None:
195
+ dir_path = os.path.dirname(os.path.realpath(__file__))
196
+ args.json_schema_path = os.path.join(dir_path,
197
+ "structured_schemas",
198
+ "structured_schema_1.json")
199
+ with open(args.json_schema_path) as f:
200
+ schema = json.load(f)
201
+ prompt = f"Generate an example of a user profile given the following schema: {json.dumps(schema)}" # noqa: E501
202
+ input_len = len(tokenizer(prompt).input_ids)
203
+ print(f"Input length of the prompt: {input_len} tokens")
204
+ requests = [
205
+ SampleRequest(prompt=prompt,
206
+ prompt_len=input_len,
207
+ expected_output_len=args.output_len,
208
+ schema=schema,
209
+ structure_type=args.structure_type)
210
+ for _ in range(args.num_prompts)
211
+ ]
212
+
213
+ elif args.dataset == "grammar":
214
+ schema = """
215
+ ?start: select_statement
216
+
217
+ ?select_statement: "SELECT " column_list " FROM " table_name
218
+
219
+ ?column_list: column_name ("," column_name)*
220
+
221
+ ?table_name: identifier
222
+
223
+ ?column_name: identifier
224
+
225
+ ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
226
+ """
227
+ prompt = "Generate an SQL query to show the 'username' \
228
+ and 'email' from the 'users' table."
229
+
230
+ input_len = len(tokenizer(prompt).input_ids)
231
+ print(f"Input length of the prompt: {input_len} tokens")
232
+ requests = [
233
+ SampleRequest(prompt=prompt,
234
+ prompt_len=input_len,
235
+ expected_output_len=args.output_len,
236
+ schema=schema,
237
+ structure_type=args.structure_type)
238
+ for _ in range(args.num_prompts)
239
+ ]
240
+
241
+ elif args.dataset == "regex":
242
+ regex = r"\w+@\w+\.com\n"
243
+ args.regex = regex
244
+ prompt = "Generate an email address for Alan Turing, \
245
+ who works in Enigma. End in .com and new line. \
246
+ Example result: alan.turing@enigma.com\n"
247
+
248
+ input_len = len(tokenizer(prompt).input_ids)
249
+ print(f"Input length of the prompt: {input_len} tokens")
250
+ requests = [
251
+ SampleRequest(prompt=prompt,
252
+ prompt_len=input_len,
253
+ expected_output_len=args.output_len,
254
+ schema=regex,
255
+ structure_type=args.structure_type)
256
+ for _ in range(args.num_prompts)
257
+ ]
258
+
259
+ elif args.dataset == "choice":
260
+ choice = ["Positive", "Negative"]
261
+ args.choice = choice
262
+ prompt = "Classify this sentiment: vLLM is wonderful!"
263
+ input_len = len(tokenizer(prompt).input_ids)
264
+ print(f"Input length of the prompt: {input_len} tokens")
265
+ requests = [
266
+ SampleRequest(prompt=prompt,
267
+ prompt_len=input_len,
268
+ expected_output_len=args.output_len,
269
+ schema=choice,
270
+ structure_type=args.structure_type)
271
+ for _ in range(args.num_prompts)
272
+ ]
273
+
274
+ elif args.dataset == "xgrammar_bench":
275
+ args.warmup = False
276
+ requests: List[SampleRequest] = []
277
+ dataset = datasets.load_dataset("NousResearch/json-mode-eval",
278
+ split="train")
279
+ print(f"dataset has {len(dataset)} entries")
280
+ len_dataset = len(dataset)
281
+ for data_point_idx in range(args.num_prompts):
282
+ idx = data_point_idx
283
+ while idx >= len_dataset:
284
+ idx -= len_dataset
285
+ schema = dataset["schema"][idx]
286
+ prompt = tokenizer.apply_chat_template(dataset["prompt"][idx],
287
+ tokenize=False)
288
+ input_len = len(tokenizer(prompt).input_ids)
289
+ completion = dataset["completion"][idx]
290
+
291
+ requests.append(
292
+ SampleRequest(prompt=prompt,
293
+ prompt_len=input_len,
294
+ expected_output_len=args.output_len,
295
+ schema=schema,
296
+ completion=completion))
297
+
298
+ return requests
299
+
300
+
301
+def evaluate(ret, args):
302
+
303
+ def _eval_correctness_json(expected, actual):
304
+ # extract json string from string using regex
305
+ import re
306
+ actual = actual.replace('\n', '').replace(' ', '').strip()
307
+ try:
308
+ actual = re.search(r'\{.*\}', actual).group()
309
+ actual = json.loads(actual)
310
+ except Exception:
311
+ return False
312
+
313
+ return True
314
+
315
+ def _eval_correctness_choice(expected, actual):
316
+ return actual in args.choice
317
+
318
+ def _eval_correctness_regex(expected, actual):
319
+ import re
320
+ return re.match(args.regex, actual) is not None
321
+
322
+ def _eval_correctness(expected, actual):
323
+ if args.structure_type == 'json':
324
+ return _eval_correctness_json(expected, actual)
325
+ elif args.structure_type == 'regex':
326
+ return _eval_correctness_regex(expected, actual)
327
+ elif args.structure_type == 'choice':
328
+ return _eval_correctness_choice(expected, actual)
329
+ else:
330
+ return None
331
+
332
+ scores = []
333
+ for res in ret:
334
+ score = _eval_correctness(res['expected'], res['generated'])
335
+ res['correctness'] = score
336
+ scores.append(score)
337
+
338
+ not_none_scores = [score for score in scores if score is not None]
339
+
340
+ return (sum(not_none_scores) / len(not_none_scores) *
341
+ 100) if len(not_none_scores) > 0 else None
342
+
343
+
344
+def main(args: argparse.Namespace):
345
+ print(args)
346
+ random.seed(args.seed)
347
+
348
+ # async engine is working for 'regex', 'choice' and 'grammar'
349
+ if args.dataset == 'grammar':
350
+ args.structure_type = 'grammar'
351
+ args.async_engine = False
352
+ elif args.dataset == 'regex':
353
+ args.structure_type = 'regex'
354
+ args.async_engine = False
355
+ elif args.dataset == 'choice':
356
+ args.structure_type = 'choice'
357
+ args.async_engine = False
358
+ else:
359
+ args.structure_type = 'json'
360
+
361
+ if args.no_guided_decoding:
362
+ args.guided_decoding_ratio = 0
363
+ if args.save_results:
364
+ result_file_name = f'{args.guided_decoding_ratio}guided'
365
+ result_file_name += f"_{args.model.split('/')[-1]}"
366
+ result_file_name += f"_{args.dataset}"
367
+ result_file_name += f"_{args.num_prompts}"
368
+ result_file_name += f"_out{args.output_len}"
369
+ result_file_name += f"_async{args.async_engine}"
370
+ result_file_name += f"_warmup{args.warmup}"
371
+ result_file_name += f"_chunkedprefill{args.enable_chunked_prefill}"
372
+ result_file_name += ".txt"
373
+ else:
374
+ result_file_name = None
375
+
376
+ # Synthesize a prompt with the given input length.
377
+ tokenizer = AutoTokenizer.from_pretrained(
378
+ args.tokenizer, trust_remote_code=args.trust_remote_code)
379
+ requests = sample_requests(tokenizer, args)
380
+
381
+ if args.async_engine:
382
+ engine_args = AsyncEngineArgs.from_cli_args(args)
383
+ elapsed_time, ret, (first_latency, next_latency) = uvloop.run(
384
+ run_vllm_async(requests, engine_args, args.n,
385
+ args.guided_decoding_ratio, args.warmup,
386
+ args.disable_frontend_multiprocessing))
387
+ else:
388
+ engine_args = EngineArgs.from_cli_args(args)
389
+ elapsed_time, ret = run_vllm(requests, engine_args, args.n,
390
+ args.guided_decoding_ratio, args.warmup)
391
+ first_latency, next_latency = None, None
392
+
393
+ score = evaluate(ret, args)
394
+ total_num_tokens = sum(request.prompt_len + request.expected_output_len
395
+ for request in requests)
396
+ total_output_tokens = sum(request.expected_output_len
397
+ for request in requests)
398
+ if first_latency is not None:
399
+ latency_breakdown = "\nFirst token latency(msecs):\n"
400
+ latency_breakdown += f"{first_latency.describe()}"
401
+ latency_breakdown += "\nNext token latency(msecs):\n"
402
+ latency_breakdown += f"{next_latency.describe()}"
403
+ print(
404
+ f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
405
+ f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
406
+ f"{total_output_tokens / elapsed_time:.2f} output tokens/s",
407
+ f"Correct rate is {score} %",
408
+ f"{latency_breakdown if first_latency is not None else ''}")
409
+
410
+ # Output JSON results if specified
411
+ if args.output_json or result_file_name:
412
+ results = {
413
+ "elapsed_time": elapsed_time,
414
+ "num_requests": len(requests),
415
+ "total_num_tokens": total_num_tokens,
416
+ "total_output_tokens": total_output_tokens,
417
+ "requests_per_second": len(requests) / elapsed_time,
418
+ "tokens_per_second": f"{total_num_tokens / elapsed_time:.2f}",
419
+ "output_tokens_per_second":
420
+ f"{total_output_tokens / elapsed_time:.2f}",
421
+ "correct_rate(%)": score
422
+ }
423
+ results = {"outputs": ret, **results}
424
+ if first_latency is not None:
425
+ results["first_token_latency(msecs)"] = first_latency.describe(
426
+ ).to_dict()
427
+ results["next_token_latency(msecs)"] = next_latency.describe(
428
+ ).to_dict()
429
+ if args.output_json:
430
+ with open(args.output_json, "w") as f:
431
+ json.dump(results, f, indent=4)
432
+ elif result_file_name:
433
+ with open(result_file_name, "w") as f:
434
+ json.dump(results, f, indent=4)
435
+
436
+
437
+if __name__ == "__main__":
438
+ parser = FlexibleArgumentParser(description="Benchmark guided decoding.")
439
+ parser = AsyncEngineArgs.add_cli_args(parser)
440
+
441
+ parser.add_argument("--output-len",
442
+ type=int,
443
+ default=512,
444
+ help="Output length for each request. Overrides the "
445
+ "output length from the dataset.")
446
+ parser.add_argument(
447
+ "--dataset",
448
+ default='json',
449
+ choices=['json', 'grammar', 'regex', 'choice', 'xgrammar_bench'])
450
+ parser.add_argument("--json_schema_path",
451
+ type=str,
452
+ default=None,
453
+ help="Path to json schema.")
454
+ parser.add_argument("--n",
455
+ type=int,
456
+ default=1,
457
+ help="Number of generated sequences per prompt.")
458
+ parser.add_argument("--num-prompts",
459
+ type=int,
460
+ default=10,
461
+ help="Number of prompts to process.")
462
+ parser.add_argument(
463
+ '--output-json',
464
+ type=str,
465
+ default=None,
466
+ help='Path to save the throughput results in JSON format.')
467
+ parser.add_argument("--async-engine",
468
+ action='store_true',
469
+ default=False,
470
+ help="Use vLLM async engine rather than LLM class.")
471
+ parser.add_argument("--no-guided-decoding",
472
+ action='store_true',
473
+ default=False,
474
+ help="Whether to disable JSON decoding or not.")
475
+ parser.add_argument("--guided-decoding-ratio",
476
+ type=float,
477
+ default=1.0,
478
+ help="Ratio of Guided Decoding requests")
479
+ parser.add_argument("--disable-frontend-multiprocessing",
480
+ action='store_true',
481
+ default=False,
482
+ help="Disable decoupled async engine frontend.")
483
+ parser.add_argument("--warmup",
484
+ action="store_true",
485
+ default=False,
486
+ help="Run warmup prompts before benchmark.")
487
+ parser.add_argument("--save-results",
488
+ action="store_true",
489
+ default=False,
490
+ help="save output results.")
491
+ args = parser.parse_args()
492
+ if args.tokenizer is None:
493
+ args.tokenizer = args.model
494
+ main(args)
benchmarks/structured_schemas/structured_schema_1.json
new
+113
@@ -0,0 +1,113 @@
1
+{
2
+ "$schema":
3
+ "https://json-schema.org/draft/2020-12/schema",
4
+ "title":
5
+ "User Profile",
6
+ "type":
7
+ "object",
8
+ "properties": {
9
+ "userId": {
10
+ "type": "string",
11
+ "description": "Unique identifier for the user."
12
+ },
13
+ "personalInfo": {
14
+ "type": "object",
15
+ "properties": {
16
+ "firstName": {
17
+ "type": "string",
18
+ "description": "The user's first name."
19
+ },
20
+ "lastName": {
21
+ "type": "string",
22
+ "description": "The user's last name."
23
+ },
24
+ "age": {
25
+ "type": "integer",
26
+ "minimum": 0,
27
+ "description": "The user's age."
28
+ },
29
+ "phoneNumbers": {
30
+ "type":
31
+ "array",
32
+ "items": {
33
+ "type": "object",
34
+ "properties": {
35
+ "type": {
36
+ "type": "string",
37
+ "enum": ["home", "work", "mobile"],
38
+ "description": "Type of phone number."
39
+ },
40
+ "number": {
41
+ "type": "string",
42
+ "pattern": "^\\+?[1-9]\\d{1,14}$",
43
+ "description": "Phone number in E.164 format."
44
+ }
45
+ },
46
+ "required": ["type", "number"]
47
+ },
48
+ "description":
49
+ "List of phone numbers associated with the user."
50
+ }
51
+ },
52
+ "required": ["firstName", "lastName"]
53
+ },
54
+ "address": {
55
+ "type": "object",
56
+ "properties": {
57
+ "street": {
58
+ "type": "string",
59
+ "description": "Street address."
60
+ },
61
+ "city": {
62
+ "type": "string",
63
+ "description": "City name."
64
+ },
65
+ "state": {
66
+ "type": "string",
67
+ "description": "State or province."
68
+ },
69
+ "postalCode": {
70
+ "type": "string",
71
+ "pattern": "^\\d{5}(-\\d{4})?$",
72
+ "description": "Postal code."
73
+ },
74
+ "country": {
75
+ "type": "string",
76
+ "description": "Country name."
77
+ }
78
+ },
79
+ "required": ["street", "city", "state", "postalCode", "country"]
80
+ },
81
+ "preferences": {
82
+ "type": "object",
83
+ "properties": {
84
+ "newsletterSubscribed": {
85
+ "type":
86
+ "boolean",
87
+ "description":
88
+ "Indicates if the user is subscribed to the newsletter."
89
+ },
90
+ "favoriteCategories": {
91
+ "type": "array",
92
+ "items": {
93
+ "type": "string"
94
+ },
95
+ "description": "List of user's favorite categories."
96
+ }
97
+ },
98
+ "required": ["newsletterSubscribed"]
99
+ },
100
+ "accountStatus": {
101
+ "type": "string",
102
+ "enum": ["active", "inactive", "suspended"],
103
+ "description": "Current status of the user's account."
104
+ },
105
+ "registrationDate": {
106
+ "type": "string",
107
+ "format": "date-time",
108
+ "description": "ISO 8601 formatted date-time of user registration."
109
+ }
110
+ },
111
+ "required":
112
+ ["userId", "personalInfo", "address", "accountStatus", "registrationDate"]
113
+}
\ No newline at end of file