@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 381ac93bb5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Benchmark] Benchmark structured output with datasets (#10557)

Signed-off-by: Aaron Pham <contact@aarnphm.xyz> Signed-off-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Aaron Pham <contact@aarnphm.xyz>

Chendi.Xue committed Dec 3, 2024 at 18:21 UTC 381ac93bb5a41347a025367bc58119cb45357095
2 files changed +607
benchmarks/benchmark_guided.py new
+494
@@ -0,0 +1,494 @@
1 +"""Benchmark guided decoding throughput."""
2 +import argparse
3 +import dataclasses
4 +import json
5 +import os
6 +import random
7 +import time
8 +from typing import List
9 +
10 +import datasets
11 +import pandas as pd
12 +import uvloop
13 +from transformers import AutoTokenizer, PreTrainedTokenizerBase
14 +
15 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
16 +from vllm.entrypoints.openai.api_server import (
17 + build_async_engine_client_from_engine_args)
18 +from vllm.sampling_params import GuidedDecodingParams
19 +from vllm.utils import FlexibleArgumentParser, merge_async_iterators
20 +
21 +
22 +@dataclasses.dataclass
23 +class SampleRequest:
24 + """A class representing a single inference request for benchmarking.
25 +
26 + Attributes:
27 + prompt: The input text prompt for the model.
28 + multi_modal_data: Optional dictionary containing multi-modal data (e.g.
29 + images).
30 + prompt_len: The length of the prompt in tokens.
31 + expected_output_len: The expected length of the output in tokens.
32 + """
33 + prompt: str
34 + prompt_len: int
35 + expected_output_len: int
36 + schema: dict
37 + structure_type: str = 'json'
38 + completion: str = None
39 +
40 +
41 +def run_vllm(requests: List[SampleRequest],
42 + engine_args: EngineArgs,
43 + n: int,
44 + guided_decoding_rate: float = 1.0,
45 + warmup: bool = False) -> float:
46 + from vllm import LLM, SamplingParams
47 + llm = LLM(**vars(engine_args))
48 +
49 + # Add the requests to the engine.
50 + prompts: List[str] = []
51 + sampling_params: List[SamplingParams] = []
52 + # create a list containing random selected true or false
53 + guided_decoding_req_idx = random.sample(
54 + range(len(requests)), int(len(requests) * guided_decoding_rate))
55 +
56 + if warmup:
57 + print(">>>>> Running warmup prompt, for the first 5")
58 + # We setup the first 5 requests to warmup FSM
59 + # if using xgrammar dataset, we will skip warmup
60 + warmup_requests = requests[:5]
61 + for i, request in enumerate(warmup_requests):
62 + prompts.append(request.prompt)
63 + sampling_params.append(
64 + SamplingParams(
65 + n=n,
66 + temperature=1.0,
67 + top_p=1.0,
68 + ignore_eos=True,
69 + max_tokens=request.expected_output_len,
70 + guided_decoding=GuidedDecodingParams(json=request.schema)
71 + if guided_decoding_rate > 0 else None,
72 + ))
73 + llm.generate(prompts, sampling_params, use_tqdm=False)
74 +
75 + print(">>>>> Benchmark started...")
76 + prompts = []
77 + sampling_params = []
78 + for i, request in enumerate(requests):
79 + prompts.append(request.prompt)
80 + sampling_params.append(
81 + SamplingParams(
82 + n=n,
83 + temperature=1.0,
84 + top_p=1.0,
85 + ignore_eos=True,
86 + max_tokens=request.expected_output_len,
87 + guided_decoding=GuidedDecodingParams(
88 + **{request.structure_type: request.schema})
89 + if i in guided_decoding_req_idx else None,
90 + ))
91 +
92 + start = time.perf_counter()
93 + outputs = llm.generate(prompts, sampling_params, use_tqdm=False)
94 + ret = []
95 + for output, request in zip(outputs, requests):
96 + generated_text = output.outputs[0].text
97 + ret.append({
98 + "generated": generated_text,
99 + "expected": request.completion
100 + })
101 + end = time.perf_counter()
102 + return end - start, ret
103 +
104 +
105 +async def run_vllm_async(
106 + requests: List[SampleRequest],
107 + engine_args: AsyncEngineArgs,
108 + n: int,
109 + guided_decoding_rate: float = 1.0,
110 + warmup: bool = False,
111 + disable_frontend_multiprocessing: bool = False) -> float:
112 + from vllm import SamplingParams
113 +
114 + async with build_async_engine_client_from_engine_args(
115 + engine_args, disable_frontend_multiprocessing) as llm:
116 +
117 + # Add the requests to the engine.
118 + prompts: List[str] = []
119 + sampling_params: List[SamplingParams] = []
120 + guided_decoding_req_idx = random.sample(
121 + range(len(requests)), int(len(requests) * guided_decoding_rate))
122 +
123 + if warmup:
124 + print(">>>>>> Running warmup prompt, for the first 5")
125 + # We setup the first 5 requests to warmup FSM
126 + # if using xgrammar dataset, we will skip warmup
127 + warmup_requests = requests[:5]
128 + for i, request in enumerate(warmup_requests):
129 + prompts.append(request.prompt)
130 + sampling_params.append(
131 + SamplingParams(
132 + n=n,
133 + temperature=1.0,
134 + top_p=1.0,
135 + ignore_eos=True,
136 + max_tokens=request.expected_output_len,
137 + guided_decoding=GuidedDecodingParams(
138 + json=request.schema)
139 + if guided_decoding_rate > 0 else None,
140 + ))
141 + generators = []
142 + for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
143 + generator = llm.generate(prompt, sp, request_id=f"test{i}")
144 + generators.append(generator)
145 + all_gens = merge_async_iterators(*generators)
146 + async for i, res in all_gens:
147 + pass
148 +
149 + print(">>>>> Benchmark started...")
150 + prompts = []
151 + sampling_params = []
152 + for i, request in enumerate(requests):
153 + prompts.append(request.prompt)
154 + sampling_params.append(
155 + SamplingParams(
156 + n=n,
157 + temperature=1.0,
158 + top_p=1.0,
159 + ignore_eos=True,
160 + max_tokens=request.expected_output_len,
161 + guided_decoding=GuidedDecodingParams(json=request.schema)
162 + if i in guided_decoding_req_idx else None,
163 + ))
164 +
165 + generators = []
166 + start_time = []
167 + latencies = []
168 + start = time.perf_counter()
169 + for i, (prompt, sp) in enumerate(zip(prompts, sampling_params)):
170 + generator = llm.generate(prompt, sp, request_id=f"test{i}")
171 + generators.append(generator)
172 + start_time.append(time.perf_counter())
173 + latencies.append([])
174 + all_gens = merge_async_iterators(*generators)
175 + generated_texts = [''] * len(requests)
176 + async for i, res in all_gens:
177 + generated_texts[i] = res.outputs[0].text
178 + lat = time.perf_counter() - start_time[i]
179 + latencies[i].append(lat)
180 + ret = [{
181 + 'generated': gt,
182 + 'expected': req.completion
183 + } for gt, req in zip(generated_texts, requests)]
184 + end = time.perf_counter()
185 + first_latency = pd.Series([lat[0] * 1000 for lat in latencies])
186 + next_latency = pd.Series([(lat[-1] - lat[0]) / len(lat[1:]) * 1000
187 + for lat in latencies])
188 + return end - start, ret, (first_latency, next_latency)
189 +
190 +
191 +def sample_requests(tokenizer: PreTrainedTokenizerBase,
192 + args: argparse.Namespace) -> List[SampleRequest]:
193 + if args.dataset == 'json':
194 + if args.json_schema_path is None:
195 + dir_path = os.path.dirname(os.path.realpath(__file__))
196 + args.json_schema_path = os.path.join(dir_path,
197 + "structured_schemas",
198 + "structured_schema_1.json")
199 + with open(args.json_schema_path) as f:
200 + schema = json.load(f)
201 + prompt = f"Generate an example of a user profile given the following schema: {json.dumps(schema)}" # noqa: E501
202 + input_len = len(tokenizer(prompt).input_ids)
203 + print(f"Input length of the prompt: {input_len} tokens")
204 + requests = [
205 + SampleRequest(prompt=prompt,
206 + prompt_len=input_len,
207 + expected_output_len=args.output_len,
208 + schema=schema,
209 + structure_type=args.structure_type)
210 + for _ in range(args.num_prompts)
211 + ]
212 +
213 + elif args.dataset == "grammar":
214 + schema = """
215 + ?start: select_statement
216 +
217 + ?select_statement: "SELECT " column_list " FROM " table_name
218 +
219 + ?column_list: column_name ("," column_name)*
220 +
221 + ?table_name: identifier
222 +
223 + ?column_name: identifier
224 +
225 + ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
226 + """
227 + prompt = "Generate an SQL query to show the 'username' \
228 + and 'email' from the 'users' table."
229 +
230 + input_len = len(tokenizer(prompt).input_ids)
231 + print(f"Input length of the prompt: {input_len} tokens")
232 + requests = [
233 + SampleRequest(prompt=prompt,
234 + prompt_len=input_len,
235 + expected_output_len=args.output_len,
236 + schema=schema,
237 + structure_type=args.structure_type)
238 + for _ in range(args.num_prompts)
239 + ]
240 +
241 + elif args.dataset == "regex":
242 + regex = r"\w+@\w+\.com\n"
243 + args.regex = regex
244 + prompt = "Generate an email address for Alan Turing, \
245 + who works in Enigma. End in .com and new line. \
246 + Example result: alan.turing@enigma.com\n"
247 +
248 + input_len = len(tokenizer(prompt).input_ids)
249 + print(f"Input length of the prompt: {input_len} tokens")
250 + requests = [
251 + SampleRequest(prompt=prompt,
252 + prompt_len=input_len,
253 + expected_output_len=args.output_len,
254 + schema=regex,
255 + structure_type=args.structure_type)
256 + for _ in range(args.num_prompts)
257 + ]
258 +
259 + elif args.dataset == "choice":
260 + choice = ["Positive", "Negative"]
261 + args.choice = choice
262 + prompt = "Classify this sentiment: vLLM is wonderful!"
263 + input_len = len(tokenizer(prompt).input_ids)
264 + print(f"Input length of the prompt: {input_len} tokens")
265 + requests = [
266 + SampleRequest(prompt=prompt,
267 + prompt_len=input_len,
268 + expected_output_len=args.output_len,
269 + schema=choice,
270 + structure_type=args.structure_type)
271 + for _ in range(args.num_prompts)
272 + ]
273 +
274 + elif args.dataset == "xgrammar_bench":
275 + args.warmup = False
276 + requests: List[SampleRequest] = []
277 + dataset = datasets.load_dataset("NousResearch/json-mode-eval",
278 + split="train")
279 + print(f"dataset has {len(dataset)} entries")
280 + len_dataset = len(dataset)
281 + for data_point_idx in range(args.num_prompts):
282 + idx = data_point_idx
283 + while idx >= len_dataset:
284 + idx -= len_dataset
285 + schema = dataset["schema"][idx]
286 + prompt = tokenizer.apply_chat_template(dataset["prompt"][idx],
287 + tokenize=False)
288 + input_len = len(tokenizer(prompt).input_ids)
289 + completion = dataset["completion"][idx]
290 +
291 + requests.append(
292 + SampleRequest(prompt=prompt,
293 + prompt_len=input_len,
294 + expected_output_len=args.output_len,
295 + schema=schema,
296 + completion=completion))
297 +
298 + return requests
299 +
300 +
301 +def evaluate(ret, args):
302 +
303 + def _eval_correctness_json(expected, actual):
304 + # extract json string from string using regex
305 + import re
306 + actual = actual.replace('\n', '').replace(' ', '').strip()
307 + try:
308 + actual = re.search(r'\{.*\}', actual).group()
309 + actual = json.loads(actual)
310 + except Exception:
311 + return False
312 +
313 + return True
314 +
315 + def _eval_correctness_choice(expected, actual):
316 + return actual in args.choice
317 +
318 + def _eval_correctness_regex(expected, actual):
319 + import re
320 + return re.match(args.regex, actual) is not None
321 +
322 + def _eval_correctness(expected, actual):
323 + if args.structure_type == 'json':
324 + return _eval_correctness_json(expected, actual)
325 + elif args.structure_type == 'regex':
326 + return _eval_correctness_regex(expected, actual)
327 + elif args.structure_type == 'choice':
328 + return _eval_correctness_choice(expected, actual)
329 + else:
330 + return None
331 +
332 + scores = []
333 + for res in ret:
334 + score = _eval_correctness(res['expected'], res['generated'])
335 + res['correctness'] = score
336 + scores.append(score)
337 +
338 + not_none_scores = [score for score in scores if score is not None]
339 +
340 + return (sum(not_none_scores) / len(not_none_scores) *
341 + 100) if len(not_none_scores) > 0 else None
342 +
343 +
344 +def main(args: argparse.Namespace):
345 + print(args)
346 + random.seed(args.seed)
347 +
348 + # async engine is working for 'regex', 'choice' and 'grammar'
349 + if args.dataset == 'grammar':
350 + args.structure_type = 'grammar'
351 + args.async_engine = False
352 + elif args.dataset == 'regex':
353 + args.structure_type = 'regex'
354 + args.async_engine = False
355 + elif args.dataset == 'choice':
356 + args.structure_type = 'choice'
357 + args.async_engine = False
358 + else:
359 + args.structure_type = 'json'
360 +
361 + if args.no_guided_decoding:
362 + args.guided_decoding_ratio = 0
363 + if args.save_results:
364 + result_file_name = f'{args.guided_decoding_ratio}guided'
365 + result_file_name += f"_{args.model.split('/')[-1]}"
366 + result_file_name += f"_{args.dataset}"
367 + result_file_name += f"_{args.num_prompts}"
368 + result_file_name += f"_out{args.output_len}"
369 + result_file_name += f"_async{args.async_engine}"
370 + result_file_name += f"_warmup{args.warmup}"
371 + result_file_name += f"_chunkedprefill{args.enable_chunked_prefill}"
372 + result_file_name += ".txt"
373 + else:
374 + result_file_name = None
375 +
376 + # Synthesize a prompt with the given input length.
377 + tokenizer = AutoTokenizer.from_pretrained(
378 + args.tokenizer, trust_remote_code=args.trust_remote_code)
379 + requests = sample_requests(tokenizer, args)
380 +
381 + if args.async_engine:
382 + engine_args = AsyncEngineArgs.from_cli_args(args)
383 + elapsed_time, ret, (first_latency, next_latency) = uvloop.run(
384 + run_vllm_async(requests, engine_args, args.n,
385 + args.guided_decoding_ratio, args.warmup,
386 + args.disable_frontend_multiprocessing))
387 + else:
388 + engine_args = EngineArgs.from_cli_args(args)
389 + elapsed_time, ret = run_vllm(requests, engine_args, args.n,
390 + args.guided_decoding_ratio, args.warmup)
391 + first_latency, next_latency = None, None
392 +
393 + score = evaluate(ret, args)
394 + total_num_tokens = sum(request.prompt_len + request.expected_output_len
395 + for request in requests)
396 + total_output_tokens = sum(request.expected_output_len
397 + for request in requests)
398 + if first_latency is not None:
399 + latency_breakdown = "\nFirst token latency(msecs):\n"
400 + latency_breakdown += f"{first_latency.describe()}"
401 + latency_breakdown += "\nNext token latency(msecs):\n"
402 + latency_breakdown += f"{next_latency.describe()}"
403 + print(
404 + f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
405 + f"{total_num_tokens / elapsed_time:.2f} total tokens/s, "
406 + f"{total_output_tokens / elapsed_time:.2f} output tokens/s",
407 + f"Correct rate is {score} %",
408 + f"{latency_breakdown if first_latency is not None else ''}")
409 +
410 + # Output JSON results if specified
411 + if args.output_json or result_file_name:
412 + results = {
413 + "elapsed_time": elapsed_time,
414 + "num_requests": len(requests),
415 + "total_num_tokens": total_num_tokens,
416 + "total_output_tokens": total_output_tokens,
417 + "requests_per_second": len(requests) / elapsed_time,
418 + "tokens_per_second": f"{total_num_tokens / elapsed_time:.2f}",
419 + "output_tokens_per_second":
420 + f"{total_output_tokens / elapsed_time:.2f}",
421 + "correct_rate(%)": score
422 + }
423 + results = {"outputs": ret, **results}
424 + if first_latency is not None:
425 + results["first_token_latency(msecs)"] = first_latency.describe(
426 + ).to_dict()
427 + results["next_token_latency(msecs)"] = next_latency.describe(
428 + ).to_dict()
429 + if args.output_json:
430 + with open(args.output_json, "w") as f:
431 + json.dump(results, f, indent=4)
432 + elif result_file_name:
433 + with open(result_file_name, "w") as f:
434 + json.dump(results, f, indent=4)
435 +
436 +
437 +if __name__ == "__main__":
438 + parser = FlexibleArgumentParser(description="Benchmark guided decoding.")
439 + parser = AsyncEngineArgs.add_cli_args(parser)
440 +
441 + parser.add_argument("--output-len",
442 + type=int,
443 + default=512,
444 + help="Output length for each request. Overrides the "
445 + "output length from the dataset.")
446 + parser.add_argument(
447 + "--dataset",
448 + default='json',
449 + choices=['json', 'grammar', 'regex', 'choice', 'xgrammar_bench'])
450 + parser.add_argument("--json_schema_path",
451 + type=str,
452 + default=None,
453 + help="Path to json schema.")
454 + parser.add_argument("--n",
455 + type=int,
456 + default=1,
457 + help="Number of generated sequences per prompt.")
458 + parser.add_argument("--num-prompts",
459 + type=int,
460 + default=10,
461 + help="Number of prompts to process.")
462 + parser.add_argument(
463 + '--output-json',
464 + type=str,
465 + default=None,
466 + help='Path to save the throughput results in JSON format.')
467 + parser.add_argument("--async-engine",
468 + action='store_true',
469 + default=False,
470 + help="Use vLLM async engine rather than LLM class.")
471 + parser.add_argument("--no-guided-decoding",
472 + action='store_true',
473 + default=False,
474 + help="Whether to disable JSON decoding or not.")
475 + parser.add_argument("--guided-decoding-ratio",
476 + type=float,
477 + default=1.0,
478 + help="Ratio of Guided Decoding requests")
479 + parser.add_argument("--disable-frontend-multiprocessing",
480 + action='store_true',
481 + default=False,
482 + help="Disable decoupled async engine frontend.")
483 + parser.add_argument("--warmup",
484 + action="store_true",
485 + default=False,
486 + help="Run warmup prompts before benchmark.")
487 + parser.add_argument("--save-results",
488 + action="store_true",
489 + default=False,
490 + help="save output results.")
491 + args = parser.parse_args()
492 + if args.tokenizer is None:
493 + args.tokenizer = args.model
494 + main(args)
benchmarks/structured_schemas/structured_schema_1.json new
+113
@@ -0,0 +1,113 @@
1 +{
2 + "$schema":
3 + "https://json-schema.org/draft/2020-12/schema",
4 + "title":
5 + "User Profile",
6 + "type":
7 + "object",
8 + "properties": {
9 + "userId": {
10 + "type": "string",
11 + "description": "Unique identifier for the user."
12 + },
13 + "personalInfo": {
14 + "type": "object",
15 + "properties": {
16 + "firstName": {
17 + "type": "string",
18 + "description": "The user's first name."
19 + },
20 + "lastName": {
21 + "type": "string",
22 + "description": "The user's last name."
23 + },
24 + "age": {
25 + "type": "integer",
26 + "minimum": 0,
27 + "description": "The user's age."
28 + },
29 + "phoneNumbers": {
30 + "type":
31 + "array",
32 + "items": {
33 + "type": "object",
34 + "properties": {
35 + "type": {
36 + "type": "string",
37 + "enum": ["home", "work", "mobile"],
38 + "description": "Type of phone number."
39 + },
40 + "number": {
41 + "type": "string",
42 + "pattern": "^\\+?[1-9]\\d{1,14}$",
43 + "description": "Phone number in E.164 format."
44 + }
45 + },
46 + "required": ["type", "number"]
47 + },
48 + "description":
49 + "List of phone numbers associated with the user."
50 + }
51 + },
52 + "required": ["firstName", "lastName"]
53 + },
54 + "address": {
55 + "type": "object",
56 + "properties": {
57 + "street": {
58 + "type": "string",
59 + "description": "Street address."
60 + },
61 + "city": {
62 + "type": "string",
63 + "description": "City name."
64 + },
65 + "state": {
66 + "type": "string",
67 + "description": "State or province."
68 + },
69 + "postalCode": {
70 + "type": "string",
71 + "pattern": "^\\d{5}(-\\d{4})?$",
72 + "description": "Postal code."
73 + },
74 + "country": {
75 + "type": "string",
76 + "description": "Country name."
77 + }
78 + },
79 + "required": ["street", "city", "state", "postalCode", "country"]
80 + },
81 + "preferences": {
82 + "type": "object",
83 + "properties": {
84 + "newsletterSubscribed": {
85 + "type":
86 + "boolean",
87 + "description":
88 + "Indicates if the user is subscribed to the newsletter."
89 + },
90 + "favoriteCategories": {
91 + "type": "array",
92 + "items": {
93 + "type": "string"
94 + },
95 + "description": "List of user's favorite categories."
96 + }
97 + },
98 + "required": ["newsletterSubscribed"]
99 + },
100 + "accountStatus": {
101 + "type": "string",
102 + "enum": ["active", "inactive", "suspended"],
103 + "description": "Current status of the user's account."
104 + },
105 + "registrationDate": {
106 + "type": "string",
107 + "format": "date-time",
108 + "description": "ISO 8601 formatted date-time of user registration."
109 + }
110 + },
111 + "required":
112 + ["userId", "personalInfo", "address", "accountStatus", "registrationDate"]
113 +}
\ No newline at end of file