@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82eb5ea8f3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

Benchmark serving structured output (#10880)

Signed-off-by: Chendi Xue <chendi.xue@intel.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>

Chendi.Xue committed Dec 4, 2024 at 15:28 UTC 82eb5ea8f3bd3aabbe5c2fd43e37d263768603c5
2 files changed +887
benchmarks/backend_request_func.py
+6
@@ -24,6 +24,7 @@ class RequestFuncInput:
24 model: str
25 best_of: int = 1
26 logprobs: Optional[int] = None
27 + extra_body: Optional[dict] = None
28 multi_modal_content: Optional[dict] = None
29 ignore_eos: bool = False
30
@@ -36,6 +37,7 @@ class RequestFuncOutput:
37 ttft: float = 0.0 # Time to first token
38 itl: List[float] = field(
39 default_factory=list) # List of inter-token latencies
40 + tpot: float = 0.0 # avg next-token latencies
41 prompt_len: int = 0
42 error: str = ""
43
@@ -242,6 +244,8 @@ async def async_request_openai_completions(
244 "stream": True,
245 "ignore_eos": request_func_input.ignore_eos,
246 }
247 + if request_func_input.extra_body:
248 + payload.update(request_func_input.extra_body)
249 headers = {
250 "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}"
251 }
@@ -336,6 +340,8 @@ async def async_request_openai_chat_completions(
340 "stream": True,
341 "ignore_eos": request_func_input.ignore_eos,
342 }
343 + if request_func_input.extra_body:
344 + payload.update(request_func_input.extra_body)
345 headers = {
346 "Content-Type": "application/json",
347 "Authorization": f"Bearer {os.environ.get('OPENAI_API_KEY')}",
benchmarks/benchmark_serving_guided.py new
+881
@@ -0,0 +1,881 @@
1 +r"""Benchmark online serving throughput with guided decoding.
2 +
3 +On the server side, run one of the following commands:
4 + (vLLM OpenAI API server)
5 + vllm serve <your_model> --disable-log-requests
6 +
7 + (TGI backend)
8 + ./launch_tgi_server.sh <your_model> <max_batch_total_tokens>
9 +
10 +On the client side, run:
11 + python benchmarks/benchmark_serving.py \
12 + --backend <backend> \
13 + --model <your_model> \
14 + --dataset json \
15 + --guided-decoding-ratio 1.0 \
16 + --guided-decoding-backend xgrammar \
17 + --request-rate 10 \
18 + --num-prompts 1000
19 +
20 + when using tgi backend, add
21 + --endpoint /generate_stream
22 + to the end of the command above.
23 +"""
24 +import argparse
25 +import asyncio
26 +import dataclasses
27 +import json
28 +import os
29 +import random
30 +import time
31 +import warnings
32 +from dataclasses import dataclass
33 +from typing import AsyncGenerator, List, Optional, Tuple
34 +
35 +import datasets
36 +import numpy as np
37 +import pandas as pd
38 +from backend_request_func import (ASYNC_REQUEST_FUNCS, RequestFuncInput,
39 + RequestFuncOutput)
40 +from tqdm.asyncio import tqdm
41 +from transformers import PreTrainedTokenizerBase
42 +
43 +try:
44 + from vllm.transformers_utils.tokenizer import get_tokenizer
45 +except ImportError:
46 + from backend_request_func import get_tokenizer
47 +
48 +try:
49 + from vllm.utils import FlexibleArgumentParser
50 +except ImportError:
51 + from argparse import ArgumentParser as FlexibleArgumentParser
52 +
53 +MILLISECONDS_TO_SECONDS_CONVERSION = 1000
54 +
55 +
56 +@dataclass
57 +class BenchmarkMetrics:
58 + completed: int
59 + total_input: int
60 + total_output: int
61 + request_throughput: float
62 + request_goodput: float
63 + output_throughput: float
64 + total_token_throughput: float
65 + mean_ttft_ms: float
66 + median_ttft_ms: float
67 + std_ttft_ms: float
68 + percentiles_ttft_ms: List[Tuple[float, float]]
69 + mean_tpot_ms: float
70 + median_tpot_ms: float
71 + std_tpot_ms: float
72 + percentiles_tpot_ms: List[Tuple[float, float]]
73 + mean_itl_ms: float
74 + median_itl_ms: float
75 + std_itl_ms: float
76 + percentiles_itl_ms: List[Tuple[float, float]]
77 + # E2EL stands for end-to-end latency per request.
78 + # It is the time taken on the client side from sending
79 + # a request to receiving a complete response.
80 + mean_e2el_ms: float
81 + median_e2el_ms: float
82 + std_e2el_ms: float
83 + percentiles_e2el_ms: List[Tuple[float, float]]
84 +
85 +
86 +@dataclasses.dataclass
87 +class SampleRequest:
88 + """A class representing a single inference request for benchmarking.
89 +
90 + Attributes:
91 + prompt: The input text prompt for the model.
92 + multi_modal_data: Optional dictionary containing multi-modal data (e.g.
93 + images).
94 + prompt_len: The length of the prompt in tokens.
95 + expected_output_len: The expected length of the output in tokens.
96 + """
97 + prompt: str
98 + prompt_len: int
99 + expected_output_len: int
100 + schema: dict
101 + structure_type: str
102 + completion: str = None
103 +
104 +
105 +def sample_requests(tokenizer: PreTrainedTokenizerBase,
106 + args: argparse.Namespace) -> List[SampleRequest]:
107 + if args.dataset == 'json':
108 + if args.json_schema_path is None:
109 + dir_path = os.path.dirname(os.path.realpath(__file__))
110 + args.json_schema_path = os.path.join(dir_path,
111 + "structured_schemas",
112 + "structured_schema_1.json")
113 + with open(args.json_schema_path) as f:
114 + schema = json.load(f)
115 + prompt = f"Generate an example of a user profile given the following schema: {json.dumps(schema)}" # noqa: E501
116 + input_len = len(tokenizer(prompt).input_ids)
117 + print(f"Input length of the prompt: {input_len} tokens")
118 + requests = [
119 + SampleRequest(prompt=prompt,
120 + prompt_len=input_len,
121 + expected_output_len=args.output_len,
122 + schema=schema,
123 + structure_type=args.structure_type)
124 + for _ in range(args.num_prompts)
125 + ]
126 +
127 + elif args.dataset == "grammar":
128 + schema = """
129 + ?start: select_statement
130 +
131 + ?select_statement: "SELECT " column_list " FROM " table_name
132 +
133 + ?column_list: column_name ("," column_name)*
134 +
135 + ?table_name: identifier
136 +
137 + ?column_name: identifier
138 +
139 + ?identifier: /[a-zA-Z_][a-zA-Z0-9_]*/
140 + """
141 + prompt = "Generate an SQL query to show the 'username' \
142 + and 'email' from the 'users' table."
143 +
144 + input_len = len(tokenizer(prompt).input_ids)
145 + print(f"Input length of the prompt: {input_len} tokens")
146 + requests = [
147 + SampleRequest(prompt=prompt,
148 + prompt_len=input_len,
149 + expected_output_len=args.output_len,
150 + schema=schema,
151 + structure_type=args.structure_type)
152 + for _ in range(args.num_prompts)
153 + ]
154 +
155 + elif args.dataset == "regex":
156 + regex = r"\w+@\w+\.com\n"
157 + args.regex = regex
158 + prompt = "Generate an email address for Alan Turing, \
159 + who works in Enigma. End in .com and new line. \
160 + Example result: alan.turing@enigma.com\n"
161 +
162 + input_len = len(tokenizer(prompt).input_ids)
163 + print(f"Input length of the prompt: {input_len} tokens")
164 + requests = [
165 + SampleRequest(prompt=prompt,
166 + prompt_len=input_len,
167 + expected_output_len=args.output_len,
168 + schema=regex,
169 + structure_type=args.structure_type)
170 + for _ in range(args.num_prompts)
171 + ]
172 +
173 + elif args.dataset == "choice":
174 + choice = ["Positive", "Negative"]
175 + args.choice = choice
176 + prompt = "Classify this sentiment: vLLM is wonderful!"
177 + input_len = len(tokenizer(prompt).input_ids)
178 + print(f"Input length of the prompt: {input_len} tokens")
179 + requests = [
180 + SampleRequest(prompt=prompt,
181 + prompt_len=input_len,
182 + expected_output_len=args.output_len,
183 + schema=choice,
184 + structure_type=args.structure_type)
185 + for _ in range(args.num_prompts)
186 + ]
187 +
188 + elif args.dataset == "xgrammar_bench":
189 + requests: List[SampleRequest] = []
190 + dataset = datasets.load_dataset("NousResearch/json-mode-eval",
191 + split="train")
192 + print(f"dataset has {len(dataset)} entries")
193 + len_dataset = len(dataset)
194 + for data_point_idx in range(args.num_prompts):
195 + idx = data_point_idx
196 + while idx >= len_dataset:
197 + idx -= len_dataset
198 + schema = dataset["schema"][idx]
199 + prompt = tokenizer.apply_chat_template(dataset["prompt"][idx],
200 + tokenize=False)
201 + input_len = len(tokenizer(prompt).input_ids)
202 + completion = dataset["completion"][idx]
203 +
204 + requests.append(
205 + SampleRequest(prompt=prompt,
206 + prompt_len=input_len,
207 + expected_output_len=args.output_len,
208 + schema=schema,
209 + structure_type=args.structure_type,
210 + completion=completion))
211 +
212 + return requests
213 +
214 +
215 +async def get_request(
216 + input_requests: List[SampleRequest],
217 + request_rate: float,
218 + burstiness: float = 1.0,
219 +) -> AsyncGenerator[Tuple[int, SampleRequest], None]:
220 + """
221 + Asynchronously generates requests at a specified rate
222 + with OPTIONAL burstiness.
223 +
224 + Args:
225 + input_requests:
226 + A list of input requests, each represented as a tuple.
227 + request_rate:
228 + The rate at which requests are generated (requests/s).
229 + burstiness (optional):
230 + The burstiness factor of the request generation.
231 + Only takes effect when request_rate is not inf.
232 + Default value is 1, which follows a Poisson process.
233 + Otherwise, the request intervals follow a gamma distribution.
234 + A lower burstiness value (0 < burstiness < 1) results
235 + in more bursty requests, while a higher burstiness value
236 + (burstiness > 1) results in a more uniform arrival of requests.
237 + """
238 + input_requests = iter(input_requests)
239 +
240 + # Calculate scale parameter theta to maintain the desired request_rate.
241 + assert burstiness > 0, (
242 + f"A positive burstiness factor is expected, but given {burstiness}.")
243 + theta = 1.0 / (request_rate * burstiness)
244 +
245 + for i, request in enumerate(input_requests):
246 + yield i, request
247 +
248 + if request_rate == float("inf"):
249 + # If the request rate is infinity, then we don't need to wait.
250 + continue
251 +
252 + # Sample the request interval from the gamma distribution.
253 + # If burstiness is 1, it follows exponential distribution.
254 + interval = np.random.gamma(shape=burstiness, scale=theta)
255 + # The next request will be sent after the interval.
256 + await asyncio.sleep(interval)
257 +
258 +
259 +def calculate_metrics(
260 + input_requests: List[Tuple[str, int, int]],
261 + outputs: List[RequestFuncOutput],
262 + dur_s: float,
263 + tokenizer: PreTrainedTokenizerBase,
264 + selected_percentile_metrics: List[str],
265 + selected_percentiles: List[float],
266 +) -> Tuple[BenchmarkMetrics, List[int]]:
267 + actual_output_lens: List[int] = []
268 + total_input = 0
269 + completed = 0
270 + good_completed = 0
271 + itls: List[float] = []
272 + tpots: List[float] = []
273 + all_tpots: List[float] = []
274 + ttfts: List[float] = []
275 + e2els: List[float] = []
276 + for i in range(len(outputs)):
277 + if outputs[i].success:
278 + # We use the tokenizer to count the number of output tokens for all
279 + # serving backends instead of looking at len(outputs[i].itl) since
280 + # multiple output tokens may be bundled together
281 + # Note : this may inflate the output token count slightly
282 + output_len = len(
283 + tokenizer(outputs[i].generated_text,
284 + add_special_tokens=False).input_ids)
285 + actual_output_lens.append(output_len)
286 + total_input += input_requests[i].prompt_len
287 + tpot = 0
288 + if output_len > 1:
289 + tpot = (outputs[i].latency - outputs[i].ttft) / (output_len -
290 + 1)
291 + tpots.append(tpot)
292 + outputs[i].tpot = sum(tpots) / len(tpots) if len(tpots) else 0
293 + # Note: if output_len <= 1, we regard tpot as 0 for goodput
294 + all_tpots.append(tpot)
295 + itls += outputs[i].itl
296 + ttfts.append(outputs[i].ttft)
297 + e2els.append(outputs[i].latency)
298 + completed += 1
299 + else:
300 + actual_output_lens.append(0)
301 +
302 + if completed == 0:
303 + warnings.warn(
304 + "All requests failed. This is likely due to a misconfiguration "
305 + "on the benchmark arguments.",
306 + stacklevel=2)
307 + metrics = BenchmarkMetrics(
308 + completed=completed,
309 + total_input=total_input,
310 + total_output=sum(actual_output_lens),
311 + request_throughput=completed / dur_s,
312 + request_goodput=good_completed / dur_s,
313 + output_throughput=sum(actual_output_lens) / dur_s,
314 + total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s,
315 + mean_ttft_ms=np.mean(ttfts or 0) *
316 + 1000, # ttfts is empty if streaming is not supported by backend
317 + std_ttft_ms=np.std(ttfts or 0) * 1000,
318 + median_ttft_ms=np.median(ttfts or 0) * 1000,
319 + percentiles_ttft_ms=[(p, np.percentile(ttfts or 0, p) * 1000)
320 + for p in selected_percentiles],
321 + mean_tpot_ms=np.mean(tpots or 0) * 1000,
322 + std_tpot_ms=np.std(tpots or 0) * 1000,
323 + median_tpot_ms=np.median(tpots or 0) * 1000,
324 + percentiles_tpot_ms=[(p, np.percentile(tpots or 0, p) * 1000)
325 + for p in selected_percentiles],
326 + mean_itl_ms=np.mean(itls or 0) * 1000,
327 + std_itl_ms=np.std(itls or 0) * 1000,
328 + median_itl_ms=np.median(itls or 0) * 1000,
329 + percentiles_itl_ms=[(p, np.percentile(itls or 0, p) * 1000)
330 + for p in selected_percentiles],
331 + mean_e2el_ms=np.mean(e2els or 0) * 1000,
332 + std_e2el_ms=np.std(e2els or 0) * 1000,
333 + median_e2el_ms=np.median(e2els or 0) * 1000,
334 + percentiles_e2el_ms=[(p, np.percentile(e2els or 0, p) * 1000)
335 + for p in selected_percentiles],
336 + )
337 +
338 + return metrics, actual_output_lens
339 +
340 +
341 +async def benchmark(
342 + backend: str,
343 + api_url: str,
344 + base_url: str,
345 + model_id: str,
346 + tokenizer: PreTrainedTokenizerBase,
347 + input_requests: List[SampleRequest],
348 + request_rate: float,
349 + burstiness: float,
350 + disable_tqdm: bool,
351 + profile: bool,
352 + selected_percentile_metrics: List[str],
353 + selected_percentiles: List[str],
354 + ignore_eos: bool,
355 + max_concurrency: Optional[int],
356 + guided_decoding_ratio: float,
357 + guided_decoding_backend: str,
358 +):
359 + if backend in ASYNC_REQUEST_FUNCS:
360 + request_func = ASYNC_REQUEST_FUNCS[backend]
361 + else:
362 + raise ValueError(f"Unknown backend: {backend}")
363 +
364 + def prepare_extra_body(request) -> dict:
365 + extra_body = {}
366 + # Add the schema to the extra_body
367 + extra_body[request.structure_type] = request.schema
368 + # Add the specific guided_decoding_backend
369 + extra_body["guided_decoding_backend"] = guided_decoding_backend
370 + return extra_body
371 +
372 + print("Starting initial single prompt test run...")
373 + guided_decoding_req_idx = random.sample(
374 + range(len(input_requests)),
375 + int(len(input_requests) * guided_decoding_ratio))
376 +
377 + test_request = input_requests[0]
378 + test_input = RequestFuncInput(
379 + model=model_id,
380 + prompt=test_request.prompt,
381 + api_url=api_url,
382 + prompt_len=test_request.prompt_len,
383 + output_len=test_request.expected_output_len,
384 + ignore_eos=ignore_eos,
385 + extra_body=prepare_extra_body(test_request),
386 + )
387 + test_output = await request_func(request_func_input=test_input)
388 + if not test_output.success:
389 + raise ValueError(
390 + "Initial test run failed - Please make sure benchmark arguments "
391 + f"are correctly specified. Error: {test_output.error}")
392 + else:
393 + print("Initial test run completed. Starting main benchmark run...")
394 +
395 + if profile:
396 + print("Starting profiler...")
397 + profile_input = RequestFuncInput(
398 + model=model_id,
399 + prompt=test_request.prompt,
400 + api_url=base_url + "/start_profile",
401 + prompt_len=test_request.prompt_len,
402 + output_len=test_request.expected_output_len,
403 + ignore_eos=ignore_eos,
404 + extra_body=prepare_extra_body(test_request),
405 + )
406 + profile_output = await request_func(request_func_input=profile_input)
407 + if profile_output.success:
408 + print("Profiler started")
409 +
410 + if burstiness == 1.0:
411 + distribution = "Poisson process"
412 + else:
413 + distribution = "Gamma distribution"
414 +
415 + print(f"Traffic request rate: {request_rate}")
416 + print(f"Burstiness factor: {burstiness} ({distribution})")
417 + print(f"Maximum request concurrency: {max_concurrency}")
418 +
419 + pbar = None if disable_tqdm else tqdm(total=len(input_requests))
420 +
421 + # This can be used once the minimum Python version is 3.10 or higher,
422 + # and it will simplify the code in limited_request_func.
423 + # semaphore = (asyncio.Semaphore(max_concurrency)
424 + # if max_concurrency else contextlib.nullcontext())
425 + semaphore = (asyncio.Semaphore(max_concurrency)
426 + if max_concurrency else None)
427 +
428 + async def limited_request_func(request_func_input, pbar):
429 + if semaphore is None:
430 + return await request_func(request_func_input=request_func_input,
431 + pbar=pbar)
432 + async with semaphore:
433 + return await request_func(request_func_input=request_func_input,
434 + pbar=pbar)
435 +
436 + benchmark_start_time = time.perf_counter()
437 + tasks: List[asyncio.Task] = []
438 + expected: List[str] = []
439 + async for i, request in get_request(input_requests, request_rate,
440 + burstiness):
441 + extra_body = prepare_extra_body(
442 + request) if i in guided_decoding_req_idx else None
443 + request_func_input = RequestFuncInput(
444 + model=model_id,
445 + prompt=request.prompt,
446 + api_url=api_url,
447 + prompt_len=request.prompt_len,
448 + output_len=request.expected_output_len,
449 + ignore_eos=ignore_eos,
450 + extra_body=extra_body,
451 + )
452 + expected.append(request.completion)
453 + tasks.append(
454 + asyncio.create_task(
455 + limited_request_func(request_func_input=request_func_input,
456 + pbar=pbar)))
457 + outputs: List[RequestFuncOutput] = await asyncio.gather(*tasks)
458 +
459 + if profile:
460 + print("Stopping profiler...")
461 + profile_input = RequestFuncInput(
462 + model=model_id,
463 + prompt=test_request.prompt,
464 + api_url=base_url + "/stop_profile",
465 + prompt_len=test_request.prompt_len,
466 + output_len=test_request.expected_output_len,
467 + extra_body={test_request.structure_type: test_request.schema},
468 + )
469 + profile_output = await request_func(request_func_input=profile_input)
470 + if profile_output.success:
471 + print("Profiler stopped")
472 +
473 + if pbar is not None:
474 + pbar.close()
475 +
476 + benchmark_duration = time.perf_counter() - benchmark_start_time
477 +
478 + metrics, actual_output_lens = calculate_metrics(
479 + input_requests=input_requests,
480 + outputs=outputs,
481 + dur_s=benchmark_duration,
482 + tokenizer=tokenizer,
483 + selected_percentile_metrics=selected_percentile_metrics,
484 + selected_percentiles=selected_percentiles,
485 + )
486 +
487 + print("{s:{c}^{n}}".format(s=' Serving Benchmark Result ', n=50, c='='))
488 + print("{:<40} {:<10}".format("Successful requests:", metrics.completed))
489 + print("{:<40} {:<10.2f}".format("Benchmark duration (s):",
490 + benchmark_duration))
491 + print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input))
492 + print("{:<40} {:<10}".format("Total generated tokens:",
493 + metrics.total_output))
494 + print("{:<40} {:<10.2f}".format("Request throughput (req/s):",
495 + metrics.request_throughput))
496 + print("{:<40} {:<10.2f}".format("Output token throughput (tok/s):",
497 + metrics.output_throughput))
498 + print("{:<40} {:<10.2f}".format("Total Token throughput (tok/s):",
499 + metrics.total_token_throughput))
500 +
501 + result = {
502 + "duration":
503 + benchmark_duration,
504 + "completed":
505 + metrics.completed,
506 + "total_input_tokens":
507 + metrics.total_input,
508 + "total_output_tokens":
509 + metrics.total_output,
510 + "request_throughput":
511 + metrics.request_throughput,
512 + "output_throughput":
513 + metrics.output_throughput,
514 + "total_token_throughput":
515 + metrics.total_token_throughput,
516 + "ttft_description":
517 + pd.Series([output.ttft for output in outputs]).describe().to_dict(),
518 + "tpot_description":
519 + pd.Series([output.tpot for output in outputs]).describe().to_dict(),
520 + "input_lens": [output.prompt_len for output in outputs],
521 + "output_lens":
522 + actual_output_lens,
523 + "ttfts": [output.ttft for output in outputs],
524 + "itls": [output.itl for output in outputs],
525 + "errors": [output.error for output in outputs],
526 + }
527 +
528 + ret = [{
529 + 'generated': output.generated_text,
530 + 'expected': gt
531 + } for output, gt in zip(outputs, expected)]
532 +
533 + def process_one_metric(
534 + # E.g., "ttft"
535 + metric_attribute_name: str,
536 + # E.g., "TTFT"
537 + metric_name: str,
538 + # E.g., "Time to First Token"
539 + metric_header: str,
540 + ):
541 + # This function prints and adds statistics of the specified
542 + # metric.
543 + if metric_attribute_name not in selected_percentile_metrics:
544 + return
545 + print("{s:{c}^{n}}".format(s=metric_header, n=50, c='-'))
546 + print("{:<40} {:<10.2f}".format(
547 + f"Mean {metric_name} (ms):",
548 + getattr(metrics, f"mean_{metric_attribute_name}_ms")))
549 + print("{:<40} {:<10.2f}".format(
550 + f"Median {metric_name} (ms):",
551 + getattr(metrics, f"median_{metric_attribute_name}_ms")))
552 + result[f"mean_{metric_attribute_name}_ms"] = getattr(
553 + metrics, f"mean_{metric_attribute_name}_ms")
554 + result[f"median_{metric_attribute_name}_ms"] = getattr(
555 + metrics, f"median_{metric_attribute_name}_ms")
556 + result[f"std_{metric_attribute_name}_ms"] = getattr(
557 + metrics, f"std_{metric_attribute_name}_ms")
558 + for p, value in getattr(metrics,
559 + f"percentiles_{metric_attribute_name}_ms"):
560 + p_word = str(int(p)) if int(p) == p else str(p)
561 + print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):",
562 + value))
563 + result[f"p{p_word}_{metric_attribute_name}_ms"] = value
564 +
565 + process_one_metric("ttft", "TTFT", "Time to First Token")
566 + process_one_metric("tpot", "TPOT",
567 + "Time per Output Token (excl. 1st token)")
568 + process_one_metric("itl", "ITL", "Inter-token Latency")
569 + process_one_metric("e2el", "E2EL", "End-to-end Latency")
570 +
571 + print("=" * 50)
572 +
573 + return result, ret
574 +
575 +
576 +def evaluate(ret, args):
577 +
578 + def _eval_correctness_json(expected, actual):
579 + # extract json string from string using regex
580 + import re
581 + actual = actual.replace('\n', '').replace(' ', '').strip()
582 + try:
583 + actual = re.search(r'\{.*\}', actual).group()
584 + actual = json.loads(actual)
585 + except Exception:
586 + return False
587 +
588 + return True
589 +
590 + def _eval_correctness_choice(expected, actual):
591 + return actual in args.choice
592 +
593 + def _eval_correctness_regex(expected, actual):
594 + import re
595 + return re.match(args.regex, actual) is not None
596 +
597 + def _eval_correctness(expected, actual):
598 + if args.structure_type == 'guided_json':
599 + return _eval_correctness_json(expected, actual)
600 + elif args.structure_type == 'guided_regex':
601 + return _eval_correctness_regex(expected, actual)
602 + elif args.structure_type == 'guided_choice':
603 + return _eval_correctness_choice(expected, actual)
604 + else:
605 + return None
606 +
607 + scores = []
608 + for res in ret:
609 + score = _eval_correctness(res['expected'], res['generated'])
610 + res['correctness'] = score
611 + scores.append(score)
612 +
613 + not_none_scores = [score for score in scores if score is not None]
614 +
615 + return (sum(not_none_scores) / len(not_none_scores) *
616 + 100) if len(not_none_scores) > 0 else None
617 +
618 +
619 +def main(args: argparse.Namespace):
620 + print(args)
621 + random.seed(args.seed)
622 + np.random.seed(args.seed)
623 +
624 + backend = args.backend
625 + model_id = args.model
626 + tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model
627 +
628 + if args.base_url is not None:
629 + api_url = f"{args.base_url}{args.endpoint}"
630 + base_url = f"{args.base_url}"
631 + else:
632 + api_url = f"http://{args.host}:{args.port}{args.endpoint}"
633 + base_url = f"http://{args.host}:{args.port}"
634 +
635 + tokenizer = get_tokenizer(tokenizer_id,
636 + trust_remote_code=args.trust_remote_code)
637 +
638 + if args.dataset == 'grammar':
639 + args.structure_type = 'guided_grammar'
640 + elif args.dataset == 'regex':
641 + args.structure_type = 'guided_regex'
642 + elif args.dataset == 'choice':
643 + args.structure_type = 'guided_choice'
644 + else:
645 + args.structure_type = 'guided_json'
646 +
647 + if args.no_guided_decoding:
648 + args.guided_decoding_ratio = 0
649 + if args.save_results:
650 + result_file_name = f'{args.guided_decoding_ratio}guided'
651 + result_file_name += f"_{backend}"
652 + result_file_name += f"_{args.request_rate}qps"
653 + result_file_name += f"_{args.model.split('/')[-1]}"
654 + result_file_name += f"_{args.dataset}"
655 + result_file_name += f"_{args.num_prompts}"
656 + result_file_name += f"_out{args.output_len}"
657 + result_file_name += ".txt"
658 + else:
659 + result_file_name = None
660 +
661 + input_requests = sample_requests(tokenizer, args)
662 +
663 + benchmark_result, ret = asyncio.run(
664 + benchmark(
665 + backend=backend,
666 + api_url=api_url,
667 + base_url=base_url,
668 + model_id=model_id,
669 + tokenizer=tokenizer,
670 + input_requests=input_requests,
671 + request_rate=args.request_rate,
672 + burstiness=args.burstiness,
673 + disable_tqdm=args.disable_tqdm,
674 + profile=args.profile,
675 + selected_percentile_metrics=args.percentile_metrics.split(","),
676 + selected_percentiles=[
677 + float(p) for p in args.metric_percentiles.split(",")
678 + ],
679 + ignore_eos=args.ignore_eos,
680 + max_concurrency=args.max_concurrency,
681 + guided_decoding_ratio=args.guided_decoding_ratio,
682 + guided_decoding_backend=args.guided_decoding_backend,
683 + ))
684 +
685 + # Save config and results to json
686 + score = evaluate(ret, args)
687 + print("correct_rate(%)", score, '\n')
688 + if args.save_results:
689 + results = {
690 + "backend":
691 + backend,
692 + "model_id":
693 + model_id,
694 + "tokenizer_id":
695 + tokenizer_id,
696 + "num_prompts":
697 + args.num_prompts,
698 + "request_rate":
699 + args.request_rate if args.request_rate < float("inf") else "inf",
700 + "burstiness":
701 + args.burstiness,
702 + "max_concurrency":
703 + args.max_concurrency,
704 + "correct_rate(%)":
705 + score
706 + }
707 + results = {"outputs": ret, **results, **benchmark_result}
708 +
709 + # Save to file
710 + if args.result_filename:
711 + result_file_name = args.result_filename
712 + if args.result_dir:
713 + result_file_name = os.path.join(args.result_dir, result_file_name)
714 + with open(result_file_name, "w", encoding='utf-8') as outfile:
715 + json.dump(results, outfile, indent=4)
716 +
717 +
718 +if __name__ == "__main__":
719 + parser = FlexibleArgumentParser(
720 + description="Benchmark the online serving throughput.")
721 + parser.add_argument(
722 + "--backend",
723 + type=str,
724 + default="vllm",
725 + choices=list(ASYNC_REQUEST_FUNCS.keys()),
726 + )
727 + parser.add_argument(
728 + "--base-url",
729 + type=str,
730 + default=None,
731 + help="Server or API base url if not using http host and port.",
732 + )
733 + parser.add_argument("--host", type=str, default="localhost")
734 + parser.add_argument("--port", type=int, default=8000)
735 + parser.add_argument(
736 + "--endpoint",
737 + type=str,
738 + default="/v1/completions",
739 + help="API endpoint.",
740 + )
741 + parser.add_argument(
742 + "--dataset",
743 + default='json',
744 + choices=['json', 'grammar', 'regex', 'choice', 'xgrammar_bench'])
745 + parser.add_argument("--json_schema_path",
746 + type=str,
747 + default=None,
748 + help="Path to json schema.")
749 + parser.add_argument(
750 + "--max-concurrency",
751 + type=int,
752 + default=None,
753 + help="Maximum number of concurrent requests. This can be used "
754 + "to help simulate an environment where a higher level component "
755 + "is enforcing a maximum number of concurrent requests. While the "
756 + "--request-rate argument controls the rate at which requests are "
757 + "initiated, this argument will control how many are actually allowed "
758 + "to execute at a time. This means that when used in combination, the "
759 + "actual request rate may be lower than specified with --request-rate, "
760 + "if the server is not processing requests fast enough to keep up.")
761 + parser.add_argument(
762 + "--model",
763 + type=str,
764 + required=True,
765 + help="Name of the model.",
766 + )
767 + parser.add_argument(
768 + "--tokenizer",
769 + type=str,
770 + help=
771 + "Name or path of the tokenizer, if not using the default tokenizer.", # noqa: E501
772 + )
773 + parser.add_argument(
774 + "--num-prompts",
775 + type=int,
776 + default=1000,
777 + help="Number of prompts to process.",
778 + )
779 + parser.add_argument(
780 + "--output-len",
781 + type=int,
782 + default=128,
783 + help="Number of output tokens.",
784 + )
785 + parser.add_argument(
786 + "--request-rate",
787 + type=float,
788 + default=float("inf"),
789 + help="Number of requests per second. If this is inf, "
790 + "then all the requests are sent at time 0. "
791 + "Otherwise, we use Poisson process or gamma distribution "
792 + "to synthesize the request arrival times.",
793 + )
794 + parser.add_argument(
795 + "--burstiness",
796 + type=float,
797 + default=1.0,
798 + help="Burstiness factor of the request generation. "
799 + "Only take effect when request_rate is not inf. "
800 + "Default value is 1, which follows Poisson process. "
801 + "Otherwise, the request intervals follow a gamma distribution. "
802 + "A lower burstiness value (0 < burstiness < 1) results in more "
803 + "bursty requests. A higher burstiness value (burstiness > 1) "
804 + "results in a more uniform arrival of requests.",
805 + )
806 + parser.add_argument("--seed", type=int, default=0)
807 + parser.add_argument(
808 + "--trust-remote-code",
809 + action="store_true",
810 + help="Trust remote code from huggingface",
811 + )
812 + parser.add_argument(
813 + "--disable-tqdm",
814 + action="store_true",
815 + help="Specify to disable tqdm progress bar.",
816 + )
817 + parser.add_argument(
818 + "--save-results",
819 + action="store_true",
820 + help="Specify to save benchmark results to a json file",
821 + )
822 + parser.add_argument(
823 + "--profile",
824 + action="store_true",
825 + help="Use Torch Profiler. The endpoint must be launched with "
826 + "VLLM_TORCH_PROFILER_DIR to enable profiler.",
827 + )
828 + parser.add_argument(
829 + "--result-dir",
830 + type=str,
831 + default=None,
832 + help="Specify directory to save benchmark json results."
833 + "If not specified, results are saved in the current directory.",
834 + )
835 + parser.add_argument(
836 + "--result-filename",
837 + type=str,
838 + default=None,
839 + help="Specify the filename to save benchmark json results."
840 + "If not specified, results will be saved in "
841 + "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
842 + " format.",
843 + )
844 + parser.add_argument(
845 + "--ignore-eos",
846 + action="store_true",
847 + help="Set ignore_eos flag when sending the benchmark request."
848 + "Warning: ignore_eos is not supported in deepspeed_mii and tgi.")
849 + parser.add_argument(
850 + "--percentile-metrics",
851 + type=str,
852 + default="ttft,tpot,itl",
853 + help="Comma-seperated list of selected metrics to report percentils. "
854 + "This argument specifies the metrics to report percentiles. "
855 + "Allowed metric names are \"ttft\", \"tpot\", \"itl\", \"e2el\". "
856 + "Default value is \"ttft,tpot,itl\".")
857 + parser.add_argument(
858 + "--metric-percentiles",
859 + type=str,
860 + default="99",
861 + help="Comma-seperated list of percentiles for selected metrics. "
862 + "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
863 + "Default value is \"99\". "
864 + "Use \"--percentile-metrics\" to select metrics.",
865 + )
866 + parser.add_argument("--no-guided-decoding",
867 + action='store_true',
868 + default=False,
869 + help="Whether to disable JSON decoding or not.")
870 + parser.add_argument("--guided-decoding-ratio",
871 + type=float,
872 + default=1.0,
873 + help="Ratio of Guided Decoding requests")
874 + parser.add_argument("--guided-decoding-backend",
875 + type=str,
876 + choices=["outlines", "lm-format-enforcer", "xgrammar"],
877 + default="xgrammar",
878 + help="Backend to use for guided decoding")
879 +
880 + args = parser.parse_args()
881 + main(args)