| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | r"""Benchmark online serving throughput with structured outputs. |
| 4 | |
| 5 | On the server side, run one of the following commands: |
| 6 | (vLLM OpenAI API server) |
| 7 | vllm serve <your_model> |
| 8 | |
| 9 | On the client side, run: |
| 10 | python benchmarks/benchmark_serving_structured_output.py \ |
| 11 | --backend <backend> \ |
| 12 | --model <your_model> \ |
| 13 | --dataset json \ |
| 14 | --structured-output-ratio 1.0 \ |
| 15 | --request-rate 10 \ |
| 16 | --num-prompts 1000 |
| 17 | |
| 18 | when using tgi backend, add |
| 19 | --endpoint /generate_stream |
| 20 | to the end of the command above. |
| 21 | """ |
| 22 | |
| 23 | import argparse |
| 24 | import asyncio |
| 25 | import copy |
| 26 | import dataclasses |
| 27 | import json |
| 28 | import os |
| 29 | import random |
| 30 | import time |
| 31 | import uuid |
| 32 | import warnings |
| 33 | from collections.abc import AsyncGenerator |
| 34 | from contextlib import nullcontext |
| 35 | from dataclasses import dataclass |
| 36 | |
| 37 | import datasets |
| 38 | import numpy as np |
| 39 | import pandas as pd |
| 40 | from backend_request_func import ( |
| 41 | ASYNC_REQUEST_FUNCS, |
| 42 | RequestFuncInput, |
| 43 | RequestFuncOutput, |
| 44 | ) |
| 45 | from tqdm.asyncio import tqdm |
| 46 | from transformers import PreTrainedTokenizerBase |
| 47 | |
| 48 | try: |
| 49 | from vllm.tokenizers import get_tokenizer |
| 50 | except ImportError: |
| 51 | from backend_request_func import get_tokenizer |
| 52 | |
| 53 | try: |
| 54 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 55 | except ImportError: |
| 56 | from argparse import ArgumentParser as FlexibleArgumentParser |
| 57 | |
| 58 | from vllm.v1.structured_output.backend_xgrammar import ( |
| 59 | has_xgrammar_unsupported_json_features, |
| 60 | ) |
| 61 | |
| 62 | MILLISECONDS_TO_SECONDS_CONVERSION = 1000 |
| 63 | |
| 64 | |
| 65 | @dataclass |
| 66 | class BenchmarkMetrics: |
| 67 | completed: int |
| 68 | total_input: int |
| 69 | total_output: int |
| 70 | request_throughput: float |
| 71 | request_goodput: float |
| 72 | output_throughput: float |
| 73 | total_token_throughput: float |
| 74 | mean_ttft_ms: float |
| 75 | median_ttft_ms: float |
| 76 | std_ttft_ms: float |
| 77 | percentiles_ttft_ms: list[tuple[float, float]] |
| 78 | mean_tpot_ms: float |
| 79 | median_tpot_ms: float |
| 80 | std_tpot_ms: float |
| 81 | percentiles_tpot_ms: list[tuple[float, float]] |
| 82 | mean_itl_ms: float |
| 83 | median_itl_ms: float |
| 84 | std_itl_ms: float |
| 85 | percentiles_itl_ms: list[tuple[float, float]] |
| 86 | # E2EL stands for end-to-end latency per request. |
| 87 | # It is the time taken on the client side from sending |
| 88 | # a request to receiving a complete response. |
| 89 | mean_e2el_ms: float |
| 90 | median_e2el_ms: float |
| 91 | std_e2el_ms: float |
| 92 | percentiles_e2el_ms: list[tuple[float, float]] |
| 93 | |
| 94 | |
| 95 | @dataclasses.dataclass |
| 96 | class SampleRequest: |
| 97 | """A class representing a single inference request for benchmarking. |
| 98 | |
| 99 | Attributes: |
| 100 | prompt: The input text prompt for the model. |
| 101 | multi_modal_data: Optional dictionary containing multi-modal data (e.g. |
| 102 | images). |
| 103 | prompt_len: The length of the prompt in tokens. |
| 104 | expected_output_len: The expected length of the output in tokens. |
| 105 | |
| 106 | """ |
| 107 | |
| 108 | prompt: str |
| 109 | prompt_len: int |
| 110 | expected_output_len: int |
| 111 | schema: dict |
| 112 | structure_type: str |
| 113 | completion: str = None |
| 114 | |
| 115 | |
| 116 | def sample_requests( |
| 117 | tokenizer: PreTrainedTokenizerBase, args: argparse.Namespace |
| 118 | ) -> list[SampleRequest]: |
| 119 | def _apply_random_prefix( |
| 120 | tokenizer: PreTrainedTokenizerBase, |
| 121 | requests: list[SampleRequest], |
| 122 | prefix_len: int, |
| 123 | seed: int, |
| 124 | ) -> list[SampleRequest]: |
| 125 | if prefix_len <= 0: |
| 126 | return requests |
| 127 | rng = np.random.default_rng(seed) |
| 128 | vocab_size = tokenizer.vocab_size |
| 129 | prohibited = getattr(tokenizer, "all_special_ids", None) or [] |
| 130 | allowed = np.array([i for i in range(vocab_size) if i not in prohibited]) |
| 131 | if len(allowed) == 0: |
| 132 | return requests |
| 133 | prefix_ids = rng.integers(0, len(allowed), size=prefix_len) |
| 134 | prefix_token_ids = allowed[prefix_ids].tolist() |
| 135 | out = [] |
| 136 | for req in requests: |
| 137 | prompt_ids = tokenizer(req.prompt, add_special_tokens=False).input_ids |
| 138 | full_ids = prefix_token_ids + prompt_ids |
| 139 | full_prompt = tokenizer.decode(full_ids, skip_special_tokens=False) |
| 140 | out.append( |
| 141 | SampleRequest( |
| 142 | prompt=full_prompt, |
| 143 | prompt_len=len(tokenizer(full_prompt).input_ids), |
| 144 | expected_output_len=req.expected_output_len, |
| 145 | schema=req.schema, |
| 146 | structure_type=req.structure_type, |
| 147 | completion=req.completion, |
| 148 | ) |
| 149 | ) |
| 150 | return out |
| 151 | |
| 152 | if args.dataset == "json" or args.dataset == "json-unique": |
| 153 | if args.json_schema_path is None: |
| 154 | dir_path = os.path.dirname(os.path.realpath(__file__)) |
| 155 | args.json_schema_path = os.path.join( |
| 156 | dir_path, "structured_schemas", "structured_schema_1.json" |
| 157 | ) |
| 158 | json_schemas = [] |
| 159 | with open(args.json_schema_path) as f: |
| 160 | schema = json.load(f) |
| 161 | |
| 162 | if args.dataset == "json-unique": |
| 163 | json_schemas = [copy.deepcopy(schema) for _ in range(args.num_prompts)] |
| 164 | for i in range(len(json_schemas)): |
| 165 | if "properties" not in json_schemas[i]: |
| 166 | json_schemas[i]["properties"] = {} |
| 167 | json_schemas[i]["properties"][f"__optional_field_{uuid.uuid4()}"] = { |
| 168 | "type": "string", |
| 169 | "description": "An unique optional field to avoid cached schemas", |
| 170 | } |
| 171 | else: |
| 172 | json_schemas = [schema] * args.num_prompts |
| 173 | |
| 174 | def gen_prompt(index: int): |
| 175 | return f"Generate an example of a brief user profile given the following schema: {json.dumps(get_schema(index))}" # noqa: E501 |
| 176 | |
| 177 | def get_schema(index: int): |
| 178 | return json_schemas[index % len(json_schemas)] |
| 179 | |
| 180 | requests = [ |
| 181 | SampleRequest( |
| 182 | prompt=gen_prompt(i), |
| 183 | prompt_len=len(tokenizer(gen_prompt(i)).input_ids), |
| 184 | expected_output_len=args.output_len, |
| 185 | schema=get_schema(i), |
| 186 | structure_type=args.structure_type, |
| 187 | ) |
| 188 | for i in range(args.num_prompts) |
| 189 | ] |
| 190 | |
| 191 | elif args.dataset == "grammar": |
| 192 | schema = """ |
| 193 | root ::= select_statement |
| 194 | |
| 195 | select_statement ::= "SELECT " column " from " table " where " condition |
| 196 | |
| 197 | column ::= "col_1 " | "col_2 " |
| 198 | |
| 199 | table ::= "table_1 " | "table_2 " |
| 200 | |
| 201 | condition ::= column "= " number |
| 202 | |
| 203 | number ::= "1 " | "2 " |
| 204 | """ |
| 205 | prompt = "Generate an SQL query to show the 'username' \ |
| 206 | and 'email' from the 'users' table." |
| 207 | |
| 208 | input_len = len(tokenizer(prompt).input_ids) |
| 209 | print(f"Input length of the prompt: {input_len} tokens") |
| 210 | requests = [ |
| 211 | SampleRequest( |
| 212 | prompt=prompt, |
| 213 | prompt_len=input_len, |
| 214 | expected_output_len=args.output_len, |
| 215 | schema=schema, |
| 216 | structure_type=args.structure_type, |
| 217 | ) |
| 218 | for _ in range(args.num_prompts) |
| 219 | ] |
| 220 | |
| 221 | elif args.dataset == "regex": |
| 222 | regex = r"\w+@\w+\.com\n" |
| 223 | args.regex = regex |
| 224 | prompt = "Generate an email address for Alan Turing, \ |
| 225 | who works in Enigma. End in .com and new line. \ |
| 226 | Example result: alan.turing@enigma.com\n" |
| 227 | |
| 228 | input_len = len(tokenizer(prompt).input_ids) |
| 229 | print(f"Input length of the prompt: {input_len} tokens") |
| 230 | requests = [ |
| 231 | SampleRequest( |
| 232 | prompt=prompt, |
| 233 | prompt_len=input_len, |
| 234 | expected_output_len=args.output_len, |
| 235 | schema=regex, |
| 236 | structure_type=args.structure_type, |
| 237 | ) |
| 238 | for _ in range(args.num_prompts) |
| 239 | ] |
| 240 | |
| 241 | elif args.dataset == "choice": |
| 242 | choice = ["Positive", "Negative"] |
| 243 | args.choice = choice |
| 244 | prompt = "Classify this sentiment: vLLM is wonderful!" |
| 245 | input_len = len(tokenizer(prompt).input_ids) |
| 246 | print(f"Input length of the prompt: {input_len} tokens") |
| 247 | requests = [ |
| 248 | SampleRequest( |
| 249 | prompt=prompt, |
| 250 | prompt_len=input_len, |
| 251 | expected_output_len=args.output_len, |
| 252 | schema=choice, |
| 253 | structure_type=args.structure_type, |
| 254 | ) |
| 255 | for _ in range(args.num_prompts) |
| 256 | ] |
| 257 | |
| 258 | elif args.dataset == "xgrammar_bench": |
| 259 | requests: list[SampleRequest] = [] |
| 260 | dataset = datasets.load_dataset("NousResearch/json-mode-eval", split="train") |
| 261 | full_dataset_len = len(dataset) |
| 262 | |
| 263 | def _filter_func(item): |
| 264 | import json |
| 265 | |
| 266 | schema = json.loads(item["schema"]) |
| 267 | return not has_xgrammar_unsupported_json_features(schema) |
| 268 | |
| 269 | dataset = dataset.filter(_filter_func) |
| 270 | num_filtered_out = full_dataset_len - len(dataset) |
| 271 | print( |
| 272 | f"dataset has {len(dataset)} entries after filtering " |
| 273 | f"out {num_filtered_out} entries with unsupported features" |
| 274 | ) |
| 275 | len_dataset = len(dataset) |
| 276 | for data_point_idx in range(args.num_prompts): |
| 277 | idx = data_point_idx |
| 278 | while idx >= len_dataset: |
| 279 | idx -= len_dataset |
| 280 | schema = dataset["schema"][idx] |
| 281 | prompt = tokenizer.apply_chat_template( |
| 282 | dataset["prompt"][idx], tokenize=False, add_generation_prompt=True |
| 283 | ) |
| 284 | input_len = len(tokenizer(prompt).input_ids) |
| 285 | completion = dataset["completion"][idx] |
| 286 | |
| 287 | requests.append( |
| 288 | SampleRequest( |
| 289 | prompt=prompt, |
| 290 | prompt_len=input_len, |
| 291 | expected_output_len=args.output_len, |
| 292 | schema=schema, |
| 293 | structure_type=args.structure_type, |
| 294 | completion=completion, |
| 295 | ) |
| 296 | ) |
| 297 | |
| 298 | requests = _apply_random_prefix( |
| 299 | tokenizer, requests, args.random_prefix_len, args.seed |
| 300 | ) |
| 301 | return requests |
| 302 | |
| 303 | |
| 304 | async def get_request( |
| 305 | input_requests: list[SampleRequest], |
| 306 | request_rate: float, |
| 307 | burstiness: float = 1.0, |
| 308 | ) -> AsyncGenerator[tuple[int, SampleRequest], None]: |
| 309 | """Asynchronously generates requests at a specified rate |
| 310 | with OPTIONAL burstiness. |
| 311 | |
| 312 | Args: |
| 313 | input_requests: |
| 314 | A list of input requests, each represented as a tuple. |
| 315 | request_rate: |
| 316 | The rate at which requests are generated (requests/s). |
| 317 | burstiness (optional): |
| 318 | The burstiness factor of the request generation. |
| 319 | Only takes effect when request_rate is not inf. |
| 320 | Default value is 1, which follows a Poisson process. |
| 321 | Otherwise, the request intervals follow a gamma distribution. |
| 322 | A lower burstiness value (0 < burstiness < 1) results |
| 323 | in more bursty requests, while a higher burstiness value |
| 324 | (burstiness > 1) results in a more uniform arrival of requests. |
| 325 | |
| 326 | """ |
| 327 | input_requests = iter(input_requests) |
| 328 | |
| 329 | # Calculate scale parameter theta to maintain the desired request_rate. |
| 330 | assert burstiness > 0, ( |
| 331 | f"A positive burstiness factor is expected, but given {burstiness}." |
| 332 | ) |
| 333 | theta = 1.0 / (request_rate * burstiness) |
| 334 | |
| 335 | for i, request in enumerate(input_requests): |
| 336 | yield i, request |
| 337 | |
| 338 | if request_rate == float("inf"): |
| 339 | # If the request rate is infinity, then we don't need to wait. |
| 340 | continue |
| 341 | |
| 342 | # Sample the request interval from the gamma distribution. |
| 343 | # If burstiness is 1, it follows exponential distribution. |
| 344 | interval = np.random.gamma(shape=burstiness, scale=theta) |
| 345 | # The next request will be sent after the interval. |
| 346 | await asyncio.sleep(interval) |
| 347 | |
| 348 | |
| 349 | def calculate_metrics( |
| 350 | input_requests: list[tuple[str, int, int]], |
| 351 | outputs: list[RequestFuncOutput], |
| 352 | dur_s: float, |
| 353 | tokenizer: PreTrainedTokenizerBase, |
| 354 | selected_percentile_metrics: list[str], |
| 355 | selected_percentiles: list[float], |
| 356 | goodput_config_dict: dict[str, float] | None = None, |
| 357 | ) -> tuple[BenchmarkMetrics, list[int]]: |
| 358 | actual_output_lens: list[int] = [] |
| 359 | total_input = 0 |
| 360 | completed = 0 |
| 361 | good_completed = 0 |
| 362 | itls: list[float] = [] |
| 363 | tpots: list[float] = [] |
| 364 | all_tpots: list[float] = [] |
| 365 | ttfts: list[float] = [] |
| 366 | e2els: list[float] = [] |
| 367 | for i in range(len(outputs)): |
| 368 | if outputs[i].success: |
| 369 | # We use the tokenizer to count the number of output tokens for all |
| 370 | # serving backends instead of looking at len(outputs[i].itl) since |
| 371 | # multiple output tokens may be bundled together |
| 372 | # Note : this may inflate the output token count slightly |
| 373 | output_len = len( |
| 374 | tokenizer(outputs[i].generated_text, add_special_tokens=False).input_ids |
| 375 | ) |
| 376 | actual_output_lens.append(output_len) |
| 377 | total_input += input_requests[i].prompt_len |
| 378 | tpot = 0 |
| 379 | if output_len > 1: |
| 380 | latency_minus_ttft = outputs[i].latency - outputs[i].ttft |
| 381 | tpot = latency_minus_ttft / (output_len - 1) |
| 382 | tpots.append(tpot) |
| 383 | outputs[i].tpot = tpot |
| 384 | # Note: if output_len <= 1, we regard tpot as 0 for goodput |
| 385 | all_tpots.append(tpot) |
| 386 | itls += outputs[i].itl |
| 387 | ttfts.append(outputs[i].ttft) |
| 388 | e2els.append(outputs[i].latency) |
| 389 | completed += 1 |
| 390 | else: |
| 391 | actual_output_lens.append(0) |
| 392 | |
| 393 | if goodput_config_dict: |
| 394 | valid_metrics = [] |
| 395 | slo_values = [] |
| 396 | |
| 397 | if "ttft" in goodput_config_dict: |
| 398 | valid_metrics.append(ttfts) |
| 399 | slo_values.append( |
| 400 | goodput_config_dict["ttft"] / MILLISECONDS_TO_SECONDS_CONVERSION |
| 401 | ) |
| 402 | if "tpot" in goodput_config_dict: |
| 403 | valid_metrics.append(all_tpots) |
| 404 | slo_values.append( |
| 405 | goodput_config_dict["tpot"] / MILLISECONDS_TO_SECONDS_CONVERSION |
| 406 | ) |
| 407 | if "e2el" in goodput_config_dict: |
| 408 | valid_metrics.append(e2els) |
| 409 | slo_values.append( |
| 410 | goodput_config_dict["e2el"] / MILLISECONDS_TO_SECONDS_CONVERSION |
| 411 | ) |
| 412 | |
| 413 | for req_metric in zip(*valid_metrics): |
| 414 | is_good_req = all([s >= r for s, r in zip(slo_values, req_metric)]) |
| 415 | if is_good_req: |
| 416 | good_completed += 1 |
| 417 | |
| 418 | if completed == 0: |
| 419 | warnings.warn( |
| 420 | "All requests failed. This is likely due to a misconfiguration " |
| 421 | "on the benchmark arguments.", |
| 422 | stacklevel=2, |
| 423 | ) |
| 424 | metrics = BenchmarkMetrics( |
| 425 | completed=completed, |
| 426 | total_input=total_input, |
| 427 | total_output=sum(actual_output_lens), |
| 428 | request_throughput=completed / dur_s, |
| 429 | request_goodput=good_completed / dur_s, |
| 430 | output_throughput=sum(actual_output_lens) / dur_s, |
| 431 | total_token_throughput=(total_input + sum(actual_output_lens)) / dur_s, |
| 432 | mean_ttft_ms=np.mean(ttfts or 0) |
| 433 | * 1000, # ttfts is empty if streaming is not supported by backend |
| 434 | std_ttft_ms=np.std(ttfts or 0) * 1000, |
| 435 | median_ttft_ms=np.median(ttfts or 0) * 1000, |
| 436 | percentiles_ttft_ms=[ |
| 437 | (p, np.percentile(ttfts or 0, p) * 1000) for p in selected_percentiles |
| 438 | ], |
| 439 | mean_tpot_ms=np.mean(tpots or 0) * 1000, |
| 440 | std_tpot_ms=np.std(tpots or 0) * 1000, |
| 441 | median_tpot_ms=np.median(tpots or 0) * 1000, |
| 442 | percentiles_tpot_ms=[ |
| 443 | (p, np.percentile(tpots or 0, p) * 1000) for p in selected_percentiles |
| 444 | ], |
| 445 | mean_itl_ms=np.mean(itls or 0) * 1000, |
| 446 | std_itl_ms=np.std(itls or 0) * 1000, |
| 447 | median_itl_ms=np.median(itls or 0) * 1000, |
| 448 | percentiles_itl_ms=[ |
| 449 | (p, np.percentile(itls or 0, p) * 1000) for p in selected_percentiles |
| 450 | ], |
| 451 | mean_e2el_ms=np.mean(e2els or 0) * 1000, |
| 452 | std_e2el_ms=np.std(e2els or 0) * 1000, |
| 453 | median_e2el_ms=np.median(e2els or 0) * 1000, |
| 454 | percentiles_e2el_ms=[ |
| 455 | (p, np.percentile(e2els or 0, p) * 1000) for p in selected_percentiles |
| 456 | ], |
| 457 | ) |
| 458 | |
| 459 | return metrics, actual_output_lens |
| 460 | |
| 461 | |
| 462 | async def benchmark( |
| 463 | backend: str, |
| 464 | api_url: str, |
| 465 | base_url: str, |
| 466 | model_id: str, |
| 467 | tokenizer: PreTrainedTokenizerBase, |
| 468 | input_requests: list[SampleRequest], |
| 469 | request_rate: float, |
| 470 | burstiness: float, |
| 471 | disable_tqdm: bool, |
| 472 | profile: bool, |
| 473 | selected_percentile_metrics: list[str], |
| 474 | selected_percentiles: list[str], |
| 475 | ignore_eos: bool, |
| 476 | max_concurrency: int | None, |
| 477 | structured_output_ratio: float, |
| 478 | goodput_config_dict: dict[str, float] | None = None, |
| 479 | ): |
| 480 | if backend in ASYNC_REQUEST_FUNCS: |
| 481 | request_func = ASYNC_REQUEST_FUNCS[backend] |
| 482 | else: |
| 483 | raise ValueError(f"Unknown backend: {backend}") |
| 484 | |
| 485 | def prepare_extra_body(request) -> dict: |
| 486 | extra_body = {} |
| 487 | # Add the schema to the extra_body |
| 488 | extra_body["structured_outputs"] = {} |
| 489 | extra_body["structured_outputs"][request.structure_type] = request.schema |
| 490 | return extra_body |
| 491 | |
| 492 | print("Starting initial single prompt test run...") |
| 493 | structured_output_req_idx = random.sample( |
| 494 | range(len(input_requests)), int(len(input_requests) * structured_output_ratio) |
| 495 | ) |
| 496 | |
| 497 | test_request = input_requests[0] |
| 498 | test_req_extra_body = ( |
| 499 | prepare_extra_body(test_request) if 0 in structured_output_req_idx else None |
| 500 | ) |
| 501 | test_input = RequestFuncInput( |
| 502 | model=model_id, |
| 503 | prompt=test_request.prompt, |
| 504 | api_url=api_url, |
| 505 | prompt_len=test_request.prompt_len, |
| 506 | output_len=test_request.expected_output_len, |
| 507 | ignore_eos=ignore_eos, |
| 508 | extra_body=test_req_extra_body, |
| 509 | ) |
| 510 | test_output = await request_func(request_func_input=test_input) |
| 511 | if not test_output.success: |
| 512 | raise ValueError( |
| 513 | "Initial test run failed - Please make sure benchmark arguments " |
| 514 | f"are correctly specified. Error: {test_output.error}" |
| 515 | ) |
| 516 | else: |
| 517 | print("Initial test run completed. Starting main benchmark run...") |
| 518 | |
| 519 | if profile: |
| 520 | print("Starting profiler...") |
| 521 | profile_input = RequestFuncInput( |
| 522 | model=model_id, |
| 523 | prompt=test_request.prompt, |
| 524 | api_url=base_url + "/start_profile", |
| 525 | prompt_len=test_request.prompt_len, |
| 526 | output_len=test_request.expected_output_len, |
| 527 | ignore_eos=ignore_eos, |
| 528 | extra_body=test_req_extra_body, |
| 529 | ) |
| 530 | profile_output = await request_func(request_func_input=profile_input) |
| 531 | if profile_output.success: |
| 532 | print("Profiler started") |
| 533 | |
| 534 | distribution = "Poisson process" if burstiness == 1.0 else "Gamma distribution" |
| 535 | |
| 536 | print(f"Traffic request rate: {request_rate}") |
| 537 | print(f"Burstiness factor: {burstiness} ({distribution})") |
| 538 | print(f"Maximum request concurrency: {max_concurrency}") |
| 539 | |
| 540 | pbar = None if disable_tqdm else tqdm(total=len(input_requests)) |
| 541 | |
| 542 | semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else nullcontext() |
| 543 | |
| 544 | async def limited_request_func(request_func_input, pbar): |
| 545 | async with semaphore: |
| 546 | return await request_func(request_func_input=request_func_input, pbar=pbar) |
| 547 | |
| 548 | benchmark_start_time = time.perf_counter() |
| 549 | tasks: list[asyncio.Task] = [] |
| 550 | expected: list[str] = [] |
| 551 | async for i, request in get_request(input_requests, request_rate, burstiness): |
| 552 | extra_body = ( |
| 553 | prepare_extra_body(request) if i in structured_output_req_idx else None |
| 554 | ) |
| 555 | request_func_input = RequestFuncInput( |
| 556 | model=model_id, |
| 557 | prompt=request.prompt, |
| 558 | api_url=api_url, |
| 559 | prompt_len=request.prompt_len, |
| 560 | output_len=request.expected_output_len, |
| 561 | ignore_eos=ignore_eos, |
| 562 | extra_body=extra_body, |
| 563 | ) |
| 564 | expected.append(request.completion) |
| 565 | tasks.append( |
| 566 | asyncio.create_task( |
| 567 | limited_request_func(request_func_input=request_func_input, pbar=pbar) |
| 568 | ) |
| 569 | ) |
| 570 | outputs: list[RequestFuncOutput] = await asyncio.gather(*tasks) |
| 571 | |
| 572 | if pbar is not None: |
| 573 | pbar.close() |
| 574 | |
| 575 | benchmark_duration = time.perf_counter() - benchmark_start_time |
| 576 | |
| 577 | metrics, actual_output_lens = calculate_metrics( |
| 578 | input_requests=input_requests, |
| 579 | outputs=outputs, |
| 580 | dur_s=benchmark_duration, |
| 581 | tokenizer=tokenizer, |
| 582 | selected_percentile_metrics=selected_percentile_metrics, |
| 583 | selected_percentiles=selected_percentiles, |
| 584 | goodput_config_dict=goodput_config_dict, |
| 585 | ) |
| 586 | |
| 587 | print("{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=50, c="=")) |
| 588 | print("{:<40} {:<10}".format("Successful requests:", metrics.completed)) |
| 589 | if max_concurrency is not None: |
| 590 | print("{:<40} {:<10}".format("Maximum request concurrency:", max_concurrency)) |
| 591 | if request_rate != float("inf"): |
| 592 | print("{:<40} {:<10.2f}".format("Request rate configured (RPS):", request_rate)) |
| 593 | print("{:<40} {:<10.2f}".format("Benchmark duration (s):", benchmark_duration)) |
| 594 | print("{:<40} {:<10}".format("Total input tokens:", metrics.total_input)) |
| 595 | print("{:<40} {:<10}".format("Total generated tokens:", metrics.total_output)) |
| 596 | print( |
| 597 | "{:<40} {:<10.2f}".format( |
| 598 | "Request throughput (req/s):", metrics.request_throughput |
| 599 | ) |
| 600 | ) |
| 601 | if goodput_config_dict: |
| 602 | print( |
| 603 | "{:<40} {:<10.2f}".format( |
| 604 | "Request goodput (req/s):", metrics.request_goodput |
| 605 | ) |
| 606 | ) |
| 607 | print( |
| 608 | "{:<40} {:<10.2f}".format( |
| 609 | "Output token throughput (tok/s):", metrics.output_throughput |
| 610 | ) |
| 611 | ) |
| 612 | print( |
| 613 | "{:<40} {:<10.2f}".format( |
| 614 | "Total token throughput (tok/s):", metrics.total_token_throughput |
| 615 | ) |
| 616 | ) |
| 617 | |
| 618 | result = { |
| 619 | "duration": benchmark_duration, |
| 620 | "completed": metrics.completed, |
| 621 | "total_input_tokens": metrics.total_input, |
| 622 | "total_output_tokens": metrics.total_output, |
| 623 | "request_throughput": metrics.request_throughput, |
| 624 | "output_throughput": metrics.output_throughput, |
| 625 | "total_token_throughput": metrics.total_token_throughput, |
| 626 | "ttft_description": pd.Series([output.ttft for output in outputs]) |
| 627 | .describe() |
| 628 | .to_dict(), |
| 629 | "tpot_description": pd.Series([output.tpot for output in outputs]) |
| 630 | .describe() |
| 631 | .to_dict(), |
| 632 | "input_lens": [output.prompt_len for output in outputs], |
| 633 | "output_lens": actual_output_lens, |
| 634 | "ttfts": [output.ttft for output in outputs], |
| 635 | "itls": [output.itl for output in outputs], |
| 636 | "errors": [output.error for output in outputs], |
| 637 | } |
| 638 | |
| 639 | ret = [ |
| 640 | {"generated": output.generated_text, "expected": gt} |
| 641 | for output, gt in zip(outputs, expected) |
| 642 | ] |
| 643 | |
| 644 | def process_one_metric( |
| 645 | # E.g., "ttft" |
| 646 | metric_attribute_name: str, |
| 647 | # E.g., "TTFT" |
| 648 | metric_name: str, |
| 649 | # E.g., "Time to First Token" |
| 650 | metric_header: str, |
| 651 | ): |
| 652 | # This function prints and adds statistics of the specified |
| 653 | # metric. |
| 654 | if metric_attribute_name not in selected_percentile_metrics: |
| 655 | return |
| 656 | print("{s:{c}^{n}}".format(s=metric_header, n=50, c="-")) |
| 657 | print( |
| 658 | "{:<40} {:<10.2f}".format( |
| 659 | f"Mean {metric_name} (ms):", |
| 660 | getattr(metrics, f"mean_{metric_attribute_name}_ms"), |
| 661 | ) |
| 662 | ) |
| 663 | print( |
| 664 | "{:<40} {:<10.2f}".format( |
| 665 | f"Median {metric_name} (ms):", |
| 666 | getattr(metrics, f"median_{metric_attribute_name}_ms"), |
| 667 | ) |
| 668 | ) |
| 669 | result[f"mean_{metric_attribute_name}_ms"] = getattr( |
| 670 | metrics, f"mean_{metric_attribute_name}_ms" |
| 671 | ) |
| 672 | result[f"median_{metric_attribute_name}_ms"] = getattr( |
| 673 | metrics, f"median_{metric_attribute_name}_ms" |
| 674 | ) |
| 675 | result[f"std_{metric_attribute_name}_ms"] = getattr( |
| 676 | metrics, f"std_{metric_attribute_name}_ms" |
| 677 | ) |
| 678 | for p, value in getattr(metrics, f"percentiles_{metric_attribute_name}_ms"): |
| 679 | p_word = str(int(p)) if int(p) == p else str(p) |
| 680 | print("{:<40} {:<10.2f}".format(f"P{p_word} {metric_name} (ms):", value)) |
| 681 | result[f"p{p_word}_{metric_attribute_name}_ms"] = value |
| 682 | |
| 683 | process_one_metric("ttft", "TTFT", "Time to First Token") |
| 684 | process_one_metric("tpot", "TPOT", "Time per Output Token (excl. 1st token)") |
| 685 | process_one_metric("itl", "ITL", "Inter-token Latency") |
| 686 | process_one_metric("e2el", "E2EL", "End-to-end Latency") |
| 687 | |
| 688 | print("=" * 50) |
| 689 | |
| 690 | if profile: |
| 691 | print("Stopping profiler...") |
| 692 | profile_input = RequestFuncInput( |
| 693 | model=model_id, |
| 694 | prompt=test_request.prompt, |
| 695 | api_url=base_url + "/stop_profile", |
| 696 | prompt_len=test_request.prompt_len, |
| 697 | output_len=test_request.expected_output_len, |
| 698 | extra_body={test_request.structure_type: test_request.schema}, |
| 699 | ) |
| 700 | profile_output = await request_func(request_func_input=profile_input) |
| 701 | if profile_output.success: |
| 702 | print("Profiler stopped") |
| 703 | |
| 704 | return result, ret |
| 705 | |
| 706 | |
| 707 | def evaluate(ret, args): |
| 708 | def _eval_correctness_json(expected, actual): |
| 709 | # extract json string from string using regex |
| 710 | import regex as re |
| 711 | |
| 712 | actual = actual.replace("\n", "").replace(" ", "").strip() |
| 713 | try: |
| 714 | actual = re.search(r"\{.*\}", actual).group() |
| 715 | actual = json.loads(actual) |
| 716 | except Exception: |
| 717 | return False |
| 718 | |
| 719 | return True |
| 720 | |
| 721 | def _eval_correctness_choice(expected, actual): |
| 722 | return actual in args.choice |
| 723 | |
| 724 | def _eval_correctness_regex(expected, actual): |
| 725 | import regex as re |
| 726 | |
| 727 | return re.match(args.regex, actual) is not None |
| 728 | |
| 729 | def _eval_correctness(expected, actual): |
| 730 | if args.structure_type == "json": |
| 731 | return _eval_correctness_json(expected, actual) |
| 732 | elif args.structure_type == "regex": |
| 733 | return _eval_correctness_regex(expected, actual) |
| 734 | elif args.structure_type == "choice": |
| 735 | return _eval_correctness_choice(expected, actual) |
| 736 | else: |
| 737 | return None |
| 738 | |
| 739 | scores = [] |
| 740 | for res in ret: |
| 741 | score = _eval_correctness(res["expected"], res["generated"]) |
| 742 | res["correctness"] = score |
| 743 | scores.append(score) |
| 744 | |
| 745 | not_none_scores = [score for score in scores if score is not None] |
| 746 | |
| 747 | return ( |
| 748 | (sum(not_none_scores) / len(not_none_scores) * 100) |
| 749 | if len(not_none_scores) > 0 |
| 750 | else None |
| 751 | ) |
| 752 | |
| 753 | |
| 754 | def parse_goodput(slo_pairs): |
| 755 | goodput_config_dict = {} |
| 756 | try: |
| 757 | for slo_pair in slo_pairs: |
| 758 | slo_name, slo_val = slo_pair.split(":") |
| 759 | goodput_config_dict[slo_name] = float(slo_val) |
| 760 | except ValueError as err: |
| 761 | raise argparse.ArgumentTypeError( |
| 762 | "Invalid format found for service level objectives. " |
| 763 | 'Specify service level objectives for goodput as "KEY:VALUE" ' |
| 764 | "pairs, where the key is a metric name, and the value is a " |
| 765 | "number in milliseconds." |
| 766 | ) from err |
| 767 | return goodput_config_dict |
| 768 | |
| 769 | |
| 770 | def check_goodput_args(args): |
| 771 | goodput_config_dict = {} |
| 772 | VALID_NAMES = ["ttft", "tpot", "e2el"] |
| 773 | if args.goodput: |
| 774 | goodput_config_dict = parse_goodput(args.goodput) |
| 775 | for slo_name, slo_val in goodput_config_dict.items(): |
| 776 | if slo_name not in VALID_NAMES: |
| 777 | raise ValueError( |
| 778 | f"Invalid metric name found, {slo_name}: {slo_val}. " |
| 779 | "The service level objective name should be one of " |
| 780 | f"{str(VALID_NAMES)}. " |
| 781 | ) |
| 782 | if slo_val < 0: |
| 783 | raise ValueError( |
| 784 | f"Invalid value found, {slo_name}: {slo_val}. " |
| 785 | "The service level objective value should be " |
| 786 | "non-negative." |
| 787 | ) |
| 788 | return goodput_config_dict |
| 789 | |
| 790 | |
| 791 | def main(args: argparse.Namespace): |
| 792 | print(args) |
| 793 | random.seed(args.seed) |
| 794 | np.random.seed(args.seed) |
| 795 | |
| 796 | backend = args.backend |
| 797 | model_id = args.model |
| 798 | tokenizer_id = args.tokenizer if args.tokenizer is not None else args.model |
| 799 | |
| 800 | if args.base_url is not None: |
| 801 | api_url = f"{args.base_url}{args.endpoint}" |
| 802 | base_url = f"{args.base_url}" |
| 803 | else: |
| 804 | api_url = f"http://{args.host}:{args.port}{args.endpoint}" |
| 805 | base_url = f"http://{args.host}:{args.port}" |
| 806 | |
| 807 | tokenizer = get_tokenizer( |
| 808 | tokenizer_id, |
| 809 | trust_remote_code=args.trust_remote_code, |
| 810 | tokenizer_mode=args.tokenizer_mode, |
| 811 | ) |
| 812 | |
| 813 | if args.dataset == "grammar": |
| 814 | args.structure_type = "grammar" |
| 815 | elif args.dataset == "regex": |
| 816 | args.structure_type = "regex" |
| 817 | elif args.dataset == "choice": |
| 818 | args.structure_type = "choice" |
| 819 | else: |
| 820 | args.structure_type = "json" |
| 821 | |
| 822 | if args.no_structured_output: |
| 823 | args.structured_output_ratio = 0 |
| 824 | if args.save_results: |
| 825 | result_file_name = f"{args.structured_output_ratio}so" |
| 826 | result_file_name += f"_{backend}" |
| 827 | result_file_name += f"_{args.request_rate}qps" |
| 828 | result_file_name += f"_{args.model.split('/')[-1]}" |
| 829 | result_file_name += f"_{args.dataset}" |
| 830 | result_file_name += f"_{args.num_prompts}" |
| 831 | result_file_name += f"_out{args.output_len}" |
| 832 | result_file_name += ".txt" |
| 833 | else: |
| 834 | result_file_name = None |
| 835 | |
| 836 | input_requests = sample_requests(tokenizer, args) |
| 837 | |
| 838 | goodput_config_dict = check_goodput_args(args) |
| 839 | |
| 840 | benchmark_result, ret = asyncio.run( |
| 841 | benchmark( |
| 842 | backend=backend, |
| 843 | api_url=api_url, |
| 844 | base_url=base_url, |
| 845 | model_id=model_id, |
| 846 | tokenizer=tokenizer, |
| 847 | input_requests=input_requests, |
| 848 | request_rate=args.request_rate, |
| 849 | burstiness=args.burstiness, |
| 850 | disable_tqdm=args.disable_tqdm, |
| 851 | profile=args.profile, |
| 852 | selected_percentile_metrics=args.percentile_metrics.split(","), |
| 853 | selected_percentiles=[float(p) for p in args.metric_percentiles.split(",")], |
| 854 | ignore_eos=args.ignore_eos, |
| 855 | max_concurrency=args.max_concurrency, |
| 856 | structured_output_ratio=args.structured_output_ratio, |
| 857 | goodput_config_dict=goodput_config_dict, |
| 858 | ) |
| 859 | ) |
| 860 | |
| 861 | # Save config and results to json |
| 862 | score = evaluate(ret, args) |
| 863 | print("correct_rate(%)", score, "\n") |
| 864 | if args.save_results: |
| 865 | results = { |
| 866 | "backend": backend, |
| 867 | "model_id": model_id, |
| 868 | "tokenizer_id": tokenizer_id, |
| 869 | "num_prompts": args.num_prompts, |
| 870 | "request_rate": args.request_rate |
| 871 | if args.request_rate < float("inf") |
| 872 | else "inf", |
| 873 | "burstiness": args.burstiness, |
| 874 | "max_concurrency": args.max_concurrency, |
| 875 | "correct_rate(%)": score, |
| 876 | } |
| 877 | results = {"outputs": ret, **results, **benchmark_result} |
| 878 | |
| 879 | # Save to file |
| 880 | if args.result_filename: |
| 881 | result_file_name = args.result_filename |
| 882 | if args.result_dir: |
| 883 | result_file_name = os.path.join(args.result_dir, result_file_name) |
| 884 | with open(result_file_name, "w", encoding="utf-8") as outfile: |
| 885 | json.dump(results, outfile, indent=4) |
| 886 | |
| 887 | |
| 888 | def create_argument_parser(): |
| 889 | parser = FlexibleArgumentParser( |
| 890 | description="Benchmark the online serving throughput." |
| 891 | ) |
| 892 | parser.add_argument( |
| 893 | "--backend", |
| 894 | type=str, |
| 895 | default="vllm", |
| 896 | choices=list(ASYNC_REQUEST_FUNCS.keys()), |
| 897 | ) |
| 898 | parser.add_argument( |
| 899 | "--base-url", |
| 900 | type=str, |
| 901 | default=None, |
| 902 | help="Server or API base url if not using http host and port.", |
| 903 | ) |
| 904 | # Use 127.0.0.1 here instead of localhost to force the use of ipv4 |
| 905 | parser.add_argument("--host", type=str, default="127.0.0.1") |
| 906 | parser.add_argument("--port", type=int, default=8000) |
| 907 | parser.add_argument( |
| 908 | "--endpoint", |
| 909 | type=str, |
| 910 | default="/v1/completions", |
| 911 | help="API endpoint.", |
| 912 | ) |
| 913 | parser.add_argument( |
| 914 | "--dataset", |
| 915 | default="json", |
| 916 | choices=["json", "json-unique", "grammar", "regex", "choice", "xgrammar_bench"], |
| 917 | ) |
| 918 | parser.add_argument( |
| 919 | "--json-schema-path", type=str, default=None, help="Path to json schema." |
| 920 | ) |
| 921 | parser.add_argument( |
| 922 | "--max-concurrency", |
| 923 | type=int, |
| 924 | default=None, |
| 925 | help="Maximum number of concurrent requests. This can be used " |
| 926 | "to help simulate an environment where a higher level component " |
| 927 | "is enforcing a maximum number of concurrent requests. While the " |
| 928 | "--request-rate argument controls the rate at which requests are " |
| 929 | "initiated, this argument will control how many are actually allowed " |
| 930 | "to execute at a time. This means that when used in combination, the " |
| 931 | "actual request rate may be lower than specified with --request-rate, " |
| 932 | "if the server is not processing requests fast enough to keep up.", |
| 933 | ) |
| 934 | parser.add_argument( |
| 935 | "--model", |
| 936 | type=str, |
| 937 | required=True, |
| 938 | help="Name of the model.", |
| 939 | ) |
| 940 | parser.add_argument( |
| 941 | "--tokenizer", |
| 942 | type=str, |
| 943 | help="Name or path of the tokenizer, if not using the default tokenizer.", |
| 944 | ) |
| 945 | parser.add_argument( |
| 946 | "--tokenizer-mode", |
| 947 | type=str, |
| 948 | default="auto", |
| 949 | help="Name or path of the tokenizer, if not using the default tokenizer.", |
| 950 | ) |
| 951 | parser.add_argument( |
| 952 | "--num-prompts", |
| 953 | type=int, |
| 954 | default=1000, |
| 955 | help="Number of prompts to process.", |
| 956 | ) |
| 957 | parser.add_argument( |
| 958 | "--output-len", |
| 959 | type=int, |
| 960 | default=128, |
| 961 | help="Number of output tokens.", |
| 962 | ) |
| 963 | parser.add_argument( |
| 964 | "--request-rate", |
| 965 | type=float, |
| 966 | default=float("inf"), |
| 967 | help="Number of requests per second. If this is inf, " |
| 968 | "then all the requests are sent at time 0. " |
| 969 | "Otherwise, we use Poisson process or gamma distribution " |
| 970 | "to synthesize the request arrival times.", |
| 971 | ) |
| 972 | parser.add_argument( |
| 973 | "--burstiness", |
| 974 | type=float, |
| 975 | default=1.0, |
| 976 | help="Burstiness factor of the request generation. " |
| 977 | "Only take effect when request_rate is not inf. " |
| 978 | "Default value is 1, which follows Poisson process. " |
| 979 | "Otherwise, the request intervals follow a gamma distribution. " |
| 980 | "A lower burstiness value (0 < burstiness < 1) results in more " |
| 981 | "bursty requests. A higher burstiness value (burstiness > 1) " |
| 982 | "results in a more uniform arrival of requests.", |
| 983 | ) |
| 984 | parser.add_argument("--seed", type=int, default=0) |
| 985 | parser.add_argument( |
| 986 | "--random-prefix-len", |
| 987 | type=int, |
| 988 | default=0, |
| 989 | help=( |
| 990 | "Number of prefix tokens to prepend to every prompt. " |
| 991 | "The same prefix is used for all prompts to enable prefix caching." |
| 992 | ), |
| 993 | ) |
| 994 | parser.add_argument( |
| 995 | "--trust-remote-code", |
| 996 | action="store_true", |
| 997 | help="Trust remote code from huggingface", |
| 998 | ) |
| 999 | parser.add_argument( |
| 1000 | "--disable-tqdm", |
| 1001 | action="store_true", |
| 1002 | help="Specify to disable tqdm progress bar.", |
| 1003 | ) |
| 1004 | parser.add_argument( |
| 1005 | "--save-results", |
| 1006 | action="store_true", |
| 1007 | help="Specify to save benchmark results to a json file", |
| 1008 | ) |
| 1009 | parser.add_argument( |
| 1010 | "--profile", |
| 1011 | action="store_true", |
| 1012 | help="Use vLLM Profiling. --profiler-config must be provided on the server.", |
| 1013 | ) |
| 1014 | parser.add_argument( |
| 1015 | "--result-dir", |
| 1016 | type=str, |
| 1017 | default=None, |
| 1018 | help="Specify directory to save benchmark json results." |
| 1019 | "If not specified, results are saved in the current directory.", |
| 1020 | ) |
| 1021 | parser.add_argument( |
| 1022 | "--result-filename", |
| 1023 | type=str, |
| 1024 | default=None, |
| 1025 | help="Specify the filename to save benchmark json results." |
| 1026 | "If not specified, results will be saved in " |
| 1027 | "{backend}-{args.request_rate}qps-{base_model_id}-{current_dt}.json" |
| 1028 | " format.", |
| 1029 | ) |
| 1030 | parser.add_argument( |
| 1031 | "--ignore-eos", |
| 1032 | action="store_true", |
| 1033 | help="Set ignore_eos flag when sending the benchmark request." |
| 1034 | "Warning: ignore_eos is not supported in deepspeed_mii and tgi.", |
| 1035 | ) |
| 1036 | parser.add_argument( |
| 1037 | "--percentile-metrics", |
| 1038 | type=str, |
| 1039 | default="ttft,tpot,itl", |
| 1040 | help="Comma-separated list of selected metrics to report percentiles. " |
| 1041 | "This argument specifies the metrics to report percentiles. " |
| 1042 | 'Allowed metric names are "ttft", "tpot", "itl", "e2el". ' |
| 1043 | 'Default value is "ttft,tpot,itl".', |
| 1044 | ) |
| 1045 | parser.add_argument( |
| 1046 | "--metric-percentiles", |
| 1047 | type=str, |
| 1048 | default="99", |
| 1049 | help="Comma-separated list of percentiles for selected metrics. " |
| 1050 | 'To report 25-th, 50-th, and 75-th percentiles, use "25,50,75". ' |
| 1051 | 'Default value is "99". ' |
| 1052 | 'Use "--percentile-metrics" to select metrics.', |
| 1053 | ) |
| 1054 | parser.add_argument( |
| 1055 | "--goodput", |
| 1056 | nargs="+", |
| 1057 | required=False, |
| 1058 | help='Specify service level objectives for goodput as "KEY:VALUE" ' |
| 1059 | "pairs, where the key is a metric name, and the value is in " |
| 1060 | 'milliseconds. Multiple "KEY:VALUE" pairs can be provided, ' |
| 1061 | "separated by spaces. Allowed request level metric names are " |
| 1062 | '"ttft", "tpot", "e2el". For more context on the definition of ' |
| 1063 | "goodput, refer to DistServe paper: https://arxiv.org/pdf/2401.09670 " |
| 1064 | "and the blog: https://hao-ai-lab.github.io/blogs/distserve", |
| 1065 | ) |
| 1066 | |
| 1067 | parser.add_argument( |
| 1068 | "--no-structured-output", |
| 1069 | action="store_true", |
| 1070 | default=False, |
| 1071 | help="Whether to disable JSON decoding or not.", |
| 1072 | ) |
| 1073 | parser.add_argument( |
| 1074 | "--structured-output-ratio", |
| 1075 | type=float, |
| 1076 | default=1.0, |
| 1077 | help="Ratio of Structured Outputs requests", |
| 1078 | ) |
| 1079 | |
| 1080 | return parser |
| 1081 | |
| 1082 | |
| 1083 | if __name__ == "__main__": |
| 1084 | parser = create_argument_parser() |
| 1085 | args = parser.parse_args() |
| 1086 | main(args) |