| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import argparse |
| 4 | import contextlib |
| 5 | import json |
| 6 | import shlex |
| 7 | from contextlib import contextmanager |
| 8 | from dataclasses import dataclass |
| 9 | from datetime import datetime |
| 10 | from pathlib import Path |
| 11 | from typing import ClassVar |
| 12 | |
| 13 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 14 | from vllm.utils.import_utils import PlaceholderModule |
| 15 | |
| 16 | from .param_sweep import ParameterSweep, ParameterSweepItem |
| 17 | from .server import ServerProcess |
| 18 | from .utils import sanitize_filename |
| 19 | |
| 20 | try: |
| 21 | import pandas as pd |
| 22 | except ImportError: |
| 23 | pd = PlaceholderModule("pandas") |
| 24 | |
| 25 | |
| 26 | @contextlib.contextmanager |
| 27 | def run_server( |
| 28 | serve_cmd: list[str], |
| 29 | after_bench_cmd: list[str], |
| 30 | *, |
| 31 | show_stdout: bool, |
| 32 | serve_overrides: ParameterSweepItem, |
| 33 | dry_run: bool, |
| 34 | server_ready_timeout: int = 300, |
| 35 | ): |
| 36 | server_cmd = serve_overrides.apply_to_cmd(serve_cmd) |
| 37 | |
| 38 | print("[BEGIN SERVER]") |
| 39 | print(f"Server overrides: {serve_overrides}") |
| 40 | print(f"Server command: {server_cmd}") |
| 41 | |
| 42 | if dry_run: |
| 43 | yield None |
| 44 | print("[END SERVER]") |
| 45 | return |
| 46 | |
| 47 | with ServerProcess(server_cmd, after_bench_cmd, show_stdout=show_stdout) as server: |
| 48 | server.wait_until_ready(timeout=server_ready_timeout) |
| 49 | yield server |
| 50 | |
| 51 | print("[END SERVER]") |
| 52 | |
| 53 | |
| 54 | def _update_run_data( |
| 55 | run_data: dict[str, object], |
| 56 | serve_overrides: ParameterSweepItem, |
| 57 | bench_overrides: ParameterSweepItem, |
| 58 | run_number: int, |
| 59 | ): |
| 60 | run_data["run_number"] = run_number |
| 61 | run_data.update(serve_overrides) |
| 62 | run_data.update(bench_overrides) |
| 63 | |
| 64 | return run_data |
| 65 | |
| 66 | |
| 67 | def run_benchmark( |
| 68 | server: ServerProcess | None, |
| 69 | bench_cmd: list[str], |
| 70 | *, |
| 71 | serve_overrides: ParameterSweepItem, |
| 72 | bench_overrides: ParameterSweepItem, |
| 73 | run_number: int, |
| 74 | output_path: Path, |
| 75 | dry_run: bool, |
| 76 | ): |
| 77 | benchmark_cmd = [ |
| 78 | *bench_overrides.apply_to_cmd(bench_cmd), |
| 79 | "--percentile-metrics", |
| 80 | "ttft,tpot,itl,e2el", |
| 81 | "--save-result", |
| 82 | "--result-dir", |
| 83 | str(output_path.parent), |
| 84 | "--result-filename", |
| 85 | output_path.name, |
| 86 | ] |
| 87 | |
| 88 | print("[BEGIN BENCHMARK]") |
| 89 | print(f"Benchmark overrides: {bench_overrides}") |
| 90 | print(f"Run Number: {run_number}") |
| 91 | print(f"Benchmark command: {benchmark_cmd}") |
| 92 | print(f"Output file: {output_path}") |
| 93 | |
| 94 | run_data: dict[str, object] |
| 95 | |
| 96 | if output_path.exists(): |
| 97 | print("Found existing results.") |
| 98 | print("[SKIPPED BENCHMARK]") |
| 99 | |
| 100 | with output_path.open("rb") as f: |
| 101 | run_data = json.load(f) |
| 102 | return _update_run_data( |
| 103 | run_data, |
| 104 | serve_overrides, |
| 105 | bench_overrides, |
| 106 | run_number, |
| 107 | ) |
| 108 | |
| 109 | if server is None: |
| 110 | if not dry_run: |
| 111 | raise ValueError(f"Cannot find results at {output_path}") |
| 112 | |
| 113 | print("[END BENCHMARK]") |
| 114 | return None |
| 115 | |
| 116 | output_path.parent.mkdir(parents=True, exist_ok=True) |
| 117 | |
| 118 | server.run_subcommand(benchmark_cmd) |
| 119 | server.after_bench() |
| 120 | |
| 121 | with output_path.open("rb") as f: |
| 122 | run_data = json.load(f) |
| 123 | |
| 124 | run_data = _update_run_data( |
| 125 | run_data, |
| 126 | serve_overrides, |
| 127 | bench_overrides, |
| 128 | run_number, |
| 129 | ) |
| 130 | |
| 131 | with output_path.open("w") as f: |
| 132 | json.dump(run_data, f, indent=4) |
| 133 | |
| 134 | print("[END BENCHMARK]") |
| 135 | |
| 136 | return run_data |
| 137 | |
| 138 | |
| 139 | def _get_comb_base_path( |
| 140 | experiment_dir: Path, |
| 141 | serve_comb: ParameterSweepItem, |
| 142 | bench_comb: ParameterSweepItem, |
| 143 | *, |
| 144 | extra_parts: tuple[str, ...] = (), |
| 145 | ): |
| 146 | parts = list[str]() |
| 147 | if serve_comb: |
| 148 | parts.extend(("SERVE-", serve_comb.name)) |
| 149 | if bench_comb: |
| 150 | parts.extend(("BENCH-", bench_comb.name)) |
| 151 | if extra_parts: |
| 152 | parts.extend(extra_parts) |
| 153 | |
| 154 | return experiment_dir / sanitize_filename("-".join(parts)) |
| 155 | |
| 156 | |
| 157 | def _get_comb_run_path(base_path: Path, run_number: int | None): |
| 158 | if run_number is None: |
| 159 | return base_path / "summary.json" |
| 160 | |
| 161 | return base_path / f"run={run_number}.json" |
| 162 | |
| 163 | |
| 164 | def _comb_needs_server( |
| 165 | serve_comb: ParameterSweepItem, |
| 166 | bench_combs: ParameterSweep, |
| 167 | experiment_dir: Path, |
| 168 | ): |
| 169 | for bench_comb in bench_combs: |
| 170 | base_path = _get_comb_base_path(experiment_dir, serve_comb, bench_comb) |
| 171 | if not _get_comb_run_path(base_path, run_number=None).exists(): |
| 172 | return True |
| 173 | |
| 174 | return False |
| 175 | |
| 176 | |
| 177 | def server_ctx( |
| 178 | serve_cmd: list[str], |
| 179 | after_bench_cmd: list[str], |
| 180 | *, |
| 181 | show_stdout: bool, |
| 182 | serve_comb: ParameterSweepItem, |
| 183 | bench_params: ParameterSweep, |
| 184 | experiment_dir: Path, |
| 185 | dry_run: bool, |
| 186 | server_ready_timeout: int = 300, |
| 187 | ): |
| 188 | if not _comb_needs_server(serve_comb, bench_params, experiment_dir): |
| 189 | return contextlib.nullcontext() |
| 190 | |
| 191 | return run_server( |
| 192 | serve_cmd, |
| 193 | after_bench_cmd, |
| 194 | show_stdout=show_stdout, |
| 195 | serve_overrides=serve_comb, |
| 196 | dry_run=dry_run, |
| 197 | server_ready_timeout=server_ready_timeout, |
| 198 | ) |
| 199 | |
| 200 | |
| 201 | def _comb_is_valid( |
| 202 | serve_comb: ParameterSweepItem, |
| 203 | bench_comb: ParameterSweepItem, |
| 204 | link_vars: list[tuple[str, str]], |
| 205 | ) -> bool: |
| 206 | return all( |
| 207 | serve_key in serve_comb |
| 208 | and bench_key in bench_comb |
| 209 | and serve_comb[serve_key] == bench_comb[bench_key] |
| 210 | for serve_key, bench_key in link_vars |
| 211 | ) |
| 212 | |
| 213 | |
| 214 | def run_comb( |
| 215 | server: ServerProcess | None, |
| 216 | bench_cmd: list[str], |
| 217 | *, |
| 218 | serve_comb: ParameterSweepItem, |
| 219 | bench_comb: ParameterSweepItem, |
| 220 | link_vars: list[tuple[str, str]], |
| 221 | base_path: Path, |
| 222 | num_runs: int, |
| 223 | dry_run: bool, |
| 224 | warmup_num_prompts: int = 0, |
| 225 | continue_on_error: bool = False, |
| 226 | ): |
| 227 | if not _comb_is_valid(serve_comb, bench_comb, link_vars): |
| 228 | return None |
| 229 | |
| 230 | comb_data = list[dict[str, object]]() |
| 231 | |
| 232 | def run_benchmark_with_error_handling( |
| 233 | *, |
| 234 | bench_overrides: ParameterSweepItem, |
| 235 | run_number: int, |
| 236 | output_path: Path, |
| 237 | ): |
| 238 | try: |
| 239 | return run_benchmark( |
| 240 | server, |
| 241 | bench_cmd, |
| 242 | serve_overrides=serve_comb, |
| 243 | bench_overrides=bench_overrides, |
| 244 | run_number=run_number, |
| 245 | output_path=output_path, |
| 246 | dry_run=dry_run, |
| 247 | ) |
| 248 | except Exception as exc: |
| 249 | if not continue_on_error: |
| 250 | raise |
| 251 | failure_path = output_path.with_name(output_path.stem + ".failure.json") |
| 252 | failure_path.parent.mkdir(parents=True, exist_ok=True) |
| 253 | failure_path.write_text( |
| 254 | json.dumps( |
| 255 | { |
| 256 | "run_number": run_number, |
| 257 | "error_type": type(exc).__name__, |
| 258 | "error": str(exc), |
| 259 | "serve_overrides": dict(serve_comb), |
| 260 | "bench_overrides": dict(bench_overrides), |
| 261 | }, |
| 262 | indent=2, |
| 263 | ) |
| 264 | + "\n", |
| 265 | encoding="utf-8", |
| 266 | ) |
| 267 | print(f"[FAILED BENCHMARK] run={run_number}: {type(exc).__name__}: {exc}") |
| 268 | print(f"Failure details: {failure_path}") |
| 269 | if server is not None: |
| 270 | with contextlib.suppress(Exception): |
| 271 | server.after_bench() |
| 272 | return None |
| 273 | |
| 274 | if warmup_num_prompts > 0: |
| 275 | warmup_overrides = bench_comb | {"num_prompts": warmup_num_prompts} |
| 276 | run_benchmark_with_error_handling( |
| 277 | bench_overrides=warmup_overrides, |
| 278 | run_number=-1, |
| 279 | output_path=base_path / "warmup.json", |
| 280 | ) |
| 281 | |
| 282 | for run_number in range(num_runs): |
| 283 | run_data = run_benchmark_with_error_handling( |
| 284 | bench_overrides=bench_comb, |
| 285 | run_number=run_number, |
| 286 | output_path=_get_comb_run_path(base_path, run_number), |
| 287 | ) |
| 288 | if run_data is not None: |
| 289 | comb_data.append(run_data) |
| 290 | |
| 291 | if dry_run: |
| 292 | return None |
| 293 | |
| 294 | if not comb_data: |
| 295 | print("[NO SUCCESSFUL MEASURED RUNS] leaving combination retryable") |
| 296 | return [] |
| 297 | |
| 298 | with _get_comb_run_path(base_path, run_number=None).open("w") as f: |
| 299 | json.dump(comb_data, f, indent=4) |
| 300 | |
| 301 | return comb_data |
| 302 | |
| 303 | |
| 304 | def run_combs( |
| 305 | serve_cmd: list[str], |
| 306 | bench_cmd: list[str], |
| 307 | after_bench_cmd: list[str], |
| 308 | *, |
| 309 | show_stdout: bool, |
| 310 | server_ready_timeout: int, |
| 311 | serve_params: ParameterSweep, |
| 312 | bench_params: ParameterSweep, |
| 313 | link_vars: list[tuple[str, str]], |
| 314 | experiment_dir: Path, |
| 315 | num_runs: int, |
| 316 | warmup_num_prompts: int, |
| 317 | dry_run: bool, |
| 318 | continue_on_error: bool = False, |
| 319 | ): |
| 320 | all_data = list[dict[str, object]]() |
| 321 | for serve_comb in serve_params: |
| 322 | try: |
| 323 | with server_ctx( |
| 324 | serve_cmd, |
| 325 | after_bench_cmd, |
| 326 | show_stdout=show_stdout, |
| 327 | serve_comb=serve_comb, |
| 328 | bench_params=bench_params, |
| 329 | experiment_dir=experiment_dir, |
| 330 | dry_run=dry_run, |
| 331 | server_ready_timeout=server_ready_timeout, |
| 332 | ) as server: |
| 333 | for bench_comb in bench_params: |
| 334 | base_path = _get_comb_base_path( |
| 335 | experiment_dir, serve_comb, bench_comb |
| 336 | ) |
| 337 | |
| 338 | comb_data = run_comb( |
| 339 | server, |
| 340 | bench_cmd, |
| 341 | serve_comb=serve_comb, |
| 342 | bench_comb=bench_comb, |
| 343 | link_vars=link_vars, |
| 344 | base_path=base_path, |
| 345 | num_runs=num_runs, |
| 346 | warmup_num_prompts=warmup_num_prompts, |
| 347 | dry_run=dry_run, |
| 348 | continue_on_error=continue_on_error, |
| 349 | ) |
| 350 | |
| 351 | if comb_data is not None: |
| 352 | all_data.extend(comb_data) |
| 353 | except Exception as exc: |
| 354 | if not continue_on_error: |
| 355 | raise |
| 356 | failure_name = sanitize_filename( |
| 357 | f"SERVE-{serve_comb.name or 'default'}-failure.json" |
| 358 | ) |
| 359 | failure_path = experiment_dir / failure_name |
| 360 | failure_path.parent.mkdir(parents=True, exist_ok=True) |
| 361 | failure_path.write_text( |
| 362 | json.dumps( |
| 363 | { |
| 364 | "error_type": type(exc).__name__, |
| 365 | "error": str(exc), |
| 366 | "serve_overrides": dict(serve_comb), |
| 367 | }, |
| 368 | indent=2, |
| 369 | ) |
| 370 | + "\n", |
| 371 | encoding="utf-8", |
| 372 | ) |
| 373 | print( |
| 374 | "[FAILED SERVER COMBINATION] " |
| 375 | f"{serve_comb.name or '(default)'}: " |
| 376 | f"{type(exc).__name__}: {exc}" |
| 377 | ) |
| 378 | print(f"Failure details: {failure_path}") |
| 379 | |
| 380 | if dry_run: |
| 381 | return None |
| 382 | |
| 383 | combined_df = pd.DataFrame.from_records(all_data) |
| 384 | combined_df.to_csv(experiment_dir / "summary.csv") |
| 385 | |
| 386 | return combined_df |
| 387 | |
| 388 | |
| 389 | @dataclass |
| 390 | class SweepServeArgs: |
| 391 | serve_cmd: list[str] |
| 392 | bench_cmd: list[str] |
| 393 | after_bench_cmd: list[str] |
| 394 | show_stdout: bool |
| 395 | server_ready_timeout: int |
| 396 | serve_params: ParameterSweep |
| 397 | bench_params: ParameterSweep |
| 398 | link_vars: list[tuple[str, str]] |
| 399 | output_dir: Path |
| 400 | experiment_name: str |
| 401 | num_runs: int |
| 402 | warmup_num_prompts: int |
| 403 | continue_on_error: bool |
| 404 | dry_run: bool |
| 405 | resume: bool |
| 406 | |
| 407 | parser_name: ClassVar[str] = "serve" |
| 408 | parser_help: ClassVar[str] = "Run vLLM server benchmark under multiple settings." |
| 409 | |
| 410 | @classmethod |
| 411 | def from_cli_args(cls, args: argparse.Namespace): |
| 412 | serve_cmd = shlex.split(args.serve_cmd) |
| 413 | bench_cmd = shlex.split(args.bench_cmd) |
| 414 | after_bench_cmd = ( |
| 415 | [] if args.after_bench_cmd is None else shlex.split(args.after_bench_cmd) |
| 416 | ) |
| 417 | |
| 418 | if args.serve_params: |
| 419 | serve_params = ParameterSweep.read_json(args.serve_params) |
| 420 | else: |
| 421 | # i.e.: run serve_cmd without any modification |
| 422 | serve_params = ParameterSweep.from_records([{}]) |
| 423 | |
| 424 | if args.bench_params: |
| 425 | bench_params = ParameterSweep.read_json(args.bench_params) |
| 426 | else: |
| 427 | # i.e.: run bench_cmd without any modification |
| 428 | bench_params = ParameterSweep.from_records([{}]) |
| 429 | |
| 430 | link_vars = cls.parse_link_vars(args.link_vars) |
| 431 | |
| 432 | if args.experiment_name: |
| 433 | experiment_name = args.experiment_name |
| 434 | else: |
| 435 | experiment_name = datetime.now().strftime("%Y%m%d_%H%M%S") |
| 436 | |
| 437 | num_runs = args.num_runs |
| 438 | if num_runs < 1: |
| 439 | raise ValueError("`num_runs` should be at least 1.") |
| 440 | warmup_num_prompts = args.warmup_num_prompts |
| 441 | if warmup_num_prompts < 0: |
| 442 | raise ValueError("`warmup_num_prompts` should be at least 0.") |
| 443 | |
| 444 | return cls( |
| 445 | serve_cmd=serve_cmd, |
| 446 | bench_cmd=bench_cmd, |
| 447 | after_bench_cmd=after_bench_cmd, |
| 448 | show_stdout=args.show_stdout, |
| 449 | serve_params=serve_params, |
| 450 | bench_params=bench_params, |
| 451 | link_vars=link_vars, |
| 452 | output_dir=Path(args.output_dir), |
| 453 | experiment_name=experiment_name, |
| 454 | num_runs=num_runs, |
| 455 | warmup_num_prompts=warmup_num_prompts, |
| 456 | continue_on_error=args.continue_on_error, |
| 457 | dry_run=args.dry_run, |
| 458 | resume=args.resume, |
| 459 | server_ready_timeout=args.server_ready_timeout, |
| 460 | ) |
| 461 | |
| 462 | @classmethod |
| 463 | def add_cli_args(cls, parser: FlexibleArgumentParser) -> FlexibleArgumentParser: |
| 464 | parser.add_argument( |
| 465 | "--serve-cmd", |
| 466 | type=str, |
| 467 | required=True, |
| 468 | help="The command used to run the server: `vllm serve ...`", |
| 469 | ) |
| 470 | parser.add_argument( |
| 471 | "--bench-cmd", |
| 472 | type=str, |
| 473 | required=True, |
| 474 | help="The command used to run the benchmark: `vllm bench serve ...`", |
| 475 | ) |
| 476 | parser.add_argument( |
| 477 | "--after-bench-cmd", |
| 478 | type=str, |
| 479 | default=None, |
| 480 | help="After a benchmark run is complete, invoke this command instead of " |
| 481 | "the default `ServerWrapper.clear_cache()`.", |
| 482 | ) |
| 483 | parser.add_argument( |
| 484 | "--show-stdout", |
| 485 | action="store_true", |
| 486 | help="If set, logs the standard output of subcommands. " |
| 487 | "Useful for debugging but can be quite spammy.", |
| 488 | ) |
| 489 | parser.add_argument( |
| 490 | "--server-ready-timeout", |
| 491 | type=int, |
| 492 | default=300, |
| 493 | help="Timeout in seconds to wait for the server to become ready.", |
| 494 | ) |
| 495 | |
| 496 | parser.add_argument( |
| 497 | "--serve-params", |
| 498 | type=str, |
| 499 | default=None, |
| 500 | help="Path to JSON file containing parameter combinations " |
| 501 | "for the `vllm serve` command. Can be either a list of dicts or a dict " |
| 502 | "where keys are benchmark names. " |
| 503 | "If both `serve_params` and `bench_params` are given, " |
| 504 | "this script will iterate over their Cartesian product.", |
| 505 | ) |
| 506 | parser.add_argument( |
| 507 | "--link-vars", |
| 508 | type=str, |
| 509 | default="", |
| 510 | help=( |
| 511 | "Comma-separated list of linked variables between serve and bench, " |
| 512 | "e.g. max_num_seqs=max_concurrency,max_model_len=random_input_len" |
| 513 | ), |
| 514 | ) |
| 515 | |
| 516 | parser.add_argument( |
| 517 | "--bench-params", |
| 518 | type=str, |
| 519 | default=None, |
| 520 | help="Path to JSON file containing parameter combinations " |
| 521 | "for the `vllm bench serve` command. Can be either a list of dicts or " |
| 522 | "a dict where keys are benchmark names. " |
| 523 | "If both `serve_params` and `bench_params` are given, " |
| 524 | "this script will iterate over their Cartesian product.", |
| 525 | ) |
| 526 | parser.add_argument( |
| 527 | "-o", |
| 528 | "--output-dir", |
| 529 | type=str, |
| 530 | default="results", |
| 531 | help="The main directory to which results are written.", |
| 532 | ) |
| 533 | parser.add_argument( |
| 534 | "-e", |
| 535 | "--experiment-name", |
| 536 | type=str, |
| 537 | default=None, |
| 538 | help="The name of this experiment (defaults to current timestamp). " |
| 539 | "Results will be stored under `output_dir/experiment_name`.", |
| 540 | ) |
| 541 | parser.add_argument( |
| 542 | "--num-runs", |
| 543 | type=int, |
| 544 | default=3, |
| 545 | help="Number of runs per parameter combination.", |
| 546 | ) |
| 547 | parser.add_argument( |
| 548 | "--warmup-num-prompts", |
| 549 | type=int, |
| 550 | default=0, |
| 551 | help=( |
| 552 | "Number of prompts in one unmeasured warmup run before each " |
| 553 | "parameter combination. The warmup is saved as warmup.json but " |
| 554 | "excluded from summary statistics." |
| 555 | ), |
| 556 | ) |
| 557 | parser.add_argument( |
| 558 | "--continue-on-error", |
| 559 | action="store_true", |
| 560 | help=( |
| 561 | "Record and skip a failed warmup or benchmark run instead of " |
| 562 | "terminating the entire sweep." |
| 563 | ), |
| 564 | ) |
| 565 | parser.add_argument( |
| 566 | "--dry-run", |
| 567 | action="store_true", |
| 568 | help="If set, prints the commands to run, " |
| 569 | "then exits without executing them.", |
| 570 | ) |
| 571 | parser.add_argument( |
| 572 | "--resume", |
| 573 | action="store_true", |
| 574 | help="Resume a previous execution of this script, i.e., only run " |
| 575 | "parameter combinations for which there are still no output files " |
| 576 | "under `output_dir/experiment_name`.", |
| 577 | ) |
| 578 | |
| 579 | return parser |
| 580 | |
| 581 | @staticmethod |
| 582 | def parse_link_vars(s: str) -> list[tuple[str, str]]: |
| 583 | if not s: |
| 584 | return [] |
| 585 | pairs = [] |
| 586 | for item in s.split(","): |
| 587 | a, b = item.split("=") |
| 588 | pairs.append((a.strip(), b.strip())) |
| 589 | return pairs |
| 590 | |
| 591 | def resolve_experiment_dir(self) -> Path: |
| 592 | experiment_dir = self.output_dir / self.experiment_name |
| 593 | |
| 594 | if self.resume: |
| 595 | if not experiment_dir.exists(): |
| 596 | raise ValueError(f"Cannot resume from non-existent {experiment_dir=}") |
| 597 | else: |
| 598 | if experiment_dir.exists(): |
| 599 | raise ValueError(f"Cannot overwrite existing {experiment_dir=}") |
| 600 | |
| 601 | return experiment_dir |
| 602 | |
| 603 | @contextmanager |
| 604 | def run_ctx(self, experiment_dir: Path): |
| 605 | if self.dry_run: |
| 606 | yield |
| 607 | print(f"Experiment will be saved at: {experiment_dir}") |
| 608 | return |
| 609 | |
| 610 | try: |
| 611 | yield |
| 612 | print(f"Experiment has been saved at: {experiment_dir}") |
| 613 | except BaseException as exc: |
| 614 | raise RuntimeError( |
| 615 | "The script was terminated early. Use `--resume` " |
| 616 | "to continue the script from its last checkpoint." |
| 617 | ) from exc |
| 618 | |
| 619 | |
| 620 | def run_main(args: SweepServeArgs): |
| 621 | experiment_dir = args.resolve_experiment_dir() |
| 622 | |
| 623 | with args.run_ctx(experiment_dir): |
| 624 | return run_combs( |
| 625 | serve_cmd=args.serve_cmd, |
| 626 | bench_cmd=args.bench_cmd, |
| 627 | link_vars=args.link_vars, |
| 628 | after_bench_cmd=args.after_bench_cmd, |
| 629 | show_stdout=args.show_stdout, |
| 630 | server_ready_timeout=args.server_ready_timeout, |
| 631 | serve_params=args.serve_params, |
| 632 | bench_params=args.bench_params, |
| 633 | experiment_dir=experiment_dir, |
| 634 | num_runs=args.num_runs, |
| 635 | warmup_num_prompts=args.warmup_num_prompts, |
| 636 | dry_run=args.dry_run, |
| 637 | continue_on_error=args.continue_on_error, |
| 638 | ) |
| 639 | |
| 640 | |
| 641 | def main(args: argparse.Namespace): |
| 642 | run_main(SweepServeArgs.from_cli_args(args)) |
| 643 | |
| 644 | |
| 645 | if __name__ == "__main__": |
| 646 | parser = FlexibleArgumentParser(description=SweepServeArgs.parser_help) |
| 647 | SweepServeArgs.add_cli_args(parser) |
| 648 | |
| 649 | main(parser.parse_args()) |