@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 79651d6085

[Tools][Recipes] Improve sweep recommendations and short-alias parsing (#53946)

Signed-off-by: louie-tsai <louie.tsai@intel.com>

Louie Tsai committed Aug 26, 2026 at 23:58 UTC 79651d6085885cf2842baf85526ece7afa87c87d

7 files changed +351 -73

docs/models/hardware_supported_models/cpu.md

+4 -4

@@ -30,7 +30,7 @@ vllm serve --config config.yaml

32 | ------------------------------------ | ---------------------------------------- | --------- | ------ |

33 -| openai/gpt-oss-20b | GptOssForCausalLM | ✅ | — |

@@ -48,13 +48,13 @@ vllm serve --config config.yaml

48 | TheBloke/TinyLlama-1.1B-Chat-v1.0-AWQ | LlamaForCausalLM | ✅ | — |

49 | TheBloke/TinyLlama-1.1B-Chat-v1.0-GPTQ | LlamaForCausalLM | ✅ | — |

51 -| Qwen/Qwen3-1.7B | Qwen3ForCausalLM | ✅ | — |

53 -| Qwen/Qwen3-8B | Qwen3ForCausalLM | ✅ | — |

57 -| Qwen/Qwen3-30B-A3B | Qwen3MoeForCausalLM | ✅ | — |

58 | Qwen/Qwen3-30B-A3B-Instruct-2507-FP8 | Qwen3MoeForCausalLM | ✅ | — |

tools/recipes/README.md

+2 -2

@@ -87,7 +87,7 @@ capacity objectives.

87 python3 tools/recipes/recipe_json_to_vllm_config.py \

88 --model meta-llama/Llama-3.1-8B-Instruct \

89 --hardware xeon6 \

90 - --input-tokens 2048 \

90 + --input-tokens 128 \

91 --output-tokens 128 \

92 --concurrency 32 \

93 --ttft-sla-ms 3000 \

@@ -111,7 +111,7 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \

111 --model meta-llama/Llama-3.1-8B-Instruct \

112 --hardware xeon6 \

113 --detect-hardware \

114 - --input-tokens 2048 \

114 + --input-tokens 128 \

115 --output-tokens 128 \

116 --concurrency 32 \

117 --generate-sweep

tools/recipes/RUNTIME_TUNING.md

+5 -5

@@ -57,7 +57,7 @@ Example:

57 python3 tools/recipes/recipe_json_to_vllm_config.py \

58 --model meta-llama/Llama-3.1-8B-Instruct \

59 --hardware xeon6 \

60 - --input-tokens 2048 \

60 + --input-tokens 128 \

61 --output-tokens 128 \

62 --concurrency 32 \

63 --ttft-sla-ms 3000 \

@@ -172,13 +172,13 @@ max-num-batched-tokens = max(

172 )

173 ```

174

175 -For example, with 2048 input tokens, 128 output tokens, and concurrency 32:

175 +For example, with 128 input tokens, 128 output tokens, and concurrency 32:

176

177 ```text

178 decode_budget = 32

179 prefills_per_step = max(1, 32 / 128) = 1

180 -prefill_budget = 2048

181 -candidate = max(2048, 32, 32 + 2048) = 2080

180 +prefill_budget = 128

181 +candidate = max(2048, 32, 32 + 128) = 2048

182 ```

183

184 If chunked prefill is explicitly disabled and `max-model-len` is available, the

@@ -261,7 +261,7 @@ python3 /recipes/recipe_json_to_vllm_config.py \

261 --model meta-llama/Llama-3.1-8B-Instruct \

262 --hardware xeon6 \

263 --detect-hardware \

264 - --input-tokens 2048 \

264 + --input-tokens 128 \

265 --output-tokens 128 \

266 --concurrency 32 \

267 --ttft-sla-ms 3000 \

tools/recipes/SWEEP_TUNING.md

+23 -10

@@ -11,7 +11,7 @@ python3 tools/recipes/recipe_json_to_vllm_config.py \

11 --model meta-llama/Llama-3.1-8B-Instruct \

12 --hardware xeon6 \

13 --detect-hardware \

14 - --input-tokens 2048 \

14 + --input-tokens 128 \

15 --output-tokens 128 \

16 --concurrency 32 \

17 --ttft-sla-ms 3000 \

@@ -30,8 +30,11 @@ sweep/

30 └── SWEEP.md

31 ```

33 -The sweep currently varies only `max-num-seqs` and

34 -`max-num-batched-tokens`.

33 +The sweep varies only `max-num-seqs` and `max-num-batched-tokens`. It uses an

34 +eight-point directed design: a batch-budget curve at the initial sequence count

35 +plus lower/higher batch interactions at three-quarters and one-half of that

36 +count. This gives broader coverage than a one-parameter-at-a-time sweep without

37 +the cost of a full Cartesian grid.

39 ## Run and Recommend

@@ -54,13 +57,23 @@ sweep/recommended-config.yml

57 sweep/recommendation.json

58 ```

57 -With TTFT/TPOT objectives, the benchmark uses vLLM `--goodput`, and the

58 -recommender selects highest mean request goodput across repeated runs. Without

59 -latency objectives it selects highest mean output-token throughput. Failed

60 -benchmark configurations are excluded.

60 +With TTFT/TPOT objectives, the benchmark uses vLLM `--goodput`. The recommender

61 +calculates duration-weighted combined compliance across repeated runs and

62 +requires both median P99 latency objectives and the minimum compliance ratio

63 +(default `0.99`). Among eligible configurations it selects the highest mean

64 +output-token throughput. Use `recommend.py --minimum-compliance VALUE` to

65 +change the compliance requirement.

62 -`recommended-config.yml` changes only the two swept scheduler parameters.

63 -`recommendation.json` records the measured evidence.

67 +If no configuration is eligible, `recommend.py` records the highest-goodput

68 +configuration as `best_effort` in `recommendation.json`, does not write a

69 +deployable `recommended-config.yml`, and exits with status 2. Without latency

70 +objectives it selects highest mean output-token throughput. Failed benchmark

71 +configurations are excluded in either mode.

72 +

73 +When an eligible configuration exists, `recommended-config.yml` changes only

74 +the two swept scheduler parameters. `recommendation.json` records mean, median,

75 +and worst-run P99 values, combined compliance, and the measured evidence for

76 +every candidate.

78 Deploy:

@@ -83,7 +96,7 @@ python3 /recipes/recipe_json_to_vllm_config.py \

96 --model meta-llama/Llama-3.1-8B-Instruct \

97 --hardware xeon6 \

98 --detect-hardware \

86 - --input-tokens 2048 \

99 + --input-tokens 128 \

100 --output-tokens 128 \

101 --concurrency 32 \

102 --ttft-sla-ms 3000 \

tools/recipes/recipe_json_to_vllm_config.py

+115 -10

@@ -59,10 +59,36 @@ except ImportError as exc:

60 DEFAULT_API_BASE = "https://recipes.vllm.ai"

62 -SHORT_ALIASES = {

63 - "-tp": "tensor-parallel-size",

62 +VALUE_SHORT_ALIASES = {

63 + "-q": "quantization",

64 "-pp": "pipeline-parallel-size",

65 + "-n": "nnodes",

66 + "-r": "node-rank",

67 + "-tp": "tensor-parallel-size",

68 + "-dcp": "decode-context-parallel-size",

69 + "-pcp": "prefill-context-parallel-size",

70 "-dp": "data-parallel-size",

71 + "-dpn": "data-parallel-rank",

72 + "-dpr": "data-parallel-start-rank",

73 + "-dpl": "data-parallel-size-local",

74 + "-dpa": "data-parallel-address",

75 + "-dpp": "data-parallel-rpc-port",

76 + "-dpb": "data-parallel-backend",

77 + "-asc": "api-server-count",

78 +}

79 +

80 +FLAG_SHORT_ALIASES = {

81 + "-dph": "data-parallel-hybrid-lb",

82 + "-dpe": "data-parallel-external-lb",

83 + "-dpm": "data-parallel-multi-port-external-lb",

84 + "-ep": "enable-expert-parallel",

85 +}

86 +

87 +DICT_SHORT_ALIASES = {

88 + "-sc": "speculative-config",

89 + "-dc": "diffusion-config",

90 + "-cc": "compilation-config",

91 + "-ac": "attention-config",

92 }

@@ -464,6 +490,11 @@ def discover_recipe_source(

490

491 def coerce(value: str) -> Any:

492 """Convert CLI string values to useful YAML scalar/object types."""

493 + # Recipes may preserve Python-style booleans in dotted config aliases,

494 + # for example: -cc.pass_config.fuse_rope_kvcache=True.

495 + if value in ("True", "False"):

496 + return value == "True"

497 +

498 try:

499 return json.loads(value)

500 except (json.JSONDecodeError, TypeError):

@@ -473,8 +504,19 @@ def coerce(value: str) -> Any:

504 def is_option_token(token: str) -> bool:

505 if token.startswith("--"):

506 return True

476 - if token in SHORT_ALIASES or token == "-O":

507 +

508 + short_name = token.split("=", 1)[0]

509 + if (

510 + short_name in VALUE_SHORT_ALIASES

511 + or short_name in FLAG_SHORT_ALIASES

512 + or short_name in DICT_SHORT_ALIASES

513 + or token == "-O"

514 + ):

515 return True

516 +

517 + if any(token.startswith(f"{alias}.") for alias in DICT_SHORT_ALIASES):

518 + return True

519 +

520 return token.startswith("-O") and len(token) > 2

521

522

@@ -554,22 +596,85 @@ def argv_to_config(argv: list[Any]) -> dict[str, Any]:

596 i += 2

597 continue

598

557 - # Selected common short aliases.

558 - if token in SHORT_ALIASES:

559 - if i + 1 >= len(argv):

560 - raise ValueError(f"{token} is missing its value")

599 + # Dotted dictionary aliases accepted by FlexibleArgumentParser:

600 + # -cc.mode=3

601 + # -cc.mode 3

602 + # -cc.pass_config.foo=True

603 + # -cc.custom_ops+ -quant_fp8

604 + # The same syntax applies to -sc, -dc, and -ac.

605 + dotted_alias = next(

606 + (alias for alias in DICT_SHORT_ALIASES if token.startswith(f"{alias}.")),

607 + None,

608 + )

609 + if dotted_alias is not None:

610 + dotted = token[len(dotted_alias) + 1 :]

611 + raw_key, separator, raw_value = dotted.partition("=")

612 +

613 + append = raw_key.endswith("+")

614 + if append:

615 + raw_key = raw_key[:-1]

616 +

617 + if not raw_key:

618 + raise ValueError(f"{token} must contain a key after {dotted_alias}.")

619 +

620 + if not separator:

621 + if i + 1 >= len(argv):

622 + raise ValueError(f"{token} is missing its value")

623 + # Consume the next token unconditionally. Dotted list values may

624 + # themselves begin with "-", for example "-quant_fp8".

625 + raw_value = argv[i + 1]

626 + i += 2

627 + else:

628 + i += 1

629 +

630 + value: Any = raw_value.split(",") if append else coerce(raw_value)

631 +

632 merge_value(

633 config,

563 - [SHORT_ALIASES[token]],

564 - coerce(argv[i + 1]),

634 + [DICT_SHORT_ALIASES[dotted_alias], *raw_key.split(".")],

635 + value,

636 )

637 + continue

638 +

639 + # Support -alias=value for value-bearing and whole-dict aliases.

640 + if "=" in token:

641 + short_name, raw_value = token.split("=", 1)

642 + if short_name in VALUE_SHORT_ALIASES:

643 + merge_value(

644 + config,

645 + [VALUE_SHORT_ALIASES[short_name]],

646 + coerce(raw_value),

647 + )

648 + i += 1

649 + continue

650 + if short_name in DICT_SHORT_ALIASES:

651 + merge_value(

652 + config,

653 + [DICT_SHORT_ALIASES[short_name]],

654 + coerce(raw_value),

655 + )

656 + i += 1

657 + continue

658 +

659 + # Boolean short aliases are flags and do not consume a value.

660 + if token in FLAG_SHORT_ALIASES:

661 + merge_value(config, [FLAG_SHORT_ALIASES[token]], True)

662 + i += 1

663 + continue

664 +

665 + # Scalar and whole-dictionary short aliases consume one value.

666 + if token in VALUE_SHORT_ALIASES or token in DICT_SHORT_ALIASES:

667 + if i + 1 >= len(argv):

668 + raise ValueError(f"{token} is missing its value")

669 + key = VALUE_SHORT_ALIASES.get(token) or DICT_SHORT_ALIASES[token]

670 + merge_value(config, [key], coerce(argv[i + 1]))

671 i += 2

672 continue

673

674 if not token.startswith("--"):

675 raise ValueError(

676 f"Unexpected positional/short argument {token!r}. "

572 - "The converter expects Recipes to emit long-form vLLM serve options."

677 + "The converter accepts vLLM serve long options and known short aliases."

678 )

679

680 # --key=value

tools/recipes/sweep_generation.py

+23 -6

@@ -43,7 +43,13 @@ def validate_sweep_workload(workload: WorkloadHints) -> None:

45 def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]:

46 - """Build a small directed sweep around the single initial suggestion."""

46 + """Build a diverse, bounded sweep around the initial suggestion.

47 +

48 + The initial five-point sweep changed only one parameter at a time and

49 + skipped the useful middle sequence count. Eight directed points cover the

50 + batch-budget curve at full concurrency plus interactions at 3/4 and 1/2 of

51 + the initial scheduler concurrency without paying for a full Cartesian grid.

52 + """

53 initial_seqs = _positive_int(config, "max-num-seqs")

54 initial_batch = _positive_int(config, "max-num-batched-tokens")

@@ -58,10 +64,15 @@ def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]:

64 minimum_batch = max(minimum_batch, max_model_len)

66 lower_seqs = max(1, (initial_seqs + 1) // 2)

67 + middle_seqs = max(lower_seqs, (3 * initial_seqs + 3) // 4)

68 lower_batch = max(

69 minimum_batch,

70 _strict_lower_power_of_two(initial_batch),

71 )

72 + smaller_batch = max(

73 + minimum_batch,

74 + _strict_lower_power_of_two(lower_batch),

75 + )

76 higher_batch = max(

77 minimum_batch,

78 _strict_upper_power_of_two(initial_batch),

@@ -90,10 +101,13 @@ def build_serve_params(config: dict[str, Any]) -> list[dict[str, Any]]:

101 # Keep the exact initial suggestion as the measured baseline. Additional

102 # values exist only in the optional sweep package.

103 add("initial", initial_seqs, initial_batch)

93 - add("lower_scheduler_concurrency", lower_seqs, initial_batch)

104 + add("smaller_batch_budget", initial_seqs, smaller_batch)

105 add("lower_batch_budget", initial_seqs, lower_batch)

95 - add("latency_focused", lower_seqs, lower_batch)

106 add("higher_batch_budget", initial_seqs, higher_batch)

107 + add("middle_seqs_lower_batch", middle_seqs, lower_batch)

108 + add("middle_seqs_higher_batch", middle_seqs, higher_batch)

109 + add("lower_seqs_lower_batch", lower_seqs, lower_batch)

110 + add("lower_seqs_higher_batch", lower_seqs, higher_batch)

111

112 return candidates

113

@@ -286,9 +300,12 @@ recommended-config.yml

300 recommendation.json

301 ```

302

289 -With TTFT/TPOT objectives it selects highest mean request goodput across

290 -repeated runs. Without latency objectives it selects highest mean output-token

291 -throughput. Configurations with failed requests are excluded.

303 +With TTFT/TPOT objectives it requires duration-weighted combined compliance of

304 +at least 99% plus median P99 TTFT/TPOT compliance, then selects highest mean

305 +output-token throughput. If no candidate qualifies, it records a best-effort

306 +candidate but does not write `recommended-config.yml`. Without latency

307 +objectives it selects highest mean output-token throughput. Configurations with

308 +failed requests are excluded.

309

310 `recommended-config.yml` copies the initial configuration and changes only

311 `max-num-seqs` and `max-num-batched-tokens`.

tools/recipes/sweep_recommendation.py

+179 -36

@@ -10,7 +10,7 @@ import argparse

10 import json

11 from collections import defaultdict

12 from pathlib import Path

13 -from statistics import mean

13 +from statistics import mean, median

14 from typing import Any

16 try:

@@ -23,6 +23,7 @@ DEFAULT_CONFIG_PATH: str | None = None

23 DEFAULT_ENV_PATH: str | None = None

24 DEFAULT_TTFT_SLA_MS: float | None = None

25 DEFAULT_TPOT_SLA_MS: float | None = None

26 +DEFAULT_MINIMUM_COMPLIANCE: float = 0.99

29 def _number(value: object) -> float | None:

@@ -37,6 +38,16 @@ def _mean(rows: list[dict[str, Any]], key: str) -> float | None:

38 return mean(numbers) if numbers else None

41 +def _percentile_summary(

42 + rows: list[dict[str, Any]], key: str

43 +) -> tuple[float | None, float | None, float | None]:

44 + values = [_number(row.get(key)) for row in rows]

45 + numbers = [value for value in values if value is not None]

46 + if not numbers:

47 + return None, None, None

48 + return mean(numbers), median(numbers), max(numbers)

49 +

50 +

51 def _positive_int(row: dict[str, Any], key: str) -> int | None:

52 value = row.get(key)

53 if isinstance(value, bool) or not isinstance(value, int) or value <= 0:

@@ -69,6 +80,9 @@ def _aggregate_candidates(

80 rows: list[dict[str, Any]],

81 *,

82 use_goodput: bool,

83 + ttft_sla_ms: float | None = None,

84 + tpot_sla_ms: float | None = None,

85 + minimum_compliance: float = DEFAULT_MINIMUM_COMPLIANCE,

86 ) -> list[dict[str, Any]]:

87 grouped: dict[tuple[int, int], list[dict[str, Any]]] = defaultdict(list)

@@ -83,9 +97,28 @@ def _aggregate_candidates(

97 for (seqs, batch), runs in sorted(grouped.items()):

98 failed_requests = sum(int(_number(run.get("failed")) or 0) for run in runs)

99 output_throughput = _mean(runs, "output_throughput")

100 + request_throughput = _mean(runs, "request_throughput")

101 request_goodput = _mean(runs, "request_goodput")

87 - p99_ttft_ms = _mean(runs, "p99_ttft_ms")

88 - p99_tpot_ms = _mean(runs, "p99_tpot_ms")

102 + mean_p99_ttft_ms, median_p99_ttft_ms, worst_p99_ttft_ms = _percentile_summary(

103 + runs, "p99_ttft_ms"

104 + )

105 + mean_p99_tpot_ms, median_p99_tpot_ms, worst_p99_tpot_ms = _percentile_summary(

106 + runs, "p99_tpot_ms"

107 + )

108 +

109 + completed_requests = sum(

110 + int(_number(run.get("completed")) or 0) for run in runs

111 + )

112 + estimated_compliant_requests = sum(

113 + (_number(run.get("request_goodput")) or 0.0)

114 + * (_number(run.get("duration")) or 0.0)

115 + for run in runs

116 + )

117 + combined_compliance_ratio = (

118 + estimated_compliant_requests / completed_requests

119 + if completed_requests > 0

120 + else None

121 + )

122

123 valid = failed_requests == 0 and output_throughput is not None

124 reason = None

@@ -96,6 +129,25 @@ def _aggregate_candidates(

129 valid = False

130 reason = "request_goodput is missing; rerun sweep with SLO goodput enabled"

131

132 + p99_sla_eligible = valid

133 + if ttft_sla_ms is not None:

134 + p99_sla_eligible = bool(

135 + p99_sla_eligible

136 + and median_p99_ttft_ms is not None

137 + and median_p99_ttft_ms <= ttft_sla_ms

138 + )

139 + if tpot_sla_ms is not None:

140 + p99_sla_eligible = bool(

141 + p99_sla_eligible

142 + and median_p99_tpot_ms is not None

143 + and median_p99_tpot_ms <= tpot_sla_ms

144 + )

145 + compliance_eligible = bool(

146 + valid

147 + and combined_compliance_ratio is not None

148 + and combined_compliance_ratio >= minimum_compliance

149 + )

150 +

151 candidates.append(

152 {

153 "max_num_seqs": seqs,

@@ -103,9 +155,26 @@ def _aggregate_candidates(

155 "run_count": len(runs),

156 "failed_requests": failed_requests,

157 "mean_request_goodput": request_goodput,

158 + "mean_request_throughput": request_throughput,

159 "mean_output_throughput": output_throughput,

107 - "mean_p99_ttft_ms": p99_ttft_ms,

108 - "mean_p99_tpot_ms": p99_tpot_ms,

160 + "mean_p99_ttft_ms": mean_p99_ttft_ms,

161 + "median_p99_ttft_ms": median_p99_ttft_ms,

162 + "worst_p99_ttft_ms": worst_p99_ttft_ms,

163 + "mean_p99_tpot_ms": mean_p99_tpot_ms,

164 + "median_p99_tpot_ms": median_p99_tpot_ms,

165 + "worst_p99_tpot_ms": worst_p99_tpot_ms,

166 + "estimated_compliant_requests": estimated_compliant_requests,

167 + "completed_requests": completed_requests,

168 + "combined_compliance_ratio": combined_compliance_ratio,

169 + "combined_compliance_percent": (

170 + combined_compliance_ratio * 100

171 + if combined_compliance_ratio is not None

172 + else None

173 + ),

174 + "benchmark_valid": valid,

175 + "p99_sla_eligible": p99_sla_eligible,

176 + "compliance_eligible": compliance_eligible,

177 + "sla_eligible": p99_sla_eligible and compliance_eligible,

178 "valid": valid,

179 "invalid_reason": reason,

180 "summary_files": sorted({str(run["_summary_path"]) for run in runs}),

@@ -119,7 +188,7 @@ def _select_candidate(

188 candidates: list[dict[str, Any]],

189 *,

190 use_goodput: bool,

122 -) -> tuple[dict[str, Any], str]:

191 +) -> tuple[dict[str, Any] | None, dict[str, Any], str]:

192 valid = [candidate for candidate in candidates if candidate["valid"]]

193 if not valid:

194 raise ValueError(

@@ -128,21 +197,37 @@ def _select_candidate(

197 )

198

199 if use_goodput:

131 - objective = "highest_mean_request_goodput"

132 - winner = max(

200 + objective = "p99_constrained_throughput"

201 + best_effort = max(

202 valid,

203 key=lambda candidate: (

204 candidate["mean_request_goodput"],

205 + candidate["combined_compliance_ratio"] or 0.0,

206 candidate["mean_output_throughput"],

207 -candidate["max_num_batched_tokens"],

208 -candidate["max_num_seqs"],

209 ),

210 )

141 - if winner["mean_request_goodput"] <= 0:

211 + if best_effort["mean_request_goodput"] <= 0:

212 raise ValueError(

213 "No sweep configuration produced non-zero request goodput for "

214 "the supplied TTFT/TPOT objectives."

215 )

216 + eligible = [candidate for candidate in valid if candidate["sla_eligible"]]

217 + winner = (

218 + max(

219 + eligible,

220 + key=lambda candidate: (

221 + candidate["mean_output_throughput"],

222 + candidate["combined_compliance_ratio"],

223 + candidate["mean_request_goodput"],

224 + -candidate["max_num_batched_tokens"],

225 + -candidate["max_num_seqs"],

226 + ),

227 + )

228 + if eligible

229 + else None

230 + )

231 else:

232 objective = "highest_mean_output_throughput"

233 winner = max(

@@ -153,8 +238,9 @@ def _select_candidate(

238 -candidate["max_num_seqs"],

239 ),

240 )

241 + best_effort = winner

242

157 - return winner, objective

243 + return winner, best_effort, objective

244

245

246 def _load_config(path: Path) -> dict[str, Any]:

@@ -219,6 +305,15 @@ def parse_args() -> argparse.Namespace:

305 default=DEFAULT_TPOT_SLA_MS,

306 help="TPOT objective used to generate request goodput.",

307 )

308 + parser.add_argument(

309 + "--minimum-compliance",

310 + type=float,

311 + default=DEFAULT_MINIMUM_COMPLIANCE,

312 + help=(

313 + "Minimum duration-weighted fraction of completed requests that must "

314 + "meet all supplied objectives (default: 0.99)."

315 + ),

316 + )

317 parser.add_argument(

318 "--output-config",

319 default="recommended-config.yml",

@@ -234,6 +329,8 @@ def parse_args() -> argparse.Namespace:

329

330 def main() -> int:

331 args = parse_args()

332 + if not 0.0 < args.minimum_compliance <= 1.0:

333 + raise ValueError("--minimum-compliance must be greater than 0 and at most 1.")

334 script_dir = Path(__file__).resolve().parent

335

336 config_path = _resolve(script_dir, args.config)

@@ -258,23 +355,64 @@ def main() -> int:

355 raise ValueError(f"No summary.json sweep results found under {results_dir}.")

356

357 use_goodput = args.ttft_sla_ms is not None or args.tpot_sla_ms is not None

261 - candidates = _aggregate_candidates(rows, use_goodput=use_goodput)

262 - winner, objective = _select_candidate(candidates, use_goodput=use_goodput)

358 + candidates = _aggregate_candidates(

359 + rows,

360 + use_goodput=use_goodput,

361 + ttft_sla_ms=args.ttft_sla_ms,

362 + tpot_sla_ms=args.tpot_sla_ms,

363 + minimum_compliance=args.minimum_compliance,

364 + )

365 + winner, best_effort, objective = _select_candidate(

366 + candidates, use_goodput=use_goodput

367 + )

368

369 initial_config = _load_config(config_path)

265 - recommended_config = dict(initial_config)

266 - recommended_config["max-num-seqs"] = winner["max_num_seqs"]

267 - recommended_config["max-num-batched-tokens"] = winner["max_num_batched_tokens"]

268 -

269 - _write_config(

270 - output_config,

271 - source_path=config_path,

272 - config=recommended_config,

273 - objective=objective,

274 - )

370 + if winner is not None:

371 + recommended_config = dict(initial_config)

372 + recommended_config["max-num-seqs"] = winner["max_num_seqs"]

373 + recommended_config["max-num-batched-tokens"] = winner["max_num_batched_tokens"]

374 + _write_config(

375 + output_config,

376 + source_path=config_path,

377 + config=recommended_config,

378 + objective=objective,

379 + )

380 + elif output_config.exists():

381 + output_config.unlink()

382 +

383 + measured = None

384 + recommended = None

385 + if winner is not None:

386 + recommended = {

387 + "max_num_seqs": winner["max_num_seqs"],

388 + "max_num_batched_tokens": winner["max_num_batched_tokens"],

389 + }

390 + measured = {

391 + key: winner[key]

392 + for key in (

393 + "run_count",

394 + "mean_request_throughput",

395 + "mean_request_goodput",

396 + "mean_output_throughput",

397 + "combined_compliance_ratio",

398 + "combined_compliance_percent",

399 + "estimated_compliant_requests",

400 + "completed_requests",

401 + "mean_p99_ttft_ms",

402 + "median_p99_ttft_ms",

403 + "worst_p99_ttft_ms",

404 + "mean_p99_tpot_ms",

405 + "median_p99_tpot_ms",

406 + "worst_p99_tpot_ms",

407 + )

408 + }

409

410 recommendation = {

411 + "status": (

412 + "sla_feasible" if winner is not None else "no_sla_feasible_configuration"

413 + ),

414 "selection_objective": objective,

415 + "minimum_compliance_ratio": args.minimum_compliance,

416 "slo": {

417 "ttft_ms": args.ttft_sla_ms,

418 "tpot_ms": args.tpot_sla_ms,

@@ -283,16 +421,14 @@ def main() -> int:

421 "max_num_seqs": initial_config.get("max-num-seqs"),

422 "max_num_batched_tokens": initial_config.get("max-num-batched-tokens"),

423 },

286 - "recommended": {

287 - "max_num_seqs": winner["max_num_seqs"],

288 - "max_num_batched_tokens": winner["max_num_batched_tokens"],

289 - },

290 - "measured": {

291 - "run_count": winner["run_count"],

292 - "mean_request_goodput": winner["mean_request_goodput"],

293 - "mean_output_throughput": winner["mean_output_throughput"],

294 - "mean_p99_ttft_ms": winner["mean_p99_ttft_ms"],

295 - "mean_p99_tpot_ms": winner["mean_p99_tpot_ms"],

424 + "recommended": recommended,

425 + "measured": measured,

426 + "best_effort": {

427 + "max_num_seqs": best_effort["max_num_seqs"],

428 + "max_num_batched_tokens": best_effort["max_num_batched_tokens"],

429 + "mean_request_goodput": best_effort["mean_request_goodput"],

430 + "combined_compliance_ratio": best_effort["combined_compliance_ratio"],

431 + "p99_sla_eligible": best_effort["p99_sla_eligible"],

432 },

433 "candidates": candidates,

434 }

@@ -301,6 +437,12 @@ def main() -> int:

437 encoding="utf-8",

438 )

439

440 + if winner is None:

441 + print("No SLA-feasible runtime configuration was found.")

442 + print(f"Wrote diagnostic evidence to {output_json}")

443 + print("No recommended config was written.")

444 + return 2

445 +

446 print("Recommended runtime configuration")

447 print()

448 print(f" max-num-seqs: {winner['max_num_seqs']}")

@@ -309,11 +451,12 @@ def main() -> int:

451 print(f"Selection objective: {objective}")

452 if use_goodput:

453 print(f" mean request goodput: {winner['mean_request_goodput']:.2f} req/s")

454 + print(f" combined compliance: {winner['combined_compliance_percent']:.2f}%")

455 print(f" mean output throughput: {winner['mean_output_throughput']:.2f} tok/s")