| 1 | #!/usr/bin/env python3 |
| 2 | # SPDX-License-Identifier: Apache-2.0 |
| 3 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 4 | |
| 5 | """Convert a vLLM Recipes per-hardware JSON rendering into: |
| 6 | |
| 7 | 1) config.yml - native `vllm serve --config` YAML |
| 8 | 2) env.sh - environment variables required by the recipe |
| 9 | |
| 10 | Examples: |
| 11 | # Interactive discovery |
| 12 | python3 recipe_json_to_vllm_config.py |
| 13 | |
| 14 | # Direct JSON URL |
| 15 | python3 recipe_json_to_vllm_config.py \ |
| 16 | https://recipes.vllm.ai/meta-llama/Llama-3.1-8B-Instruct/hw/xeon6.json |
| 17 | |
| 18 | # Non-interactive discovery (recommended strategy) |
| 19 | python3 recipe_json_to_vllm_config.py \ |
| 20 | --model meta-llama/Llama-3.1-8B-Instruct --hardware xeon6 |
| 21 | |
| 22 | # Promoted variant + explicit strategy |
| 23 | python3 recipe_json_to_vllm_config.py \ |
| 24 | --model nvidia/Llama-3.1-8B-Instruct-FP8 \ |
| 25 | --hardware arc_pro_b70 \ |
| 26 | --strategy single_node_tp |
| 27 | |
| 28 | Then: |
| 29 | source env.sh |
| 30 | vllm serve --config config.yml |
| 31 | |
| 32 | Variants with a distinct Hugging Face model ID are promoted by the Recipes API |
| 33 | and selected through --model; this converter does not need a separate --variant |
| 34 | argument. Strategy selection follows the exact `alternatives` JSON links emitted |
| 35 | by the Recipes API instead of synthesizing strategy URLs locally. |
| 36 | |
| 37 | The converter intentionally targets a single `vllm serve` process. If the |
| 38 | recipe rendering is multi-node, PD-disaggregated, or another multi-process |
| 39 | deployment, it exits instead of silently generating an incomplete config. |
| 40 | |
| 41 | """ |
| 42 | |
| 43 | from __future__ import annotations |
| 44 | |
| 45 | import argparse |
| 46 | import difflib |
| 47 | import json |
| 48 | import shlex |
| 49 | import sys |
| 50 | import urllib.parse |
| 51 | import urllib.request |
| 52 | from pathlib import Path |
| 53 | from typing import Any |
| 54 | |
| 55 | try: |
| 56 | import yaml |
| 57 | except ImportError as exc: |
| 58 | raise SystemExit("PyYAML is required. Install it with: pip install pyyaml") from exc |
| 59 | |
| 60 | DEFAULT_API_BASE = "https://recipes.vllm.ai" |
| 61 | |
| 62 | VALUE_SHORT_ALIASES = { |
| 63 | "-q": "quantization", |
| 64 | "-pp": "pipeline-parallel-size", |
| 65 | "-n": "nnodes", |
| 66 | "-r": "node-rank", |
| 67 | "-tp": "tensor-parallel-size", |
| 68 | "-dcp": "decode-context-parallel-size", |
| 69 | "-pcp": "prefill-context-parallel-size", |
| 70 | "-dp": "data-parallel-size", |
| 71 | "-dpn": "data-parallel-rank", |
| 72 | "-dpr": "data-parallel-start-rank", |
| 73 | "-dpl": "data-parallel-size-local", |
| 74 | "-dpa": "data-parallel-address", |
| 75 | "-dpp": "data-parallel-rpc-port", |
| 76 | "-dpb": "data-parallel-backend", |
| 77 | "-asc": "api-server-count", |
| 78 | } |
| 79 | |
| 80 | FLAG_SHORT_ALIASES = { |
| 81 | "-dph": "data-parallel-hybrid-lb", |
| 82 | "-dpe": "data-parallel-external-lb", |
| 83 | "-dpm": "data-parallel-multi-port-external-lb", |
| 84 | "-ep": "enable-expert-parallel", |
| 85 | } |
| 86 | |
| 87 | DICT_SHORT_ALIASES = { |
| 88 | "-sc": "speculative-config", |
| 89 | "-dc": "diffusion-config", |
| 90 | "-cc": "compilation-config", |
| 91 | "-ac": "attention-config", |
| 92 | } |
| 93 | |
| 94 | |
| 95 | def parse_args() -> argparse.Namespace: |
| 96 | p = argparse.ArgumentParser( |
| 97 | description="Convert vLLM Recipes JSON to native vllm serve config.yml + env.sh" |
| 98 | ) |
| 99 | p.add_argument( |
| 100 | "source", |
| 101 | nargs="?", |
| 102 | help=( |
| 103 | "Per-hardware recipe JSON URL or local JSON file. " |
| 104 | "If omitted, discover a recipe from the Recipes API." |
| 105 | ), |
| 106 | ) |
| 107 | p.add_argument( |
| 108 | "--model", |
| 109 | help=( |
| 110 | "Model hf_id or search text for recipe discovery. Promoted recipe " |
| 111 | "variants are selected by their own hf_id. Use with --hardware for " |
| 112 | "non-interactive operation." |
| 113 | ), |
| 114 | ) |
| 115 | p.add_argument( |
| 116 | "--hardware", |
| 117 | help=( |
| 118 | "Hardware ID for recipe discovery, for example xeon6, b200, or arc_pro_b70." |
| 119 | ), |
| 120 | ) |
| 121 | p.add_argument( |
| 122 | "--strategy", |
| 123 | help=( |
| 124 | "Serving strategy for recipe discovery, for example single_node_tp. " |
| 125 | "When omitted with --model and --hardware, use the Recipes-recommended " |
| 126 | "strategy. Interactive discovery offers the recommended strategy and " |
| 127 | "all generated alternatives." |
| 128 | ), |
| 129 | ) |
| 130 | p.add_argument( |
| 131 | "--api-base", |
| 132 | default=DEFAULT_API_BASE, |
| 133 | help=f"Recipes API base URL (default: {DEFAULT_API_BASE})", |
| 134 | ) |
| 135 | p.add_argument( |
| 136 | "--config-out", |
| 137 | default="config.yml", |
| 138 | help="Output vLLM YAML config (default: config.yml)", |
| 139 | ) |
| 140 | p.add_argument( |
| 141 | "--env-out", |
| 142 | default="env.sh", |
| 143 | help="Output shell environment file (default: env.sh)", |
| 144 | ) |
| 145 | |
| 146 | tuning = p.add_argument_group( |
| 147 | "optional runtime tuning", |
| 148 | ( |
| 149 | "Refine the recipe baseline only when additional information is supplied. " |
| 150 | "Runtime tuning is enabled only for recipe hardware with a registered " |
| 151 | "policy (currently: xeon6)." |
| 152 | ), |
| 153 | ) |
| 154 | tuning.add_argument( |
| 155 | "--detect-hardware", |
| 156 | action="store_true", |
| 157 | help=( |
| 158 | "Detect effective CPU/NUMA/memory resources and allow the selected " |
| 159 | "recipe-hardware policy to override recipe runtime arguments." |
| 160 | ), |
| 161 | ) |
| 162 | tuning.add_argument( |
| 163 | "--input-tokens", |
| 164 | type=int, |
| 165 | help="Expected input-token length. Optional workload hint.", |
| 166 | ) |
| 167 | tuning.add_argument( |
| 168 | "--output-tokens", |
| 169 | type=int, |
| 170 | help="Expected output-token length. Optional workload hint.", |
| 171 | ) |
| 172 | tuning.add_argument( |
| 173 | "--concurrency", |
| 174 | type=int, |
| 175 | help="Expected maximum concurrent requests. Optional workload hint.", |
| 176 | ) |
| 177 | tuning.add_argument( |
| 178 | "--ttft-sla-ms", |
| 179 | type=float, |
| 180 | help="Optional time-to-first-token objective in milliseconds.", |
| 181 | ) |
| 182 | tuning.add_argument( |
| 183 | "--tpot-sla-ms", |
| 184 | type=float, |
| 185 | help="Optional time-per-output-token objective in milliseconds.", |
| 186 | ) |
| 187 | tuning.add_argument( |
| 188 | "--target-qps", |
| 189 | type=float, |
| 190 | help="Optional capacity target for future DP/capacity tuning.", |
| 191 | ) |
| 192 | |
| 193 | sweep = p.add_argument_group( |
| 194 | "optional performance sweep", |
| 195 | ("Generate benchmark files after creating one initial runtime suggestion."), |
| 196 | ) |
| 197 | sweep.add_argument( |
| 198 | "--generate-sweep", |
| 199 | action="store_true", |
| 200 | help=( |
| 201 | "Generate an optional vllm bench sweep package around the single " |
| 202 | "initial runtime suggestion." |
| 203 | ), |
| 204 | ) |
| 205 | sweep.add_argument( |
| 206 | "--sweep-out-dir", |
| 207 | default="sweep", |
| 208 | help="Output directory for optional sweep files (default: sweep).", |
| 209 | ) |
| 210 | return p.parse_args() |
| 211 | |
| 212 | |
| 213 | def api_url(base: str, path: str) -> str: |
| 214 | return urllib.parse.urljoin(base.rstrip("/") + "/", path.lstrip("/")) |
| 215 | |
| 216 | |
| 217 | def load_json(source: str) -> dict[str, Any] | list[Any]: |
| 218 | if source.startswith(("http://", "https://")): |
| 219 | req = urllib.request.Request( |
| 220 | source, |
| 221 | headers={"User-Agent": "vllm-recipe-config-converter/1.0"}, |
| 222 | ) |
| 223 | with urllib.request.urlopen(req, timeout=30) as response: |
| 224 | return json.load(response) |
| 225 | |
| 226 | with open(source, encoding="utf-8") as f: |
| 227 | return json.load(f) |
| 228 | |
| 229 | |
| 230 | def prompt(text: str) -> str: |
| 231 | try: |
| 232 | return input(text).strip() |
| 233 | except EOFError as exc: |
| 234 | raise ValueError( |
| 235 | "Interactive input is unavailable. Pass --model and --hardware, " |
| 236 | "or provide a recipe JSON URL/file." |
| 237 | ) from exc |
| 238 | |
| 239 | |
| 240 | def model_label(model: dict[str, Any]) -> str: |
| 241 | label = str(model.get("hf_id", "")) |
| 242 | title = model.get("title") |
| 243 | provider = model.get("provider") |
| 244 | if title and title != label: |
| 245 | label += f" — {title}" |
| 246 | if provider: |
| 247 | label += f" [{provider}]" |
| 248 | return label |
| 249 | |
| 250 | |
| 251 | def search_models( |
| 252 | models: list[dict[str, Any]], query: str, limit: int = 20 |
| 253 | ) -> list[dict[str, Any]]: |
| 254 | q = query.strip().lower() |
| 255 | if not q: |
| 256 | return [] |
| 257 | |
| 258 | exact = [model for model in models if str(model.get("hf_id", "")).lower() == q] |
| 259 | if exact: |
| 260 | return exact |
| 261 | |
| 262 | tokens = [token for token in q.replace("/", " ").replace("-", " ").split() if token] |
| 263 | scored: list[tuple[float, dict[str, Any]]] = [] |
| 264 | |
| 265 | for model in models: |
| 266 | hf_id = str(model.get("hf_id", "")).lower() |
| 267 | title = str(model.get("title", "")).lower() |
| 268 | provider = str(model.get("provider", "")).lower() |
| 269 | text = f"{hf_id} {title} {provider}" |
| 270 | |
| 271 | if q in hf_id: |
| 272 | score = 1000.0 |
| 273 | elif q in title: |
| 274 | score = 900.0 |
| 275 | elif tokens and all(token in text for token in tokens): |
| 276 | score = 800.0 |
| 277 | elif tokens and any(token in text for token in tokens): |
| 278 | score = 500.0 |
| 279 | else: |
| 280 | ratio = difflib.SequenceMatcher(None, q, hf_id).ratio() |
| 281 | if ratio < 0.30: |
| 282 | continue |
| 283 | score = ratio * 100.0 |
| 284 | |
| 285 | scored.append((score, model)) |
| 286 | |
| 287 | scored.sort(key=lambda pair: (-pair[0], str(pair[1].get("hf_id", "")).lower())) |
| 288 | return [model for _, model in scored[:limit]] |
| 289 | |
| 290 | |
| 291 | def choose_from_menu(items: list[Any], label_fn, prompt_text: str) -> Any: |
| 292 | if not items: |
| 293 | raise ValueError("No selectable items found.") |
| 294 | |
| 295 | if len(items) == 1: |
| 296 | print(f"Selected: {label_fn(items[0])}") |
| 297 | return items[0] |
| 298 | |
| 299 | for index, item in enumerate(items, start=1): |
| 300 | print(f" [{index}] {label_fn(item)}") |
| 301 | |
| 302 | while True: |
| 303 | answer = prompt(prompt_text) |
| 304 | try: |
| 305 | index = int(answer) |
| 306 | except ValueError: |
| 307 | print(f"Enter a number from 1 to {len(items)}.") |
| 308 | continue |
| 309 | if 1 <= index <= len(items): |
| 310 | return items[index - 1] |
| 311 | print(f"Enter a number from 1 to {len(items)}.") |
| 312 | |
| 313 | |
| 314 | def select_model(models: list[dict[str, Any]], requested: str | None) -> dict[str, Any]: |
| 315 | query = requested |
| 316 | while True: |
| 317 | if not query: |
| 318 | query = prompt("Model search (for example: llama 3.1): ") |
| 319 | |
| 320 | matches = search_models(models, query) |
| 321 | if matches: |
| 322 | print("\nMatching models:") |
| 323 | return choose_from_menu(matches, model_label, "Select model: ") |
| 324 | |
| 325 | if requested: |
| 326 | raise ValueError(f"No recipe model matched {requested!r}.") |
| 327 | |
| 328 | print(f"No recipe model matched {query!r}. Try again.") |
| 329 | query = None |
| 330 | |
| 331 | |
| 332 | def select_hardware( |
| 333 | by_hardware: dict[str, str], requested: str | None |
| 334 | ) -> tuple[str, str]: |
| 335 | hardware_ids = sorted(by_hardware) |
| 336 | |
| 337 | if requested: |
| 338 | selected = next( |
| 339 | (hw for hw in hardware_ids if hw.lower() == requested.lower()), |
| 340 | None, |
| 341 | ) |
| 342 | if selected is None: |
| 343 | raise ValueError( |
| 344 | f"Hardware {requested!r} is not available for this model. " |
| 345 | f"Available: {', '.join(hardware_ids)}" |
| 346 | ) |
| 347 | return selected, by_hardware[selected] |
| 348 | |
| 349 | print("\nAvailable hardware:") |
| 350 | selected = choose_from_menu(hardware_ids, lambda value: value, "Select hardware: ") |
| 351 | return selected, by_hardware[selected] |
| 352 | |
| 353 | |
| 354 | def strategy_sources( |
| 355 | api_base: str, hardware_json_url: str, recipe: dict[str, Any] |
| 356 | ) -> tuple[str, dict[str, str]]: |
| 357 | recommended = recipe.get("strategy") |
| 358 | if not isinstance(recommended, str) or not recommended: |
| 359 | raise ValueError( |
| 360 | "Hardware recipe JSON does not contain a usable `strategy` field." |
| 361 | ) |
| 362 | |
| 363 | sources = {recommended: hardware_json_url} |
| 364 | raw_alternatives = recipe.get("alternatives") or {} |
| 365 | if not isinstance(raw_alternatives, dict): |
| 366 | raise ValueError( |
| 367 | "Hardware recipe JSON `alternatives` must be an object when present." |
| 368 | ) |
| 369 | |
| 370 | for strategy, path in raw_alternatives.items(): |
| 371 | if isinstance(strategy, str) and strategy and isinstance(path, str) and path: |
| 372 | sources[strategy] = api_url(api_base, path) |
| 373 | |
| 374 | return recommended, sources |
| 375 | |
| 376 | |
| 377 | def select_strategy( |
| 378 | api_base: str, |
| 379 | hardware_json_url: str, |
| 380 | recipe: dict[str, Any], |
| 381 | requested: str | None, |
| 382 | interactive: bool, |
| 383 | ) -> tuple[str, str]: |
| 384 | recommended, sources = strategy_sources(api_base, hardware_json_url, recipe) |
| 385 | strategies = [recommended, *sorted(s for s in sources if s != recommended)] |
| 386 | |
| 387 | if requested: |
| 388 | selected = next( |
| 389 | ( |
| 390 | strategy |
| 391 | for strategy in strategies |
| 392 | if strategy.lower() == requested.lower() |
| 393 | ), |
| 394 | None, |
| 395 | ) |
| 396 | if selected is None: |
| 397 | raise ValueError( |
| 398 | f"Strategy {requested!r} is not available for this model/hardware. " |
| 399 | f"Available: {', '.join(strategies)}" |
| 400 | ) |
| 401 | return selected, sources[selected] |
| 402 | |
| 403 | if not interactive: |
| 404 | return recommended, hardware_json_url |
| 405 | |
| 406 | print("\nAvailable strategies:") |
| 407 | |
| 408 | def strategy_label(strategy: str) -> str: |
| 409 | if strategy == recommended: |
| 410 | return f"{strategy} (recommended)" |
| 411 | return strategy |
| 412 | |
| 413 | selected = choose_from_menu(strategies, strategy_label, "Select strategy: ") |
| 414 | return selected, sources[selected] |
| 415 | |
| 416 | |
| 417 | def discover_recipe_source( |
| 418 | api_base: str, |
| 419 | requested_model: str | None, |
| 420 | requested_hardware: str | None, |
| 421 | requested_strategy: str | None, |
| 422 | ) -> str: |
| 423 | print("No recipe JSON supplied; starting Recipes API discovery.") |
| 424 | |
| 425 | models_url = api_url(api_base, "/models.json") |
| 426 | models_data = load_json(models_url) |
| 427 | if not isinstance(models_data, list): |
| 428 | raise ValueError(f"{models_url} did not return a model list.") |
| 429 | |
| 430 | models = [model for model in models_data if isinstance(model, dict)] |
| 431 | model = select_model(models, requested_model) |
| 432 | |
| 433 | model_json_path = model.get("json") |
| 434 | if not isinstance(model_json_path, str) or not model_json_path: |
| 435 | raise ValueError(f"Selected model {model.get('hf_id')!r} has no JSON API path.") |
| 436 | |
| 437 | model_json_url = api_url(api_base, model_json_path) |
| 438 | model_data = load_json(model_json_url) |
| 439 | if not isinstance(model_data, dict): |
| 440 | raise ValueError(f"{model_json_url} did not return a JSON object.") |
| 441 | |
| 442 | recommended = model_data.get("recommended_command") |
| 443 | if not isinstance(recommended, dict): |
| 444 | raise ValueError( |
| 445 | f"Model {model.get('hf_id')!r} has no rendered " |
| 446 | "recommended_command in the Recipes API." |
| 447 | ) |
| 448 | |
| 449 | raw_by_hardware = recommended.get("by_hardware") |
| 450 | if not isinstance(raw_by_hardware, dict) or not raw_by_hardware: |
| 451 | raise ValueError( |
| 452 | f"Model {model.get('hf_id')!r} has no per-hardware renderings " |
| 453 | "in recommended_command.by_hardware." |
| 454 | ) |
| 455 | |
| 456 | by_hardware = { |
| 457 | str(hw): path |
| 458 | for hw, path in raw_by_hardware.items() |
| 459 | if isinstance(path, str) and path |
| 460 | } |
| 461 | if not by_hardware: |
| 462 | raise ValueError( |
| 463 | f"Model {model.get('hf_id')!r} has no usable hardware JSON paths." |
| 464 | ) |
| 465 | |
| 466 | hardware, path = select_hardware(by_hardware, requested_hardware) |
| 467 | hardware_json_url = api_url(api_base, path) |
| 468 | hardware_data = load_json(hardware_json_url) |
| 469 | if not isinstance(hardware_data, dict): |
| 470 | raise ValueError(f"{hardware_json_url} did not return a JSON object.") |
| 471 | |
| 472 | interactive = requested_model is None or requested_hardware is None |
| 473 | strategy, resolved = select_strategy( |
| 474 | api_base, |
| 475 | hardware_json_url, |
| 476 | hardware_data, |
| 477 | requested_strategy, |
| 478 | interactive, |
| 479 | ) |
| 480 | |
| 481 | print("\nResolved recipe:") |
| 482 | print(f" Model: {model.get('hf_id')}") |
| 483 | print(f" Hardware: {hardware}") |
| 484 | print(f" Strategy: {strategy}") |
| 485 | print(f" JSON: {resolved}") |
| 486 | print() |
| 487 | |
| 488 | return resolved |
| 489 | |
| 490 | |
| 491 | def coerce(value: str) -> Any: |
| 492 | """Convert CLI string values to useful YAML scalar/object types.""" |
| 493 | # Recipes may preserve Python-style booleans in dotted config aliases, |
| 494 | # for example: -cc.pass_config.fuse_rope_kvcache=True. |
| 495 | if value in ("True", "False"): |
| 496 | return value == "True" |
| 497 | |
| 498 | try: |
| 499 | return json.loads(value) |
| 500 | except (json.JSONDecodeError, TypeError): |
| 501 | return value |
| 502 | |
| 503 | |
| 504 | def is_option_token(token: str) -> bool: |
| 505 | if token.startswith("--"): |
| 506 | return True |
| 507 | |
| 508 | short_name = token.split("=", 1)[0] |
| 509 | if ( |
| 510 | short_name in VALUE_SHORT_ALIASES |
| 511 | or short_name in FLAG_SHORT_ALIASES |
| 512 | or short_name in DICT_SHORT_ALIASES |
| 513 | or token == "-O" |
| 514 | ): |
| 515 | return True |
| 516 | |
| 517 | if any(token.startswith(f"{alias}.") for alias in DICT_SHORT_ALIASES): |
| 518 | return True |
| 519 | |
| 520 | return token.startswith("-O") and len(token) > 2 |
| 521 | |
| 522 | |
| 523 | def merge_value(dst: dict[str, Any], path: list[str], value: Any) -> None: |
| 524 | """Merge dotted CLI args into nested YAML dictionaries.""" |
| 525 | cur = dst |
| 526 | for part in path[:-1]: |
| 527 | existing = cur.get(part) |
| 528 | if existing is None: |
| 529 | cur[part] = {} |
| 530 | elif not isinstance(existing, dict): |
| 531 | raise ValueError( |
| 532 | f"Cannot merge nested option {'.'.join(path)!r}: " |
| 533 | f"{part!r} is already a scalar" |
| 534 | ) |
| 535 | cur = cur[part] |
| 536 | |
| 537 | leaf = path[-1] |
| 538 | if leaf not in cur: |
| 539 | cur[leaf] = value |
| 540 | return |
| 541 | |
| 542 | # Repeated CLI option. Preserve all values. |
| 543 | old = cur[leaf] |
| 544 | if not isinstance(old, list): |
| 545 | old = [old] |
| 546 | if isinstance(value, list): |
| 547 | old.extend(value) |
| 548 | else: |
| 549 | old.append(value) |
| 550 | cur[leaf] = old |
| 551 | |
| 552 | |
| 553 | def normalize_key(raw_key: str) -> list[str]: |
| 554 | """Convert the CLI key to config-file spelling. |
| 555 | |
| 556 | Only the top-level CLI option name gets underscore -> dash normalization. |
| 557 | Nested JSON field names after a dot are preserved. |
| 558 | """ |
| 559 | parts = raw_key.split(".") |
| 560 | parts[0] = parts[0].replace("_", "-") |
| 561 | return parts |
| 562 | |
| 563 | |
| 564 | def argv_to_config(argv: list[Any]) -> dict[str, Any]: |
| 565 | argv = [str(x) for x in argv] |
| 566 | |
| 567 | if len(argv) < 3 or argv[0:2] != ["vllm", "serve"]: |
| 568 | raise ValueError( |
| 569 | "Expected recipe argv to start with: ['vllm', 'serve', MODEL, ...]. " |
| 570 | f"Got: {argv[:4]!r}" |
| 571 | ) |
| 572 | |
| 573 | model = argv[2] |
| 574 | if model.startswith("-"): |
| 575 | raise ValueError(f"Expected model after 'vllm serve', got {model!r}") |
| 576 | |
| 577 | config: dict[str, Any] = {"model": model} |
| 578 | |
| 579 | i = 3 |
| 580 | while i < len(argv): |
| 581 | token = argv[i] |
| 582 | |
| 583 | # -O3 / -O=3 |
| 584 | if token.startswith("-O") and token != "-O": |
| 585 | value = token[3:] if token.startswith("-O=") else token[2:] |
| 586 | merge_value(config, ["optimization-level"], coerce(value)) |
| 587 | i += 1 |
| 588 | continue |
| 589 | |
| 590 | # -O 3 |
| 591 | if token == "-O": |
| 592 | if i + 1 >= len(argv): |
| 593 | raise ValueError("-O is missing its value") |
| 594 | merge_value(config, ["optimization-level"], coerce(argv[i + 1])) |
| 595 | i += 2 |
| 596 | continue |
| 597 | |
| 598 | # Dotted dictionary aliases accepted by FlexibleArgumentParser: |
| 599 | # -cc.mode=3 |
| 600 | # -cc.mode 3 |
| 601 | # -cc.pass_config.foo=True |
| 602 | # -cc.custom_ops+ -quant_fp8 |
| 603 | # The same syntax applies to -sc, -dc, and -ac. |
| 604 | dotted_alias = next( |
| 605 | (alias for alias in DICT_SHORT_ALIASES if token.startswith(f"{alias}.")), |
| 606 | None, |
| 607 | ) |
| 608 | if dotted_alias is not None: |
| 609 | dotted = token[len(dotted_alias) + 1 :] |
| 610 | raw_key, separator, raw_value = dotted.partition("=") |
| 611 | |
| 612 | append = raw_key.endswith("+") |
| 613 | if append: |
| 614 | raw_key = raw_key[:-1] |
| 615 | |
| 616 | if not raw_key: |
| 617 | raise ValueError(f"{token} must contain a key after {dotted_alias}.") |
| 618 | |
| 619 | if not separator: |
| 620 | if i + 1 >= len(argv): |
| 621 | raise ValueError(f"{token} is missing its value") |
| 622 | # Consume the next token unconditionally. Dotted list values may |
| 623 | # themselves begin with "-", for example "-quant_fp8". |
| 624 | raw_value = argv[i + 1] |
| 625 | i += 2 |
| 626 | else: |
| 627 | i += 1 |
| 628 | |
| 629 | value: Any = raw_value.split(",") if append else coerce(raw_value) |
| 630 | |
| 631 | merge_value( |
| 632 | config, |
| 633 | [DICT_SHORT_ALIASES[dotted_alias], *raw_key.split(".")], |
| 634 | value, |
| 635 | ) |
| 636 | continue |
| 637 | |
| 638 | # Support -alias=value for value-bearing and whole-dict aliases. |
| 639 | if "=" in token: |
| 640 | short_name, raw_value = token.split("=", 1) |
| 641 | if short_name in VALUE_SHORT_ALIASES: |
| 642 | merge_value( |
| 643 | config, |
| 644 | [VALUE_SHORT_ALIASES[short_name]], |
| 645 | coerce(raw_value), |
| 646 | ) |
| 647 | i += 1 |
| 648 | continue |
| 649 | if short_name in DICT_SHORT_ALIASES: |
| 650 | merge_value( |
| 651 | config, |
| 652 | [DICT_SHORT_ALIASES[short_name]], |
| 653 | coerce(raw_value), |
| 654 | ) |
| 655 | i += 1 |
| 656 | continue |
| 657 | |
| 658 | # Boolean short aliases are flags and do not consume a value. |
| 659 | if token in FLAG_SHORT_ALIASES: |
| 660 | merge_value(config, [FLAG_SHORT_ALIASES[token]], True) |
| 661 | i += 1 |
| 662 | continue |
| 663 | |
| 664 | # Scalar and whole-dictionary short aliases consume one value. |
| 665 | if token in VALUE_SHORT_ALIASES or token in DICT_SHORT_ALIASES: |
| 666 | if i + 1 >= len(argv): |
| 667 | raise ValueError(f"{token} is missing its value") |
| 668 | key = VALUE_SHORT_ALIASES.get(token) or DICT_SHORT_ALIASES[token] |
| 669 | merge_value(config, [key], coerce(argv[i + 1])) |
| 670 | i += 2 |
| 671 | continue |
| 672 | |
| 673 | if not token.startswith("--"): |
| 674 | raise ValueError( |
| 675 | f"Unexpected positional/short argument {token!r}. " |
| 676 | "The converter accepts vLLM serve long options and known short aliases." |
| 677 | ) |
| 678 | |
| 679 | # --key=value |
| 680 | if "=" in token: |
| 681 | key, raw_value = token[2:].split("=", 1) |
| 682 | merge_value(config, normalize_key(key), coerce(raw_value)) |
| 683 | i += 1 |
| 684 | continue |
| 685 | |
| 686 | key = token[2:] |
| 687 | i += 1 |
| 688 | |
| 689 | # Gather values until the next option. This supports both scalar and |
| 690 | # nargs-style vLLM options. |
| 691 | raw_values: list[str] = [] |
| 692 | while i < len(argv) and not is_option_token(argv[i]): |
| 693 | raw_values.append(argv[i]) |
| 694 | i += 1 |
| 695 | |
| 696 | if not raw_values: |
| 697 | # Store flags exactly by their long-form spelling. |
| 698 | # Example: |
| 699 | # --trust-remote-code -> trust-remote-code: true |
| 700 | # --no-enable-prefix-caching -> no-enable-prefix-caching: true |
| 701 | value: Any = True |
| 702 | elif len(raw_values) == 1: |
| 703 | value = coerce(raw_values[0]) |
| 704 | else: |
| 705 | value = [coerce(v) for v in raw_values] |
| 706 | |
| 707 | merge_value(config, normalize_key(key), value) |
| 708 | |
| 709 | return config |
| 710 | |
| 711 | |
| 712 | def recipe_argv(recipe: dict[str, Any]) -> list[Any]: |
| 713 | deploy_type = recipe.get("deploy_type") |
| 714 | |
| 715 | # Current Recipes API's single-node rendering exposes `argv`. |
| 716 | if isinstance(recipe.get("argv"), list): |
| 717 | if deploy_type not in (None, "single_node"): |
| 718 | raise ValueError( |
| 719 | f"Recipe deploy_type={deploy_type!r} is not a single-node deployment. " |
| 720 | "A single config.yml would not fully represent this deployment." |
| 721 | ) |
| 722 | return recipe["argv"] |
| 723 | |
| 724 | # Give a useful failure for other known rendered shapes. |
| 725 | multi_process_fields = [ |
| 726 | k |
| 727 | for k in ( |
| 728 | "head_argv", |
| 729 | "worker_argv", |
| 730 | "worker_argvs", |
| 731 | "prefill", |
| 732 | "decode", |
| 733 | "vllm_argv", |
| 734 | ) |
| 735 | if k in recipe |
| 736 | ] |
| 737 | if multi_process_fields: |
| 738 | raise ValueError( |
| 739 | "This recipe is a multi-process deployment and cannot be represented " |
| 740 | "by one config.yml. Found fields: " + ", ".join(multi_process_fields) |
| 741 | ) |
| 742 | |
| 743 | raise ValueError("Recipe JSON does not contain an `argv` field") |
| 744 | |
| 745 | |
| 746 | def write_config( |
| 747 | path: str, |
| 748 | source: str, |
| 749 | recipe: dict[str, Any], |
| 750 | config: dict[str, Any], |
| 751 | ) -> None: |
| 752 | metadata = [ |
| 753 | "# Generated from vLLM Recipes JSON.", |
| 754 | f"# Source: {source}", |
| 755 | ] |
| 756 | for key in ("hardware", "strategy", "variant", "deploy_type"): |
| 757 | if recipe.get(key) is not None: |
| 758 | metadata.append(f"# {key}: {recipe[key]}") |
| 759 | |
| 760 | body = yaml.safe_dump( |
| 761 | config, |
| 762 | sort_keys=False, |
| 763 | default_flow_style=False, |
| 764 | allow_unicode=True, |
| 765 | ) |
| 766 | Path(path).write_text("\n".join(metadata) + "\n" + body, encoding="utf-8") |
| 767 | |
| 768 | |
| 769 | def write_env(path: str, source: str, recipe: dict[str, Any]) -> None: |
| 770 | env = recipe.get("env") or {} |
| 771 | if not isinstance(env, dict): |
| 772 | raise ValueError(f"Recipe `env` must be an object, got {type(env).__name__}") |
| 773 | |
| 774 | lines = [ |
| 775 | "#!/usr/bin/env bash", |
| 776 | "# Generated from vLLM Recipes JSON.", |
| 777 | f"# Source: {source}", |
| 778 | "", |
| 779 | ] |
| 780 | |
| 781 | if env: |
| 782 | for key, value in env.items(): |
| 783 | lines.append(f"export {key}={shlex.quote(str(value))}") |
| 784 | else: |
| 785 | lines.append("# No recipe-specific environment variables.") |
| 786 | |
| 787 | lines.append("") |
| 788 | Path(path).write_text("\n".join(lines), encoding="utf-8") |
| 789 | Path(path).chmod(Path(path).stat().st_mode | 0o111) |
| 790 | |
| 791 | |
| 792 | def main() -> int: |
| 793 | args = parse_args() |
| 794 | |
| 795 | try: |
| 796 | source = args.source |
| 797 | if source is None: |
| 798 | source = discover_recipe_source( |
| 799 | args.api_base, args.model, args.hardware, args.strategy |
| 800 | ) |
| 801 | elif args.model or args.hardware or args.strategy: |
| 802 | raise ValueError( |
| 803 | "Do not combine a positional recipe JSON source with " |
| 804 | "--model/--hardware/--strategy discovery options." |
| 805 | ) |
| 806 | |
| 807 | recipe = load_json(source) |
| 808 | if not isinstance(recipe, dict): |
| 809 | raise ValueError("Recipe JSON must be a JSON object.") |
| 810 | |
| 811 | argv = recipe_argv(recipe) |
| 812 | config = argv_to_config(argv) |
| 813 | |
| 814 | tuning_requested = ( |
| 815 | args.detect_hardware |
| 816 | or args.generate_sweep |
| 817 | or any( |
| 818 | value is not None |
| 819 | for value in ( |
| 820 | args.input_tokens, |
| 821 | args.output_tokens, |
| 822 | args.concurrency, |
| 823 | args.ttft_sla_ms, |
| 824 | args.tpot_sla_ms, |
| 825 | args.target_qps, |
| 826 | ) |
| 827 | ) |
| 828 | ) |
| 829 | |
| 830 | tuning = None |
| 831 | workload = None |
| 832 | sweep_writer = None |
| 833 | if tuning_requested: |
| 834 | # Keep plain Recipes conversion lightweight. vLLM-specific modules |
| 835 | # are imported only for optional runtime tuning or sweep generation. |
| 836 | from runtime_tuning import ( |
| 837 | WorkloadHints, |
| 838 | finetune_runtime_config, |
| 839 | get_runtime_tuning_policies, |
| 840 | ) |
| 841 | |
| 842 | workload = WorkloadHints( |
| 843 | input_tokens=args.input_tokens, |
| 844 | output_tokens=args.output_tokens, |
| 845 | concurrency=args.concurrency, |
| 846 | ttft_sla_ms=args.ttft_sla_ms, |
| 847 | tpot_sla_ms=args.tpot_sla_ms, |
| 848 | target_qps=args.target_qps, |
| 849 | ) |
| 850 | |
| 851 | if args.generate_sweep: |
| 852 | from sweep_generation import ( |
| 853 | validate_sweep_workload, |
| 854 | write_sweep_files, |
| 855 | ) |
| 856 | |
| 857 | validate_sweep_workload(workload) |
| 858 | sweep_writer = write_sweep_files |
| 859 | |
| 860 | recipe_hardware = recipe.get("hardware") |
| 861 | policies = get_runtime_tuning_policies(recipe_hardware) |
| 862 | |
| 863 | hardware = None |
| 864 | if args.detect_hardware: |
| 865 | from hardware_detection import detect_hardware |
| 866 | |
| 867 | hardware = detect_hardware() |
| 868 | |
| 869 | tuning = finetune_runtime_config( |
| 870 | config, |
| 871 | hardware=hardware, |
| 872 | workload=workload, |
| 873 | policies=policies, |
| 874 | ) |
| 875 | config.update(tuning.overrides) |
| 876 | |
| 877 | write_config(args.config_out, source, recipe, config) |
| 878 | write_env(args.env_out, source, recipe) |
| 879 | |
| 880 | sweep_files: list[Path] = [] |
| 881 | if args.generate_sweep: |
| 882 | assert workload is not None |
| 883 | assert sweep_writer is not None |
| 884 | sweep_files = sweep_writer( |
| 885 | args.sweep_out_dir, |
| 886 | config_path=args.config_out, |
| 887 | env_path=args.env_out, |
| 888 | config=config, |
| 889 | workload=workload, |
| 890 | ) |
| 891 | |
| 892 | if tuning is not None: |
| 893 | if tuning.overrides: |
| 894 | print("Initial runtime suggestion:") |
| 895 | for key, value in tuning.overrides.items(): |
| 896 | print(f" {key}: {value}") |
| 897 | for note in tuning.notes: |
| 898 | print(f" tuning: {note}") |
| 899 | except Exception as exc: |
| 900 | print(f"ERROR: {exc}", file=sys.stderr) |
| 901 | return 1 |
| 902 | |
| 903 | print(f"Wrote {args.config_out}") |
| 904 | print(f"Wrote {args.env_out}") |
| 905 | if sweep_files: |
| 906 | print(f"Wrote optional sweep package under {args.sweep_out_dir}/") |
| 907 | print() |
| 908 | print("Run:") |
| 909 | print(f" source {shlex.quote(args.env_out)}") |
| 910 | print(f" vllm serve --config {shlex.quote(args.config_out)}") |
| 911 | if sweep_files: |
| 912 | sweep_dir = Path(args.sweep_out_dir) |
| 913 | run_sweep = sweep_dir / "run_sweep.sh" |
| 914 | recommend = sweep_dir / "recommend.py" |
| 915 | print() |
| 916 | print("Optional performance sweep:") |
| 917 | print(f" {shlex.quote(str(run_sweep))} --dry-run") |
| 918 | print(f" {shlex.quote(str(run_sweep))}") |
| 919 | print() |
| 920 | print("After the sweep:") |
| 921 | print(f" {shlex.quote(str(recommend))}") |
| 922 | return 0 |
| 923 | |
| 924 | |
| 925 | if __name__ == "__main__": |
| 926 | raise SystemExit(main()) |