| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """Argument parsing utilities for vLLM.""" |
| 4 | |
| 5 | import argparse |
| 6 | import json |
| 7 | import sys |
| 8 | import textwrap |
| 9 | from argparse import ( |
| 10 | Action, |
| 11 | ArgumentDefaultsHelpFormatter, |
| 12 | ArgumentParser, |
| 13 | ArgumentTypeError, |
| 14 | Namespace, |
| 15 | RawDescriptionHelpFormatter, |
| 16 | _ArgumentGroup, |
| 17 | ) |
| 18 | from collections import defaultdict |
| 19 | from typing import Any, NoReturn |
| 20 | |
| 21 | import regex as re |
| 22 | import yaml |
| 23 | |
| 24 | from vllm.logger import init_logger |
| 25 | |
| 26 | logger = init_logger(__name__) |
| 27 | |
| 28 | |
| 29 | def human_readable_int(value: str) -> int: |
| 30 | """Parse human-readable integers like '1k', '2M', etc. |
| 31 | Including decimal values with decimal multipliers. |
| 32 | |
| 33 | Examples: |
| 34 | - '1k' -> 1,000 |
| 35 | - '1K' -> 1,024 |
| 36 | - '25.6k' -> 25,600 |
| 37 | |
| 38 | """ |
| 39 | value = value.strip() |
| 40 | |
| 41 | match = re.fullmatch(r"(\d+(?:\.\d+)?)([kKmMgGtT])", value) |
| 42 | if match: |
| 43 | decimal_multiplier = { |
| 44 | "k": 10**3, |
| 45 | "m": 10**6, |
| 46 | "g": 10**9, |
| 47 | "t": 10**12, |
| 48 | } |
| 49 | binary_multiplier = { |
| 50 | "K": 2**10, |
| 51 | "M": 2**20, |
| 52 | "G": 2**30, |
| 53 | "T": 2**40, |
| 54 | } |
| 55 | |
| 56 | number, suffix = match.groups() |
| 57 | if suffix in decimal_multiplier: |
| 58 | mult = decimal_multiplier[suffix] |
| 59 | return int(float(number) * mult) |
| 60 | elif suffix in binary_multiplier: |
| 61 | mult = binary_multiplier[suffix] |
| 62 | # Do not allow decimals with binary multipliers |
| 63 | try: |
| 64 | return int(number) * mult |
| 65 | except ValueError as e: |
| 66 | raise argparse.ArgumentTypeError( |
| 67 | "Decimals are not allowed " |
| 68 | f"with binary suffixes like {suffix}. Did you mean to use " |
| 69 | f"{number}{suffix.lower()} instead?" |
| 70 | ) from e |
| 71 | |
| 72 | # Regular plain number. |
| 73 | return int(value) |
| 74 | |
| 75 | |
| 76 | def human_readable_int_or_auto(value: str) -> int: |
| 77 | """Parse human-readable integers like '1k', '2M', etc. |
| 78 | Including decimal values with decimal multipliers. |
| 79 | Also accepts -1 or 'auto' as a special value for auto-detection. |
| 80 | |
| 81 | Examples: |
| 82 | - '1k' -> 1,000 |
| 83 | - '1K' -> 1,024 |
| 84 | - '25.6k' -> 25,600 |
| 85 | - '-1' or 'auto' -> -1 (special value for auto-detection) |
| 86 | |
| 87 | """ |
| 88 | value = value.strip() |
| 89 | |
| 90 | if value == "-1" or value.lower() == "auto": |
| 91 | return -1 |
| 92 | |
| 93 | return human_readable_int(value) |
| 94 | |
| 95 | |
| 96 | def summary_line(description: str | None) -> str: |
| 97 | """Return the first paragraph of a config docstring as a single line.""" |
| 98 | return " ".join((description or "").split("\n\n", 1)[0].split()) |
| 99 | |
| 100 | |
| 101 | class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpFormatter): |
| 102 | """SortedHelpFormatter that sorts arguments by their option strings.""" |
| 103 | |
| 104 | def _split_lines(self, text, width): |
| 105 | """1. Sentences split across lines have their single newlines removed. |
| 106 | 2. Paragraphs and lists are split into separate lines. |
| 107 | 3. Each line is wrapped to the specified width (width of terminal). |
| 108 | """ |
| 109 | # The pattern also includes whitespace after the newline |
| 110 | newlines_to_remove = re.compile(r"(?<!\n)\n(?!\n)(?!\s*(-|\*|\+|\d+\.))\s*") |
| 111 | lines = newlines_to_remove.sub(" ", text).splitlines() |
| 112 | return sum([textwrap.wrap(line, width) for line in lines], []) |
| 113 | |
| 114 | def add_arguments(self, actions): |
| 115 | actions = sorted(actions, key=lambda x: x.option_strings) |
| 116 | super().add_arguments(actions) |
| 117 | |
| 118 | |
| 119 | class FlexibleArgumentParser(ArgumentParser): |
| 120 | """ArgumentParser that allows both underscore and dash in names.""" |
| 121 | |
| 122 | _deprecated: set[Action] = set() |
| 123 | _json_tip: str = ( |
| 124 | "When passing JSON CLI arguments, the following sets of arguments " |
| 125 | "are equivalent:\n" |
| 126 | ' --json-arg \'{"key1": "value1", "key2": {"key3": "value2"}}\'\n' |
| 127 | " --json-arg.key1 value1 --json-arg.key2.key3 value2\n\n" |
| 128 | "Additionally, list elements can be passed individually using +:\n" |
| 129 | ' --json-arg \'{"key4": ["value3", "value4", "value5"]}\'\n' |
| 130 | " --json-arg.key4+ value3 --json-arg.key4+='value4,value5'\n\n" |
| 131 | ) |
| 132 | _search_keyword: str | None = None |
| 133 | |
| 134 | def __init__(self, *args, **kwargs): |
| 135 | # Set the default "formatter_class" to SortedHelpFormatter |
| 136 | if "formatter_class" not in kwargs: |
| 137 | kwargs["formatter_class"] = SortedHelpFormatter |
| 138 | # Pop kwarg "add_json_tip" to control whether to add the JSON tip |
| 139 | self.add_json_tip = kwargs.pop("add_json_tip", True) |
| 140 | super().__init__(*args, **kwargs) |
| 141 | self._show_serve_task_hint = False |
| 142 | |
| 143 | def error(self, message: str) -> NoReturn: |
| 144 | if ( |
| 145 | (self.prog.endswith(" serve") or self._show_serve_task_hint) |
| 146 | and message.startswith("unrecognized arguments:") |
| 147 | and re.search(r"(^|\s)--task(=|\s|$)", message) |
| 148 | ): |
| 149 | message += ( |
| 150 | "\n\nHint: --task is not a vllm serve option. " |
| 151 | "For embedding, reranking, or reward models, use " |
| 152 | "--runner pooling. To adapt a generative model for pooling, " |
| 153 | "also use --convert embed, --convert classify, or " |
| 154 | "--convert reward." |
| 155 | ) |
| 156 | super().error(message) |
| 157 | |
| 158 | if sys.version_info < (3, 13): |
| 159 | # Enable the deprecated kwarg for Python 3.12 and below |
| 160 | |
| 161 | def parse_known_args(self, args=None, namespace=None): |
| 162 | namespace, args = super().parse_known_args(args, namespace) |
| 163 | for action in FlexibleArgumentParser._deprecated: |
| 164 | if ( |
| 165 | hasattr(namespace, dest := action.dest) |
| 166 | and getattr(namespace, dest) != action.default |
| 167 | ): |
| 168 | logger.warning_once("argument '%s' is deprecated", dest) |
| 169 | return namespace, args |
| 170 | |
| 171 | def add_argument(self, *args, **kwargs): |
| 172 | deprecated = kwargs.pop("deprecated", False) |
| 173 | action = super().add_argument(*args, **kwargs) |
| 174 | if deprecated: |
| 175 | FlexibleArgumentParser._deprecated.add(action) |
| 176 | return action |
| 177 | |
| 178 | class _FlexibleArgumentGroup(_ArgumentGroup): |
| 179 | def add_argument(self, *args, **kwargs): |
| 180 | deprecated = kwargs.pop("deprecated", False) |
| 181 | action = super().add_argument(*args, **kwargs) |
| 182 | if deprecated: |
| 183 | FlexibleArgumentParser._deprecated.add(action) |
| 184 | return action |
| 185 | |
| 186 | def add_argument_group(self, *args, **kwargs): |
| 187 | group = self._FlexibleArgumentGroup(self, *args, **kwargs) |
| 188 | self._action_groups.append(group) |
| 189 | return group |
| 190 | |
| 191 | def format_help(self): |
| 192 | # Only use custom help formatting for bottom level parsers |
| 193 | if self._subparsers is not None: |
| 194 | return super().format_help() |
| 195 | |
| 196 | formatter = self._get_formatter() |
| 197 | |
| 198 | # Handle keyword search of the args |
| 199 | if (search_keyword := self._search_keyword) is not None: |
| 200 | # Normalise the search keyword |
| 201 | search_keyword = search_keyword.lower().replace("_", "-") |
| 202 | # Return full help if searching for 'all' |
| 203 | if search_keyword == "all": |
| 204 | self.epilog = self._json_tip |
| 205 | return super().format_help() |
| 206 | |
| 207 | # Return group help if searching for a group title |
| 208 | for group in self._action_groups: |
| 209 | if group.title and group.title.lower() == search_keyword: |
| 210 | formatter.start_section(group.title) |
| 211 | formatter.add_text(summary_line(group.description)) |
| 212 | formatter.add_arguments(group._group_actions) |
| 213 | formatter.end_section() |
| 214 | formatter.add_text(self._json_tip) |
| 215 | return formatter.format_help() |
| 216 | |
| 217 | # Return matched args if searching for an arg name |
| 218 | matched_actions = [] |
| 219 | for group in self._action_groups: |
| 220 | for action in group._group_actions: |
| 221 | # search option name |
| 222 | if any( |
| 223 | search_keyword in opt.lower() for opt in action.option_strings |
| 224 | ): |
| 225 | matched_actions.append(action) |
| 226 | if matched_actions: |
| 227 | formatter.start_section(f"Arguments matching '{search_keyword}'") |
| 228 | formatter.add_arguments(matched_actions) |
| 229 | formatter.end_section() |
| 230 | formatter.add_text(self._json_tip) |
| 231 | return formatter.format_help() |
| 232 | |
| 233 | # No match found |
| 234 | formatter.add_text( |
| 235 | f"No group or arguments matching '{search_keyword}'.\n" |
| 236 | "Use '--help' to see available groups or " |
| 237 | "'--help=all' to see all available parameters." |
| 238 | ) |
| 239 | return formatter.format_help() |
| 240 | |
| 241 | # usage |
| 242 | formatter.add_usage(self.usage, self._actions, self._mutually_exclusive_groups) |
| 243 | |
| 244 | # description |
| 245 | formatter.add_text(self.description) |
| 246 | |
| 247 | # positionals, optionals and user-defined groups |
| 248 | formatter.start_section("Config Groups") |
| 249 | config_groups = "" |
| 250 | groups = [g for g in self._action_groups if g._group_actions] |
| 251 | titles = [g.title or "" for g in groups] |
| 252 | title_width = max((len(title) for title in titles), default=0) + 1 |
| 253 | width = max(formatter._width - formatter._current_indent, title_width + 20) |
| 254 | for title, group in zip(titles, groups): |
| 255 | lines = textwrap.wrap( |
| 256 | summary_line(group.description), |
| 257 | width, |
| 258 | initial_indent=f"{title: <{title_width}}", |
| 259 | subsequent_indent=" " * title_width, |
| 260 | ) |
| 261 | config_groups += ("\n".join(lines) or title) + "\n" |
| 262 | formatter.add_text(config_groups) |
| 263 | formatter.end_section() |
| 264 | |
| 265 | # epilog |
| 266 | formatter.add_text(self.epilog) |
| 267 | |
| 268 | # determine help from format above |
| 269 | return formatter.format_help() |
| 270 | |
| 271 | def parse_args( # type: ignore[override] |
| 272 | self, |
| 273 | args: list[str] | None = None, |
| 274 | namespace: Namespace | None = None, |
| 275 | ): |
| 276 | if args is None: |
| 277 | args = sys.argv[1:] |
| 278 | self._show_serve_task_hint = args[:1] == ["serve"] and any( |
| 279 | re.match(r"^--task(=.+|$)", arg) for arg in args[1:] |
| 280 | ) |
| 281 | |
| 282 | if args and args[0] == "serve": |
| 283 | # Check for --model in command line arguments first |
| 284 | try: |
| 285 | model_idx = next( |
| 286 | i for i, arg in enumerate(args) if re.match(r"^--model(=.+|$)", arg) |
| 287 | ) |
| 288 | logger.warning( |
| 289 | "With `vllm serve`, you should provide the model as a " |
| 290 | "positional argument or in a config file instead of via " |
| 291 | "the `--model` option. " |
| 292 | "The `--model` option will be removed in a future version." |
| 293 | ) |
| 294 | |
| 295 | if args[model_idx] == "--model": |
| 296 | model_tag = args[model_idx + 1] |
| 297 | rest_start_idx = model_idx + 2 |
| 298 | else: |
| 299 | model_tag = args[model_idx].removeprefix("--model=") |
| 300 | rest_start_idx = model_idx + 1 |
| 301 | |
| 302 | # Move <model> to the front, e,g: |
| 303 | # [Before] |
| 304 | # vllm serve -tp 2 --model <model> --enforce-eager --port 8001 |
| 305 | # [After] |
| 306 | # vllm serve <model> -tp 2 --enforce-eager --port 8001 |
| 307 | args = [ |
| 308 | "serve", |
| 309 | model_tag, |
| 310 | *args[1:model_idx], |
| 311 | *args[rest_start_idx:], |
| 312 | ] |
| 313 | except StopIteration: |
| 314 | pass |
| 315 | # Check for --served-model-name without a positional model argument |
| 316 | if ( |
| 317 | len(args) > 1 |
| 318 | and args[1].startswith("-") |
| 319 | and not any(re.match(r"^--config(=.+|$)", arg) for arg in args) |
| 320 | and any( |
| 321 | re.match(r"^--served[-_]model[-_]name(=.+|$)", arg) for arg in args |
| 322 | ) |
| 323 | ): |
| 324 | raise ValueError( |
| 325 | "`model` should be provided as the first positional argument when " |
| 326 | "using `vllm serve`. i.e. `vllm serve <model> --<arg> <value>`." |
| 327 | ) |
| 328 | |
| 329 | if "--config" in args: |
| 330 | args = self._pull_args_from_config(args) |
| 331 | |
| 332 | def repl(match: re.Match) -> str: |
| 333 | """Replaces underscores with dashes in the matched string.""" |
| 334 | return match.group(0).replace("_", "-") |
| 335 | |
| 336 | # Everything between the first -- and the first . |
| 337 | pattern = re.compile(r"(?<=--)[^\.]*") |
| 338 | |
| 339 | # Convert underscores to dashes and vice versa in argument names |
| 340 | processed_args = list[str]() |
| 341 | for i, arg in enumerate(args): |
| 342 | if arg.startswith("--help="): |
| 343 | FlexibleArgumentParser._search_keyword = arg.split("=", 1)[-1].lower() |
| 344 | processed_args.append("--help") |
| 345 | elif arg.startswith("--"): |
| 346 | if "=" in arg: |
| 347 | key, value = arg.split("=", 1) |
| 348 | key = pattern.sub(repl, key, count=1) |
| 349 | processed_args.append(f"{key}={value}") |
| 350 | else: |
| 351 | key = pattern.sub(repl, arg, count=1) |
| 352 | processed_args.append(key) |
| 353 | elif arg.startswith("-O") and arg != "-O": |
| 354 | # allow -O flag to be used without space, e.g. -O3 or -Odecode |
| 355 | # also handle -O=<optimization_level> here |
| 356 | optimization_level = arg[3:] if arg[2] == "=" else arg[2:] |
| 357 | processed_args += ["--optimization-level", optimization_level] |
| 358 | elif ( |
| 359 | arg == "-O" |
| 360 | and i + 1 < len(args) |
| 361 | and args[i + 1] in {"0", "1", "2", "3"} |
| 362 | ): |
| 363 | # Convert -O <n> to --optimization-level <n> |
| 364 | processed_args.append("--optimization-level") |
| 365 | else: |
| 366 | processed_args.append(arg) |
| 367 | |
| 368 | def create_nested_dict(keys: list[str], value: str) -> dict[str, Any]: |
| 369 | """Creates a nested dictionary from a list of keys and a value. |
| 370 | |
| 371 | For example, `keys = ["a", "b", "c"]` and `value = 1` will create: |
| 372 | `{"a": {"b": {"c": 1}}}` |
| 373 | """ |
| 374 | nested_dict: Any = value |
| 375 | for key in reversed(keys): |
| 376 | nested_dict = {key: nested_dict} |
| 377 | return nested_dict |
| 378 | |
| 379 | def recursive_dict_update( |
| 380 | original: dict[str, Any], |
| 381 | update: dict[str, Any], |
| 382 | ) -> set[str]: |
| 383 | """Recursively updates a dictionary with another dictionary. |
| 384 | Returns a set of duplicate keys that were overwritten. |
| 385 | """ |
| 386 | duplicates = set[str]() |
| 387 | for k, v in update.items(): |
| 388 | if isinstance(v, dict) and isinstance(original.get(k), dict): |
| 389 | nested_duplicates = recursive_dict_update(original[k], v) |
| 390 | duplicates |= {f"{k}.{d}" for d in nested_duplicates} |
| 391 | elif isinstance(v, list) and isinstance(original.get(k), list): |
| 392 | original[k] += v |
| 393 | else: |
| 394 | if k in original: |
| 395 | duplicates.add(k) |
| 396 | original[k] = v |
| 397 | return duplicates |
| 398 | |
| 399 | delete = set[int]() |
| 400 | dict_args = defaultdict[str, dict[str, Any]](dict) |
| 401 | duplicates = set[str]() |
| 402 | # Track regular arguments (non-dict args) for duplicate detection |
| 403 | regular_args_seen = set[str]() |
| 404 | for i, processed_arg in enumerate(processed_args): |
| 405 | if i in delete: # skip if value from previous arg |
| 406 | continue |
| 407 | |
| 408 | if processed_arg.startswith("--") and "." not in processed_arg: |
| 409 | if "=" in processed_arg: |
| 410 | arg_name = processed_arg.split("=", 1)[0] |
| 411 | else: |
| 412 | arg_name = processed_arg |
| 413 | |
| 414 | if arg_name in regular_args_seen: |
| 415 | duplicates.add(arg_name) |
| 416 | else: |
| 417 | regular_args_seen.add(arg_name) |
| 418 | continue |
| 419 | |
| 420 | if processed_arg.startswith("-") and "." in processed_arg: |
| 421 | if "=" in processed_arg: |
| 422 | processed_arg, value_str = processed_arg.split("=", 1) |
| 423 | if "." not in processed_arg: |
| 424 | # False positive, '.' was only in the value |
| 425 | continue |
| 426 | else: |
| 427 | value_str = processed_args[i + 1] |
| 428 | delete.add(i + 1) |
| 429 | |
| 430 | if processed_arg.endswith("+"): |
| 431 | processed_arg = processed_arg[:-1] |
| 432 | value_str = json.dumps(list(value_str.split(","))) |
| 433 | |
| 434 | key, *keys = processed_arg.split(".") |
| 435 | try: |
| 436 | value = json.loads(value_str) |
| 437 | except json.decoder.JSONDecodeError: |
| 438 | # Support human-readable suffixes (e.g. 1k, 80g) for |
| 439 | # dotted config args like --config.field 80g |
| 440 | try: |
| 441 | value = human_readable_int(value_str) # type: ignore[assignment] |
| 442 | except (ValueError, ArgumentTypeError): |
| 443 | value = value_str |
| 444 | |
| 445 | # Merge all values with the same key into a single dict |
| 446 | arg_dict = create_nested_dict(keys, value) |
| 447 | arg_duplicates = recursive_dict_update(dict_args[key], arg_dict) |
| 448 | duplicates |= {f"{key}.{d}" for d in arg_duplicates} |
| 449 | delete.add(i) |
| 450 | # Filter out the dict args we set to None |
| 451 | processed_args = [a for i, a in enumerate(processed_args) if i not in delete] |
| 452 | if duplicates: |
| 453 | logger.warning("Found duplicate keys %s", ", ".join(duplicates)) |
| 454 | |
| 455 | # Add the dict args back as if they were originally passed as JSON |
| 456 | for dict_arg, dict_value in dict_args.items(): |
| 457 | processed_args.append(dict_arg) |
| 458 | processed_args.append(json.dumps(dict_value)) |
| 459 | |
| 460 | return super().parse_args(processed_args, namespace) |
| 461 | |
| 462 | def check_port(self, value): |
| 463 | try: |
| 464 | value = int(value) |
| 465 | except ValueError: |
| 466 | msg = "Port must be an integer" |
| 467 | raise ArgumentTypeError(msg) from None |
| 468 | |
| 469 | if not (1024 <= value <= 65535): |
| 470 | raise ArgumentTypeError("Port must be between 1024 and 65535") |
| 471 | |
| 472 | return value |
| 473 | |
| 474 | def _pull_args_from_config(self, args: list[str]) -> list[str]: |
| 475 | """Method to pull arguments specified in the config file |
| 476 | into the command-line args variable. |
| 477 | |
| 478 | The arguments in config file will be inserted between |
| 479 | the argument list. |
| 480 | |
| 481 | Example: |
| 482 | ```yaml |
| 483 | port: 12323 |
| 484 | tensor-parallel-size: 4 |
| 485 | ``` |
| 486 | ```python |
| 487 | $: vllm {serve,chat,complete} "facebook/opt-12B" \ |
| 488 | --config config.yaml -tp 2 |
| 489 | $: args = [ |
| 490 | "serve,chat,complete", |
| 491 | "facebook/opt-12B", |
| 492 | '--config', 'config.yaml', |
| 493 | '-tp', '2' |
| 494 | ] |
| 495 | $: args = [ |
| 496 | "serve,chat,complete", |
| 497 | "facebook/opt-12B", |
| 498 | '--port', '12323', |
| 499 | '--tensor-parallel-size', '4', |
| 500 | '-tp', '2' |
| 501 | ] |
| 502 | ``` |
| 503 | |
| 504 | Please note how the config args are inserted after the sub command. |
| 505 | this way the order of priorities is maintained when these are args |
| 506 | parsed by super(). |
| 507 | |
| 508 | """ |
| 509 | assert args.count("--config") <= 1, "More than one config file specified!" |
| 510 | |
| 511 | index = args.index("--config") |
| 512 | if index == len(args) - 1: |
| 513 | raise ValueError( |
| 514 | "No config file specified! Please check your command-line arguments." |
| 515 | ) |
| 516 | |
| 517 | file_path = args[index + 1] |
| 518 | |
| 519 | config_args = self.load_config_file(file_path) |
| 520 | |
| 521 | # 0th index might be the sub command {serve,chat,complete,...} |
| 522 | # optionally followed by model_tag (serve or snapshot create) |
| 523 | # followed by config args |
| 524 | # followed by rest of cli args. |
| 525 | # maintaining this order will enforce the precedence |
| 526 | # of cli > config > defaults |
| 527 | if args[0].startswith("-"): |
| 528 | # No sub command (e.g., api_server entry point) |
| 529 | args = config_args + args[0:index] + args[index + 2 :] |
| 530 | elif args[0] == "serve" or args[:2] == ["snapshot", "create"]: |
| 531 | model_index = 1 if args[0] == "serve" else 2 |
| 532 | model_in_cli = len(args) > model_index and not args[model_index].startswith( |
| 533 | "-" |
| 534 | ) |
| 535 | model_in_config = any(arg == "--model" for arg in config_args) |
| 536 | |
| 537 | if not model_in_cli and not model_in_config: |
| 538 | raise ValueError( |
| 539 | "No model specified! Please specify model either " |
| 540 | "as a positional argument or in a config file." |
| 541 | ) |
| 542 | |
| 543 | if model_in_cli: |
| 544 | # Model specified as positional arg, keep CLI version |
| 545 | args = ( |
| 546 | args[: model_index + 1] |
| 547 | + config_args |
| 548 | + args[model_index + 1 : index] |
| 549 | + args[index + 2 :] |
| 550 | ) |
| 551 | else: |
| 552 | # No model in CLI, use config if available |
| 553 | args = ( |
| 554 | args[:model_index] |
| 555 | + config_args |
| 556 | + args[model_index:index] |
| 557 | + args[index + 2 :] |
| 558 | ) |
| 559 | else: |
| 560 | args = [args[0]] + config_args + args[1:index] + args[index + 2 :] |
| 561 | |
| 562 | return args |
| 563 | |
| 564 | def load_config_file(self, file_path: str) -> list[str]: |
| 565 | """Loads a yaml file and returns the key value pairs as a |
| 566 | flattened list with argparse like pattern. |
| 567 | |
| 568 | Supports both flat configs and nested YAML structures. |
| 569 | |
| 570 | Flat config example: |
| 571 | ```yaml |
| 572 | port: 12323 |
| 573 | tensor-parallel-size: 4 |
| 574 | ``` |
| 575 | returns: |
| 576 | ['--port', '12323', '--tensor-parallel-size', '4'] |
| 577 | |
| 578 | Nested config example: |
| 579 | ```yaml |
| 580 | compilation-config: |
| 581 | pass_config: |
| 582 | fuse_allreduce_rms: true |
| 583 | speculative-config: |
| 584 | model: "nvidia/gpt-oss-120b-Eagle3-v2" |
| 585 | num_speculative_tokens: 3 |
| 586 | ``` |
| 587 | returns: |
| 588 | ['--compilation-config', '{"pass_config": {"fuse_allreduce_rms": true}}', |
| 589 | '--speculative-config', '{"model": "nvidia/gpt-oss-120b-Eagle3-v2", ...}'] |
| 590 | """ |
| 591 | extension: str = file_path.split(".")[-1] |
| 592 | if extension not in ("yaml", "yml"): |
| 593 | raise ValueError( |
| 594 | f"Config file must be of a yaml/yml type. {extension} supplied" |
| 595 | ) |
| 596 | |
| 597 | # Supports both flat configs and nested dicts |
| 598 | processed_args: list[str] = [] |
| 599 | |
| 600 | config: dict[str, Any] = {} |
| 601 | try: |
| 602 | with open(file_path) as config_file: |
| 603 | config = yaml.safe_load(config_file) |
| 604 | except Exception as ex: |
| 605 | logger.error( |
| 606 | "Unable to read the config file at %s. Check path correctness", |
| 607 | file_path, |
| 608 | ) |
| 609 | raise ex |
| 610 | |
| 611 | for key, value in config.items(): |
| 612 | if isinstance(value, bool): |
| 613 | if value: |
| 614 | processed_args.append("--" + key) |
| 615 | elif (no_key := f"--no-{key}") in self._option_string_actions: |
| 616 | processed_args.append(no_key) |
| 617 | elif isinstance(value, list): |
| 618 | if value: |
| 619 | processed_args.append("--" + key) |
| 620 | for item in value: |
| 621 | processed_args.append(str(item)) |
| 622 | elif isinstance(value, dict): |
| 623 | # Convert nested dicts to JSON strings so they can be parsed |
| 624 | # by the existing JSON argument parsing machinery. |
| 625 | processed_args.append("--" + key) |
| 626 | processed_args.append(json.dumps(value)) |
| 627 | else: |
| 628 | processed_args.append("--" + key) |
| 629 | processed_args.append(str(value)) |
| 630 | |
| 631 | return processed_args |