| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import importlib.metadata |
| 4 | import importlib.util |
| 5 | import inspect |
| 6 | import logging |
| 7 | import sys |
| 8 | import textwrap |
| 9 | import traceback |
| 10 | from argparse import SUPPRESS, Action, HelpFormatter |
| 11 | from collections.abc import Callable, Iterable |
| 12 | from importlib.machinery import ModuleSpec |
| 13 | from pathlib import Path |
| 14 | from typing import TYPE_CHECKING |
| 15 | from unittest.mock import MagicMock, patch |
| 16 | |
| 17 | import mkdocs_gen_files |
| 18 | import regex as re |
| 19 | from pydantic_core import core_schema |
| 20 | |
| 21 | logger = logging.getLogger("mkdocs") |
| 22 | |
| 23 | ROOT_DIR = Path(__file__).parent.parent.parent.parent |
| 24 | |
| 25 | sys.path.insert(0, str(ROOT_DIR)) |
| 26 | sys.path.insert(0, str(Path(__file__).parent)) |
| 27 | |
| 28 | from generated_content import fill_markers # noqa: E402 |
| 29 | |
| 30 | HAS_TORCH = importlib.util.find_spec("torch") is not None |
| 31 | |
| 32 | |
| 33 | def mock_if_no_torch(mock_module: str, mock: MagicMock): |
| 34 | if not HAS_TORCH: |
| 35 | sys.modules[mock_module] = mock |
| 36 | |
| 37 | |
| 38 | class PydanticMagicMock(MagicMock): |
| 39 | """`MagicMock` that's able to generate pydantic-core schemas.""" |
| 40 | |
| 41 | def __init__(self, *args, **kwargs): |
| 42 | name = kwargs.get("name") |
| 43 | super().__init__(*args, **kwargs) |
| 44 | self.__spec__ = ModuleSpec(name, None) |
| 45 | |
| 46 | def __get_pydantic_core_schema__(self, source_type, handler): |
| 47 | return core_schema.any_schema() |
| 48 | |
| 49 | |
| 50 | # Mock custom op code |
| 51 | class MockCustomOp: |
| 52 | @staticmethod |
| 53 | def register(*args, **kwargs): |
| 54 | def decorator(cls): |
| 55 | return cls |
| 56 | |
| 57 | return decorator |
| 58 | |
| 59 | |
| 60 | class MockPluggableLayer: |
| 61 | @staticmethod |
| 62 | def register(*args, **kwargs): |
| 63 | def decorator(cls): |
| 64 | return cls |
| 65 | |
| 66 | return decorator |
| 67 | |
| 68 | |
| 69 | mock_if_no_torch("vllm._C", MagicMock()) |
| 70 | mock_if_no_torch("vllm._C_stable_libtorch", MagicMock()) |
| 71 | mock_if_no_torch( |
| 72 | "vllm.model_executor.custom_op", |
| 73 | MagicMock(CustomOp=MockCustomOp, PluggableLayer=MockPluggableLayer), |
| 74 | ) |
| 75 | mock_if_no_torch( |
| 76 | "vllm.utils.torch_utils", MagicMock(direct_register_custom_op=lambda *a, **k: None) |
| 77 | ) |
| 78 | |
| 79 | |
| 80 | # Mock any version checks by reading from compiled CI requirements |
| 81 | with open(ROOT_DIR / "requirements/test/cuda.txt") as f: |
| 82 | VERSIONS = dict(line.strip().split("==") for line in f if "==" in line) |
| 83 | importlib.metadata.version = lambda name: VERSIONS.get(name) or "0.0.0" |
| 84 | |
| 85 | |
| 86 | # Real class because a `MagicMock` base conflicts with `ABCMeta` |
| 87 | class MockModule: |
| 88 | pass |
| 89 | |
| 90 | |
| 91 | # Make torch.nn.Parameter and torch.nn.Module safe to inherit from. |
| 92 | # `import torch.nn` resolves the attribute on `torch`, so mock both. |
| 93 | mock_nn = MagicMock(Parameter=object, Module=MockModule) |
| 94 | mock_if_no_torch("torch.nn", mock_nn) |
| 95 | mock_if_no_torch("torch", PydanticMagicMock(name="torch", nn=mock_nn)) |
| 96 | |
| 97 | |
| 98 | # Mock torch.library.infer_schema for vllm.ir.ops.IrOpInplaceOverload.__init__ |
| 99 | # We need to return the corresponding number of inputs, as IR infra will assert it |
| 100 | def get_outputs(native_fn: Callable) -> str: |
| 101 | """Extract output schema from function's return type annotation, |
| 102 | e.g. 'Tensor' or 'Tensor, Tensor'. |
| 103 | """ |
| 104 | import typing |
| 105 | |
| 106 | return_type = typing.get_type_hints(native_fn)["return"] |
| 107 | origin = typing.get_origin(return_type) |
| 108 | arg_name = lambda a: a.__name__ if hasattr(a, "__name__") else str(a) |
| 109 | if origin is tuple: |
| 110 | args = typing.get_args(return_type) |
| 111 | return ", ".join(arg_name(arg) for arg in args) |
| 112 | else: |
| 113 | return f"{arg_name(return_type)}" |
| 114 | |
| 115 | |
| 116 | mock_if_no_torch( |
| 117 | "torch.library", |
| 118 | MagicMock(infer_schema=lambda fn, **k: f"(Tensor x) -> {get_outputs(fn)}"), |
| 119 | ) |
| 120 | |
| 121 | |
| 122 | def auto_mock(module_name: str, attr: str, max_mocks: int = 100): |
| 123 | """Function that automatically mocks missing modules during imports.""" |
| 124 | logger.info("Importing %s from %s", attr, module_name) |
| 125 | |
| 126 | for _ in range(max_mocks): |
| 127 | try: |
| 128 | module = importlib.import_module(module_name) |
| 129 | |
| 130 | # First treat attr as an attr, then as a submodule |
| 131 | if hasattr(module, attr): |
| 132 | return getattr(module, attr) |
| 133 | |
| 134 | return importlib.import_module(f"{module_name}.{attr}") |
| 135 | except ModuleNotFoundError as e: |
| 136 | assert e.name is not None |
| 137 | logger.info("Mocking %s for argparse doc generation", e.name) |
| 138 | sys.modules[e.name] = PydanticMagicMock(name=e.name) |
| 139 | except Exception: |
| 140 | logger.exception("Failed to import %s.%s", module_name, attr) |
| 141 | raise |
| 142 | |
| 143 | raise ImportError( |
| 144 | f"Failed to import {module_name}.{attr} after mocking {max_mocks} imports" |
| 145 | ) |
| 146 | |
| 147 | |
| 148 | bench_latency = auto_mock("vllm.benchmarks", "latency") |
| 149 | bench_serve = auto_mock("vllm.benchmarks", "serve") |
| 150 | bench_startup = auto_mock("vllm.benchmarks", "startup") |
| 151 | bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs") |
| 152 | bench_sweep_plot_pareto = auto_mock( |
| 153 | "vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs" |
| 154 | ) |
| 155 | bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs") |
| 156 | bench_sweep_serve_workload = auto_mock( |
| 157 | "vllm.benchmarks.sweep.serve_workload", "SweepServeWorkloadArgs" |
| 158 | ) |
| 159 | bench_sweep_startup = auto_mock("vllm.benchmarks.sweep.startup", "SweepStartupArgs") |
| 160 | bench_throughput = auto_mock("vllm.benchmarks", "throughput") |
| 161 | AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs") |
| 162 | EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs") |
| 163 | ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand") |
| 164 | CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand") |
| 165 | BenchmarkSubcommand = auto_mock( |
| 166 | "vllm.entrypoints.cli.benchmark.main", "BenchmarkSubcommand" |
| 167 | ) |
| 168 | import_bench_subcommands = auto_mock( |
| 169 | "vllm.entrypoints.cli.benchmark.main", "_import_bench_subcommand_modules" |
| 170 | ) |
| 171 | BenchmarkSubcommandBase = auto_mock( |
| 172 | "vllm.entrypoints.cli.benchmark.base", "BenchmarkSubcommandBase" |
| 173 | ) |
| 174 | BenchmarkMMProcessorSubcommand = auto_mock( |
| 175 | "vllm.entrypoints.cli.benchmark.mm_processor", "BenchmarkMMProcessorSubcommand" |
| 176 | ) |
| 177 | LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase") |
| 178 | launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION") |
| 179 | RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand") |
| 180 | PreloadSubcommand = auto_mock("vllm.entrypoints.cli.preload", "PreloadSubcommand") |
| 181 | sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS") |
| 182 | openai_cli_args = auto_mock("vllm.entrypoints.launchers", "cli_args") |
| 183 | run_batch = auto_mock("vllm.entrypoints.launchers", "run_batch") |
| 184 | |
| 185 | if TYPE_CHECKING: |
| 186 | from vllm.utils.argparse_utils import FlexibleArgumentParser |
| 187 | else: |
| 188 | FlexibleArgumentParser = auto_mock( |
| 189 | "vllm.utils.argparse_utils", "FlexibleArgumentParser" |
| 190 | ) |
| 191 | |
| 192 | |
| 193 | class MarkdownFormatter(HelpFormatter): |
| 194 | """Custom formatter that generates markdown for argument groups.""" |
| 195 | |
| 196 | def __init__(self, prog: str, starting_heading_level: int = 3): |
| 197 | super().__init__(prog, max_help_position=sys.maxsize, width=sys.maxsize) |
| 198 | |
| 199 | self._section_heading_prefix = "#" * starting_heading_level |
| 200 | self._argument_heading_prefix = "#" * (starting_heading_level + 1) |
| 201 | self._markdown_output = [] |
| 202 | |
| 203 | def start_section(self, heading: str): |
| 204 | if heading not in {"positional arguments", "options"}: |
| 205 | heading_md = f"\n{self._section_heading_prefix} {heading}\n\n" |
| 206 | self._markdown_output.append(heading_md) |
| 207 | |
| 208 | def end_section(self): |
| 209 | pass |
| 210 | |
| 211 | def add_text(self, text: str): |
| 212 | if text: |
| 213 | self._markdown_output.append(f"{inspect.cleandoc(text)}\n\n") |
| 214 | |
| 215 | def add_usage(self, usage, actions, groups, prefix=None): |
| 216 | pass |
| 217 | |
| 218 | def add_arguments(self, actions: Iterable[Action]): |
| 219 | for action in actions: |
| 220 | if len(action.option_strings) == 0 or "--help" in action.option_strings: |
| 221 | continue |
| 222 | |
| 223 | option_strings = f"`{'`, `'.join(action.option_strings)}`" |
| 224 | heading_md = f"{self._argument_heading_prefix} {option_strings}\n\n" |
| 225 | self._markdown_output.append(heading_md) |
| 226 | |
| 227 | if action.choices or isinstance(action.metavar, list | tuple): |
| 228 | choices_iterable = action.choices or action.metavar |
| 229 | choices = f"`{'`, `'.join(str(c) for c in choices_iterable)}`" |
| 230 | self._markdown_output.append(f": Possible choices: {choices}\n\n") |
| 231 | |
| 232 | if action.help: |
| 233 | help_dd = ":" + textwrap.indent(action.help, " ")[1:] |
| 234 | self._markdown_output.append(f"{help_dd}\n\n") |
| 235 | |
| 236 | # None usually means the default is determined at runtime |
| 237 | if (default := action.default) != SUPPRESS and default is not None: |
| 238 | # Make empty string defaults visible |
| 239 | if default == "": |
| 240 | default = '""' |
| 241 | self._markdown_output.append(f": Default: `{default}`\n\n") |
| 242 | |
| 243 | def format_help(self): |
| 244 | """Return the formatted help as markdown.""" |
| 245 | return "".join(self._markdown_output) |
| 246 | |
| 247 | |
| 248 | def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser: |
| 249 | """Create a parser for the given class with markdown formatting. |
| 250 | |
| 251 | Args: |
| 252 | add_cli_args: The `add_cli_args` classmethod to build the parser from. |
| 253 | **kwargs: Additional keyword arguments to pass to `cls.add_cli_args`. |
| 254 | |
| 255 | Returns: |
| 256 | FlexibleArgumentParser: A parser with markdown formatting for the class. |
| 257 | |
| 258 | """ |
| 259 | try: |
| 260 | parser = FlexibleArgumentParser(add_json_tip=False) |
| 261 | parser.formatter_class = MarkdownFormatter |
| 262 | with patch("vllm.config.DeviceConfig.__post_init__"): |
| 263 | _parser = add_cli_args(parser, **kwargs) |
| 264 | except ModuleNotFoundError as e: |
| 265 | # Auto-mock runtime imports |
| 266 | if tb_list := traceback.extract_tb(e.__traceback__): |
| 267 | path = Path(tb_list[-1].filename).relative_to(ROOT_DIR) |
| 268 | auto_mock(module_name=".".join(path.parent.parts), attr=path.stem) |
| 269 | return create_parser(add_cli_args, **kwargs) |
| 270 | else: |
| 271 | raise e |
| 272 | # add_cli_args might be in-place so return parser if _parser is None |
| 273 | return _parser or parser |
| 274 | |
| 275 | |
| 276 | def format_help(parser: FlexibleArgumentParser) -> str: |
| 277 | """Format a parser's help as markdown using `MarkdownFormatter`.""" |
| 278 | return super(type(parser), parser).format_help() |
| 279 | |
| 280 | |
| 281 | # Absolute docs URLs are kept in the help text because they are useful in the |
| 282 | # terminal. Wrap them as markdown links so the `url_schemes` hook can rewrite |
| 283 | # them into doc-relative links / cross-references at render time. |
| 284 | _DOCS_URL = re.compile(r"https://docs\.vllm\.ai/en/[^/\s]+/[^\s)>]+") |
| 285 | |
| 286 | |
| 287 | def linkify_docs_urls(text: str) -> str: |
| 288 | """Wrap bare docs.vllm.ai URLs in help text as markdown links.""" |
| 289 | return _DOCS_URL.sub(lambda m: f"[{m.group()}]({m.group()})", text) |
| 290 | |
| 291 | |
| 292 | logger.info("Generating argparse documentation") |
| 293 | logger.debug("Root directory: %s", ROOT_DIR.resolve()) |
| 294 | |
| 295 | # The JSON tip is always rendered immediately before generated argument content, |
| 296 | # and the generator is its only consumer, so it lives here rather than in a |
| 297 | # separate snippet file. (The runtime terminal equivalent is |
| 298 | # `FlexibleArgumentParser._json_tip` in vllm/utils/argparse_utils.py.) |
| 299 | JSON_TIP = """## JSON CLI Arguments |
| 300 | |
| 301 | When passing JSON CLI arguments, the following sets of arguments are equivalent: |
| 302 | |
| 303 | - `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'` |
| 304 | - `--json-arg.key1 value1 --json-arg.key2.key3 value2` |
| 305 | |
| 306 | Additionally, list elements can be passed individually using `+`: |
| 307 | |
| 308 | - `--json-arg '{"key4": ["value3", "value4", "value5"]}'` |
| 309 | - `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'` |
| 310 | |
| 311 | """ |
| 312 | |
| 313 | # Argument sections filled into `gen:` markers on handwritten pages |
| 314 | engine_args = create_parser(EngineArgs.add_cli_args) |
| 315 | async_engine_args = create_parser(AsyncEngineArgs.add_cli_args, async_args_only=True) |
| 316 | fill_markers( |
| 317 | "configuration/engine_args.md", |
| 318 | { |
| 319 | "engine-args": ( |
| 320 | f"{JSON_TIP}## `EngineArgs`\n\n" |
| 321 | f"{linkify_docs_urls(format_help(engine_args))}" |
| 322 | f"## `AsyncEngineArgs`\n\n" |
| 323 | f"{linkify_docs_urls(format_help(async_engine_args))}" |
| 324 | ) |
| 325 | }, |
| 326 | ) |
| 327 | |
| 328 | # CLI reference pages generated entirely from their parser: page -> (parser, JSON tip) |
| 329 | pages = { |
| 330 | "cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True), |
| 331 | "cli/chat.md": (create_parser(ChatCommand.add_cli_args), False), |
| 332 | "cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False), |
| 333 | "cli/run-batch.md": (create_parser(run_batch.make_arg_parser), True), |
| 334 | "cli/preload.md": (create_parser(PreloadSubcommand.add_cli_args), True), |
| 335 | "cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True), |
| 336 | "cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True), |
| 337 | # URL kept as `mm_processor` for back-compat; command name is `mm-processor` |
| 338 | "cli/bench/mm_processor.md": ( |
| 339 | create_parser(BenchmarkMMProcessorSubcommand.add_cli_args), |
| 340 | True, |
| 341 | ), |
| 342 | "cli/bench/serve.md": (create_parser(bench_serve.add_cli_args), True), |
| 343 | "cli/bench/startup.md": (create_parser(bench_startup.add_cli_args), True), |
| 344 | "cli/bench/throughput.md": (create_parser(bench_throughput.add_cli_args), True), |
| 345 | "cli/bench/sweep/plot.md": (create_parser(bench_sweep_plot.add_cli_args), True), |
| 346 | "cli/bench/sweep/plot_pareto.md": ( |
| 347 | create_parser(bench_sweep_plot_pareto.add_cli_args), |
| 348 | True, |
| 349 | ), |
| 350 | "cli/bench/sweep/serve.md": (create_parser(bench_sweep_serve.add_cli_args), True), |
| 351 | "cli/bench/sweep/serve_workload.md": ( |
| 352 | create_parser(bench_sweep_serve_workload.add_cli_args), |
| 353 | True, |
| 354 | ), |
| 355 | "cli/bench/sweep/startup.md": ( |
| 356 | create_parser(bench_sweep_startup.add_cli_args), |
| 357 | True, |
| 358 | ), |
| 359 | } |
| 360 | |
| 361 | # Command name for pages whose file stem differs (URL kept for back-compat). |
| 362 | COMMAND_NAMES = {"cli/bench/mm_processor.md": "mm-processor"} |
| 363 | |
| 364 | for doc_path, (parser, json_tip) in pages.items(): |
| 365 | segments = Path(doc_path).relative_to("cli").with_suffix("").parts |
| 366 | label = COMMAND_NAMES.get(doc_path, segments[-1]) |
| 367 | command = " ".join([*segments[:-1], label]) |
| 368 | # `title` frontmatter keeps the nav label to just this command's segment, |
| 369 | # while the H1 stays the full `vllm ...` command for the page heading. |
| 370 | content = f"---\ntitle: {label}\n---\n\n" |
| 371 | content += f"# vllm {command}\n\n" |
| 372 | if parser.description: |
| 373 | content += f"## Overview\n\n{parser.description}\n\n" |
| 374 | # Rendered above instead of at the top of the Arguments section |
| 375 | parser.description = None |
| 376 | if json_tip: |
| 377 | content += JSON_TIP |
| 378 | content += f"## Arguments\n\n{linkify_docs_urls(format_help(parser))}" |
| 379 | with mkdocs_gen_files.open(doc_path, "w") as f: |
| 380 | f.write(content) |
| 381 | logger.debug("CLI reference generated: %s", doc_path) |
| 382 | |
| 383 | logger.info("Total argparse docs generated: %d", len(pages) + 2) |
| 384 | |
| 385 | |
| 386 | # --- Bare subcommand (group) pages ------------------------------------------- |
| 387 | # Mirror `vllm <group> --help`: an overview plus a table of child subcommands, |
| 388 | # each linked to its reference page. Children are read from the CLI registries |
| 389 | # so the listing can never drift from the actual subcommands. Each page is the |
| 390 | # `README.md` of its command directory so it becomes that section's index and is |
| 391 | # picked up by the existing nav globs. |
| 392 | import_bench_subcommands() # populate BenchmarkSubcommandBase.__subclasses__() |
| 393 | bench_subcommands = BenchmarkSubcommandBase.__subclasses__() |
| 394 | bench_children = [(cmd.name, cmd.help) for cmd in bench_subcommands] |
| 395 | |
| 396 | groups = { |
| 397 | "cli/bench/README.md": (BenchmarkSubcommand.help, bench_children), |
| 398 | "cli/launch/README.md": ( |
| 399 | launch_description, |
| 400 | [(cmd.name, cmd.help) for cmd in LaunchSubcommandBase.__subclasses__()], |
| 401 | ), |
| 402 | "cli/bench/sweep/README.md": ( |
| 403 | dict(bench_children).get("sweep"), |
| 404 | [(args.parser_name, args.parser_help) for args, _ in sweep_subcommands], |
| 405 | ), |
| 406 | } |
| 407 | |
| 408 | # Doc paths that exist, so we only link a child that has a reference page. |
| 409 | existing_pages = set(pages) | set(groups) |
| 410 | |
| 411 | |
| 412 | def child_link(group_doc: str, name: str) -> str | None: |
| 413 | group_dir = Path(group_doc).parent # cli/bench/README.md -> cli/bench |
| 414 | for stem in (name, name.replace("-", "_")): |
| 415 | # A leaf page (bench/latency.md) or a nested group index (sweep/README.md) |
| 416 | for candidate in (group_dir / f"{stem}.md", group_dir / stem / "README.md"): |
| 417 | if candidate.as_posix() in existing_pages: |
| 418 | return candidate.relative_to(group_dir).as_posix() |
| 419 | return None |
| 420 | |
| 421 | |
| 422 | for doc_path, (overview, children) in groups.items(): |
| 423 | title = "vllm " + Path(doc_path).parent.relative_to("cli").as_posix() |
| 424 | lines = [f"# {title.replace('/', ' ')}", ""] |
| 425 | if overview: |
| 426 | lines += ["## Overview", "", overview.strip(), ""] |
| 427 | lines += ["## Subcommands", "", "| Command | Description |", "| --- | --- |"] |
| 428 | for name, summary in children: |
| 429 | link = child_link(doc_path, name) |
| 430 | command = f"[`{name}`]({link})" if link else f"`{name}`" |
| 431 | lines.append(f"| {command} | {(summary or '').strip()} |") |
| 432 | with mkdocs_gen_files.open(doc_path, "w") as f: |
| 433 | f.write("\n".join(lines) + "\n") |
| 434 | logger.debug("CLI group reference generated: %s", doc_path) |
| 435 | |
| 436 | logger.info("CLI group reference pages generated: %d", len(groups)) |