| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | """MkDocs hook to automatically convert inline code references to API doc links. |
| 4 | |
| 5 | For example, `WeightTransferConfig` becomes |
| 6 | [`WeightTransferConfig`][vllm.config.WeightTransferConfig] |
| 7 | |
| 8 | This works with the `autorefs` plugin to create clickable cross-references |
| 9 | to API documentation pages generated by `mkdocstrings`. |
| 10 | |
| 11 | The hook builds an index of all documented public Python names (classes and |
| 12 | functions with docstrings) from the vllm package at startup using AST parsing, |
| 13 | then substitutes matching inline code spans on each page. Names without |
| 14 | docstrings are excluded because mkdocstrings will not generate a page for them. |
| 15 | """ |
| 16 | |
| 17 | import ast |
| 18 | import logging |
| 19 | from pathlib import Path |
| 20 | |
| 21 | import regex as re |
| 22 | from mkdocs.config.defaults import MkDocsConfig |
| 23 | from mkdocs.structure.files import Files |
| 24 | from mkdocs.structure.pages import Page |
| 25 | |
| 26 | logger = logging.getLogger("mkdocs") |
| 27 | |
| 28 | ROOT_DIR = Path(__file__).parent.parent.parent.parent.resolve() |
| 29 | VLLM_DIR = ROOT_DIR / "vllm" |
| 30 | |
| 31 | # Maps short name -> qualified name (e.g. "ModelConfig" -> "vllm.config.ModelConfig") |
| 32 | _name_index: dict[str, str] = {} |
| 33 | |
| 34 | # Fenced code block pattern (``` or ~~~, with optional language specifier). |
| 35 | _FENCED_BLOCK = re.compile( |
| 36 | r"(?:^|\n)(?P<fence>`{3,}|~{3,})[^\n]*\n.*?(?:\n(?P=fence))", re.DOTALL |
| 37 | ) |
| 38 | |
| 39 | # Inline code that is NOT already part of a markdown link. |
| 40 | # Matches `Name` but not [`Name`] and not [`Name`][...] or [`Name`](...). |
| 41 | _INLINE_CODE = re.compile( |
| 42 | r"(?<!\[)" # not preceded by [ |
| 43 | r"`(?P<name>[A-Za-z0-9_]*)`" # `UpperCamelCase` or `UPPER_SNAKE` |
| 44 | r"(?!\])" # not followed by ] |
| 45 | ) |
| 46 | |
| 47 | |
| 48 | def _has_docstring(node: ast.AST) -> bool: |
| 49 | """Check if a class or function node has a docstring.""" |
| 50 | if not isinstance(node, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef): |
| 51 | return False |
| 52 | return ast.get_docstring(node, clean=False) is not None |
| 53 | |
| 54 | |
| 55 | def _module_path(filepath: Path) -> str: |
| 56 | """Convert a filesystem path to a dotted module path.""" |
| 57 | rel = filepath.relative_to(ROOT_DIR) |
| 58 | parts = list(rel.with_suffix("").parts) |
| 59 | if parts[-1] == "__init__": |
| 60 | parts = parts[:-1] |
| 61 | return ".".join(parts) |
| 62 | |
| 63 | |
| 64 | def _index_file(filepath: Path) -> dict[str, str]: |
| 65 | """Extract documented public names from a Python file using AST parsing. |
| 66 | |
| 67 | Only classes and functions with docstrings are included, since |
| 68 | mkdocstrings won't generate a page for undocumented symbols. |
| 69 | """ |
| 70 | names: dict[str, str] = {} |
| 71 | try: |
| 72 | source = filepath.read_text(encoding="utf-8") |
| 73 | tree = ast.parse(source, filename=str(filepath)) |
| 74 | except (SyntaxError, UnicodeDecodeError): |
| 75 | return names |
| 76 | |
| 77 | module = _module_path(filepath) |
| 78 | |
| 79 | for node in ast.iter_child_nodes(tree): |
| 80 | if ( |
| 81 | # Class definitions (with docstring) |
| 82 | isinstance(node, ast.ClassDef) |
| 83 | and not node.name.startswith("_") |
| 84 | and _has_docstring(node) |
| 85 | ) or ( |
| 86 | # Function definitions (with docstring, only uppercase/CamelCase) |
| 87 | isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef) |
| 88 | and not node.name.startswith("_") |
| 89 | and node.name[0].isupper() |
| 90 | and _has_docstring(node) |
| 91 | ): |
| 92 | names[node.name] = f"{module}.{node.name}" |
| 93 | |
| 94 | return names |
| 95 | |
| 96 | |
| 97 | def _build_index() -> dict[str, str]: |
| 98 | """Walk the vllm package and build a name -> qualified path index.""" |
| 99 | index: dict[str, str] = {} |
| 100 | # Track conflicts: if multiple modules define the same name, |
| 101 | # prefer shallower modules (more likely to be the public API). |
| 102 | depth: dict[str, int] = {} |
| 103 | |
| 104 | for filepath in sorted(VLLM_DIR.rglob("*.py")): |
| 105 | # Skip internal/private modules |
| 106 | if any(part.startswith("_") and part != "__init__" for part in filepath.parts): |
| 107 | continue |
| 108 | # Skip third-party vendored code |
| 109 | rel = filepath.relative_to(VLLM_DIR) |
| 110 | if rel.parts and rel.parts[0] in ("third_party", "vllm_flash_attn"): |
| 111 | continue |
| 112 | |
| 113 | module_depth = len(filepath.relative_to(ROOT_DIR).parts) |
| 114 | file_names = _index_file(filepath) |
| 115 | |
| 116 | for name, qualified in file_names.items(): |
| 117 | if name not in index or module_depth < depth[name]: |
| 118 | index[name] = qualified |
| 119 | depth[name] = module_depth |
| 120 | |
| 121 | return index |
| 122 | |
| 123 | |
| 124 | def on_startup(*, command: str, dirty: bool) -> None: |
| 125 | """Build the name index once at startup.""" |
| 126 | global _name_index |
| 127 | _name_index = _build_index() |
| 128 | logger.info("autoref_code: indexed %d names from vllm/", len(_name_index)) |
| 129 | |
| 130 | |
| 131 | def on_page_markdown( |
| 132 | markdown: str, *, page: Page, config: MkDocsConfig, files: Files |
| 133 | ) -> str: |
| 134 | """Replace inline code references with autoref links.""" |
| 135 | if not _name_index: |
| 136 | return markdown |
| 137 | |
| 138 | # Skip API reference pages to avoid circular/redundant links. |
| 139 | if page.file.src_path.startswith("api/"): |
| 140 | return markdown |
| 141 | |
| 142 | # Step 1: Mask fenced code blocks so we don't touch code inside them. |
| 143 | masks: list[str] = [] |
| 144 | |
| 145 | def _mask_block(match: re.Match) -> str: |
| 146 | masks.append(match.group(0)) |
| 147 | return f"\ue000CODEBLOCK{len(masks) - 1}\ue000" |
| 148 | |
| 149 | masked = _FENCED_BLOCK.sub(_mask_block, markdown) |
| 150 | |
| 151 | # Step 2: Replace inline code references. |
| 152 | def _replace(match: re.Match) -> str: |
| 153 | name = match.group("name") |
| 154 | qualified = _name_index.get(name) |
| 155 | if qualified is None: |
| 156 | return match.group(0) |
| 157 | logger.debug("autoref_code: linking `%s` to [%s]", name, qualified) |
| 158 | return f"[`{name}`][{qualified}]" |
| 159 | |
| 160 | result = _INLINE_CODE.sub(_replace, masked) |
| 161 | |
| 162 | # Step 3: Restore masked code blocks. |
| 163 | result = re.sub( |
| 164 | r"\ue000CODEBLOCK(\d+)\ue000", lambda m: masks[int(m.group(1))], result |
| 165 | ) |
| 166 | return result |