| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import ast |
| 4 | import logging |
| 5 | import sys |
| 6 | from pathlib import Path |
| 7 | |
| 8 | sys.path.insert(0, str(Path(__file__).parent)) |
| 9 | |
| 10 | from generated_content import fill_markers # noqa: E402 |
| 11 | |
| 12 | logger = logging.getLogger("mkdocs") |
| 13 | |
| 14 | ROOT_DIR = Path(__file__).parent.parent.parent.parent |
| 15 | |
| 16 | # Files to scan for metric definitions - each fills a `gen:` marker in |
| 17 | # docs/usage/metrics.md with its table (the section heading and any preamble |
| 18 | # live in the tracked page next to the marker). |
| 19 | METRIC_SOURCE_FILES = [ |
| 20 | {"path": "vllm/v1/metrics/loggers.py", "key": "metrics-general"}, |
| 21 | {"path": "vllm/v1/spec_decode/metrics.py", "key": "metrics-spec-decode"}, |
| 22 | { |
| 23 | "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py", |
| 24 | "key": "metrics-nixl", |
| 25 | }, |
| 26 | { |
| 27 | "path": "vllm/v1/simple_kv_offload/metrics.py", |
| 28 | "key": "metrics-simple-kv-offload", |
| 29 | }, |
| 30 | {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"}, |
| 31 | ] |
| 32 | |
| 33 | |
| 34 | class MetricExtractor(ast.NodeVisitor): |
| 35 | """AST visitor to extract metric definitions.""" |
| 36 | |
| 37 | def __init__(self): |
| 38 | self.metrics: list[dict[str, str]] = [] |
| 39 | |
| 40 | def visit_Call(self, node: ast.Call) -> None: |
| 41 | """Visit function calls to find metric class instantiations.""" |
| 42 | metric_type = self._get_metric_type(node) |
| 43 | if metric_type: |
| 44 | name = self._extract_kwarg(node, "name") |
| 45 | documentation = self._extract_kwarg(node, "documentation") |
| 46 | |
| 47 | if name: |
| 48 | self.metrics.append( |
| 49 | { |
| 50 | "name": name, |
| 51 | "type": metric_type, |
| 52 | "documentation": documentation or "", |
| 53 | } |
| 54 | ) |
| 55 | |
| 56 | self.generic_visit(node) |
| 57 | |
| 58 | def _get_metric_type(self, node: ast.Call) -> str | None: |
| 59 | """Determine if this call creates a metric and return its type.""" |
| 60 | metric_type_map = { |
| 61 | "_gauge_cls": "gauge", |
| 62 | "_counter_cls": "counter", |
| 63 | "_histogram_cls": "histogram", |
| 64 | } |
| 65 | if isinstance(node.func, ast.Attribute): |
| 66 | return metric_type_map.get(node.func.attr) |
| 67 | return None |
| 68 | |
| 69 | def _extract_kwarg(self, node: ast.Call, key: str) -> str | None: |
| 70 | """Extract a keyword argument value from a function call.""" |
| 71 | for keyword in node.keywords: |
| 72 | if keyword.arg == key: |
| 73 | return self._get_string_value(keyword.value) |
| 74 | return None |
| 75 | |
| 76 | def _get_string_value(self, node: ast.AST) -> str | None: |
| 77 | """Extract string value from an AST node.""" |
| 78 | if isinstance(node, ast.Constant): |
| 79 | return str(node.value) if node.value is not None else None |
| 80 | return None |
| 81 | |
| 82 | |
| 83 | def extract_metrics_from_file(filepath: Path) -> list[dict[str, str]]: |
| 84 | """Parse a Python file and extract all metric definitions.""" |
| 85 | try: |
| 86 | with open(filepath, encoding="utf-8") as f: |
| 87 | source = f.read() |
| 88 | |
| 89 | tree = ast.parse(source, filename=str(filepath)) |
| 90 | extractor = MetricExtractor() |
| 91 | extractor.visit(tree) |
| 92 | return extractor.metrics |
| 93 | except Exception as e: |
| 94 | raise RuntimeError(f"Failed to parse {filepath}: {e}") from e |
| 95 | |
| 96 | |
| 97 | def generate_markdown_table(metrics: list[dict[str, str]]) -> str: |
| 98 | """Generate a markdown table from extracted metrics.""" |
| 99 | if not metrics: |
| 100 | return "No metrics found.\n" |
| 101 | |
| 102 | # Sort by type, then by name |
| 103 | metrics_sorted = sorted(metrics, key=lambda m: (m["type"], m["name"])) |
| 104 | |
| 105 | lines = [] |
| 106 | lines.append("| Metric Name | Type | Description |") |
| 107 | lines.append("|-------------|------|-------------|") |
| 108 | |
| 109 | for metric in metrics_sorted: |
| 110 | name = metric["name"] |
| 111 | metric_type = metric["type"].capitalize() |
| 112 | doc = metric["documentation"].replace("\n", " ").strip() |
| 113 | lines.append(f"| `{name}` | {metric_type} | {doc} |") |
| 114 | |
| 115 | return "\n".join(lines) + "\n" |
| 116 | |
| 117 | |
| 118 | logger.info("Generating metrics documentation") |
| 119 | |
| 120 | blocks = {} |
| 121 | total_metrics = 0 |
| 122 | for source_config in METRIC_SOURCE_FILES: |
| 123 | source_path = source_config["path"] |
| 124 | |
| 125 | filepath = ROOT_DIR / source_path |
| 126 | if not filepath.exists(): |
| 127 | raise FileNotFoundError(f"Metrics source file not found: {filepath}") |
| 128 | |
| 129 | logger.debug("Extracting metrics from: %s", source_path) |
| 130 | metrics = extract_metrics_from_file(filepath) |
| 131 | logger.debug("Found %d metrics in %s", len(metrics), source_path) |
| 132 | |
| 133 | blocks[source_config["key"]] = generate_markdown_table(metrics).strip() |
| 134 | total_metrics += len(metrics) |
| 135 | |
| 136 | fill_markers("usage/metrics.md", blocks) |
| 137 | logger.info( |
| 138 | "Total metrics generated: %d across %d files", |
| 139 | total_metrics, |
| 140 | len(METRIC_SOURCE_FILES), |
| 141 | ) |