A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 17:30 UTC
[Docs] Generate full list of metrics in user docs (#30388)
Signed-off-by: Mark McLoughlin <markmc@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Mark McLoughlin committed
Dec 10, 2025 at 16:09 UTC
2dcbac9077ecadff0aa78b7c282f9e147a260e86
4 files changed
+163
-15
docs/design/metrics.md
+1
-11
@@ -21,30 +21,20 @@ The mental model is that server-level metrics help explain the values of request
21
22
### v1 Metrics
23
24
-In v1, the following metrics are exposed via a Prometheus-compatible `/metrics` endpoint using the `vllm:` prefix:
24
+In v1, an extensive set of metrics are exposed via a Prometheus-compatible `/metrics` endpoint using the `vllm:` prefix, for example:
25
26
- `vllm:num_requests_running` (Gauge) - Number of requests currently running.
27
-- `vllm:num_requests_waiting` (Gauge) - Number of requests currently waiting.
27
- `vllm:kv_cache_usage_perc` (Gauge) - Fraction of used KV cache blocks (0–1).
28
- `vllm:prefix_cache_queries` (Counter) - Number of prefix cache queries.
29
- `vllm:prefix_cache_hits` (Counter) - Number of prefix cache hits.
31
-- `vllm:mm_cache_queries` (Counter) - (For multimodal models) Number of multimodal cache queries.
32
-- `vllm:mm_cache_hits` (Counter) - (For multimodal models) Number of multimodal cache hits.
33
-- `vllm:num_preemptions_total` (Counter) - Number of preemptions.
30
- `vllm:prompt_tokens_total` (Counter) - Total number of prompt tokens processed.
31
- `vllm:generation_tokens_total` (Counter) - Total number of generated tokens.
36
-- `vllm:iteration_tokens_total` (Histogram) - Histogram of tokens processed in each engine step.
37
-- `vllm:cache_config_info` (Gauge) - Information about the cache configuration.
32
- `vllm:request_success_total` (Counter) - Number of finished requests (by finish reason).
33
- `vllm:request_prompt_tokens` (Histogram) - Histogram of input prompt token counts.
34
- `vllm:request_generation_tokens` (Histogram) - Histogram of generation token counts.
41
-- `vllm:request_params_n` (Histogram) - Histogram of request parameter n.
42
-- `vllm:request_params_max_tokens` - (Histogram) - Histogram of max_tokens parameter in requests.
35
- `vllm:time_to_first_token_seconds` (Histogram) - Time to first token (TTFT).
36
- `vllm:inter_token_latency_seconds` (Histogram) - Inter-token latency.
37
- `vllm:e2e_request_latency_seconds` (Histogram) - End-to-end request latency.
46
-- `vllm:request_queue_time_seconds` (Histogram) - Time spent in the queue.
47
-- `vllm:request_inference_time_seconds` (Histogram) - Request inference time.
38
- `vllm:request_prefill_time_seconds` (Histogram) - Request prefill time.
39
- `vllm:request_decode_time_seconds` (Histogram) - Request decode time.
40
docs/mkdocs/hooks/generate_metrics.py
new
+149
@@ -0,0 +1,149 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import ast
4
+import logging
5
+from pathlib import Path
6
+from typing import Literal
7
+
8
+logger = logging.getLogger("mkdocs")
9
+
10
+ROOT_DIR = Path(__file__).parent.parent.parent.parent
11
+DOCS_DIR = ROOT_DIR / "docs"
12
+GENERATED_METRICS_DIR = DOCS_DIR / "generated" / "metrics"
13
+
14
+# Files to scan for metric definitions - each will generate a separate table
15
+METRIC_SOURCE_FILES = [
16
+ {"path": "vllm/v1/metrics/loggers.py", "output": "general.md"},
17
+ {
18
+ "path": "vllm/v1/spec_decode/metrics.py",
19
+ "output": "spec_decode.md",
20
+ },
21
+ {
22
+ "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl_connector.py",
23
+ "output": "nixl_connector.md",
24
+ },
25
+]
26
+
27
+
28
+class MetricExtractor(ast.NodeVisitor):
29
+ """AST visitor to extract metric definitions."""
30
+
31
+ def __init__(self):
32
+ self.metrics: list[dict[str, str]] = []
33
+
34
+ def visit_Call(self, node: ast.Call) -> None:
35
+ """Visit function calls to find metric class instantiations."""
36
+ metric_type = self._get_metric_type(node)
37
+ if metric_type:
38
+ name = self._extract_kwarg(node, "name")
39
+ documentation = self._extract_kwarg(node, "documentation")
40
+
41
+ if name:
42
+ self.metrics.append(
43
+ {
44
+ "name": name,
45
+ "type": metric_type,
46
+ "documentation": documentation or "",
47
+ }
48
+ )
49
+
50
+ self.generic_visit(node)
51
+
52
+ def _get_metric_type(self, node: ast.Call) -> str | None:
53
+ """Determine if this call creates a metric and return its type."""
54
+ metric_type_map = {
55
+ "_gauge_cls": "gauge",
56
+ "_counter_cls": "counter",
57
+ "_histogram_cls": "histogram",
58
+ }
59
+ if isinstance(node.func, ast.Attribute):
60
+ return metric_type_map.get(node.func.attr)
61
+ return None
62
+
63
+ def _extract_kwarg(self, node: ast.Call, key: str) -> str | None:
64
+ """Extract a keyword argument value from a function call."""
65
+ for keyword in node.keywords:
66
+ if keyword.arg == key:
67
+ return self._get_string_value(keyword.value)
68
+ return None
69
+
70
+ def _get_string_value(self, node: ast.AST) -> str | None:
71
+ """Extract string value from an AST node."""
72
+ if isinstance(node, ast.Constant):
73
+ return str(node.value) if node.value is not None else None
74
+ return None
75
+
76
+
77
+def extract_metrics_from_file(filepath: Path) -> list[dict[str, str]]:
78
+ """Parse a Python file and extract all metric definitions."""
79
+ try:
80
+ with open(filepath, encoding="utf-8") as f:
81
+ source = f.read()
82
+
83
+ tree = ast.parse(source, filename=str(filepath))
84
+ extractor = MetricExtractor()
85
+ extractor.visit(tree)
86
+ return extractor.metrics
87
+ except Exception as e:
88
+ raise RuntimeError(f"Failed to parse {filepath}: {e}") from e
89
+
90
+
91
+def generate_markdown_table(metrics: list[dict[str, str]]) -> str:
92
+ """Generate a markdown table from extracted metrics."""
93
+ if not metrics:
94
+ return "No metrics found.\n"
95
+
96
+ # Sort by type, then by name
97
+ metrics_sorted = sorted(metrics, key=lambda m: (m["type"], m["name"]))
98
+
99
+ lines = []
100
+ lines.append("| Metric Name | Type | Description |")
101
+ lines.append("|-------------|------|-------------|")
102
+
103
+ for metric in metrics_sorted:
104
+ name = metric["name"]
105
+ metric_type = metric["type"].capitalize()
106
+ doc = metric["documentation"].replace("\n", " ").strip()
107
+ lines.append(f"| `{name}` | {metric_type} | {doc} |")
108
+
109
+ return "\n".join(lines) + "\n"
110
+
111
+
112
+def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
113
+ """Generate metrics documentation tables from source files."""
114
+ logger.info("Generating metrics documentation")
115
+
116
+ # Create generated directory if it doesn't exist
117
+ GENERATED_METRICS_DIR.mkdir(parents=True, exist_ok=True)
118
+
119
+ total_metrics = 0
120
+ for source_config in METRIC_SOURCE_FILES:
121
+ source_path = source_config["path"]
122
+ output_file = source_config["output"]
123
+
124
+ filepath = ROOT_DIR / source_path
125
+ if not filepath.exists():
126
+ raise FileNotFoundError(f"Metrics source file not found: {filepath}")
127
+
128
+ logger.debug("Extracting metrics from: %s", source_path)
129
+ metrics = extract_metrics_from_file(filepath)
130
+ logger.debug("Found %d metrics in %s", len(metrics), source_path)
131
+
132
+ # Generate and write the markdown table for this source
133
+ table_content = generate_markdown_table(metrics)
134
+ output_path = GENERATED_METRICS_DIR / output_file
135
+ with open(output_path, "w", encoding="utf-8") as f:
136
+ f.write(table_content)
137
+
138
+ total_metrics += len(metrics)
139
+ logger.info(
140
+ "Generated metrics table: %s (%d metrics)",
141
+ output_path.relative_to(ROOT_DIR),
142
+ len(metrics),
143
+ )
144
+
145
+ logger.info(
146
+ "Total metrics generated: %d across %d files",
147
+ total_metrics,
148
+ len(METRIC_SOURCE_FILES),
149
+ )
docs/usage/metrics.md
+12
-4
@@ -33,11 +33,19 @@ Then query the endpoint to get the latest metrics from the server:
33
34
The following metrics are exposed:
35
36
-??? code
36
+## General Metrics
37
38
- ```python
39
- --8<-- "vllm/engine/metrics.py:metrics-definitions"
40
- ```
38
+--8<-- "docs/generated/metrics/general.md"
39
+
40
+## Speculative Decoding Metrics
41
+
42
+--8<-- "docs/generated/metrics/spec_decode.md"
43
+
44
+## NIXL KV Connector Metrics
45
+
46
+--8<-- "docs/generated/metrics/nixl_connector.md"
47
+
48
+## Deprecation Policy
49
50
Note: when metrics are deprecated in version `X.Y`, they are hidden in version `X.Y+1`
51
but can be re-enabled using the `--show-hidden-metrics-for-version=X.Y` escape hatch,
mkdocs.yaml
+1
@@ -51,6 +51,7 @@ hooks:
51
- docs/mkdocs/hooks/remove_announcement.py
52
- docs/mkdocs/hooks/generate_examples.py
53
- docs/mkdocs/hooks/generate_argparse.py
54
+ - docs/mkdocs/hooks/generate_metrics.py
55
- docs/mkdocs/hooks/url_schemes.py
56
57
plugins: