@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 3e74c60b9c

[Docs] Use `gen-files` for generated docs content (#49587)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jul 25, 2026 at 15:43 UTC 3e74c60b9c74b9bdb4a4adf9ebd010c7b22104db

34 files changed +522 -868

.gitignore

-3

@@ -173,9 +173,6 @@ venv.bak/

173

174 # mkdocs documentation

175 /site

176 -docs/argparse

177 -docs/examples/*

178 -!docs/examples/README.md

176

177 # mypy

178 .mypy_cache/

.markdownlint.yaml

@@ -3,6 +3,9 @@ MD007:

3 MD013: false

4 MD024:

5 siblings_only: true

6 +MD025:

7 + # Allow front matter title to be different from the first heading in the document.

8 + front_matter_title: ""

9 MD031:

10 list_items: false

11 MD033: false

.pre-commit-config.yaml

-4

@@ -260,10 +260,6 @@ repos:

260 files: ^docker/(Dockerfile|versions\.json)$

261 pass_filenames: false

262 additional_dependencies: [dockerfile-parse]

263 - - id: attention-backend-docs

264 - name: Check attention backend documentation is up to date

265 - entry: python tools/pre_commit/generate_attention_backend_docs.py --check

266 - language: python

263 - id: check-boolean-context-manager

264 name: Check for boolean ops in with-statements

265 entry: python tools/pre_commit/check_boolean_context_manager.py

docs/.nav.yml

+3 -1

@@ -56,7 +56,9 @@ nav:

56 - API Reference:

57 - api/README.md

58 - api/vllm

59 - - CLI Reference: cli

59 + - CLI Reference:

60 + - cli/README.md

61 + - vllm: cli

62 - Community:

63 - community/*

64 - Governance: governance

docs/cli/.nav.yml

+7 -9

@@ -1,10 +1,8 @@

1 nav:

2 - - README.md

3 - - serve.md

4 - - chat.md

5 - - complete.md

6 - - run-batch.md

7 - - vllm bench:

8 - - bench/**/*.md

9 - - vllm launch:

10 - - launch/**/*.md

2 + - "*.md"

3 + - bench:

4 + - bench/*.md

5 + - sweep:

6 + - bench/sweep/*.md

7 + - launch:

8 + - launch/*.md

docs/cli/bench/latency.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench latency

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_latency.inc.md"

docs/cli/bench/mm_processor.md deleted

-55

@@ -1,55 +0,0 @@

1 -# vllm bench mm-processor

2 -

3 -## Overview

4 -

5 -`vllm bench mm-processor` profiles the multimodal input processor pipeline of

6 -vision-language models. It measures per-stage latency from the HuggingFace

7 -processor through to the encoder forward pass, helping you identify

8 -preprocessing bottlenecks and understand how different image resolutions or

9 -item counts affect end-to-end request time.

10 -

11 -The benchmark supports two data sources: synthetic random multimodal inputs

12 -(`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before

13 -measurement to ensure stable results.

14 -

15 -## Quick Start

16 -

17 -```bash

18 -vllm bench mm-processor \

19 - --model Qwen/Qwen2-VL-7B-Instruct \

20 - --dataset-name random-mm \

21 - --num-prompts 50 \

22 - --random-input-len 300 \

23 - --random-output-len 40 \

24 - --random-mm-base-items-per-request 2 \

25 - --random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \

26 - --random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'

27 -```

28 -

29 -## Measured Stages

30 -

31 -| Stage | Description |

32 -| ----- | ----------- |

33 -| `get_mm_hashes_secs` | Time spent hashing multimodal inputs |

34 -| `get_cache_missing_items_secs` | Time spent looking up the processor cache |

35 -| `apply_hf_processor_secs` | Time spent in the HuggingFace processor |

36 -| `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |

37 -| `apply_prompt_updates_secs` | Time spent updating prompt tokens |

38 -| `preprocessor_total_secs` | Total preprocessing time |

39 -| `encoder_forward_secs` | Time spent in the encoder model forward pass |

40 -| `num_encoder_calls` | Number of encoder invocations per request |

41 -

42 -The benchmark also reports end-to-end latency (TTFT + decode time) per

43 -request. Use `--metric-percentiles` to select which percentiles to report

44 -(default: p99) and `--output-json` to save results.

45 -

46 -For more examples (HF datasets, warmup, JSON output), see

47 -[Benchmarking CLI — Multimodal Processor Benchmark](../../benchmarking/cli.md#multimodal-processor-benchmark).

48 -

49 -## JSON CLI Arguments

50 -

51 ---8<-- "docs/cli/json_tip.inc.md"

52 -

53 -## Arguments

54 -

55 ---8<-- "docs/generated/argparse/bench_mm_processor.inc.md"

docs/cli/bench/serve.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench serve

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_serve.inc.md"

docs/cli/bench/sweep/plot.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench sweep plot

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_sweep_plot.inc.md"

docs/cli/bench/sweep/plot_pareto.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench sweep plot_pareto

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_sweep_plot_pareto.inc.md"

docs/cli/bench/sweep/serve.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench sweep serve

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_sweep_serve.inc.md"

docs/cli/bench/sweep/serve_workload.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench sweep serve_workload

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_sweep_serve_workload.inc.md"

docs/cli/bench/throughput.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm bench throughput

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/bench_throughput.inc.md"

docs/cli/chat.md deleted

-5

@@ -1,5 +0,0 @@

1 -# vllm chat

2 -

3 -## Arguments

4 -

5 ---8<-- "docs/generated/argparse/chat.inc.md"

docs/cli/complete.md deleted

-5

@@ -1,5 +0,0 @@

1 -# vllm complete

2 -

3 -## Arguments

4 -

5 ---8<-- "docs/generated/argparse/complete.inc.md"

docs/cli/json_tip.inc.md deleted

-10

@@ -1,10 +0,0 @@

1 -

2 -When passing JSON CLI arguments, the following sets of arguments are equivalent:

3 -

4 -- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`

5 -- `--json-arg.key1 value1 --json-arg.key2.key3 value2`

6 -

7 -Additionally, list elements can be passed individually using `+`:

8 -

9 -- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`

10 -- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`

docs/cli/launch/render.md deleted

-22

@@ -1,22 +0,0 @@

1 -# vllm launch render

2 -

3 -## Overview

4 -

5 -`vllm launch render` starts a GPU-less rendering server for preprocessing and

6 -postprocessing only.

7 -

8 -```bash

9 -vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100

10 -```

11 -

12 -This command reuses the standard serving parser, so model, frontend,

13 -networking, and related CLI options follow the same conventions as

14 -[`vllm serve`](../serve.md).

15 -

16 -## JSON CLI Arguments

17 -

18 ---8<-- "docs/cli/json_tip.inc.md"

19 -

20 -## Arguments

21 -

22 ---8<-- "docs/generated/argparse/launch_render.inc.md"

docs/cli/run-batch.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm run-batch

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/run-batch.inc.md"

docs/cli/serve.md deleted

-9

@@ -1,9 +0,0 @@

1 -# vllm serve

2 -

3 -## JSON CLI Arguments

4 -

5 ---8<-- "docs/cli/json_tip.inc.md"

6 -

7 -## Arguments

8 -

9 ---8<-- "docs/generated/argparse/serve.inc.md"

docs/configuration/engine_args.md

+1 -9

@@ -11,12 +11,4 @@ Engine arguments control the behavior of the vLLM engine.

12 The engine argument classes, [EngineArgs][vllm.engine.arg_utils.EngineArgs] and [AsyncEngineArgs][vllm.engine.arg_utils.AsyncEngineArgs], are a combination of the configuration classes defined in [vllm.config][]. Therefore, if you are interested in developer documentation, we recommend looking at these configuration classes as they are the source of truth for types, defaults and docstrings.

14 ---8<-- "docs/cli/json_tip.inc.md"

15 -

16 -## `EngineArgs`

17 -

18 ---8<-- "docs/generated/argparse/engine_args.inc.md"

19 -

20 -## `AsyncEngineArgs`

21 -

22 ---8<-- "docs/generated/argparse/async_engine_args.inc.md"

14 +--8<-- "gen:engine-args"

docs/design/attention_backends.md

+11 -89

@@ -1,15 +1,9 @@

1 # Attention Backend Feature Support

3 -This document is auto-generated by `tools/pre_commit/generate_attention_backend_docs.py`.

4 -It shows the feature support for each registered attention backend

5 -based on the checks in `AttentionBackend.validate_configuration()`.

6 -

7 -**Do not edit this file manually.** Run the following command to

8 -regenerate it:

9 -

10 -```bash

11 -python tools/pre_commit/generate_attention_backend_docs.py

12 -```

3 +The priority and feature tables on this page are auto-generated from the

4 +attention backend registry by

5 +`docs/mkdocs/gen_files/generate_attention_backends.py`, based on the checks in

6 +`AttentionBackend.validate_configuration()`.

8 ## Setting the Attention Backend

@@ -98,40 +92,11 @@ Priority is **1 = highest** (tried first).

93 ### Standard Attention (MHA, MQA, GQA)

101 -**Blackwell (SM 10.x):**

102 -

103 -| Priority | Backend |

104 -| -------- | ------- |

105 -| 1 | `FLASHINFER` |

106 -| 2 | `FLASH_ATTN` |

107 -| 3 | `TRITON_ATTN` |

108 -| 4 | `FLEX_ATTENTION` |

109 -| 5 | `TURBOQUANT` |

110 -

111 -**Ampere/Hopper (SM 8.x-9.x):**

112 -

113 -| Priority | Backend |

114 -| -------- | ------- |

115 -| 1 | `FLASH_ATTN` |

116 -| 2 | `FLASHINFER` |

117 -| 3 | `TRITON_ATTN` |

118 -| 4 | `FLEX_ATTENTION` |

119 -| 5 | `TURBOQUANT` |

95 +--8<-- "gen:priority-standard"

97 ### MLA Attention (DeepSeek-style)

123 -**Blackwell (SM 10.x):**

124 -

125 -| Priority | Backend |

126 -| -------- | ------- |

127 -| 1 | `FLASHINFER_MLA` |

128 -| 2 | `TOKENSPEED_MLA` |

129 -| 3 | `CUTLASS_MLA` |

130 -| 4 | `FLASH_ATTN_MLA` |

131 -| 5 | `FLASHMLA` |

132 -| 6 | `TRITON_MLA` |

133 -| 7 | `FLASHINFER_MLA_SPARSE`**\*** |

134 -| 8 | `FLASHMLA_SPARSE` |

99 +--8<-- "gen:priority-mla"

100

101 > **\*** For sparse MLA, FP8 KV cache always prefers `FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` is preferred for low query-head counts (<= 16), while `FLASHMLA_SPARSE` is preferred otherwise.

102 >

@@ -157,24 +122,7 @@ Priority is **1 = highest** (tried first).

122

123 ## Standard Attention (MHA, MQA, GQA) Backends

124

161 -| ------- | ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |

162 -| `CPU_ATTN` | | fp16, bf16, fp32 | `auto`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 112, 128, 160, 192, 224, 256, 512 | ❌ | ✅ | ❌ | ❌ | All | N/A |

163 -| `FLASHINFER` | Native† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ✅ | ❌ | ✅ | Decoder | 8.x-9.x |

164 -| `FLASHINFER` | XQA† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ❌ | ❌ | ❌ | ✅ | Decoder | 9.0 |

165 -| `FLASHINFER` | trtllm-gen† | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `nvfp4` | 16, 32, 64, 128, 256, 512, 1024 | 64, 128, 256, 512 | ✅ | ✅ | ❌ | ✅ | Decoder | 10.x |

166 -| `FLASH_ATTN` | FA2* | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ❌ | ✅ | All | ≥8.0 |

167 -| `FLASH_ATTN` | FA3* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | 9.x |

168 -| `FLASH_ATTN` | FA4* | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ✅ | ✅ | ❌ | ✅ | All | ≥10.0 |

169 -| `FLASH_ATTN_DIFFKV` | | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ✅ | Decoder | Any |

170 -| `FLEX_ATTENTION` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder Only | Any |

171 -| `HPC_ATTN` | | fp16, bf16 | `auto`, `bfloat16`, `fp8_e4m3` | 64 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | ≥9.0 |

172 -| `ROCM_AITER_FA` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 16, 32 | 64, 128, 256 | ✅ | ✅ | ❌ | ❌ | Decoder | N/A |

173 -| `ROCM_AITER_UNIFIED_ATTN` | | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | Any | ✅ | ❌ | ✅ | ❌ | All | N/A |

174 -| `ROCM_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %16 | 32, 64, 80, 96, 128, 160, 192, 224, 256 | ❌ | ✅ | ✅ | ❌ | Decoder, Encoder, Encoder Only | N/A |

175 -| `TRITON_ATTN` | | fp16, bf16, fp32 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2`, `int4_per_token_head`, `int8_per_token_head`, `fp8_per_token_head` | %16 | Any | ✅ | ✅ | ✅ | ❌ | All | Any |

176 -| `TRITON_ATTN_DIFFKV` | | fp16, bf16 | `auto`, `bfloat16` | Any | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |

177 -| `TURBOQUANT` | | fp16, bf16 | `turboquant_k8v4`, `turboquant_4bit_nc`, `turboquant_k3v4_nc`, `turboquant_3bit_nc` | 16, 32, 64, 128 | Any | ❌ | ❌ | ❌ | ❌ | Decoder | Any |

125 +--8<-- "gen:table-standard"

126

127 > **†** FlashInfer Native is the regular FlashInfer path. XQA is the SM90 decode path exposed through FlashInfer's TRTLLM decode API. trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen via `--attention-config.use_trtllm_attention=0`.

128 >

@@ -188,9 +136,7 @@ automatic priority lists above. A lightning indexer scores KV blocks, the

136 top-k blocks (plus fixed init/local blocks) are selected, and attention

137 attends only to those blocks; index keys live in a separate side cache.

138

192 -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | --------- | --- | --------------- | ------------ |

193 -| `MINIMAX_M3_SPARSE` | bf16, fp16 | `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | 128 | 128 | ❌ | ❌ | ❌ | ❌ | Decoder | Any |

139 +--8<-- "gen:table-minimax"

140

141 ## MLA (Multi-head Latent Attention) Backends

142

@@ -203,12 +149,7 @@ To explicitly select a prefill backend, use

149 Otherwise, the prefill backend is selected automatically at runtime based on

150 hardware and configuration.

151

207 -| ------- | ----------- | ------ | ------------ | ----- |

208 -| `FLASH_ATTN`‡ | FlashAttention varlen (FA2/FA3/FA4) | fp16, bf16 | Any | (qk_nope_head_dim=128, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=64, qk_rope_head_dim=64, v_head_dim=128) (FA2/FA3/FA4) or (qk_nope_head_dim=192, qk_rope_head_dim=64, v_head_dim=256) (FA2/FA3 only) |

152 +--8<-- "gen:table-mla-prefill"

153

154 > **‡** Automatic selection tries FlashAttention first. On Blackwell

155 > (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then

@@ -219,22 +160,7 @@ hardware and configuration.

160 MLA decode backends are selected using the standard

161 `-ac.backend=<BACKEND>` argument (e.g., `FLASHMLA`, `TRITON_MLA`).

162

223 -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |

224 -| `CUTLASS_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 128 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |

225 -| `FLASHINFER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |

226 -| `FLASHINFER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |

227 -| `FLASHINFER_MLA_SPARSE_SM120` | bf16 | `auto`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 64, 256 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | 12.x |

228 -| `FLASHMLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x-10.x |

229 -| `FLASHMLA_SPARSE` | bf16 | `auto`, `bfloat16`, `fp8_ds_mla` | 64 | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |

230 -| `FLASH_ATTN_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 9.x |

231 -| `FLASH_ATTN_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x |

232 -| `ROCM_AITER_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_e5m2` | %1 | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |

233 -| `ROCM_AITER_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | 1, 64 | Any | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | N/A |

234 -| `ROCM_AITER_TRITON_MLA` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |

235 -| `TOKENSPEED_MLA` | fp16, bf16 | `fp8`, `fp8_e4m3` | 32, 64 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | 10.x |

236 -| `TRITON_MLA` | fp16, bf16 | `auto`, `float16`, `bfloat16`, `fp8`, `fp8_e4m3` | %16 | Any | ❌ | ❌ | ❌ | ❌ | ✅ | Decoder | Any |

237 -| `XPU_MLA_SPARSE` | fp16, bf16 | `auto`, `float16`, `bfloat16` | Any | 576 | ❌ | ❌ | ✅ | ❌ | ❌ | Decoder | Any |

163 +--8<-- "gen:table-mla-decode"

164

165 ### DeepSeek V4 Decode Backends

166

@@ -245,8 +171,4 @@ pipeline (compressor + SWA + indexer, 256-token blocks, head 512);

171 default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and

172 `FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures.

173

249 -| ------- | ------ | --------- | ----------- | ---------- | ---- | ---------- | ------ | --------- | --- | --------------- | ------------ |

250 -| `FLASHINFER_MLA_SPARSE_DSV4` | bf16 | `auto`, `bfloat16`, `fp8`, `fp8_e4m3`, `fp8_ds_mla` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 10.x, 12.x |

251 -| `FLASHMLA_SPARSE_DSV4` | bf16 | `auto`, `fp8_ds_mla`, `fp8` | 256 | 512 | ✅ | ❌ | ✅ | ❌ | ❌ | Decoder | 9.x-10.x |

252 -| `ROCM_FLASHMLA_SPARSE_DSV4` | fp16, bf16 | `auto` | Any | Any | ❌ | ❌ | ❌ | ❌ | ❌ | Decoder | N/A |

174 +--8<-- "gen:table-mla-v4-decode"

docs/mkdocs/gen_files/generate_argparse.py renamed

+185 -50

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import importlib.metadata

4 import importlib.util

5 +import inspect

6 import logging

7 import sys

8 import textwrap

@@ -10,17 +11,21 @@ from argparse import SUPPRESS, Action, HelpFormatter

11 from collections.abc import Callable, Iterable

12 from importlib.machinery import ModuleSpec

13 from pathlib import Path

13 -from typing import TYPE_CHECKING, Literal

14 +from typing import TYPE_CHECKING

15 from unittest.mock import MagicMock, patch

17 +import mkdocs_gen_files

18 +import regex as re

19 from pydantic_core import core_schema

21 logger = logging.getLogger("mkdocs")

23 ROOT_DIR = Path(__file__).parent.parent.parent.parent

21 -ARGPARSE_DOC_DIR = ROOT_DIR / "docs/generated/argparse"

25 sys.path.insert(0, str(ROOT_DIR))

26 +sys.path.insert(0, str(Path(__file__).parent))

27 +

28 +from generated_content import fill_markers # noqa: E402

31 def mock_if_no_torch(mock_module: str, mock: MagicMock):

@@ -132,8 +137,8 @@ def auto_mock(module_name: str, attr: str, max_mocks: int = 100):

137

138

139 bench_latency = auto_mock("vllm.benchmarks", "latency")

135 -bench_mm_processor = auto_mock("vllm.benchmarks", "mm_processor")

140 bench_serve = auto_mock("vllm.benchmarks", "serve")

141 +bench_startup = auto_mock("vllm.benchmarks", "startup")

142 bench_sweep_plot = auto_mock("vllm.benchmarks.sweep.plot", "SweepPlotArgs")

143 bench_sweep_plot_pareto = auto_mock(

144 "vllm.benchmarks.sweep.plot_pareto", "SweepPlotParetoArgs"

@@ -142,12 +147,28 @@ bench_sweep_serve = auto_mock("vllm.benchmarks.sweep.serve", "SweepServeArgs")

147 bench_sweep_serve_workload = auto_mock(

148 "vllm.benchmarks.sweep.serve_workload", "SweepServeWorkloadArgs"

149 )

150 +bench_sweep_startup = auto_mock("vllm.benchmarks.sweep.startup", "SweepStartupArgs")

151 bench_throughput = auto_mock("vllm.benchmarks", "throughput")

152 AsyncEngineArgs = auto_mock("vllm.engine.arg_utils", "AsyncEngineArgs")

153 EngineArgs = auto_mock("vllm.engine.arg_utils", "EngineArgs")

154 ChatCommand = auto_mock("vllm.entrypoints.cli.openai", "ChatCommand")

155 CompleteCommand = auto_mock("vllm.entrypoints.cli.openai", "CompleteCommand")

156 +BenchmarkSubcommand = auto_mock(

157 + "vllm.entrypoints.cli.benchmark.main", "BenchmarkSubcommand"

158 +)

159 +import_bench_subcommands = auto_mock(

160 + "vllm.entrypoints.cli.benchmark.main", "_import_bench_subcommand_modules"

161 +)

162 +BenchmarkSubcommandBase = auto_mock(

163 + "vllm.entrypoints.cli.benchmark.base", "BenchmarkSubcommandBase"

164 +)

165 +BenchmarkMMProcessorSubcommand = auto_mock(

166 + "vllm.entrypoints.cli.benchmark.mm_processor", "BenchmarkMMProcessorSubcommand"

167 +)

168 +LaunchSubcommandBase = auto_mock("vllm.entrypoints.cli.launch", "LaunchSubcommandBase")

169 +launch_description = auto_mock("vllm.entrypoints.cli.launch", "DESCRIPTION")

170 RenderSubcommand = auto_mock("vllm.entrypoints.cli.launch", "RenderSubcommand")

171 +sweep_subcommands = auto_mock("vllm.benchmarks.sweep.cli", "SUBCOMMANDS")

172 openai_cli_args = auto_mock("vllm.entrypoints.openai", "cli_args")

173 openai_run_batch = auto_mock("vllm.entrypoints.openai", "run_batch")

174

@@ -179,7 +200,7 @@ class MarkdownFormatter(HelpFormatter):

200

201 def add_text(self, text: str):

202 if text:

182 - self._markdown_output.append(f"{text.strip()}\n\n")

203 + self._markdown_output.append(f"{inspect.cleandoc(text)}\n\n")

204

205 def add_usage(self, usage, actions, groups, prefix=None):

206 pass

@@ -241,49 +262,163 @@ def create_parser(add_cli_args, **kwargs) -> FlexibleArgumentParser:

262 return _parser or parser

263

264

244 -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):

245 - logger.info("Generating argparse documentation")

246 - logger.debug("Root directory: %s", ROOT_DIR.resolve())

247 - logger.debug("Output directory: %s", ARGPARSE_DOC_DIR.resolve())

248 -

249 - # Create the ARGPARSE_DOC_DIR if it doesn't exist

250 - if not ARGPARSE_DOC_DIR.exists():

251 - ARGPARSE_DOC_DIR.mkdir(parents=True)

252 -

253 - # Create parsers to document

254 - parsers = {

255 - # Engine args

256 - "engine_args": create_parser(EngineArgs.add_cli_args),

257 - "async_engine_args": create_parser(

258 - AsyncEngineArgs.add_cli_args, async_args_only=True

259 - ),

260 - # CLI

261 - "serve": create_parser(openai_cli_args.make_arg_parser),

262 - "chat": create_parser(ChatCommand.add_cli_args),

263 - "complete": create_parser(CompleteCommand.add_cli_args),

264 - "launch_render": create_parser(RenderSubcommand.add_cli_args),

265 - "run-batch": create_parser(openai_run_batch.make_arg_parser),

266 - # Benchmark CLI

267 - "bench_latency": create_parser(bench_latency.add_cli_args),

268 - "bench_mm_processor": create_parser(bench_mm_processor.add_cli_args),

269 - "bench_serve": create_parser(bench_serve.add_cli_args),

270 - "bench_sweep_plot": create_parser(bench_sweep_plot.add_cli_args),

271 - "bench_sweep_plot_pareto": create_parser(bench_sweep_plot_pareto.add_cli_args),

272 - "bench_sweep_serve": create_parser(bench_sweep_serve.add_cli_args),

273 - "bench_sweep_serve_workload": create_parser(

274 - bench_sweep_serve_workload.add_cli_args

275 - ),

276 - "bench_throughput": create_parser(bench_throughput.add_cli_args),

277 - }

278 -

279 - # Generate documentation for each parser

280 - for stem, parser in parsers.items():

281 - doc_path = ARGPARSE_DOC_DIR / f"{stem}.inc.md"

282 - # Specify encoding for building on Windows

283 - with open(doc_path, "w", encoding="utf-8") as f:

284 - f.write(super(type(parser), parser).format_help())

285 - logger.info("Argparse generated: %s", doc_path.relative_to(ROOT_DIR))

286 -

287 -

288 -if __name__ == "__main__":

289 - on_startup("build", False)

265 +def format_help(parser: FlexibleArgumentParser) -> str:

266 + """Format a parser's help as markdown using `MarkdownFormatter`."""

267 + return super(type(parser), parser).format_help()

268 +

269 +

270 +# Absolute docs URLs are kept in the help text because they are useful in the

271 +# terminal. Wrap them as markdown links so the `url_schemes` hook can rewrite

272 +# them into doc-relative links / cross-references at render time.

273 +_DOCS_URL = re.compile(r"https://docs\.vllm\.ai/en/[^/\s]+/[^\s)>]+")

274 +

275 +

276 +def linkify_docs_urls(text: str) -> str:

277 + """Wrap bare docs.vllm.ai URLs in help text as markdown links."""

278 + return _DOCS_URL.sub(lambda m: f"[{m.group()}]({m.group()})", text)

279 +

280 +

281 +logger.info("Generating argparse documentation")

282 +logger.debug("Root directory: %s", ROOT_DIR.resolve())

283 +

284 +# The JSON tip is always rendered immediately before generated argument content,

285 +# and the generator is its only consumer, so it lives here rather than in a

286 +# separate snippet file. (The runtime terminal equivalent is

287 +# `FlexibleArgumentParser._json_tip` in vllm/utils/argparse_utils.py.)

288 +JSON_TIP = """## JSON CLI Arguments

289 +

290 +When passing JSON CLI arguments, the following sets of arguments are equivalent:

291 +

292 +- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`

293 +- `--json-arg.key1 value1 --json-arg.key2.key3 value2`

294 +

295 +Additionally, list elements can be passed individually using `+`:

296 +

297 +- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`

298 +- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`

299 +

300 +"""

301 +

302 +# Argument sections filled into `gen:` markers on handwritten pages

303 +engine_args = create_parser(EngineArgs.add_cli_args)

304 +async_engine_args = create_parser(AsyncEngineArgs.add_cli_args, async_args_only=True)

305 +fill_markers(

306 + "configuration/engine_args.md",

307 + {

308 + "engine-args": (

309 + f"{JSON_TIP}## `EngineArgs`\n\n"

310 + f"{linkify_docs_urls(format_help(engine_args))}"

311 + f"## `AsyncEngineArgs`\n\n"

312 + f"{linkify_docs_urls(format_help(async_engine_args))}"

313 + )

314 + },

315 +)

316 +

317 +# CLI reference pages generated entirely from their parser: page -> (parser, JSON tip)

318 +pages = {

319 + "cli/serve.md": (create_parser(openai_cli_args.make_arg_parser), True),

320 + "cli/chat.md": (create_parser(ChatCommand.add_cli_args), False),

321 + "cli/complete.md": (create_parser(CompleteCommand.add_cli_args), False),

322 + "cli/run-batch.md": (create_parser(openai_run_batch.make_arg_parser), True),

323 + "cli/launch/render.md": (create_parser(RenderSubcommand.add_cli_args), True),

324 + "cli/bench/latency.md": (create_parser(bench_latency.add_cli_args), True),

325 + # URL kept as `mm_processor` for back-compat; command name is `mm-processor`

326 + "cli/bench/mm_processor.md": (

327 + create_parser(BenchmarkMMProcessorSubcommand.add_cli_args),

328 + True,

329 + ),

330 + "cli/bench/serve.md": (create_parser(bench_serve.add_cli_args), True),

331 + "cli/bench/startup.md": (create_parser(bench_startup.add_cli_args), True),

332 + "cli/bench/throughput.md": (create_parser(bench_throughput.add_cli_args), True),

333 + "cli/bench/sweep/plot.md": (create_parser(bench_sweep_plot.add_cli_args), True),

334 + "cli/bench/sweep/plot_pareto.md": (

335 + create_parser(bench_sweep_plot_pareto.add_cli_args),

336 + True,

337 + ),

338 + "cli/bench/sweep/serve.md": (create_parser(bench_sweep_serve.add_cli_args), True),

339 + "cli/bench/sweep/serve_workload.md": (

340 + create_parser(bench_sweep_serve_workload.add_cli_args),

341 + True,

342 + ),

343 + "cli/bench/sweep/startup.md": (

344 + create_parser(bench_sweep_startup.add_cli_args),

345 + True,

346 + ),

347 +}

348 +

349 +# Command name for pages whose file stem differs (URL kept for back-compat).

350 +COMMAND_NAMES = {"cli/bench/mm_processor.md": "mm-processor"}

351 +

352 +for doc_path, (parser, json_tip) in pages.items():

353 + segments = Path(doc_path).relative_to("cli").with_suffix("").parts

354 + label = COMMAND_NAMES.get(doc_path, segments[-1])

355 + command = " ".join([*segments[:-1], label])

356 + # `title` frontmatter keeps the nav label to just this command's segment,

357 + # while the H1 stays the full `vllm ...` command for the page heading.

358 + content = f"---\ntitle: {label}\n---\n\n"

359 + content += f"# vllm {command}\n\n"

360 + if parser.description:

361 + content += f"## Overview\n\n{parser.description}\n\n"

362 + # Rendered above instead of at the top of the Arguments section

363 + parser.description = None

364 + if json_tip:

365 + content += JSON_TIP

366 + content += f"## Arguments\n\n{linkify_docs_urls(format_help(parser))}"

367 + with mkdocs_gen_files.open(doc_path, "w") as f:

368 + f.write(content)

369 + logger.debug("CLI reference generated: %s", doc_path)

370 +

371 +logger.info("Total argparse docs generated: %d", len(pages) + 2)

372 +

373 +

374 +# --- Bare subcommand (group) pages -------------------------------------------

375 +# Mirror `vllm <group> --help`: an overview plus a table of child subcommands,

376 +# each linked to its reference page. Children are read from the CLI registries

377 +# so the listing can never drift from the actual subcommands. Each page is the

378 +# `README.md` of its command directory so it becomes that section's index and is

379 +# picked up by the existing nav globs.

380 +import_bench_subcommands() # populate BenchmarkSubcommandBase.__subclasses__()

381 +bench_subcommands = BenchmarkSubcommandBase.__subclasses__()

382 +bench_children = [(cmd.name, cmd.help) for cmd in bench_subcommands]

383 +

384 +groups = {

385 + "cli/bench/README.md": (BenchmarkSubcommand.help, bench_children),

386 + "cli/launch/README.md": (

387 + launch_description,

388 + [(cmd.name, cmd.help) for cmd in LaunchSubcommandBase.__subclasses__()],

389 + ),

390 + "cli/bench/sweep/README.md": (

391 + dict(bench_children).get("sweep"),

392 + [(args.parser_name, args.parser_help) for args, _ in sweep_subcommands],

393 + ),

394 +}

395 +

396 +# Doc paths that exist, so we only link a child that has a reference page.

397 +existing_pages = set(pages) | set(groups)

398 +

399 +

400 +def child_link(group_doc: str, name: str) -> str | None:

401 + group_dir = Path(group_doc).parent # cli/bench/README.md -> cli/bench

402 + for stem in (name, name.replace("-", "_")):

403 + # A leaf page (bench/latency.md) or a nested group index (sweep/README.md)

404 + for candidate in (group_dir / f"{stem}.md", group_dir / stem / "README.md"):

405 + if candidate.as_posix() in existing_pages:

406 + return candidate.relative_to(group_dir).as_posix()

407 + return None

408 +

409 +

410 +for doc_path, (overview, children) in groups.items():

411 + title = "vllm " + Path(doc_path).parent.relative_to("cli").as_posix()

412 + lines = [f"# {title.replace('/', ' ')}", ""]

413 + if overview:

414 + lines += ["## Overview", "", overview.strip(), ""]

415 + lines += ["## Subcommands", "", "| Command | Description |", "| --- | --- |"]

416 + for name, summary in children:

417 + link = child_link(doc_path, name)

418 + command = f"[`{name}`]({link})" if link else f"`{name}`"

419 + lines.append(f"| {command} | {(summary or '').strip()} |")

420 + with mkdocs_gen_files.open(doc_path, "w") as f:

421 + f.write("\n".join(lines) + "\n")

422 + logger.debug("CLI group reference generated: %s", doc_path)

423 +

424 +logger.info("CLI group reference pages generated: %d", len(groups))

docs/mkdocs/gen_files/generate_attention_backends.py renamed

+58 -401

@@ -9,33 +9,28 @@ based on the checks in AttentionBackend.validate_configuration().

10 This approach avoids requiring CUDA/ROCm/GPU libraries to be installed.

12 -When used as a pre-commit hook, this script receives filenames as arguments

13 -and only runs the check if any of the relevant files were modified.

12 +It runs as an mkdocs-gen-files script, so the page is generated at docs build

13 +time rather than being committed to the repository.

14 """

16 -import argparse

16 import ast

18 -import fnmatch

17 +import logging

18 import sys

19 from collections.abc import Callable

20 from pathlib import Path

21 from typing import Any

23 +sys.path.insert(0, str(Path(__file__).parent))

24 +

25 +from generated_content import fill_markers # noqa: E402

26 +

27 +logger = logging.getLogger("mkdocs")

28 +

29 # ---------------------------------------------------------------------------

30 # Constants and file paths

31 # ---------------------------------------------------------------------------

28 -REPO_ROOT = Path(__file__).parent.parent.parent

29 -

30 -RELEVANT_PATTERNS = [

31 - "vllm/v1/attention/backends/*.py",

32 - "vllm/v1/attention/backends/**/*.py",

33 - "vllm/models/minimax_m3/common/sparse_attention.py",

34 - "vllm/model_executor/layers/attention/mla_attention.py",

35 - "vllm/platforms/cuda.py",

36 - "tools/pre_commit/generate_attention_backend_docs.py",

37 - "docs/design/attention_backends.md",

38 -]

33 +REPO_ROOT = Path(__file__).parent.parent.parent.parent

35 BACKENDS_DIR = REPO_ROOT / "vllm" / "v1" / "attention" / "backends"

36 REGISTRY_FILE = BACKENDS_DIR / "registry.py"

@@ -55,19 +50,6 @@ BACKEND_KV_DTYPE_EXCLUDES: dict[str, set[str]] = {

50 }

58 -def is_relevant_file(filepath: str) -> bool:

59 - """Check if a file matches any of the relevant patterns."""

60 - path = Path(filepath)

61 - if path.is_absolute():

62 - try:

63 - path = path.relative_to(REPO_ROOT)

64 - except ValueError:

65 - return False

66 - path_str = str(path)

67 -

68 - return any(fnmatch.fnmatch(path_str, pattern) for pattern in RELEVANT_PATTERNS)

69 -

70 -

53 MLA_PREFILL_DIR = BACKENDS_DIR / "mla" / "prefill"

54 MLA_PREFILL_REGISTRY_FILE = MLA_PREFILL_DIR / "registry.py"

55 MLA_PREFILL_SELECTOR_FILE = MLA_PREFILL_DIR / "selector.py"

@@ -960,7 +942,7 @@ def analyze_backend(backend_name: str, class_path: str) -> dict[str, Any] | None

942 try:

943 tree = ast.parse(file_path.read_text())

944 except Exception as e:

963 - print(f" Warning: Could not parse {file_path}: {e}", file=sys.stderr)

945 + logger.warning("Could not parse %s: %s", file_path, e)

946 return None

947

948 class_name = class_path.rsplit(".", 1)[1]

@@ -1657,113 +1639,12 @@ def _render_table(

1639 return lines

1640

1641

1660 -def generate_markdown_table(

1661 - backends: list[dict[str, Any]], title: str, is_mla_table: bool = False

1662 -) -> str:

1663 - """Generate a titled markdown table from backend info."""

1664 - if not backends:

1665 - return f"## {title}\n\nNo backends found.\n"

1666 - has_versions = any(b.get("version") for b in backends)

1667 - columns = _build_columns(is_mla_table, has_versions)

1668 - lines = [f"## {title}", ""]

1669 - lines.extend(_render_table(columns, backends))

1670 - lines.append("")

1671 - return "\n".join(lines)

1672 -

1673 -

1674 -# ---------------------------------------------------------------------------

1675 -# Markdown section generators (usage, priority, legend, MLA)

1676 -# ---------------------------------------------------------------------------

1677 -

1678 -

1679 -def generate_usage_section() -> str:

1680 - """Generate the usage documentation section."""

1681 - return """## Setting the Attention Backend

1682 -

1683 -### Command Line

1684 -

1685 -There are two ways to specify the backend from the command line:

1686 -

1687 -**Option 1: Using `--attention-backend` (simple)**

1688 -

1689 -```bash

1690 -vllm serve <model> --attention-backend FLASH_ATTN

1691 -```

1692 -

1693 -**Option 2: Using `--attention-config.backend` / `-ac.backend` (structured config)**

1694 -

1695 -```bash

1696 -# Dot notation

1697 -vllm serve <model> --attention-config.backend FLASH_ATTN

1698 -vllm serve <model> -ac.backend FLASH_ATTN

1699 -

1700 -# JSON format

1701 -vllm serve <model> --attention-config '{"backend": "FLASH_ATTN"}'

1702 -vllm serve <model> -ac '{"backend": "FLASH_ATTN"}'

1703 -```

1704 -

1705 -> **Note:** `--attention-backend` and `--attention-config.backend` are mutually

1706 -> exclusive. Use one or the other, not both.

1707 -

1708 -### Python API

1709 -

1710 -Use `AttentionConfig` with the `LLM` class:

1711 -

1712 -```python

1713 -from vllm import LLM

1714 -from vllm.config import AttentionConfig

1715 -from vllm.v1.attention.backends.registry import AttentionBackendEnum

1716 -

1717 -# Method 1: Using AttentionConfig with enum

1718 -llm = LLM(

1719 - model="Qwen/Qwen3-0.6B",

1720 - attention_config=AttentionConfig(backend=AttentionBackendEnum.FLASH_ATTN),

1721 -)

1722 -

1723 -# Method 2: Using attention_backend parameter with string

1724 -llm = LLM(

1725 - model="Qwen/Qwen3-0.6B",

1726 - attention_backend="FLASH_ATTN",

1727 -)

1728 -```

1729 -

1730 -## Backend Selection Behavior

1731 -

1732 -### Manual Selection

1733 -

1734 -When you explicitly set a backend via `--attention-backend` or `AttentionConfig`:

1735 -

1736 -1. The backend is **validated** against your configuration (model dtype, head

1737 - size, compute capability, etc.)

1738 -2. If the backend **doesn't support** your configuration, an error is raised

1739 - with the specific reason

1740 -3. If valid, the backend is used

1741 -

1742 -Example error when selecting an incompatible backend:

1743 -

1744 -```text

1745 -ValueError: Selected backend FLASHMLA is not valid for this configuration.

1746 -Reason: ['compute capability not supported']

1747 -```

1748 -

1749 -### Automatic Selection

1750 -

1751 -When no backend is specified (the default):

1752 -

1753 -1. vLLM iterates through backends in **priority order** (see tables below)

1754 -2. Each backend is validated against your configuration

1755 -3. The **first compatible backend** is selected

1756 -4. If no backend is compatible, an error is raised listing all backends and

1757 - their incompatibility reasons

1758 -"""

1759 -

1760 -

1642 def _priority_table(

1643 title: str,

1644 backends: list[str],

1645 annotations: dict[str, str] | None = None,

1646 ) -> list[str]:

1766 - """Generate a priority table for a list of backends."""

1647 + """Render a priority table for a list of backends."""

1648

1649 def _fmt(b: str) -> str:

1650 suffix = annotations.get(b, "") if annotations else ""

@@ -1779,102 +1660,38 @@ def _priority_table(

1660 ]

1661

1662

1782 -def generate_priority_section(priorities: dict[str, list[str]]) -> str:

1783 - """Generate the priority ranking section."""

1784 - lines = [

1785 - "## Backend Priority (CUDA)",

1786 - "",

1787 - "When no backend is explicitly selected, vLLM chooses the first",

1788 - "compatible backend from these priority-ordered lists.",

1789 - "",

1790 - "Priority is **1 = highest** (tried first).",

1791 - "",

1792 - "### Standard Attention (MHA, MQA, GQA)",

1793 - "",

1794 - ]

1795 -

1796 - sm100 = "Blackwell (SM 10.x)"

1797 - ampere = "Ampere/Hopper (SM 8.x-9.x)"

1798 -

1799 - if "standard_sm100" in priorities:

1800 - lines.extend(_priority_table(sm100, priorities["standard_sm100"]))

1801 - if "standard_default" in priorities:

1802 - lines.extend(_priority_table(ampere, priorities["standard_default"]))

1663 +_SM100 = "Blackwell (SM 10.x)"

1664 +_AMPERE = "Ampere/Hopper (SM 8.x-9.x)"

1665

1804 - lines.extend(["### MLA Attention (DeepSeek-style)", ""])

1666

1806 - mla_sm100_annotations = {

1807 - "FLASHINFER_MLA_SPARSE": "**\\***",

1808 - }

1809 - if "mla_sm100" in priorities:

1810 - lines.extend(

1811 - _priority_table(sm100, priorities["mla_sm100"], mla_sm100_annotations)

1812 - )

1813 - if "mla_default" in priorities:

1814 - lines.extend(_priority_table(ampere, priorities["mla_default"]))

1815 -

1816 - if "mla_sm100" in priorities:

1817 - lines.append(

1818 - "> **\\*** For sparse MLA, FP8 KV cache always prefers "

1819 - "`FLASHINFER_MLA_SPARSE`. With BF16 KV cache, `FLASHINFER_MLA_SPARSE` "

1820 - "is preferred for low query-head counts (<= 16), while "

1821 - "`FLASHMLA_SPARSE` is preferred otherwise."

1822 - )

1823 - lines.append(">")

1824 -

1825 - lines.append(

1826 - "> **Note:** ROCm and CPU platforms have their own selection logic. "

1827 - "See the platform-specific documentation for details."

1828 - )

1829 - lines.append("")

1830 -

1831 - return "\n".join(lines)

1667 +def _priority_block(

1668 + priorities: dict[str, list[str]],

1669 + sm100_key: str,

1670 + default_key: str,

1671 + sm100_annotations: dict[str, str] | None = None,

1672 +) -> str:

1673 + """Render whichever priority tables exist for one attention category."""

1674 + lines: list[str] = []

1675 + if sm100_key in priorities:

1676 + lines += _priority_table(_SM100, priorities[sm100_key], sm100_annotations)

1677 + if default_key in priorities:

1678 + lines += _priority_table(_AMPERE, priorities[default_key])

1679 + return "\n".join(lines).strip()

1680

1681

1834 -def generate_legend() -> str:

1835 - """Generate a legend explaining the table columns."""

1836 - return """## Legend

1837 -

1838 -| Column | Description |

1839 -| ------ | ----------- |

1840 -| **Dtypes** | Supported model data types (fp16, bf16, fp32) |

1841 -| **KV Dtypes** | Supported KV cache data types (`auto`, `fp8`, `fp8_e4m3`, etc.) |

1842 -| **Block Sizes** | Supported KV cache block sizes (%N means multiples of N) |

1843 -| **Head Sizes** | Supported attention head sizes |

1844 -| **Sink** | Attention sink support (for StreamingLLM) |

1845 -| **Non-Causal** | Non-causal (bidirectional) attention support for decoder models |

1846 -| **Sparse** | Sparse attention support (MLA only) |

1847 -| **MM Prefix** | Multimodal prefix full attention support |

1848 -| **DCP** | Decode Context Parallelism support (`--decode-context-parallel-size`) |

1849 -| **Attention Types** | Supported attention patterns (Decoder, Encoder, Enc-Dec) |

1850 -| **Compute Cap.** | Required CUDA compute capability (N/A for non-CUDA backends) |

1851 -

1852 -**Symbols:** ✅ = Supported, ❌ = Not supported

1853 -"""

1682 +def _feature_table(backends: list[dict[str, Any]], is_mla: bool) -> str:

1683 + """Render a backend feature table (header, separator, one row per backend)."""

1684 + has_versions = any(b.get("version") for b in backends)

1685 + columns = _build_columns(is_mla, has_versions)

1686 + return "\n".join(_render_table(columns, backends))

1687

1688

1856 -def generate_mla_section(

1857 - prefill_backends: list[dict[str, Any]],

1858 - decode_backends: list[dict[str, Any]],

1859 - v4_decode_backends: list[dict[str, Any]] | None = None,

1860 -) -> str:

1861 - """Generate the complete MLA section with prefill and decode tables."""

1689 +def _mla_prefill_table(prefill_backends: list[dict[str, Any]]) -> str:

1690 + """Render the MLA prefill backend table."""

1691 lines = [

1863 - "## MLA (Multi-head Latent Attention) Backends",

1864 - "",

1865 - "MLA uses separate backends for prefill and decode phases.",

1866 - "",

1867 - "### Prefill Backends",

1868 - "",

1869 - "To explicitly select a prefill backend, use",

1870 - "`-ac.mla_prefill_backend=<BACKEND>` (e.g., `FLASH_ATTN`, `FLASHINFER`).",

1871 - "Otherwise, the prefill backend is selected automatically at runtime based on",

1872 - "hardware and configuration.",

1873 - "",

1693 "| ------- | ----------- | ------ | ------------ | ----- |",

1694 ]

1877 -

1695 for backend in prefill_backends:

1696 row = "| `{}`{} | {} | {} | {} | {} |".format(

1697 backend["name"],

@@ -1885,87 +1702,21 @@ def generate_mla_section(

1702 backend.get("notes", ""),

1703 )

1704 lines.append(row.replace(" ", " "))

1888 -

1889 - lines.extend(

1890 - [

1891 - "",

1892 - "> **‡** Automatic selection tries FlashAttention first. On Blackwell",

1893 - "> (SM100), the fallback order is TRT-LLM Ragged, FlashInfer, then",

1894 - "> TokenSpeed MLA. On other GPUs, only FlashAttention is considered.",

1895 - "",

1896 - "### Decode Backends",

1897 - "",

1898 - "MLA decode backends are selected using the standard",

1899 - "`-ac.backend=<BACKEND>` argument (e.g., `FLASHMLA`, `TRITON_MLA`).",

1900 - "",

1901 - ]

1902 - )

1903 -

1904 - # Reuse data-driven table rendering for decode backends

1905 - columns = _build_columns(is_mla=True, has_versions=False)

1906 - lines.extend(_render_table(columns, decode_backends))

1907 -

1908 - if v4_decode_backends:

1909 - lines.extend(

1910 - [

1911 - "",

1912 - "### DeepSeek V4 Decode Backends",

1913 - "",

1914 - "DeepSeek V4 sparse MLA uses its own decode backends, selected via",

1915 - "`--attention-backend=<BACKEND>` (e.g., `FLASHMLA_SPARSE_DSV4`,",

1916 - "`FLASHINFER_MLA_SPARSE_DSV4`). They share the V4 sparse-index",

1917 - "pipeline (compressor + SWA + indexer, 256-token blocks, head 512);",

1918 - "default on NVIDIA is `FLASHINFER_MLA_SPARSE_DSV4` on SM12x and",

1919 - "`FLASHMLA_SPARSE_DSV4` on other supported CUDA architectures.",

1920 - "",

1921 - ]

1922 - )

1923 - lines.extend(_render_table(columns, v4_decode_backends))

1924 -

1925 - lines.append("")

1705 return "\n".join(lines)

1706

1707

1929 -def generate_minimax_section(backends: list[dict[str, Any]]) -> str:

1930 - """Generate the MiniMax M3 sparse attention section."""

1931 - lines = [

1932 - "## MiniMax M3 Sparse Attention Backends",

1933 - "",

1934 - 'Block-sparse GQA backend used by MiniMax M3 sparse ("lightning indexer")',

1935 - "layers. It is wired in directly by the model and is not part of the",

1936 - "automatic priority lists above. A lightning indexer scores KV blocks, the",

1937 - "top-k blocks (plus fixed init/local blocks) are selected, and attention",

1938 - "attends only to those blocks; index keys live in a separate side cache.",

1939 - "",

1940 - ]

1941 - columns = _build_columns(is_mla=False, has_versions=False)

1942 - lines.extend(_render_table(columns, backends))

1943 - lines.append("")

1944 - return "\n".join(lines)

1945 -

1946 -

1947 -# ---------------------------------------------------------------------------

1948 -# Top-level orchestration

1949 -# ---------------------------------------------------------------------------

1708 +def build_blocks() -> dict[str, str]:

1709 + """Build the generated table blocks keyed by their `gen:` marker name.

1710

1951 -

1952 -def generate_docs() -> str:

1953 - """Generate the complete documentation."""

1711 + Only the tables are generated here; the surrounding prose lives in the

1712 + handwritten ``docs/design/attention_backends.md`` page.

1713 + """

1714 attention_backends_map = parse_registry()

1955 -

1956 - # Parse priority lists from cuda.py

1715 priorities = parse_cuda_priority_lists()

1958 -

1959 - # Parse FlashAttention FA2/FA3 feature differences

1716 fa_features = parse_flash_attn_features()

1961 -

1962 - # Parse FlashInfer TRTLLM feature differences (native vs TRTLLM on Blackwell)

1717 fi_features = parse_flashinfer_trtllm_features()

1964 -

1965 - # Parse MLA prefill backends

1718 mla_prefill_backends = parse_mla_prefill_backends()

1719

1968 - # Collect backend info

1720 all_backends = []

1721 for backend_name, class_path in attention_backends_map.items():

1722 if backend_name in SKIP_BACKENDS:

@@ -1973,17 +1724,14 @@ def generate_docs() -> str:

1724 info = analyze_backend(backend_name, class_path)

1725 if info:

1726 all_backends.append(info)

1976 -

1977 - # Expand backends into version variants

1727 if fa_features:

1728 all_backends = _expand_flash_attn_variants(all_backends, fa_features)

1729 if fi_features:

1730 all_backends = _expand_flashinfer_variants(all_backends, fi_features)

1731

1983 - # DeepSeek V4 (*_DSV4) decode backends and MiniMax M3 sparse backends each

1984 - # get their own subsection rather than mixing into the main MLA / standard

1985 - # tables (the ROCm V4 backend isn't flagged is_mla by the AST heuristic, so

1986 - # filter purely on the name).

1732 + # DeepSeek V4 (*_DSV4) and MiniMax M3 sparse backends get their own tables

1733 + # rather than mixing into the main MLA / standard tables (the ROCm V4 backend

1734 + # isn't flagged is_mla by the AST heuristic, so filter purely on the name).

1735 def _is_v4(b: dict[str, Any]) -> bool:

1736 return b["name"].endswith("_DSV4")

1737

@@ -1999,112 +1747,21 @@ def generate_docs() -> str:

1747 if not b["is_mla"] and not _is_v4(b) and not _is_minimax(b)

1748 ]

1749

2002 - # Generate documentation

2003 - script_path = "tools/pre_commit/generate_attention_backend_docs.py"

2004 - doc_lines = [

2005 - "# Attention Backend Feature Support",

2006 - "",

2007 - f"This document is auto-generated by `{script_path}`.",

2008 - "It shows the feature support for each registered attention backend",

2009 - "based on the checks in `AttentionBackend.validate_configuration()`.",

2010 - "",

2011 - "**Do not edit this file manually.** Run the following command to",

2012 - "regenerate it:",

2013 - "",

2014 - "```bash",

2015 - f"python {script_path}",

2016 - "```",

2017 - "",

2018 - ]

2019 -

2020 - # Add usage documentation

2021 - doc_lines.append(generate_usage_section())

2022 -

2023 - # Add priority section

2024 - doc_lines.append(generate_priority_section(priorities))

2025 -

2026 - # Add legend and feature tables

2027 - doc_lines.append(generate_legend())

2028 - standard_title = "Standard Attention (MHA, MQA, GQA) Backends"

2029 - doc_lines.append(

2030 - generate_markdown_table(non_mla_backends, standard_title, is_mla_table=False)

2031 - )

2032 - # Add footnotes for version/variant distinctions (in table order)

2033 - footnotes = []

2034 - if fi_features:

2035 - footnotes.append(

2036 - "> **†** FlashInfer Native is the regular FlashInfer path. XQA is the "

2037 - "SM90 decode path exposed through FlashInfer's TRTLLM decode API. "

2038 - "trtllm-gen is used on SM100 and supports sinks. Disable XQA/trtllm-gen "

2039 - "via `--attention-config.use_trtllm_attention=0`."

2040 - )

2041 - if fa_features:

2042 - footnotes.append(

2043 - "> **\\*** Specify the FlashAttention version via "

2044 - "`--attention-config.flash_attn_version=2`, `3`, or `4`. "

2045 - "Default is FA4 on SM100+ (Blackwell), FA3 on SM90 (Hopper), "

2046 - "FA2 otherwise."

2047 - )

2048 - if footnotes:

2049 - doc_lines.append("\n>\n".join(footnotes) + "\n")

2050 -

2051 - # Add MiniMax M3 sparse section (separate category after standard GQA)

2052 - if minimax_backends:

2053 - doc_lines.append(generate_minimax_section(minimax_backends))

2054 -

2055 - # Add MLA section with prefill and decode backends

2056 - doc_lines.append(

2057 - generate_mla_section(mla_prefill_backends, mla_backends, v4_decode_backends)

2058 - )

2059 -

2060 - return "\n".join(doc_lines)

2061 -

2062 -

2063 -def main():

2064 - parser = argparse.ArgumentParser(

2065 - description="Generate attention backend documentation table"

2066 - )

2067 - parser.add_argument(

2068 - "--output",

2069 - "-o",

2070 - type=str,

2071 - default=str(REPO_ROOT / "docs" / "design" / "attention_backends.md"),

2072 - help="Output file path (default: docs/design/attention_backends.md)",

2073 - )

2074 - parser.add_argument(

2075 - "--check",

2076 - action="store_true",

2077 - help="Check if the documentation is up to date (for pre-commit)",

2078 - )

2079 - parser.add_argument(

2080 - "files",

2081 - nargs="*",

2082 - help="Files to check (passed by pre-commit). If none are relevant, skip.",

2083 - )

2084 - args = parser.parse_args()

2085 -

2086 - if args.files and not any(is_relevant_file(f) for f in args.files):

2087 - sys.exit(0)

2088 -

2089 - output_path = Path(args.output)

2090 - new_content = generate_docs()

2091 -

2092 - if args.check:

2093 - needs_update = (

2094 - not output_path.exists() or output_path.read_text() != new_content

2095 - )

2096 - if needs_update:

2097 - output_path.parent.mkdir(parents=True, exist_ok=True)

2098 - output_path.write_text(new_content)

2099 - print(f"🔄 Regenerated: {output_path}")

2100 - sys.exit(1)

2101 - print(f"✅ Up to date: {output_path}")

2102 - sys.exit(0)

2103 -

2104 - output_path.parent.mkdir(parents=True, exist_ok=True)

2105 - output_path.write_text(new_content)

2106 - print(f"Generated: {output_path}")

1750 + mla_sm100_annotations = {"FLASHINFER_MLA_SPARSE": "**\\***"}

1751 + return {

1752 + "priority-standard": _priority_block(

1753 + priorities, "standard_sm100", "standard_default"

1754 + ),

1755 + "priority-mla": _priority_block(

1756 + priorities, "mla_sm100", "mla_default", mla_sm100_annotations

1757 + ),

1758 + "table-standard": _feature_table(non_mla_backends, is_mla=False),

1759 + "table-minimax": _feature_table(minimax_backends, is_mla=False),

1760 + "table-mla-prefill": _mla_prefill_table(mla_prefill_backends),

1761 + "table-mla-decode": _feature_table(mla_backends, is_mla=True),

1762 + "table-mla-v4-decode": _feature_table(v4_decode_backends, is_mla=True),

1763 + }

1764

1765

2109 -if __name__ == "__main__":

2110 - main()

1766 +logger.info("Generating attention backend documentation")

1767 +fill_markers("design/attention_backends.md", build_blocks())

docs/mkdocs/gen_files/generate_examples.py renamed

+37 -44

@@ -5,16 +5,15 @@ import logging

5 from dataclasses import dataclass

6 from functools import cached_property

7 from pathlib import Path

8 -from typing import Literal

9 +import mkdocs_awesome_nav.nav.directory as _nav_dir

10 +import mkdocs_gen_files

11 import regex as re

13 logger = logging.getLogger("mkdocs")

15 ROOT_DIR = Path(__file__).parent.parent.parent.parent

15 -ROOT_DIR_RELATIVE = "../../../../.."

16 EXAMPLE_DIR = ROOT_DIR / "examples"

17 -EXAMPLE_DOC_DIR = ROOT_DIR / "docs/examples"

19 def title(text: str) -> str:

@@ -197,44 +196,38 @@ class Example:

196 return content

197

198

200 -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):

201 - # Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are

202 - # title-cased (e.g. "Offline Inference" instead of "Offline inference").

203 - import mkdocs_awesome_nav.nav.directory as _nav_dir

204 -

205 - _nav_dir.dirname_to_title = title

206 - logger.info("Generating example documentation")

207 - logger.debug("Root directory: %s", ROOT_DIR.resolve())

208 - logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())

209 - logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve())

210 -

211 - # Create the EXAMPLE_DOC_DIR if it doesn't exist

212 - if not EXAMPLE_DOC_DIR.exists():

213 - EXAMPLE_DOC_DIR.mkdir(parents=True)

214 -

215 - categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())

216 -

217 - examples = []

218 - glob_patterns = ["*.py", "*.md", "*.sh"]

219 - # Find categorised examples

220 - for category in categories:

221 - logger.info("Processing category: %s", category.stem)

222 - globs = [category.glob(pattern) for pattern in glob_patterns]

223 - for path in itertools.chain(*globs):

224 - examples.append(Example(path, category.stem))

225 - # Find examples in subdirectories

226 - globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]

227 - for path in itertools.chain(*globs):

228 - examples.append(Example(path.parent, category.stem))

229 -

230 - # Generate the example documentation

231 - for example in sorted(examples, key=lambda e: e.path.stem):

232 - example_name = f"{example.path.stem}.md"

233 - doc_path = EXAMPLE_DOC_DIR / example.category / example_name

234 - if not doc_path.parent.exists():

235 - doc_path.parent.mkdir(parents=True)

236 - # Specify encoding for building on Windows

237 - with open(doc_path, "w+", encoding="utf-8") as f:

238 - f.write(example.generate())

239 - logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))

240 - logger.info("Total examples generated: %d", len(examples))

199 +# Monkey-patch dirname_to_title in awesome-nav so that sub-directory names are

200 +# title-cased (e.g. "Offline Inference" instead of "Offline inference").

201 +_nav_dir.dirname_to_title = title

202 +logger.info("Generating example documentation")

203 +logger.debug("Root directory: %s", ROOT_DIR.resolve())

204 +logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())

205 +

206 +categories = sorted(

207 + p for p in EXAMPLE_DIR.iterdir() if p.is_dir() and not p.name.startswith(".")

208 +)

209 +

210 +examples = []

211 +glob_patterns = ["*.py", "*.md", "*.sh"]

212 +# Find categorised examples

213 +for category in categories:

214 + logger.info("Processing category: %s", category.stem)

215 + globs = [category.glob(pattern) for pattern in glob_patterns]

216 + for path in itertools.chain(*globs):

217 + examples.append(Example(path, category.stem))

218 + # Find examples in subdirectories

219 + globs = [category.glob(f"*/{pattern}") for pattern in glob_patterns]

220 + for path in itertools.chain(*globs):

221 + examples.append(Example(path.parent, category.stem))

222 +

223 +# Generate the example documentation

224 +for example in sorted(examples, key=lambda e: e.path.stem):

225 + doc_path = f"examples/{example.category}/{example.path.stem}.md"

226 + with mkdocs_gen_files.open(doc_path, "w") as f:

227 + f.write(example.generate())

228 + if example.main_file is not None:

229 + # Point the edit button at the example's source file

230 + edit_path = Path("..") / example.main_file.relative_to(ROOT_DIR)

231 + mkdocs_gen_files.set_edit_path(doc_path, str(edit_path))

232 + logger.debug("Example generated: %s", doc_path)

233 +logger.info("Total examples generated: %d", len(examples))

docs/mkdocs/gen_files/generate_metrics.py renamed

+36 -49

@@ -2,27 +2,28 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import ast

4 import logging

5 +import sys

6 from pathlib import Path

6 -from typing import Literal

7 +

8 +sys.path.insert(0, str(Path(__file__).parent))

9 +

10 +from generated_content import fill_markers # noqa: E402

12 logger = logging.getLogger("mkdocs")

14 ROOT_DIR = Path(__file__).parent.parent.parent.parent

11 -DOCS_DIR = ROOT_DIR / "docs"

12 -GENERATED_METRICS_DIR = DOCS_DIR / "generated" / "metrics"

14 -# Files to scan for metric definitions - each will generate a separate table

16 +# Files to scan for metric definitions - each fills a `gen:` marker in

17 +# docs/usage/metrics.md with its table (the section heading and any preamble

18 +# live in the tracked page next to the marker).

19 METRIC_SOURCE_FILES = [

16 - {"path": "vllm/v1/metrics/loggers.py", "output": "general.inc.md"},

17 - {

18 - "path": "vllm/v1/spec_decode/metrics.py",

19 - "output": "spec_decode.inc.md",

20 - },

20 + {"path": "vllm/v1/metrics/loggers.py", "key": "metrics-general"},

21 + {"path": "vllm/v1/spec_decode/metrics.py", "key": "metrics-spec-decode"},

22 {

23 "path": "vllm/distributed/kv_transfer/kv_connector/v1/nixl/stats.py",

23 - "output": "nixl_connector.inc.md",

24 + "key": "metrics-nixl",

25 },

25 - {"path": "vllm/v1/metrics/perf.py", "output": "perf.inc.md"},

26 + {"path": "vllm/v1/metrics/perf.py", "key": "metrics-mfu"},

27 ]

@@ -110,41 +111,27 @@ def generate_markdown_table(metrics: list[dict[str, str]]) -> str:

111 return "\n".join(lines) + "\n"

112

113

113 -def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):

114 - """Generate metrics documentation tables from source files."""

115 - logger.info("Generating metrics documentation")

116 -

117 - # Create generated directory if it doesn't exist

118 - GENERATED_METRICS_DIR.mkdir(parents=True, exist_ok=True)

119 -

120 - total_metrics = 0

121 - for source_config in METRIC_SOURCE_FILES:

122 - source_path = source_config["path"]

123 - output_file = source_config["output"]

124 -

125 - filepath = ROOT_DIR / source_path

126 - if not filepath.exists():

127 - raise FileNotFoundError(f"Metrics source file not found: {filepath}")

128 -

129 - logger.debug("Extracting metrics from: %s", source_path)

130 - metrics = extract_metrics_from_file(filepath)

131 - logger.debug("Found %d metrics in %s", len(metrics), source_path)

132 -

133 - # Generate and write the markdown table for this source

134 - table_content = generate_markdown_table(metrics)

135 - output_path = GENERATED_METRICS_DIR / output_file

136 - with open(output_path, "w", encoding="utf-8") as f:

137 - f.write(table_content)

138 -

139 - total_metrics += len(metrics)

140 - logger.info(

141 - "Generated metrics table: %s (%d metrics)",

142 - output_path.relative_to(ROOT_DIR),

143 - len(metrics),

144 - )

145 -

146 - logger.info(

147 - "Total metrics generated: %d across %d files",

148 - total_metrics,

149 - len(METRIC_SOURCE_FILES),

150 - )

114 +logger.info("Generating metrics documentation")

115 +

116 +blocks = {}

117 +total_metrics = 0

118 +for source_config in METRIC_SOURCE_FILES:

119 + source_path = source_config["path"]

120 +

121 + filepath = ROOT_DIR / source_path

122 + if not filepath.exists():

123 + raise FileNotFoundError(f"Metrics source file not found: {filepath}")

124 +

125 + logger.debug("Extracting metrics from: %s", source_path)

126 + metrics = extract_metrics_from_file(filepath)

127 + logger.debug("Found %d metrics in %s", len(metrics), source_path)

128 +

129 + blocks[source_config["key"]] = generate_markdown_table(metrics).strip()

130 + total_metrics += len(metrics)

131 +

132 +fill_markers("usage/metrics.md", blocks)

133 +logger.info(

134 + "Total metrics generated: %d across %d files",

135 + total_metrics,

136 + len(METRIC_SOURCE_FILES),

137 +)

docs/mkdocs/gen_files/generated_content.py new

+56

@@ -0,0 +1,56 @@

1 +# SPDX-License-Identifier: Apache-2.0

2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 +"""Inline build-time generated content into existing docs pages.

4 +

5 +Source pages mark where generated content goes with a snippet-style marker,

6 +`--8<-- "gen:<key>"`, so the insertion point is explicit and readable. The

7 +substitution happens here (at gen-files time, before mkdocs-gen-files shadows

8 +the page), not via pymdownx.snippets, so the content can be generated at build

9 +time without living in a real file on disk.

10 +

11 +The `gen:` prefix keeps these markers distinct from real pymdownx.snippets

12 +includes, and `fill_markers` fails loudly if a marker is missing or left behind

13 +(pymdownx.snippets would otherwise silently drop an unsubstituted marker).

14 +"""

15 +

16 +from pathlib import Path

17 +

18 +import mkdocs_gen_files

19 +import regex as re

20 +

21 +DOCS_DIR = Path(__file__).parent.parent.parent

22 +

23 +_MARKER = '--8<-- "gen:{key}"'

24 +_ANY_MARKER = re.compile(r'--8<-- "gen:[^"]*"')

25 +

26 +

27 +def fill_markers(doc_path: str, blocks: dict[str, str]) -> None:

28 + """Replace `--8<-- "gen:<key>"` markers in a docs page with generated content.

29 +

30 + Args:

31 + doc_path: Docs-relative path of the source page to fill.

32 + blocks: Mapping of marker key to the markdown to insert in its place.

33 +

34 + Raises:

35 + FileNotFoundError: If the source page does not exist.

36 + ValueError: If an expected marker is missing, or any `gen:` marker is

37 + left unsubstituted after filling.

38 + """

39 + source = DOCS_DIR / doc_path

40 + if not source.exists():

41 + raise FileNotFoundError(f"Cannot fill markers in missing page: {doc_path}")

42 +

43 + text = source.read_text()

44 + for key, content in blocks.items():

45 + marker = _MARKER.format(key=key)

46 + if marker not in text:

47 + raise ValueError(f"{doc_path}: missing marker {marker}")

48 + text = text.replace(marker, content)

49 +

50 + if leftover := _ANY_MARKER.search(text):

51 + raise ValueError(f"{doc_path}: unsubstituted marker {leftover.group()}")

52 +

53 + with mkdocs_gen_files.open(doc_path, "w") as f:

54 + f.write(text)

55 + # Keep the edit button pointing at the real source page

56 + mkdocs_gen_files.set_edit_path(doc_path, doc_path)

docs/mkdocs/hooks/url_schemes.py

+39 -4

@@ -19,6 +19,7 @@ The on_page_markdown hook passes the current page context to the preprocessor be

19 each page is converted.

20 """

22 +import posixpath

23 from pathlib import Path

25 import regex as re

@@ -38,18 +39,22 @@ TITLE = r"(?P<title>[^\[\]<>]+?)"

39 REPO = r"(?P<repo>.+?/.+?)"

40 TYPE = r"(?P<type>issues|pull|projects)"

41 NUMBER = r"(?P<number>\d+)"

42 +VERSION = r"[^/\s]+"

43 PATH = r"(?P<path>[^\s]+?)"

44 FRAGMENT = r"(?P<fragment>#[^\s]+)?"

43 -URL = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"

45 +URL_GITHUB = f"https://github.com/{REPO}/{TYPE}/{NUMBER}{FRAGMENT}"

46 RELATIVE = rf"(?!(https?|ftp)://|#){PATH}{FRAGMENT}"

47 +URL_DOCS = f"https://docs.vllm.ai/en/{VERSION}/{PATH}{FRAGMENT}"

49 # Common titles to use for GitHub links when none is provided in the link.

50 TITLES = {"issues": "Issue ", "pull": "Pull Request ", "projects": "Project "}

52 # Regex to match GitHub issue, PR, and project links with optional titles.

50 -github_link = re.compile(rf"(\[{TITLE}\]$|<){URL}($|>)")

53 +github_link = re.compile(rf"(\[{TITLE}\]$|<){URL_GITHUB}($|>)")

54 # Regex to match relative file links with optional titles.

55 relative_link = re.compile(rf"\[{TITLE}\]${RELATIVE}$")

56 +# Regex to match absolute docs.vllm.ai links (should only exist in CLI).

57 +docs_link = re.compile(rf"\[{TITLE}\]${URL_DOCS}$")

60 class UrlSchemesPreprocessor(Preprocessor):

@@ -61,7 +66,8 @@ class UrlSchemesPreprocessor(Preprocessor):

67 def run(self, lines):

68 page = self.ext.page

64 - if page is None or getattr(page.file, "abs_src_path", None) is None:

69 + files = self.ext.files

70 + if page is None:

71 return lines

73 def replace_relative_link(match: re.Match) -> str:

@@ -70,7 +76,7 @@ class UrlSchemesPreprocessor(Preprocessor):

76 """

77 title = match.group("title")

78 path = match.group("path")

73 - path = (Path(page.file.abs_src_path).parent / path).resolve()

79 + path = ((DOC_DIR / page.file.src_uri).parent / path).resolve()

80 fragment = match.group("fragment") or ""

82 # Check if the path exists and is outside the docs dir

@@ -105,9 +111,36 @@ class UrlSchemesPreprocessor(Preprocessor):

111 url = f"https://github.com/{repo}/{type}/{number}{fragment}"

112 return f"[{gh_icon} {title}]({url})"

113

114 + def replace_docs_link(match: re.Match) -> str:

115 + """Rewrite absolute docs.vllm.ai links as doc-relative links."""

116 + title = match.group("title")

117 + path = match.group("path").rstrip("/")

118 + fragment = match.group("fragment") or ""

119 +

120 + # vllm.config.<Class> API reference -> mkdocstrings cross-reference

121 + if path == "api/vllm/config" and re.fullmatch(

122 + r"#vllm\.config\.\w+", fragment

123 + ):

124 + ident = fragment[1:]

125 + return f"[`{ident}`][{ident}]"

126 +

127 + # Other docs pages -> link relative to the current page, but only

128 + # when the target is a known docs page (real or generated); leave

129 + # unknown/external URLs untouched. This is correct even when the same

130 + # docstring is also rendered on its API reference page.

131 + src = f"{path.removesuffix('.html')}.md"

132 + if files.get_file_from_path(src) is None:

133 + return match.group(0)

134 + rel = posixpath.relpath(src, posixpath.dirname(page.file.src_uri))

135 + # Auto-wrapped bare URLs use the URL as their title; make it readable.

136 + if title.startswith("http"):

137 + title = path.removesuffix(".html")

138 + return f"[{title}]({rel}{fragment})"

139 +

140 markdown = "\n".join(lines)

141 markdown = github_link.sub(replace_github_link, markdown)

142 markdown = relative_link.sub(replace_relative_link, markdown)

143 + markdown = docs_link.sub(replace_docs_link, markdown)

144 return markdown.split("\n")

145

146

@@ -116,6 +149,7 @@ class UrlSchemesExtension(Extension):

149

150 def __init__(self, **kwargs):

151 self.page = None

152 + self.files = None

153 super().__init__(**kwargs)

154

155 def extendMarkdown(self, md):

@@ -138,4 +172,5 @@ def on_page_markdown(

172 ) -> str:

173 """Pass the current page context to the preprocessor."""

174 _ext.page = page

175 + _ext.files = files

176 return markdown

docs/pre_run_check.sh

+1 -1

@@ -28,7 +28,7 @@ DOCS_PATHS=(

28 docs/ # Actual docs content

29 examples/ # Examples are rendered in docs

30 vllm/ # API & CLI reference

31 - requirements/test/cuda.txt # CLI reference (see docs/mkdocs/hooks/generate_argparse.py)

31 + requirements/test/cuda.txt # CLI reference (see docs/mkdocs/gen_files/generate_argparse.py)

32 mkdocs.yaml # Affects build process

33 .readthedocs.yaml # Affects build process

34 requirements/docs.txt # Affects build process

docs/usage/metrics.md

+4 -4

@@ -35,21 +35,21 @@ The following metrics are exposed:

36 ## General Metrics

38 ---8<-- "docs/generated/metrics/general.inc.md"

38 +--8<-- "gen:metrics-general"

40 ## Speculative Decoding Metrics

42 ---8<-- "docs/generated/metrics/spec_decode.inc.md"

42 +--8<-- "gen:metrics-spec-decode"

44 ## NIXL KV Connector Metrics

46 ---8<-- "docs/generated/metrics/nixl_connector.inc.md"

46 +--8<-- "gen:metrics-nixl"

48 ## Model Flops Utilization (MFU) Performance Metrics

50 These metrics are available via `--enable-mfu-metrics`:

52 ---8<-- "docs/generated/metrics/perf.inc.md"

52 +--8<-- "gen:metrics-mfu"

54 ## Deprecation Policy

mkdocs.yaml

+7 -10

@@ -3,7 +3,6 @@ site_url: !ENV READTHEDOCS_CANONICAL_URL

3 repo_url: https://github.com/vllm-project/vllm

4 edit_uri: edit/main/docs/

5 exclude_docs: |

6 - argparse

6 *.inc.md

7 *.template.md

8 theme:

@@ -50,24 +49,22 @@ theme:

50 hooks:

51 - docs/mkdocs/hooks/remove_announcement.py

53 - - docs/mkdocs/hooks/generate_examples.py

54 - - docs/mkdocs/hooks/generate_argparse.py

55 - - docs/mkdocs/hooks/generate_metrics.py

52 - docs/mkdocs/hooks/url_schemes.py

53 - docs/mkdocs/hooks/autoref_code.py

55 plugins:

56 - meta

57 - search

58 + - gen-files:

59 + scripts:

60 + - docs/mkdocs/gen_files/generate_examples.py

61 + - docs/mkdocs/gen_files/generate_argparse.py

62 + - docs/mkdocs/gen_files/generate_metrics.py

63 + - docs/mkdocs/gen_files/generate_attention_backends.py

64 - autorefs

65 - awesome-nav

66 - glightbox

65 - - git-revision-date-localized:

66 - # exclude autogenerated files

67 - exclude:

68 - - api/*

69 - - examples/*

70 - - generated/*

67 + - git-revision-date-localized

68 - minify:

69 minify_html: true

70 minify_js: true

vllm/config/cache.py

+4 -4

@@ -137,13 +137,13 @@ class CacheConfig:

137 still be controlled by mamba_cache_dtype). If set to 'auto', the data type

138 for the ssm state will be determined by mamba_cache_dtype."""

139 mamba_cache_mode: MambaCacheMode = "none"

140 - """The cache strategy for Mamba layers.

140 + """The cache strategy for Mamba layers:

141 +

142 - "none": set when prefix caching is disabled.

143 - "all": cache the mamba state of all tokens at position i * block_size. This is

143 - the default behavior (for models that support it) when prefix caching is

144 - enabled.

144 + the default behavior (for models that support it) when prefix caching is enabled.

145 - "align": only cache the mamba state of the last token of each scheduler step and

146 - when the token is at position i * block_size.

146 + when the token is at position i * block_size.

147 """

148 replayssm_buffer_len: int = Field(default=16, gt=0)

149 """ReplaySSM history buffer length B: with use_replayssm, standard decode

vllm/engine/arg_utils.py

+6 -4

@@ -249,17 +249,19 @@ def get_type_hints(type_hint: TypeHint) -> set[TypeHint]:

249

250 NEEDS_HELP = (

251 any("--help" in arg for arg in sys.argv) # vllm SUBCOMMAND --help

252 - or (argv0 := sys.argv[0]).endswith("mkdocs") # mkdocs SUBCOMMAND

253 - or argv0.endswith("mkdocs/__main__.py") # python -m mkdocs SUBCOMMAND

252 + or "mkdocs" in sys.modules # mkdocs SUBCOMMAND

253 )

254

255

256 def _maybe_add_docs_url(cls: Any) -> str:

257 """Generate API docs URL for a vllm config class."""

259 - if not cls.__module__.startswith("vllm.config"):

258 + import vllm.config

259 +

260 + name = cls.__name__

261 + if getattr(vllm.config, name, None) is not cls:

262 return ""

263 version = f"v{VLLM_VERSION}" if "dev" not in VLLM_VERSION else "latest"

262 - return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{cls.__name__}"

264 + return f"\n\nAPI docs: https://docs.vllm.ai/en/{version}/api/vllm/config/#vllm.config.{name}"

265

266

267 def _expand_json_human_readable_numbers(val: str) -> str:

vllm/entrypoints/cli/benchmark/mm_processor.py

+48 -1

@@ -1,6 +1,7 @@

1 # SPDX-License-Identifier: Apache-2.0

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

3 import argparse

4 +import inspect

6 from vllm.benchmarks.mm_processor import add_cli_args, main

7 from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase

@@ -8,13 +9,59 @@ from vllm.utils.argparse_utils import FlexibleArgumentParser

11 class BenchmarkMMProcessorSubcommand(BenchmarkSubcommandBase):

11 - """The `mm-processor` subcommand for `vllm bench`."""

12 + r"""`vllm bench mm-processor` profiles the multimodal input processor pipeline of

13 + vision-language models. It measures per-stage latency from the HuggingFace

14 + processor through to the encoder forward pass, helping you identify

15 + preprocessing bottlenecks and understand how different image resolutions or

16 + item counts affect end-to-end request time.

17 +

18 + The benchmark supports two data sources: synthetic random multimodal inputs

19 + (`random-mm`) and HuggingFace datasets (`hf`). Warmup requests are run before

20 + measurement to ensure stable results.

21 +

22 + ## Quick Start

23 +

24 + ```bash

25 + vllm bench mm-processor \

26 + --model Qwen/Qwen2-VL-7B-Instruct \

27 + --dataset-name random-mm \

28 + --num-prompts 50 \

29 + --random-input-len 300 \

30 + --random-output-len 40 \

31 + --random-mm-base-items-per-request 2 \

32 + --random-mm-limit-mm-per-prompt '{"image": 3, "video": 0}' \

33 + --random-mm-bucket-config '{(256, 256, 1): 0.7, (720, 1280, 1): 0.3}'

34 + ```

35 +

36 + ## Measured Stages

37 +

38 + | Stage | Description |

39 + | ----- | ----------- |

40 + | `get_mm_hashes_secs` | Time spent hashing multimodal inputs |

41 + | `get_cache_missing_items_secs` | Time spent looking up the processor cache |

42 + | `apply_hf_processor_secs` | Time spent in the HuggingFace processor |

43 + | `merge_mm_kwargs_secs` | Time spent merging multimodal kwargs |

44 + | `apply_prompt_updates_secs` | Time spent updating prompt tokens |

45 + | `preprocessor_total_secs` | Total preprocessing time |

46 + | `encoder_forward_secs` | Time spent in the encoder model forward pass |

47 + | `num_encoder_calls` | Number of encoder invocations per request |

48 +

49 + The benchmark also reports end-to-end latency (TTFT + decode time) per

50 + request. Use `--metric-percentiles` to select which percentiles to report

51 + (default: p99) and `--output-json` to save results.

52 +

53 + For more examples (HF datasets, warmup, JSON output), see the

54 + [multimodal processor benchmark guide](https://docs.vllm.ai/en/latest/benchmarking/cli/#multimodal-processor-benchmark).

55 + """

57 name = "mm-processor"

58 help = "Benchmark multimodal processor latency across different configurations."

60 @classmethod

61 def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:

62 + # The class docstring is the page overview / `--help` description.

63 + if cls.__doc__:

64 + parser.description = inspect.cleandoc(cls.__doc__)

65 add_cli_args(parser)

67 @staticmethod

vllm/entrypoints/cli/launch.py

+16 -3

@@ -2,6 +2,7 @@

2 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project

4 import argparse

5 +import inspect

6 import signal

8 import uvloop

@@ -36,9 +37,12 @@ class LaunchSubcommandBase(CLISubcommand):

37 def add_cli_args(cls, parser: FlexibleArgumentParser) -> None:

38 """Add the CLI arguments to the parser.

39 - By default, adds the standard vLLM serving arguments.

40 + By default, uses the subcommand's docstring as the description and adds

41 + the standard vLLM serving arguments.

42 Subclasses can override to add component-specific arguments.

43 """

44 + if cls.__doc__:

45 + parser.description = inspect.cleandoc(cls.__doc__)

46 make_arg_parser(parser)

48 @staticmethod

@@ -47,7 +51,17 @@ class LaunchSubcommandBase(CLISubcommand):

53 class RenderSubcommand(LaunchSubcommandBase):

50 - """The `render` subcommand for `vllm launch`."""

54 + """`vllm launch render` starts a GPU-less rendering server for preprocessing

55 + and postprocessing only.

56 +

57 + ```bash

58 + vllm launch render meta-llama/Llama-3.2-1B-Instruct --port 8100

59 + ```

60 +

61 + This command reuses the standard serving parser, so model, frontend,

62 + networking, and related CLI options follow the same conventions as

63 + [`vllm serve`](https://docs.vllm.ai/en/latest/cli/serve/).

64 + """

66 name = "render"

67 help = "Launch a GPU-less rendering server (preprocessing and postprocessing only)."

@@ -93,7 +107,6 @@ class LaunchSubcommand(CLISubcommand):

107 cmd_subparser = launch_subparsers.add_parser(

108 cmd_cls.name,

109 help=cmd_cls.help,

96 - description=cmd_cls.help,

110 usage=f"vllm {self.name} {cmd_cls.name} [options]",

111 )

112 cmd_subparser.set_defaults(launch_command=cmd_cls.cmd)