A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 22:15 UTC
Remove need for explicit `\n` in docstring lists for `--help` formatting (#38350)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Harry Mellor committed
Mar 27, 2026 at 15:38 UTC
21d2b53f88d99f9ab369444f6d53ed2b9c260e4f
10 files changed
+89
-82
vllm/config/cache.py
+12
-13
@@ -66,22 +66,21 @@ class CacheConfig:
66
enable_prefix_caching: bool = True
67
"""Whether to enable prefix caching."""
68
prefix_caching_hash_algo: PrefixCachingHashAlgo = "sha256"
69
- """Set the hash algorithm for prefix caching:\n
70
- - "sha256" uses Pickle for object serialization before hashing. This is the
71
- current default, as SHA256 is the most secure choice to avoid potential
72
- hash collisions.\n
69
+ """Set the hash algorithm for prefix caching:
70
+
71
+ - "sha256" uses Pickle for object serialization before hashing. This is the current
72
+ default, as SHA256 is the most secure choice to avoid potential hash collisions.
73
- "sha256_cbor" provides a reproducible, cross-language compatible hash. It
74
- serializes objects using canonical CBOR and hashes them with SHA-256.\n
74
+ serializes objects using canonical CBOR and hashes them with SHA-256.
75
- "xxhash" uses Pickle serialization with xxHash (128-bit) for faster,
76
- non-cryptographic hashing. Requires the optional ``xxhash`` package.
77
- IMPORTANT: Use of a hashing algorithm that is not considered
78
- cryptographically secure theoretically increases the risk of hash collisions,
79
- which can cause undefined behavior or even leak private information in
80
- multi-tenant environments. Even if collisions are still very unlikely, it is
81
- important to consider your security risk tolerance against the performance
82
- benefits before turning this on.\n
76
+ non-cryptographic hashing. Requires the optional ``xxhash`` package.
77
+ IMPORTANT: Use of a hashing algorithm that is not considered cryptographically
78
+ secure theoretically increases the risk of hash collisions, which can cause
79
+ undefined behavior or even leak private information in multi-tenant environments.
80
+ Even if collisions are still very unlikely, it is important to consider your
81
+ security risk tolerance against the performance benefits before turning this on.
82
- "xxhash_cbor" combines canonical CBOR serialization with xxHash for
84
- reproducible hashing. Requires the optional ``xxhash`` package."""
83
+ reproducible hashing. Requires the optional ``xxhash`` package."""
84
calculate_kv_scales: bool = False
85
"""Deprecated: This option is deprecated and will be removed in v0.19.
86
It enables dynamic calculation of `k_scale` and `v_scale` when
vllm/config/kernel.py
+8
-8
@@ -32,14 +32,14 @@ class KernelConfig:
32
moe_backend: MoEBackend = "auto"
33
"""Backend for MoE expert computation kernels. Available options:
34
35
- - "auto": Automatically select the best backend based on model and hardware\n
36
- - "triton": Use Triton-based fused MoE kernels\n
37
- - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)\n
38
- - "cutlass": Use vLLM CUTLASS kernels\n
39
- - "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels\n
40
- - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels\n
41
- - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)\n
42
- - "marlin": Use Marlin kernels (weight-only quantization)\n
35
+ - "auto": Automatically select the best backend based on model and hardware
36
+ - "triton": Use Triton-based fused MoE kernels
37
+ - "deep_gemm": Use DeepGEMM kernels (FP8 block-quantized only)
38
+ - "cutlass": Use vLLM CUTLASS kernels
39
+ - "flashinfer_trtllm": Use FlashInfer with TRTLLM-GEN kernels
40
+ - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
41
+ - "flashinfer_cutedsl": Use FlashInfer with CuteDSL kernels (FP4 only)
42
+ - "marlin": Use Marlin kernels (weight-only quantization)
43
- "aiter": Use AMD AITer kernels (ROCm only)"""
44
45
@field_validator("moe_backend", mode="before")
vllm/config/load.py
+1
-1
@@ -51,7 +51,7 @@ class LoadConfig:
51
- "gguf" will load weights from GGUF format files (details specified in
52
https://github.com/ggml-org/ggml/blob/master/docs/gguf.md).
53
- "mistral" will load weights from consolidated safetensors files used by
54
- Mistral models.\n
54
+ Mistral models.
55
- Other custom values can be supported via plugins.
56
"""
57
download_dir: str | None = None
vllm/config/model.py
+41
-35
@@ -125,26 +125,28 @@ class ModelConfig:
125
"""Name or path of the Hugging Face tokenizer to use. If unspecified, model
126
name or path will be used."""
127
tokenizer_mode: TokenizerMode | str = "auto"
128
- """Tokenizer mode:\n
128
+ """Tokenizer mode:
129
+
130
- "auto" will use the tokenizer from `mistral_common` for Mistral models
130
- if available, otherwise it will use the "hf" tokenizer.\n
131
- - "hf" will use the fast tokenizer if available.\n
132
- - "slow" will always use the slow tokenizer.\n
133
- - "mistral" will always use the tokenizer from `mistral_common`.\n
134
- - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n
135
- - "qwen_vl" will always use the tokenizer from `qwen_vl`.\n
131
+ if available, otherwise it will use the "hf" tokenizer.
132
+ - "hf" will use the fast tokenizer if available.
133
+ - "slow" will always use the slow tokenizer.
134
+ - "mistral" will always use the tokenizer from `mistral_common`.
135
+ - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.
136
+ - "qwen_vl" will always use the tokenizer from `qwen_vl`.
137
- Other custom values can be supported via plugins."""
138
trust_remote_code: bool = False
139
"""Trust remote code (e.g., from HuggingFace) when downloading the model
140
and tokenizer."""
141
dtype: ModelDType | torch.dtype = "auto"
141
- """Data type for model weights and activations:\n
142
+ """Data type for model weights and activations:
143
+
144
- "auto" will use FP16 precision for FP32 and FP16 models, and BF16
143
- precision for BF16 models.\n
144
- - "half" for FP16. Recommended for AWQ quantization.\n
145
- - "float16" is the same as "half".\n
146
- - "bfloat16" for a balance between precision and range.\n
147
- - "float" is shorthand for FP32 precision.\n
145
+ precision for BF16 models.
146
+ - "half" for FP16. Recommended for AWQ quantization.
147
+ - "float16" is the same as "half".
148
+ - "bfloat16" for a balance between precision and range.
149
+ - "float" is shorthand for FP32 precision.
150
- "float32" for FP32 precision."""
151
seed: int = 0
152
"""Random seed for reproducibility.
@@ -182,13 +184,14 @@ class ModelConfig:
184
automatically derived from the model config.
185
186
When passing via `--max-model-len`, supports k/m/g/K/M/G in human-readable
185
- format. Examples:\n
186
- - 1k -> 1000\n
187
- - 1K -> 1024\n
188
- - 25.6k -> 25,600\n
187
+ format. Examples:
188
+
189
+ - 1k -> 1000
190
+ - 1K -> 1024
191
+ - 25.6k -> 25,600
192
- -1 or 'auto' -> Automatically choose the maximum model length that fits in
190
- GPU memory. This will use the model's maximum context length if it fits,
191
- otherwise it will find the largest length that can be accommodated."""
193
+ GPU memory. This will use the model's maximum context length if it fits,
194
+ otherwise it will find the largest length that can be accommodated."""
195
spec_target_max_model_len: int | None = None
196
"""Specify the maximum length for spec decoding draft models."""
197
quantization: QuantizationMethods | str | None = None
@@ -248,10 +251,11 @@ class ModelConfig:
251
prometheus metrics, if multiple names provided, metrics tag will take the
252
first one."""
253
config_format: str | ConfigFormat = "auto"
251
- """The format of the model config to load:\n
254
+ """The format of the model config to load:
255
+
256
- "auto" will try to load the config in hf format if available after trying
253
- to load in mistral format.\n
254
- - "hf" will load the config in hf format.\n
257
+ to load in mistral format.
258
+ - "hf" will load the config in hf format.
259
- "mistral" will load the config in mistral format."""
260
hf_token: bool | str | None = None
261
"""The token to use as HTTP bearer authorization for remote files . If
@@ -276,12 +280,12 @@ class ModelConfig:
280
"""Enable sleep mode for the engine (only cuda and
281
hip platforms are supported)."""
282
model_impl: str | ModelImpl = "auto"
279
- """Which implementation of the model to use:\n
280
- - "auto" will try to use the vLLM implementation, if it exists, and fall
281
- back to the Transformers implementation if no vLLM implementation is
282
- available.\n
283
- - "vllm" will use the vLLM model implementation.\n
284
- - "transformers" will use the Transformers model implementation.\n
283
+ """Which implementation of the model to use:
284
+
285
+ - "auto" will try to use the vLLM implementation, if it exists, and fall back to the
286
+ Transformers implementation if no vLLM implementation is available.
287
+ - "vllm" will use the vLLM model implementation.
288
+ - "transformers" will use the Transformers model implementation.
289
- "terratorch" will use the TerraTorch model implementation.
290
"""
291
override_attention_dtype: str | None = None
@@ -1512,10 +1516,11 @@ class ModelConfig:
1516
@property
1517
def score_type(self) -> ScoreType:
1518
"""
1515
- Scoring API handles score/rerank for:\n
1516
- - "classify" task (score_type: cross-encoder models)\n
1517
- - "embed" task (score_type: bi-encoder models)\n
1518
- - "token_embed" task (score_type: late interaction models)\n
1519
+ Scoring API handles score/rerank for:
1520
+
1521
+ - "classify" task (score_type: cross-encoder models)
1522
+ - "embed" task (score_type: bi-encoder models)
1523
+ - "token_embed" task (score_type: late interaction models)
1524
"""
1525
# fixme: self._model_info.score_type is the score type before
1526
# as_seq_cls_model, which is "bi-encoder", rather than the
@@ -1593,9 +1598,10 @@ class ModelConfig:
1598
such as the lm_head in a generation model,
1599
or the score or classifier in a classification model.
1600
1596
- `head_dtype` currently only supports pooling models.\n
1597
- - The pooling model defaults to using fp32 head,
1598
- you can use --hf-overrides '{"head_dtype": "model"}' to disable it.
1601
+ `head_dtype` currently only supports pooling models.
1602
+
1603
+ - The pooling model defaults to using fp32 head, you can use
1604
+ --hf-overrides '{"head_dtype": "model"}' to disable it.
1605
"""
1606
1607
head_dtype = _get_head_dtype(
vllm/config/multimodal.py
+7
-7
@@ -146,14 +146,14 @@ class MultiModalConfig:
146
parallelism (TP).
147
148
- `"weights"`: Within the same vLLM engine, split the weights of
149
- each layer across TP ranks. (default TP behavior)\n
149
+ each layer across TP ranks. (default TP behavior)
150
- `"data"`: Within the same vLLM engine, split the batched input data
151
- across TP ranks to process the data in parallel, while hosting
152
- the full weights on each TP rank.
153
- This batch-level DP is not to be confused with API request-level
154
- DP (which is controlled by `--data-parallel-size`).
155
- This is only supported on a per-model basis and falls back to
156
- `"weights"` if the encoder does not support DP."""
151
+ across TP ranks to process the data in parallel, while hosting
152
+ the full weights on each TP rank.
153
+ This batch-level DP is not to be confused with API request-level
154
+ DP (which is controlled by `--data-parallel-size`).
155
+ This is only supported on a per-model basis and falls back to
156
+ `"weights"` if the encoder does not support DP."""
157
mm_encoder_attn_backend: AttentionBackendEnum | None = None
158
"""Optional override for the multi-modal encoder attention backend when
159
using vision transformers. Accepts any value from
vllm/config/parallel.py
+8
-7
@@ -148,10 +148,11 @@ class ParallelConfig:
148
eplb_config: EPLBConfig = Field(default_factory=EPLBConfig)
149
"""Expert parallelism configuration."""
150
expert_placement_strategy: ExpertPlacementStrategy = "linear"
151
- """The expert placement strategy for MoE layers:\n
151
+ """The expert placement strategy for MoE layers:
152
+
153
- "linear": Experts are placed in a contiguous manner. For example, with 4
154
experts and 2 ranks, rank 0 will have experts [0, 1] and rank 1 will have
154
- experts [2, 3].\n
155
+ experts [2, 3].
156
- "round_robin": Experts are placed in a round-robin manner. For example,
157
with 4 experts and 2 ranks, rank 0 will have experts [0, 2] and rank 1
158
will have experts [1, 3]. This strategy can help improve load balancing
@@ -159,11 +160,11 @@ class ParallelConfig:
160
all2all_backend: All2AllBackend = "allgather_reducescatter"
161
"""All2All backend for MoE expert parallel communication. Available options:
162
162
- - "allgather_reducescatter": All2all based on allgather and reducescatter\n
163
- - "deepep_high_throughput": Use deepep high-throughput kernels\n
164
- - "deepep_low_latency": Use deepep low-latency kernels\n
165
- - "mori": Use mori kernels\n
166
- - "nixl_ep": Use nixl-ep kernels\n
163
+ - "allgather_reducescatter": All2all based on allgather and reducescatter
164
+ - "deepep_high_throughput": Use deepep high-throughput kernels
165
+ - "deepep_low_latency": Use deepep low-latency kernels
166
+ - "mori": Use mori kernels
167
+ - "nixl_ep": Use nixl-ep kernels
168
- "flashinfer_nvlink_two_sided": Use flashinfer two-sided kernels for mnnvl
169
- "flashinfer_nvlink_one_sided": Use flashinfer high-throughput a2a kernels"""
170
vllm/config/profiler.py
+1
-1
@@ -37,7 +37,7 @@ class ProfilerConfig:
37
profiler: ProfilerKind | None = None
38
"""Which profiler to use. Defaults to None. Options are:
39
40
- - 'torch': Use PyTorch profiler.\n
40
+ - 'torch': Use PyTorch profiler.
41
- 'cuda': Use CUDA profiler."""
42
43
torch_profiler_dir: str = ""
vllm/config/scheduler.py
+5
-4
@@ -106,11 +106,12 @@ class SchedulerConfig:
106
max_num_batched_tokens in case max multimodal embedding size is larger."""
107
108
policy: SchedulerPolicy = "fcfs"
109
- """The scheduling policy to use:\n
110
- - "fcfs" means first come first served, i.e. requests are handled in order
111
- of arrival.\n
109
+ """The scheduling policy to use:
110
+
111
+ - "fcfs" means first come first served, i.e. requests are handled in order
112
+ of arrival.
113
- "priority" means requests are handled based on given priority (lower
113
- value means earlier handling) and time of arrival deciding any ties)."""
114
+ value means earlier handling) and time of arrival deciding any ties)."""
115
116
disable_chunked_mm_input: bool = False
117
"""If set to true and chunked prefill is enabled, we do not want to
vllm/config/utils.py
+2
@@ -37,10 +37,12 @@ ConfigT = TypeVar("ConfigT", bound=DataclassInstance)
37
38
39
@overload
40
+@dataclass_transform(field_specifiers=(PydanticField,))
41
def config(cls: type[ConfigT]) -> type[ConfigT]: ...
42
43
44
@overload
45
+@dataclass_transform(field_specifiers=(PydanticField,))
46
def config(
47
*, config: ConfigDict | None = None, **kwargs: Any
48
) -> Callable[[type[ConfigT]], type[ConfigT]]: ...
vllm/utils/argparse_utils.py
+4
-6
@@ -31,14 +31,12 @@ class SortedHelpFormatter(ArgumentDefaultsHelpFormatter, RawDescriptionHelpForma
31
def _split_lines(self, text, width):
32
"""
33
1. Sentences split across lines have their single newlines removed.
34
- 2. Paragraphs and explicit newlines are split into separate lines.
34
+ 2. Paragraphs and lists are split into separate lines.
35
3. Each line is wrapped to the specified width (width of terminal).
36
"""
37
- # The patterns also include whitespace after the newline
38
- single_newline = re.compile(r"(?<!\n)\n(?!\n)\s*")
39
- multiple_newlines = re.compile(r"\n{2,}\s*")
40
- text = single_newline.sub(" ", text)
41
- lines = re.split(multiple_newlines, text)
37
+ # The pattern also includes whitespace after the newline
38
+ newlines_to_remove = re.compile(r"(?<!\n)\n(?!\n)(?!\s*(-|\*|\+|\d+\.))\s*")
39
+ lines = newlines_to_remove.sub(" ", text).splitlines()
40
return sum([textwrap.wrap(line, width) for line in lines], [])
41
42
def add_arguments(self, actions):