A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 02:15 UTC
[Doc] Add engine args back in to the docs (#20674)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Jul 10, 2025 at 16:02 UTC
3482fd7e4e52848644a85d70e20d468d8f44e12a
14 files changed
+218
-40
.gitignore
+1
@@ -146,6 +146,7 @@ venv.bak/
146
147
# mkdocs documentation
148
/site
149
+docs/argparse
150
docs/examples
151
152
# mypy
docs/configuration/engine_args.md
+10
-5
@@ -1,3 +1,7 @@
1
+---
2
+toc_depth: 3
3
+---
4
+
5
# Engine Arguments
6
7
Engine arguments control the behavior of the vLLM engine.
@@ -5,11 +9,12 @@ Engine arguments control the behavior of the vLLM engine.
9
- For [offline inference](../serving/offline_inference.md), they are part of the arguments to [LLM][vllm.LLM] class.
10
- For [online serving](../serving/openai_compatible_server.md), they are part of the arguments to `vllm serve`.
11
8
-You can look at [EngineArgs][vllm.engine.arg_utils.EngineArgs] and [AsyncEngineArgs][vllm.engine.arg_utils.AsyncEngineArgs] to see the available engine arguments.
12
+The engine argument classes, [EngineArgs][vllm.engine.arg_utils.EngineArgs] and [AsyncEngineArgs][vllm.engine.arg_utils.AsyncEngineArgs], are a combination of the configuration classes defined in [vllm.config][]. Therefore, if you are interested in developer documentation, we recommend looking at these configuration classes as they are the source of truth for types, defaults and docstrings.
13
+
14
+## `EngineArgs`
15
10
-However, these classes are a combination of the configuration classes defined in [vllm.config][]. Therefore, we would recommend you read about them there where they are best documented.
16
+--8<-- "docs/argparse/engine_args.md"
17
12
-For offline inference you will have access to these configuration classes and for online serving you can cross-reference the configs with `vllm serve --help`, which has its arguments grouped by config.
18
+## `AsyncEngineArgs`
19
14
-!!! note
15
- Additional arguments are available to the [AsyncLLMEngine][vllm.engine.async_llm_engine.AsyncLLMEngine] which is used for online serving. These can be found by running `vllm serve --help`
20
+--8<-- "docs/argparse/async_engine_args.md"
docs/mkdocs/hooks/generate_argparse.py
new
+105
@@ -0,0 +1,105 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import logging
4
+import sys
5
+from argparse import SUPPRESS, HelpFormatter
6
+from pathlib import Path
7
+from typing import Literal
8
+from unittest.mock import MagicMock, patch
9
+
10
+ROOT_DIR = Path(__file__).parent.parent.parent.parent
11
+ARGPARSE_DOC_DIR = ROOT_DIR / "docs/argparse"
12
+
13
+sys.path.insert(0, str(ROOT_DIR))
14
+sys.modules["aiohttp"] = MagicMock()
15
+sys.modules["blake3"] = MagicMock()
16
+sys.modules["vllm._C"] = MagicMock()
17
+
18
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs # noqa: E402
19
+from vllm.utils import FlexibleArgumentParser # noqa: E402
20
+
21
+logger = logging.getLogger("mkdocs")
22
+
23
+
24
+class MarkdownFormatter(HelpFormatter):
25
+ """Custom formatter that generates markdown for argument groups."""
26
+
27
+ def __init__(self, prog):
28
+ super().__init__(prog,
29
+ max_help_position=float('inf'),
30
+ width=float('inf'))
31
+ self._markdown_output = []
32
+
33
+ def start_section(self, heading):
34
+ if heading not in {"positional arguments", "options"}:
35
+ self._markdown_output.append(f"\n### {heading}\n\n")
36
+
37
+ def end_section(self):
38
+ pass
39
+
40
+ def add_text(self, text):
41
+ if text:
42
+ self._markdown_output.append(f"{text.strip()}\n\n")
43
+
44
+ def add_usage(self, usage, actions, groups, prefix=None):
45
+ pass
46
+
47
+ def add_arguments(self, actions):
48
+ for action in actions:
49
+
50
+ option_strings = f'`{"`, `".join(action.option_strings)}`'
51
+ self._markdown_output.append(f"#### {option_strings}\n\n")
52
+
53
+ if choices := action.choices:
54
+ choices = f'`{"`, `".join(str(c) for c in choices)}`'
55
+ self._markdown_output.append(
56
+ f"Possible choices: {choices}\n\n")
57
+
58
+ self._markdown_output.append(f"{action.help}\n\n")
59
+
60
+ if (default := action.default) != SUPPRESS:
61
+ self._markdown_output.append(f"Default: `{default}`\n\n")
62
+
63
+ def format_help(self):
64
+ """Return the formatted help as markdown."""
65
+ return "".join(self._markdown_output)
66
+
67
+
68
+def create_parser(cls, **kwargs) -> FlexibleArgumentParser:
69
+ """Create a parser for the given class with markdown formatting.
70
+
71
+ Args:
72
+ cls: The class to create a parser for
73
+ **kwargs: Additional keyword arguments to pass to `cls.add_cli_args`.
74
+
75
+ Returns:
76
+ FlexibleArgumentParser: A parser with markdown formatting for the class.
77
+ """
78
+ parser = FlexibleArgumentParser()
79
+ parser.formatter_class = MarkdownFormatter
80
+ with patch("vllm.config.DeviceConfig.__post_init__"):
81
+ return cls.add_cli_args(parser, **kwargs)
82
+
83
+
84
+def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
85
+ logger.info("Generating argparse documentation")
86
+ logger.debug("Root directory: %s", ROOT_DIR.resolve())
87
+ logger.debug("Output directory: %s", ARGPARSE_DOC_DIR.resolve())
88
+
89
+ # Create the ARGPARSE_DOC_DIR if it doesn't exist
90
+ if not ARGPARSE_DOC_DIR.exists():
91
+ ARGPARSE_DOC_DIR.mkdir(parents=True)
92
+
93
+ # Create parsers to document
94
+ parsers = {
95
+ "engine_args": create_parser(EngineArgs),
96
+ "async_engine_args": create_parser(AsyncEngineArgs,
97
+ async_args_only=True),
98
+ }
99
+
100
+ # Generate documentation for each parser
101
+ for stem, parser in parsers.items():
102
+ doc_path = ARGPARSE_DOC_DIR / f"{stem}.md"
103
+ with open(doc_path, "w") as f:
104
+ f.write(parser.format_help())
105
+ logger.info("Argparse generated: %s", doc_path.relative_to(ROOT_DIR))
docs/mkdocs/hooks/generate_examples.py
+1
-1
@@ -161,8 +161,8 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
161
for example in sorted(examples, key=lambda e: e.path.stem):
162
example_name = f"{example.path.stem}.md"
163
doc_path = EXAMPLE_DOC_DIR / example.category / example_name
164
- logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
164
if not doc_path.parent.exists():
165
doc_path.parent.mkdir(parents=True)
166
with open(doc_path, "w+") as f:
167
f.write(example.generate())
168
+ logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
docs/mkdocs/overrides/partials/toc-item.html
new
+21
@@ -0,0 +1,21 @@
1
+<!-- Enables the use of toc_depth in document frontmatter https://github.com/squidfunk/mkdocs-material/issues/4827#issuecomment-1869812019 -->
2
+<li class="md-nav__item">
3
+ <a href="{{ toc_item.url }}" class="md-nav__link">
4
+ <span class="md-ellipsis">
5
+ {{ toc_item.title }}
6
+ </span>
7
+ </a>
8
+
9
+ <!-- Table of contents list -->
10
+ {% if toc_item.children %}
11
+ <nav class="md-nav" aria-label="{{ toc_item.title | striptags }}">
12
+ <ul class="md-nav__list">
13
+ {% for toc_item in toc_item.children %}
14
+ {% if not page.meta.toc_depth or toc_item.level <= page.meta.toc_depth %}
15
+ {% include "partials/toc-item.html" %}
16
+ {% endif %}
17
+ {% endfor %}
18
+ </ul>
19
+ </nav>
20
+ {% endif %}
21
+ </li>
\ No newline at end of file
mkdocs.yaml
+2
@@ -3,6 +3,7 @@ site_url: https://docs.vllm.ai
3
repo_url: https://github.com/vllm-project/vllm
4
edit_uri: edit/main/docs/
5
exclude_docs: |
6
+ argparse
7
*.inc.md
8
*.template.md
9
theme:
@@ -47,6 +48,7 @@ theme:
48
hooks:
49
- docs/mkdocs/hooks/remove_announcement.py
50
- docs/mkdocs/hooks/generate_examples.py
51
+ - docs/mkdocs/hooks/generate_argparse.py
52
- docs/mkdocs/hooks/url_schemes.py
53
54
# Required to stop api-autonav from raising an error
requirements/docs.txt
+15
@@ -7,3 +7,18 @@ mkdocs-awesome-nav
7
python-markdown-math
8
regex
9
ruff
10
+
11
+# Required for argparse hook only
12
+-f https://download.pytorch.org/whl/cpu
13
+cachetools
14
+cloudpickle
15
+fastapi
16
+msgspec
17
+openai
18
+pillow
19
+psutil
20
+pybase64
21
+pydantic
22
+torch
23
+transformers
24
+zmq
vllm/engine/arg_utils.py
+26
-16
@@ -12,8 +12,9 @@ import threading
12
import warnings
13
from dataclasses import MISSING, dataclass, fields, is_dataclass
14
from itertools import permutations
15
-from typing import (Annotated, Any, Callable, Dict, List, Literal, Optional,
16
- Type, TypeVar, Union, cast, get_args, get_origin)
15
+from typing import (TYPE_CHECKING, Annotated, Any, Callable, Dict, List,
16
+ Literal, Optional, Type, TypeVar, Union, cast, get_args,
17
+ get_origin)
18
19
import regex as re
20
import torch
@@ -33,20 +34,26 @@ from vllm.config import (BlockSize, CacheConfig, CacheDType, CompilationConfig,
34
SchedulerConfig, SchedulerPolicy, SpeculativeConfig,
35
TaskOption, TokenizerMode, TokenizerPoolConfig,
36
VllmConfig, get_attr_docs, get_field)
36
-from vllm.executor.executor_base import ExecutorBase
37
from vllm.logger import init_logger
38
-from vllm.model_executor.layers.quantization import QuantizationMethods
38
from vllm.platforms import CpuArchEnum, current_platform
39
from vllm.plugins import load_general_plugins
40
from vllm.reasoning import ReasoningParserManager
41
from vllm.test_utils import MODEL_WEIGHTS_S3_BUCKET, MODELS_ON_S3
42
from vllm.transformers_utils.utils import check_gguf_file
44
-from vllm.usage.usage_lib import UsageContext
43
from vllm.utils import (STR_DUAL_CHUNK_FLASH_ATTN_VAL, FlexibleArgumentParser,
44
GiB_bytes, get_ip, is_in_ray_actor)
45
46
# yapf: enable
47
48
+if TYPE_CHECKING:
49
+ from vllm.executor.executor_base import ExecutorBase
50
+ from vllm.model_executor.layers.quantization import QuantizationMethods
51
+ from vllm.usage.usage_lib import UsageContext
52
+else:
53
+ ExecutorBase = Any
54
+ QuantizationMethods = Any
55
+ UsageContext = Any
56
+
57
logger = init_logger(__name__)
58
59
# object is used to allow for special typing forms
@@ -200,14 +207,17 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, Any]:
207
kwargs[name] = {"default": default, "help": help}
208
209
# Set other kwargs based on the type hints
203
- json_tip = """\n\nShould either be a valid JSON string or JSON keys
204
- passed individually. For example, the following sets of arguments are
205
- equivalent:\n\n
206
- - `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`\n
207
- - `--json-arg.key1 value1 --json-arg.key2.key3 value2`\n
208
- Additionally, list elements can be passed individually using '+':
209
- - `--json-arg '{"key4": ["value3", "value4", "value5"]}'`\n
210
- - `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`\n\n"""
210
+ json_tip = """Should either be a valid JSON string or JSON keys
211
+passed individually. For example, the following sets of arguments are
212
+equivalent:
213
+
214
+- `--json-arg '{"key1": "value1", "key2": {"key3": "value2"}}'`\n
215
+- `--json-arg.key1 value1 --json-arg.key2.key3 value2`
216
+
217
+Additionally, list elements can be passed individually using `+`:
218
+
219
+- `--json-arg '{"key4": ["value3", "value4", "value5"]}'`\n
220
+- `--json-arg.key4+ value3 --json-arg.key4+='value4,value5'`"""
221
if dataclass_cls is not None:
222
223
def parse_dataclass(val: str, cls=dataclass_cls) -> Any:
@@ -219,7 +229,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, Any]:
229
raise argparse.ArgumentTypeError(repr(e)) from e
230
231
kwargs[name]["type"] = parse_dataclass
222
- kwargs[name]["help"] += json_tip
232
+ kwargs[name]["help"] += f"\n\n{json_tip}"
233
elif contains_type(type_hints, bool):
234
# Creates --no-<name> and --<name> flags
235
kwargs[name]["action"] = argparse.BooleanOptionalAction
@@ -255,7 +265,7 @@ def _compute_kwargs(cls: ConfigType) -> dict[str, Any]:
265
kwargs[name]["type"] = union_dict_and_str
266
elif contains_type(type_hints, dict):
267
kwargs[name]["type"] = parse_type(json.loads)
258
- kwargs[name]["help"] += json_tip
268
+ kwargs[name]["help"] += f"\n\n{json_tip}"
269
elif (contains_type(type_hints, str)
270
or any(is_not_builtin(th) for th in type_hints)):
271
kwargs[name]["type"] = str
@@ -1545,7 +1555,6 @@ class EngineArgs:
1555
# Enable chunked prefill by default for long context (> 32K)
1556
# models to avoid OOM errors in initial memory profiling phase.
1557
elif use_long_context:
1548
- from vllm.platforms import current_platform
1558
is_gpu = current_platform.is_cuda()
1559
use_sliding_window = (model_config.get_sliding_window()
1560
is not None)
@@ -1653,6 +1662,7 @@ class EngineArgs:
1662
# NOTE(Kuntai): Setting large `max_num_batched_tokens` for A100 reduces
1663
# throughput, see PR #17885 for more details.
1664
# So here we do an extra device name check to prevent such regression.
1665
+ from vllm.usage.usage_lib import UsageContext
1666
if device_memory >= 70 * GiB_bytes and "a100" not in device_name:
1667
# For GPUs like H100 and MI300x, use larger default values.
1668
default_max_num_batched_tokens = {
vllm/entrypoints/chat_utils.py
+1
-1
@@ -38,7 +38,6 @@ from typing_extensions import Required, TypeAlias, TypedDict
38
39
from vllm.config import ModelConfig
40
from vllm.logger import init_logger
41
-from vllm.model_executor.model_loader import get_model_cls
41
from vllm.model_executor.models import SupportsMultiModal
42
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict
43
from vllm.multimodal.utils import MediaConnector
@@ -524,6 +523,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
523
524
@cached_property
525
def model_cls(self):
526
+ from vllm.model_executor.model_loader import get_model_cls
527
return get_model_cls(self.model_config)
528
529
@property
vllm/inputs/registry.py
+14
-7
@@ -13,7 +13,6 @@ from typing_extensions import TypeVar
13
from vllm.jsontree import JSONTree, json_map_leaves
14
from vllm.logger import init_logger
15
from vllm.transformers_utils.processor import cached_processor_from_config
16
-from vllm.transformers_utils.tokenizer import AnyTokenizer
16
from vllm.utils import resolve_mm_processor_kwargs
17
18
if TYPE_CHECKING:
@@ -21,6 +20,14 @@ if TYPE_CHECKING:
20
from vllm.multimodal import (MultiModalDataDict, MultiModalPlaceholderDict,
21
MultiModalRegistry)
22
from vllm.sequence import SequenceData
23
+ from vllm.transformers_utils.tokenizer import AnyTokenizer
24
+else:
25
+ ModelConfig = Any
26
+ MultiModalDataDict = Any
27
+ MultiModalPlaceholderDict = Any
28
+ MultiModalRegistry = Any
29
+ SequenceData = Any
30
+ AnyTokenizer = Any
31
32
_T = TypeVar("_T")
33
_C = TypeVar("_C", bound=PretrainedConfig, default=PretrainedConfig)
@@ -36,7 +43,7 @@ class InputContext:
43
modify the inputs.
44
"""
45
39
- model_config: "ModelConfig"
46
+ model_config: ModelConfig
47
"""The configuration of the model."""
48
49
def get_hf_config(
@@ -200,9 +207,9 @@ class DummyData(NamedTuple):
207
Note: This is only used in V0.
208
"""
209
203
- seq_data: "SequenceData"
204
- multi_modal_data: Optional["MultiModalDataDict"] = None
205
- multi_modal_placeholders: Optional["MultiModalPlaceholderDict"] = None
210
+ seq_data: SequenceData
211
+ multi_modal_data: Optional[MultiModalDataDict] = None
212
+ multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
213
214
215
class InputRegistry:
@@ -212,9 +219,9 @@ class InputRegistry:
219
220
def dummy_data_for_profiling(
221
self,
215
- model_config: "ModelConfig",
222
+ model_config: ModelConfig,
223
seq_len: int,
217
- mm_registry: "MultiModalRegistry",
224
+ mm_registry: MultiModalRegistry,
225
is_encoder_data: bool = False,
226
) -> DummyData:
227
"""
vllm/model_executor/models/registry.py
+1
-1
@@ -16,7 +16,6 @@ from dataclasses import dataclass, field
16
from functools import lru_cache
17
from typing import Callable, Optional, TypeVar, Union
18
19
-import cloudpickle
19
import torch.nn as nn
20
21
from vllm.logger import init_logger
@@ -598,6 +597,7 @@ def _run_in_subprocess(fn: Callable[[], _T]) -> _T:
597
output_filepath = os.path.join(tempdir, "registry_output.tmp")
598
599
# `cloudpickle` allows pickling lambda functions directly
600
+ import cloudpickle
601
input_bytes = cloudpickle.dumps((fn, output_filepath))
602
603
# cannot use `sys.executable __file__` here because the script
vllm/platforms/cpu.py
+1
-1
@@ -7,7 +7,6 @@ import sys
7
from importlib.util import find_spec
8
from typing import TYPE_CHECKING, Optional
9
10
-import psutil
10
import torch
11
12
from vllm.logger import init_logger
@@ -73,6 +72,7 @@ class CpuPlatform(Platform):
72
73
@classmethod
74
def get_device_total_memory(cls, device_id: int = 0) -> int:
75
+ import psutil
76
return psutil.virtual_memory().total
77
78
@classmethod
vllm/reasoning/abs_reasoning_parsers.py
+12
-4
@@ -7,14 +7,22 @@ import os
7
from abc import abstractmethod
8
from collections.abc import Sequence
9
from functools import cached_property
10
-from typing import Callable, Optional, Union
10
+from typing import TYPE_CHECKING, Any, Callable, Optional, Union
11
12
-from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
13
- DeltaMessage, ResponsesRequest)
12
from vllm.logger import init_logger
15
-from vllm.transformers_utils.tokenizer import AnyTokenizer
13
from vllm.utils import import_from_path, is_list_of
14
15
+if TYPE_CHECKING:
16
+ from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
17
+ DeltaMessage,
18
+ ResponsesRequest)
19
+ from vllm.transformers_utils.tokenizer import AnyTokenizer
20
+else:
21
+ ChatCompletionRequest = Any
22
+ DeltaMessage = Any
23
+ ResponsesRequest = Any
24
+ AnyTokenizer = Any
25
+
26
logger = init_logger(__name__)
27
28
vllm/transformers_utils/tokenizer.py
+8
-4
@@ -16,15 +16,18 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
16
17
from vllm import envs
18
from vllm.logger import init_logger
19
-from vllm.lora.request import LoRARequest
20
-from vllm.transformers_utils.tokenizer_base import (TokenizerBase,
21
- TokenizerRegistry)
19
from vllm.transformers_utils.tokenizers import MistralTokenizer
20
from vllm.transformers_utils.utils import check_gguf_file
21
from vllm.utils import make_async
22
23
if TYPE_CHECKING:
24
from vllm.config import ModelConfig
25
+ from vllm.lora.request import LoRARequest
26
+ from vllm.transformers_utils.tokenizer_base import TokenizerBase
27
+else:
28
+ ModelConfig = Any
29
+ LoRARequest = Any
30
+ TokenizerBase = Any
31
32
logger = init_logger(__name__)
33
@@ -222,6 +225,7 @@ def get_tokenizer(
225
tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
226
revision=revision)
227
elif tokenizer_mode == "custom":
228
+ from vllm.transformers_utils.tokenizer_base import TokenizerRegistry
229
tokenizer = TokenizerRegistry.get_tokenizer(str(tokenizer_name),
230
*args,
231
revision=revision,
@@ -271,7 +275,7 @@ cached_get_tokenizer = lru_cache(get_tokenizer)
275
276
277
def cached_tokenizer_from_config(
274
- model_config: "ModelConfig",
278
+ model_config: ModelConfig,
279
**kwargs: Any,
280
):
281
return cached_get_tokenizer(