A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
Add full API docs and improve the UX of navigating them (#17485)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
May 4, 2025 at 03:42 UTC
d6484ef3c3a01dc89176ed49874a327303356bee
101 files changed
+900
-1008
.buildkite/test-pipeline.yaml
+1
-1
@@ -39,7 +39,7 @@ steps:
39
- pip install -r ../../requirements/docs.txt
40
- SPHINXOPTS=\"-W\" make html
41
# Check API reference (if it fails, you may have missing mock imports)
42
- - grep \"sig sig-object py\" build/html/api/inference_params.html
42
+ - grep \"sig sig-object py\" build/html/api/vllm/vllm.sampling_params.html
43
44
- label: Async Engine, Inputs, Utils, Worker Test # 24min
45
source_file_dependencies:
.gitignore
+1
@@ -80,6 +80,7 @@ instance/
80
# Sphinx documentation
81
docs/_build/
82
docs/source/getting_started/examples/
83
+docs/source/api/vllm
84
85
# PyBuilder
86
.pybuilder/
docs/Makefile
+1
@@ -22,3 +22,4 @@ help:
22
clean:
23
@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
24
rm -rf "$(SOURCEDIR)/getting_started/examples"
25
+ rm -rf "$(SOURCEDIR)/api/vllm"
docs/source/api/engine/async_llm_engine.md
deleted
-7
@@ -1,7 +0,0 @@
1
-# AsyncLLMEngine
2
-
3
-```{eval-rst}
4
-.. autoclass:: vllm.AsyncLLMEngine
5
- :members:
6
- :show-inheritance:
7
-```
docs/source/api/engine/index.md
deleted
-17
@@ -1,17 +0,0 @@
1
-# vLLM Engine
2
-
3
-```{eval-rst}
4
-.. automodule:: vllm.engine
5
-```
6
-
7
-```{eval-rst}
8
-.. currentmodule:: vllm.engine
9
-```
10
-
11
-:::{toctree}
12
-:caption: Engines
13
-:maxdepth: 2
14
-
15
-llm_engine
16
-async_llm_engine
17
-:::
docs/source/api/engine/llm_engine.md
deleted
-7
@@ -1,7 +0,0 @@
1
-# LLMEngine
2
-
3
-```{eval-rst}
4
-.. autoclass:: vllm.LLMEngine
5
- :members:
6
- :show-inheritance:
7
-```
docs/source/api/inference_params.md
deleted
-21
@@ -1,21 +0,0 @@
1
-# Inference Parameters
2
-
3
-Inference parameters for vLLM APIs.
4
-
5
-(sampling-params)=
6
-
7
-## Sampling Parameters
8
-
9
-```{eval-rst}
10
-.. autoclass:: vllm.SamplingParams
11
- :members:
12
-```
13
-
14
-(pooling-params)=
15
-
16
-## Pooling Parameters
17
-
18
-```{eval-rst}
19
-.. autoclass:: vllm.PoolingParams
20
- :members:
21
-```
docs/source/api/model/adapters.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Model Adapters
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.model_executor.models.adapters
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/model/index.md
deleted
-11
@@ -1,11 +0,0 @@
1
-# Model Development
2
-
3
-## Submodules
4
-
5
-:::{toctree}
6
-:maxdepth: 1
7
-
8
-interfaces_base
9
-interfaces
10
-adapters
11
-:::
docs/source/api/model/interfaces.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Optional Interfaces
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.model_executor.models.interfaces
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/model/interfaces_base.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Base Model Interfaces
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.model_executor.models.interfaces_base
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/multimodal/index.md
deleted
-28
@@ -1,28 +0,0 @@
1
-(multi-modality)=
2
-
3
-# Multi-Modality
4
-
5
-vLLM provides experimental support for multi-modal models through the {mod}`vllm.multimodal` package.
6
-
7
-Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
8
-via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
9
-
10
-Looking to add your own multi-modal model? Please follow the instructions listed [here](#supports-multimodal).
11
-
12
-## Module Contents
13
-
14
-```{eval-rst}
15
-.. autodata:: vllm.multimodal.MULTIMODAL_REGISTRY
16
-```
17
-
18
-## Submodules
19
-
20
-:::{toctree}
21
-:maxdepth: 1
22
-
23
-inputs
24
-parse
25
-processing
26
-profiling
27
-registry
28
-:::
docs/source/api/multimodal/inputs.md
deleted
-49
@@ -1,49 +0,0 @@
1
-# Input Definitions
2
-
3
-## User-facing inputs
4
-
5
-```{eval-rst}
6
-.. autodata:: vllm.multimodal.inputs.MultiModalDataDict
7
-```
8
-
9
-## Internal data structures
10
-
11
-```{eval-rst}
12
-.. autoclass:: vllm.multimodal.inputs.PlaceholderRange
13
- :members:
14
- :show-inheritance:
15
-```
16
-
17
-```{eval-rst}
18
-.. autodata:: vllm.multimodal.inputs.NestedTensors
19
-```
20
-
21
-```{eval-rst}
22
-.. autoclass:: vllm.multimodal.inputs.MultiModalFieldElem
23
- :members:
24
- :show-inheritance:
25
-```
26
-
27
-```{eval-rst}
28
-.. autoclass:: vllm.multimodal.inputs.MultiModalFieldConfig
29
- :members:
30
- :show-inheritance:
31
-```
32
-
33
-```{eval-rst}
34
-.. autoclass:: vllm.multimodal.inputs.MultiModalKwargsItem
35
- :members:
36
- :show-inheritance:
37
-```
38
-
39
-```{eval-rst}
40
-.. autoclass:: vllm.multimodal.inputs.MultiModalKwargs
41
- :members:
42
- :show-inheritance:
43
-```
44
-
45
-```{eval-rst}
46
-.. autoclass:: vllm.multimodal.inputs.MultiModalInputs
47
- :members:
48
- :show-inheritance:
49
-```
docs/source/api/multimodal/parse.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Data Parsing
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.multimodal.parse
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/multimodal/processing.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Data Processing
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.multimodal.processing
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/multimodal/profiling.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Memory Profiling
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.multimodal.profiling
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/multimodal/registry.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Registry
2
-
3
-## Module Contents
4
-
5
-```{eval-rst}
6
-.. automodule:: vllm.multimodal.registry
7
- :members:
8
- :member-order: bysource
9
-```
docs/source/api/offline_inference/index.md
deleted
-9
@@ -1,9 +0,0 @@
1
-# Offline Inference
2
-
3
-:::{toctree}
4
-:caption: Contents
5
-:maxdepth: 1
6
-
7
-llm
8
-llm_inputs
9
-:::
docs/source/api/offline_inference/llm.md
deleted
-7
@@ -1,7 +0,0 @@
1
-# LLM Class
2
-
3
-```{eval-rst}
4
-.. autoclass:: vllm.LLM
5
- :members:
6
- :show-inheritance:
7
-```
docs/source/api/offline_inference/llm_inputs.md
deleted
-19
@@ -1,19 +0,0 @@
1
-# LLM Inputs
2
-
3
-```{eval-rst}
4
-.. autodata:: vllm.inputs.PromptType
5
-```
6
-
7
-```{eval-rst}
8
-.. autoclass:: vllm.inputs.TextPrompt
9
- :show-inheritance:
10
- :members:
11
- :member-order: bysource
12
-```
13
-
14
-```{eval-rst}
15
-.. autoclass:: vllm.inputs.TokensPrompt
16
- :show-inheritance:
17
- :members:
18
- :member-order: bysource
19
-```
docs/source/api/summary.md
new
+133
@@ -0,0 +1,133 @@
1
+# Summary
2
+
3
+(configuration)=
4
+
5
+## Configuration
6
+
7
+API documentation for vLLM's configuration classes.
8
+
9
+```{autodoc2-summary}
10
+ vllm.config.ModelConfig
11
+ vllm.config.CacheConfig
12
+ vllm.config.TokenizerPoolConfig
13
+ vllm.config.LoadConfig
14
+ vllm.config.ParallelConfig
15
+ vllm.config.SchedulerConfig
16
+ vllm.config.DeviceConfig
17
+ vllm.config.SpeculativeConfig
18
+ vllm.config.LoRAConfig
19
+ vllm.config.PromptAdapterConfig
20
+ vllm.config.MultiModalConfig
21
+ vllm.config.PoolerConfig
22
+ vllm.config.DecodingConfig
23
+ vllm.config.ObservabilityConfig
24
+ vllm.config.KVTransferConfig
25
+ vllm.config.CompilationConfig
26
+ vllm.config.VllmConfig
27
+```
28
+
29
+(offline-inference-api)=
30
+
31
+## Offline Inference
32
+
33
+LLM Class.
34
+
35
+```{autodoc2-summary}
36
+ vllm.LLM
37
+```
38
+
39
+LLM Inputs.
40
+
41
+```{autodoc2-summary}
42
+ vllm.inputs.PromptType
43
+ vllm.inputs.TextPrompt
44
+ vllm.inputs.TokensPrompt
45
+```
46
+
47
+## vLLM Engines
48
+
49
+Engine classes for offline and online inference.
50
+
51
+```{autodoc2-summary}
52
+ vllm.LLMEngine
53
+ vllm.AsyncLLMEngine
54
+```
55
+
56
+## Inference Parameters
57
+
58
+Inference parameters for vLLM APIs.
59
+
60
+(sampling-params)=
61
+(pooling-params)=
62
+
63
+```{autodoc2-summary}
64
+ vllm.SamplingParams
65
+ vllm.PoolingParams
66
+```
67
+
68
+(multi-modality)=
69
+
70
+## Multi-Modality
71
+
72
+vLLM provides experimental support for multi-modal models through the {mod}`vllm.multimodal` package.
73
+
74
+Multi-modal inputs can be passed alongside text and token prompts to [supported models](#supported-mm-models)
75
+via the `multi_modal_data` field in {class}`vllm.inputs.PromptType`.
76
+
77
+Looking to add your own multi-modal model? Please follow the instructions listed [here](#supports-multimodal).
78
+
79
+```{autodoc2-summary}
80
+ vllm.multimodal.MULTIMODAL_REGISTRY
81
+```
82
+
83
+### Inputs
84
+
85
+User-facing inputs.
86
+
87
+```{autodoc2-summary}
88
+ vllm.multimodal.inputs.MultiModalDataDict
89
+```
90
+
91
+Internal data structures.
92
+
93
+```{autodoc2-summary}
94
+ vllm.multimodal.inputs.PlaceholderRange
95
+ vllm.multimodal.inputs.NestedTensors
96
+ vllm.multimodal.inputs.MultiModalFieldElem
97
+ vllm.multimodal.inputs.MultiModalFieldConfig
98
+ vllm.multimodal.inputs.MultiModalKwargsItem
99
+ vllm.multimodal.inputs.MultiModalKwargs
100
+ vllm.multimodal.inputs.MultiModalInputs
101
+```
102
+
103
+### Data Parsing
104
+
105
+```{autodoc2-summary}
106
+ vllm.multimodal.parse
107
+```
108
+
109
+### Data Processing
110
+
111
+```{autodoc2-summary}
112
+ vllm.multimodal.processing
113
+```
114
+
115
+### Memory Profiling
116
+
117
+```{autodoc2-summary}
118
+ vllm.multimodal.profiling
119
+```
120
+
121
+### Registry
122
+
123
+```{autodoc2-summary}
124
+ vllm.multimodal.registry
125
+```
126
+
127
+## Model Development
128
+
129
+```{autodoc2-summary}
130
+ vllm.model_executor.models.interfaces_base
131
+ vllm.model_executor.models.interfaces
132
+ vllm.model_executor.models.adapters
133
+```
docs/source/autodoc2_docstring_parser.py
new
+21
@@ -0,0 +1,21 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+from docutils import nodes
3
+from myst_parser.parsers.sphinx_ import MystParser
4
+from sphinx.ext.napoleon import docstring
5
+
6
+
7
+class NapoleonParser(MystParser):
8
+
9
+ def parse(self, input_string: str, document: nodes.document) -> None:
10
+ # Get the Sphinx configuration
11
+ config = document.settings.env.config
12
+
13
+ parsed_content = str(
14
+ docstring.GoogleDocstring(
15
+ str(docstring.NumpyDocstring(input_string, config)),
16
+ config,
17
+ ))
18
+ return super().parse(parsed_content, document)
19
+
20
+
21
+Parser = NapoleonParser
docs/source/conf.py
+66
-70
@@ -13,16 +13,17 @@
13
# documentation root, use os.path.abspath to make it absolute, like shown here.
14
15
import datetime
16
-import inspect
16
import logging
17
import os
18
+import re
19
import sys
20
+from pathlib import Path
21
22
import requests
22
-from sphinx.ext import autodoc
23
24
logger = logging.getLogger(__name__)
25
-sys.path.append(os.path.abspath("../.."))
25
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent
26
+sys.path.append(os.path.abspath(REPO_ROOT))
27
28
# -- Project information -----------------------------------------------------
29
@@ -40,8 +41,7 @@ extensions = [
41
"sphinx.ext.linkcode",
42
"sphinx.ext.intersphinx",
43
"sphinx_copybutton",
43
- "sphinx.ext.autodoc",
44
- "sphinx.ext.autosummary",
44
+ "autodoc2",
45
"myst_parser",
46
"sphinxarg.ext",
47
"sphinx_design",
@@ -49,7 +49,22 @@ extensions = [
49
]
50
myst_enable_extensions = [
51
"colon_fence",
52
+ "fieldlist",
53
]
54
+autodoc2_packages = [
55
+ {
56
+ "path": "../../vllm",
57
+ "exclude_dirs": ["__pycache__", "third_party"],
58
+ },
59
+]
60
+autodoc2_output_dir = "api"
61
+autodoc2_render_plugin = "myst"
62
+autodoc2_hidden_objects = ["dunder", "private", "inherited"]
63
+autodoc2_docstring_parser_regexes = [
64
+ (".*", "docs.source.autodoc2_docstring_parser"),
65
+]
66
+autodoc2_sort_names = True
67
+autodoc2_index_template = None
68
69
# Add any paths that contain templates here, relative to this directory.
70
templates_path = ['_templates']
@@ -77,6 +92,11 @@ html_theme_options = {
92
'repository_url': 'https://github.com/vllm-project/vllm',
93
'use_repository_button': True,
94
'use_edit_page_button': True,
95
+ # Prevents the full API being added to the left sidebar of every page.
96
+ # Reduces build time by 2.5x and reduces build size from ~225MB to ~95MB.
97
+ 'collapse_navbar': True,
98
+ # Makes API visible in the right sidebar on API reference pages.
99
+ 'show_toc_level': 3,
100
}
101
# Add any paths that contain custom static files (such as style sheets) here,
102
# relative to this directory. They are copied after the builtin static files,
@@ -164,73 +184,64 @@ def linkcode_resolve(domain, info):
184
return None
185
if not info['module']:
186
return None
167
- filename = info['module'].replace('.', '/')
168
- module = info['module']
169
-
170
- # try to determine the correct file and line number to link to
171
- obj = sys.modules[module]
172
-
173
- # get as specific as we can
174
- lineno: int = 0
175
- filename: str = ""
176
- try:
177
- for part in info['fullname'].split('.'):
178
- obj = getattr(obj, part)
179
-
180
- # Skip decorator wrappers by checking if the object is a function
181
- # and has a __wrapped__ attribute (which decorators typically set)
182
- while hasattr(obj, '__wrapped__'):
183
- obj = obj.__wrapped__
184
-
185
- if not (inspect.isclass(obj) or inspect.isfunction(obj)
186
- or inspect.ismethod(obj)):
187
- obj = obj.__class__ # Get the class of the instance
188
-
189
- lineno = inspect.getsourcelines(obj)[1]
190
- filename = (inspect.getsourcefile(obj)
191
- or f"{filename}.py").split("vllm/", 1)[1]
192
- except Exception:
193
- # For some things, like a class member, won't work, so
194
- # we'll use the line number of the parent (the class)
195
- pass
196
-
197
- if filename.startswith("checkouts/"):
187
+
188
+ # Get path from module name
189
+ file = Path(f"{info['module'].replace('.', '/')}.py")
190
+ path = REPO_ROOT / file
191
+ if not path.exists():
192
+ path = REPO_ROOT / file.with_suffix("") / "__init__.py"
193
+ if not path.exists():
194
+ return None
195
+
196
+ # Get the line number of the object
197
+ with open(path) as f:
198
+ lines = f.readlines()
199
+ name = info['fullname'].split(".")[-1]
200
+ pattern = fr"^( {{4}})*((def|class) )?{name}\b.*"
201
+ for lineno, line in enumerate(lines, 1):
202
+ if not line or line.startswith("#"):
203
+ continue
204
+ if re.match(pattern, line):
205
+ break
206
+
207
+ # If the line number is not found, return None
208
+ if lineno == len(lines):
209
+ return None
210
+
211
+ # If the line number is found, create the URL
212
+ filename = path.relative_to(REPO_ROOT)
213
+ if "checkouts" in path.parts:
214
# a PR build on readthedocs
199
- pr_number = filename.split("/")[1]
200
- filename = filename.split("/", 2)[2]
215
+ pr_number = REPO_ROOT.name
216
base, branch = get_repo_base_and_branch(pr_number)
217
if base and branch:
218
return f"https://github.com/{base}/blob/{branch}/{filename}#L{lineno}"
204
-
219
# Otherwise, link to the source file on the main branch
220
return f"https://github.com/vllm-project/vllm/blob/main/{filename}#L{lineno}"
221
222
209
-# Mock out external dependencies here, otherwise the autodoc pages may be blank.
223
+# Mock out external dependencies here, otherwise sphinx-argparse won't work.
224
autodoc_mock_imports = [
225
+ "huggingface_hub",
226
+ "pydantic",
227
+ "zmq",
228
+ "cloudpickle",
229
+ "aiohttp",
230
+ "starlette",
231
"blake3",
212
- "compressed_tensors",
232
"cpuinfo",
214
- "cv2",
215
- "torch",
233
"transformers",
234
"psutil",
218
- "prometheus_client",
219
- "sentencepiece",
235
"vllm._C",
236
"PIL",
237
"numpy",
223
- 'triton',
238
"tqdm",
225
- "tensorizer",
226
- "pynvml",
227
- "outlines",
228
- "xgrammar",
229
- "librosa",
230
- "soundfile",
231
- "gguf",
232
- "lark",
233
- "decord",
239
+ # The mocks below are required by
240
+ # docs/source/serving/openai_compatible_server.md's
241
+ # vllm.entrypoints.openai.cli_args
242
+ "openai",
243
+ "fastapi",
244
+ "partial_json_parser",
245
]
246
247
for mock_target in autodoc_mock_imports:
@@ -241,18 +252,6 @@ for mock_target in autodoc_mock_imports:
252
"been loaded into sys.modules when the sphinx build starts.",
253
mock_target)
254
244
-
245
-class MockedClassDocumenter(autodoc.ClassDocumenter):
246
- """Remove note about base class when a class is derived from object."""
247
-
248
- def add_line(self, line: str, source: str, *lineno: int) -> None:
249
- if line == " Bases: :py:class:`object`":
250
- return
251
- super().add_line(line, source, *lineno)
252
-
253
-
254
-autodoc.ClassDocumenter = MockedClassDocumenter
255
-
255
intersphinx_mapping = {
256
"python": ("https://docs.python.org/3", None),
257
"typing_extensions":
@@ -264,7 +263,4 @@ intersphinx_mapping = {
263
"psutil": ("https://psutil.readthedocs.io/en/stable", None),
264
}
265
267
-autodoc_preserve_defaults = True
268
-autodoc_warningiserror = True
269
-
266
navigation_with_keys = False
docs/source/design/arch_overview.md
+2
-2
@@ -52,8 +52,8 @@ for output in outputs:
52
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
53
```
54
55
-More API details can be found in the {doc}`Offline Inference
56
-</api/offline_inference/index>` section of the API docs.
55
+More API details can be found in the [Offline Inference]
56
+(#offline-inference-api) section of the API docs.
57
58
The code for the `LLM` class can be found in <gh-file:vllm/entrypoints/llm.py>.
59
docs/source/features/compatibility_matrix.md
+3
-3
@@ -42,7 +42,7 @@ Check the ❌ or 🟠 with links to see tracking issue for unsupported feature/h
42
* [APC](#automatic-prefix-caching)
43
* [LoRA](#lora-adapter)
44
* <abbr title="Prompt Adapter">prmpt adptr</abbr>
45
- * [SD](#spec_decode)
45
+ * [SD](#spec-decode)
46
* CUDA graph
47
* <abbr title="Pooling Models">pooling</abbr>
48
* <abbr title="Encoder-Decoder Models">enc-dec</abbr>
@@ -122,7 +122,7 @@ Check the ❌ or 🟠 with links to see tracking issue for unsupported feature/h
122
*
123
*
124
*
125
-- * [SD](#spec_decode)
125
+- * [SD](#spec-decode)
126
* ✅
127
* ✅
128
* ❌
@@ -377,7 +377,7 @@ Check the ❌ or 🟠 with links to see tracking issue for unsupported feature/h
377
* ✅
378
* [❌](gh-issue:8475)
379
* ✅
380
-- * [SD](#spec_decode)
380
+- * [SD](#spec-decode)
381
* ✅
382
* ✅
383
* ✅
docs/source/index.md
+2
-5
@@ -194,11 +194,8 @@ contributing/vulnerability_management
194
:caption: API Reference
195
:maxdepth: 2
196
197
-api/offline_inference/index
198
-api/engine/index
199
-api/inference_params
200
-api/multimodal/index
201
-api/model/index
197
+api/summary
198
+api/vllm/vllm
199
:::
200
201
% Latest news and acknowledgements
docs/source/models/generative_models.md
+1
-1
@@ -14,7 +14,7 @@ Usually, this is automatically inferred so you don't have to specify it.
14
## Offline Inference
15
16
The {class}`~vllm.LLM` class provides various methods for offline inference.
17
-See [Engine Arguments](#engine-args) for a list of options when initializing the model.
17
+See <project:#configuration> for a list of options when initializing the model.
18
19
### `LLM.generate`
20
docs/source/models/pooling_models.md
+1
-1
@@ -60,7 +60,7 @@ which takes priority over both the model's and Sentence Transformers's defaults.
60
## Offline Inference
61
62
The {class}`~vllm.LLM` class provides various methods for offline inference.
63
-See [Engine Arguments](#engine-args) for a list of options when initializing the model.
63
+See <project:#configuration> for a list of options when initializing the model.
64
65
### `LLM.encode`
66
docs/source/serving/offline_inference.md
+2
-2
@@ -25,7 +25,7 @@ The available APIs depend on the type of model that is being run:
25
Please refer to the above pages for more details about each API.
26
27
:::{seealso}
28
-[API Reference](/api/offline_inference/index)
28
+[API Reference](#offline-inference-api)
29
:::
30
31
(configuration-options)=
@@ -33,7 +33,7 @@ Please refer to the above pages for more details about each API.
33
## Configuration Options
34
35
This section lists the most common options for running the vLLM engine.
36
-For a full list, refer to the [Engine Arguments](#engine-args) page.
36
+For a full list, refer to the <project:#configuration> page.
37
38
(model-resolution)=
39
examples/offline_inference/profiling.py
+1
-1
@@ -14,7 +14,7 @@ import tqdm
14
15
from vllm import LLM, SamplingParams
16
from vllm.engine.arg_utils import EngineArgs
17
-from vllm.profiler import layerwise_profile
17
+from vllm.profiler.layerwise_profile import layerwise_profile
18
from vllm.utils import FlexibleArgumentParser
19
20
BATCH_SIZE_DEFAULT = 1
requirements/docs.txt
+2
-14
@@ -1,27 +1,15 @@
1
sphinx==8.2.3
2
sphinx-argparse==0.5.2
3
+sphinx-autodoc2==0.5.0
4
sphinx-book-theme==1.1.4
5
sphinx-copybutton==0.5.2
6
sphinx-design==0.6.1
7
sphinx-togglebutton==0.3.2
8
myst-parser==4.0.1
9
msgspec
9
-cloudpickle
10
commonmark # Required by sphinx-argparse when using :markdownhelp:
11
12
# packages to install to build the documentation
13
cachetools
14
-pydantic >= 2.8
14
-f https://download.pytorch.org/whl/cpu
16
-torch
17
-py-cpuinfo
18
-transformers
19
-mistral_common >= 1.5.4
20
-aiohttp
21
-starlette
22
-scipy
23
-openai # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
24
-fastapi # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
25
-partial-json-parser # Required by docs/source/serving/openai_compatible_server.md's vllm.entrypoints.openai.cli_args
26
-requests
27
-zmq
15
+torch
\ No newline at end of file
tests/conftest.py
+4
-4
@@ -112,11 +112,11 @@ class AudioTestAssets(list[AudioAsset]):
112
113
114
IMAGE_ASSETS = ImageTestAssets()
115
-"""Singleton instance of :class:`ImageTestAssets`."""
115
+"""Singleton instance of {class}`ImageTestAssets`."""
116
VIDEO_ASSETS = VideoTestAssets()
117
-"""Singleton instance of :class:`VideoTestAssets`."""
117
+"""Singleton instance of {class}`VideoTestAssets`."""
118
AUDIO_ASSETS = AudioTestAssets()
119
-"""Singleton instance of :class:`AudioTestAssets`."""
119
+"""Singleton instance of {class}`AudioTestAssets`."""
120
121
122
@pytest.fixture(scope="function", autouse=True)
@@ -724,7 +724,7 @@ def hf_runner():
724
class VllmRunner:
725
"""
726
The default value of some arguments have been modified from
727
- :class:`~vllm.LLM` as follows:
727
+ {class}`~vllm.LLM` as follows:
728
729
- `trust_remote_code`: Set to `True` instead of `False` for convenience.
730
- `seed`: Set to `0` instead of `None` for test reproducibility.
tests/tokenization/test_get_eos.py
+1
-1
@@ -2,7 +2,7 @@
2
"""
3
This test file includes some cases where it is inappropriate to
4
only get the `eos_token_id` from the tokenizer as defined by
5
-:meth:`vllm.LLMEngine._get_eos_token_id`.
5
+{meth}`vllm.LLMEngine._get_eos_token_id`.
6
"""
7
from vllm.transformers_utils.config import try_get_generation_config
8
from vllm.transformers_utils.tokenizer import get_tokenizer
tests/utils.py
+1
-1
@@ -952,7 +952,7 @@ def get_client_text_logprob_generations(
952
completions: list[Completion]) -> list[TextTextLogprobs]:
953
'''Operates on the output of a request made to an Open-AI-protocol
954
completions endpoint; obtains top-rank logprobs for each token in
955
- each :class:`SequenceGroup`
955
+ each {class}`SequenceGroup`
956
'''
957
text_generations = get_client_text_generations(completions)
958
text = ''.join(text_generations)
tests/v1/core/test_scheduler.py
+1
-1
@@ -44,7 +44,7 @@ def create_scheduler(
44
(None)
45
46
Returns:
47
- :class:`Scheduler` instance
47
+ {class}`Scheduler` instance
48
'''
49
if max_model_len is None:
50
max_model_len = max_num_batched_tokens
vllm/attention/backends/mla/common.py
+2
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""
3
+# MLA Common Components
4
+
5
This file implements common components for MLA implementations.
6
7
First we define:
vllm/attention/backends/utils.py
+1
-1
@@ -550,7 +550,7 @@ def get_num_prefill_decode_query_kv_tokens(
550
based on the attention metadata and the specified attention type.
551
552
Args:
553
- attn_metadata (FlashAttentionMetadata): Attention Metadata object.
553
+ attn_metadata (AttentionMetadata): Attention Metadata object.
554
attn_type (AttentionType): The type of attention being used.
555
Returns:
556
Tuple[int, int, int]: A tuple containing three integers:
vllm/compilation/compiler_interface.py
+1
-1
@@ -39,7 +39,7 @@ class CompilerInterface:
39
Gather all the relevant information from the vLLM config,
40
to compute a hash so that we can cache the compiled model.
41
42
- See :meth:`VllmConfig.compute_hash` to check what information
42
+ See {meth}`VllmConfig.compute_hash` to check what information
43
is already considered by default. This function should only
44
consider the information that is specific to the compiler.
45
"""
vllm/config.py
+5
-5
@@ -1911,10 +1911,10 @@ class SchedulerConfig:
1911
1912
cuda_graph_sizes: list[int] = field(default_factory=lambda: [512])
1913
"""Cuda graph capture sizes, default is 512.
1914
- 1. if one value is provided, then the capture list would follow the pattern:
1915
- [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)]
1916
- 2. more than one value (e.g. 1 2 128) is provided,
1917
- then the capture list will follow the provided list."""
1914
+ 1. if one value is provided, then the capture list would follow the
1915
+ pattern: [1, 2, 4] + [i for i in range(8, cuda_graph_sizes + 1, 8)]
1916
+ 2. more than one value (e.g. 1 2 128) is provided, then the capture list
1917
+ will follow the provided list."""
1918
1919
delay_factor: float = 0.0
1920
"""Apply a delay (of delay factor multiplied by previous
@@ -2888,7 +2888,7 @@ class PoolerConfig:
2888
pooling_type: Optional[str] = None
2889
"""
2890
The pooling method of the pooling model. This should be a key in
2891
- :class:`vllm.model_executor.layers.pooler.PoolingType`.
2891
+ {class}`vllm.model_executor.layers.pooler.PoolingType`.
2892
"""
2893
2894
normalize: Optional[bool] = None
vllm/connections.py
+1
-1
@@ -167,4 +167,4 @@ class HTTPConnection:
167
168
169
global_http_connection = HTTPConnection()
170
-"""The global :class:`HTTPConnection` instance used by vLLM."""
170
+"""The global {class}`HTTPConnection` instance used by vLLM."""
vllm/distributed/kv_transfer/__init__.py
+1
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
+from vllm.distributed.kv_transfer.kv_connector.base import KVConnectorBaseType
4
from vllm.distributed.kv_transfer.kv_transfer_state import (
5
ensure_kv_transfer_initialized, get_kv_transfer_group,
6
has_kv_transfer_group, is_v1_kv_transfer_group)
vllm/engine/async_llm_engine.py
+50
-48
@@ -475,7 +475,7 @@ class _AsyncLLMEngine(LLMEngine):
475
*,
476
inputs: Optional[PromptType] = None, # DEPRECATED
477
) -> None:
478
- """Async version of :meth:`add_request`."""
478
+ """Async version of {meth}`add_request`."""
479
if inputs is not None:
480
prompt = inputs
481
assert prompt is not None and params is not None
@@ -582,20 +582,20 @@ async def build_guided_decoding_logits_processor_async(
582
583
584
class AsyncLLMEngine(EngineClient):
585
- """An asynchronous wrapper for :class:`LLMEngine`.
585
+ """An asynchronous wrapper for {class}`LLMEngine`.
586
587
- This class is used to wrap the :class:`LLMEngine` class to make it
587
+ This class is used to wrap the {class}`LLMEngine` class to make it
588
asynchronous. It uses asyncio to create a background loop that keeps
589
- processing incoming requests. The :class:`LLMEngine` is kicked by the
589
+ processing incoming requests. The {class}`LLMEngine` is kicked by the
590
generate method when there are requests in the waiting queue. The generate
591
- method yields the outputs from the :class:`LLMEngine` to the caller.
591
+ method yields the outputs from the {class}`LLMEngine` to the caller.
592
593
Args:
594
log_requests: Whether to log the requests.
595
start_engine_loop: If True, the background task to run the engine
596
will be automatically started in the generate call.
597
- *args: Arguments for :class:`LLMEngine`.
598
- **kwargs: Arguments for :class:`LLMEngine`.
597
+ *args: Arguments for {class}`LLMEngine`.
598
+ **kwargs: Arguments for {class}`LLMEngine`.
599
"""
600
601
_engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
@@ -985,7 +985,7 @@ class AsyncLLMEngine(EngineClient):
985
from the LLMEngine to the caller.
986
987
Args:
988
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
988
+ prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
989
for more details about the format of each input.
990
sampling_params: The sampling parameters of the request.
991
request_id: The unique id of the request.
@@ -1003,7 +1003,7 @@ class AsyncLLMEngine(EngineClient):
1003
Details:
1004
- If the engine is not running, start the background loop,
1005
which iteratively invokes
1006
- :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1006
+ {meth}`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1007
to process the waiting requests.
1008
- Add the request to the engine's `RequestTracker`.
1009
On the next background loop, this request will be sent to
@@ -1075,7 +1075,7 @@ class AsyncLLMEngine(EngineClient):
1075
from the LLMEngine to the caller.
1076
1077
Args:
1078
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
1078
+ prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
1079
for more details about the format of each input.
1080
pooling_params: The pooling parameters of the request.
1081
request_id: The unique id of the request.
@@ -1089,46 +1089,48 @@ class AsyncLLMEngine(EngineClient):
1089
for the request.
1090
1091
Details:
1092
- - If the engine is not running, start the background loop,
1093
- which iteratively invokes
1094
- :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1095
- to process the waiting requests.
1096
- - Add the request to the engine's `RequestTracker`.
1097
- On the next background loop, this request will be sent to
1098
- the underlying engine.
1099
- Also, a corresponding `AsyncStream` will be created.
1100
- - Wait for the request outputs from `AsyncStream` and yield them.
1092
+ - If the engine is not running, start the background loop,
1093
+ which iteratively invokes
1094
+ {meth}`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1095
+ to process the waiting requests.
1096
+ - Add the request to the engine's `RequestTracker`.
1097
+ On the next background loop, this request will be sent to
1098
+ the underlying engine.
1099
+ Also, a corresponding `AsyncStream` will be created.
1100
+ - Wait for the request outputs from `AsyncStream` and yield them.
1101
1102
Example:
1103
- >>> # Please refer to entrypoints/api_server.py for
1104
- >>> # the complete example.
1105
- >>>
1106
- >>> # initialize the engine and the example input
1107
- >>> # note that engine_args here is AsyncEngineArgs instance
1108
- >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
1109
- >>> example_input = {
1110
- >>> "input": "What is LLM?",
1111
- >>> "request_id": 0,
1112
- >>> }
1113
- >>>
1114
- >>> # start the generation
1115
- >>> results_generator = engine.encode(
1116
- >>> example_input["input"],
1117
- >>> PoolingParams(),
1118
- >>> example_input["request_id"])
1119
- >>>
1120
- >>> # get the results
1121
- >>> final_output = None
1122
- >>> async for request_output in results_generator:
1123
- >>> if await request.is_disconnected():
1124
- >>> # Abort the request if the client disconnects.
1125
- >>> await engine.abort(request_id)
1126
- >>> # Return or raise an error
1127
- >>> ...
1128
- >>> final_output = request_output
1129
- >>>
1130
- >>> # Process and return the final output
1131
- >>> ...
1103
+ ```
1104
+ # Please refer to entrypoints/api_server.py for
1105
+ # the complete example.
1106
+
1107
+ # initialize the engine and the example input
1108
+ # note that engine_args here is AsyncEngineArgs instance
1109
+ engine = AsyncLLMEngine.from_engine_args(engine_args)
1110
+ example_input = {
1111
+ "input": "What is LLM?",
1112
+ "request_id": 0,
1113
+ }
1114
+
1115
+ # start the generation
1116
+ results_generator = engine.encode(
1117
+ example_input["input"],
1118
+ PoolingParams(),
1119
+ example_input["request_id"])
1120
+
1121
+ # get the results
1122
+ final_output = None
1123
+ async for request_output in results_generator:
1124
+ if await request.is_disconnected():
1125
+ # Abort the request if the client disconnects.
1126
+ await engine.abort(request_id)
1127
+ # Return or raise an error
1128
+ ...
1129
+ final_output = request_output
1130
+
1131
+ # Process and return the final output
1132
+ ...
1133
+ ```
1134
"""
1135
try:
1136
async for output in await self.add_request(
vllm/engine/llm_engine.py
+54
-51
@@ -130,11 +130,11 @@ class LLMEngine:
130
iteration-level scheduling and efficient memory management to maximize the
131
serving throughput.
132
133
- The :class:`~vllm.LLM` class wraps this class for offline batched inference
134
- and the :class:`AsyncLLMEngine` class wraps this class for online serving.
133
+ The {class}`~vllm.LLM` class wraps this class for offline batched inference
134
+ and the {class}`AsyncLLMEngine` class wraps this class for online serving.
135
136
- The config arguments are derived from :class:`~vllm.EngineArgs`. (See
137
- :ref:`engine-args`)
136
+ The config arguments are derived from {class}`~vllm.EngineArgs`. (See
137
+ {ref}`engine-args`)
138
139
Args:
140
model_config: The configuration related to the LLM model.
@@ -694,11 +694,11 @@ class LLMEngine:
694
695
Args:
696
request_id: The unique ID of the request.
697
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
697
+ prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
698
for more details about the format of each input.
699
params: Parameters for sampling or pooling.
700
- :class:`~vllm.SamplingParams` for text generation.
701
- :class:`~vllm.PoolingParams` for pooling.
700
+ {class}`~vllm.SamplingParams` for text generation.
701
+ {class}`~vllm.PoolingParams` for pooling.
702
arrival_time: The arrival time of the request. If None, we use
703
the current monotonic time.
704
lora_request: The LoRA request to add.
@@ -710,10 +710,10 @@ class LLMEngine:
710
Details:
711
- Set arrival_time to the current time if it is None.
712
- Set prompt_token_ids to the encoded prompt if it is None.
713
- - Create `n` number of :class:`~vllm.Sequence` objects.
714
- - Create a :class:`~vllm.SequenceGroup` object
715
- from the list of :class:`~vllm.Sequence`.
716
- - Add the :class:`~vllm.SequenceGroup` object to the scheduler.
713
+ - Create `n` number of {class}`~vllm.Sequence` objects.
714
+ - Create a {class}`~vllm.SequenceGroup` object
715
+ from the list of {class}`~vllm.Sequence`.
716
+ - Add the {class}`~vllm.SequenceGroup` object to the scheduler.
717
718
Example:
719
>>> # initialize engine
@@ -861,8 +861,8 @@ class LLMEngine:
861
862
Details:
863
- Refer to the
864
- :meth:`~vllm.core.scheduler.Scheduler.abort_seq_group`
865
- from class :class:`~vllm.core.scheduler.Scheduler`.
864
+ {meth}`~vllm.core.scheduler.Scheduler.abort_seq_group`
865
+ from class {class}`~vllm.core.scheduler.Scheduler`.
866
867
Example:
868
>>> # initialize engine and add a request with request_id
@@ -1258,53 +1258,56 @@ class LLMEngine:
1258
def step(self) -> List[Union[RequestOutput, PoolingRequestOutput]]:
1259
"""Performs one decoding iteration and returns newly generated results.
1260
1261
- .. figure:: https://i.imgur.com/sv2HssD.png
1262
- :alt: Overview of the step function
1263
- :align: center
1261
+ :::{figure} https://i.imgur.com/sv2HssD.png
1262
+ :alt: Overview of the step function
1263
+ :align: center
1264
1265
- Overview of the step function.
1265
+ Overview of the step function.
1266
+ :::
1267
1268
Details:
1268
- - Step 1: Schedules the sequences to be executed in the next
1269
- iteration and the token blocks to be swapped in/out/copy.
1269
+ - Step 1: Schedules the sequences to be executed in the next
1270
+ iteration and the token blocks to be swapped in/out/copy.
1271
1271
- - Depending on the scheduling policy,
1272
- sequences may be `preempted/reordered`.
1273
- - A Sequence Group (SG) refer to a group of sequences
1274
- that are generated from the same prompt.
1272
+ - Depending on the scheduling policy,
1273
+ sequences may be `preempted/reordered`.
1274
+ - A Sequence Group (SG) refer to a group of sequences
1275
+ that are generated from the same prompt.
1276
1276
- - Step 2: Calls the distributed executor to execute the model.
1277
- - Step 3: Processes the model output. This mainly includes:
1277
+ - Step 2: Calls the distributed executor to execute the model.
1278
+ - Step 3: Processes the model output. This mainly includes:
1279
1279
- - Decodes the relevant outputs.
1280
- - Updates the scheduled sequence groups with model outputs
1281
- based on its `sampling parameters` (`use_beam_search` or not).
1282
- - Frees the finished sequence groups.
1280
+ - Decodes the relevant outputs.
1281
+ - Updates the scheduled sequence groups with model outputs
1282
+ based on its `sampling parameters` (`use_beam_search` or not).
1283
+ - Frees the finished sequence groups.
1284
1284
- - Finally, it creates and returns the newly generated results.
1285
+ - Finally, it creates and returns the newly generated results.
1286
1287
Example:
1287
- >>> # Please see the example/ folder for more detailed examples.
1288
- >>>
1289
- >>> # initialize engine and request arguments
1290
- >>> engine = LLMEngine.from_engine_args(engine_args)
1291
- >>> example_inputs = [(0, "What is LLM?",
1292
- >>> SamplingParams(temperature=0.0))]
1293
- >>>
1294
- >>> # Start the engine with an event loop
1295
- >>> while True:
1296
- >>> if example_inputs:
1297
- >>> req_id, prompt, sampling_params = example_inputs.pop(0)
1298
- >>> engine.add_request(str(req_id),prompt,sampling_params)
1299
- >>>
1300
- >>> # continue the request processing
1301
- >>> request_outputs = engine.step()
1302
- >>> for request_output in request_outputs:
1303
- >>> if request_output.finished:
1304
- >>> # return or show the request output
1305
- >>>
1306
- >>> if not (engine.has_unfinished_requests() or example_inputs):
1307
- >>> break
1288
+ ```
1289
+ # Please see the example/ folder for more detailed examples.
1290
+
1291
+ # initialize engine and request arguments
1292
+ engine = LLMEngine.from_engine_args(engine_args)
1293
+ example_inputs = [(0, "What is LLM?",
1294
+ SamplingParams(temperature=0.0))]
1295
+
1296
+ # Start the engine with an event loop
1297
+ while True:
1298
+ if example_inputs:
1299
+ req_id, prompt, sampling_params = example_inputs.pop(0)
1300
+ engine.add_request(str(req_id),prompt,sampling_params)
1301
+
1302
+ # continue the request processing
1303
+ request_outputs = engine.step()
1304
+ for request_output in request_outputs:
1305
+ if request_output.finished:
1306
+ # return or show the request output
1307
+
1308
+ if not (engine.has_unfinished_requests() or example_inputs):
1309
+ break
1310
+ ```
1311
"""
1312
if self.parallel_config.pipeline_parallel_size > 1:
1313
raise NotImplementedError(
vllm/engine/multiprocessing/client.py
+2
-2
@@ -491,7 +491,7 @@ class MQLLMEngineClient(EngineClient):
491
from the LLMEngine to the caller.
492
493
Args:
494
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
494
+ prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
495
for more details about the format of each input.
496
sampling_params: The sampling parameters of the request.
497
request_id: The unique id of the request.
@@ -560,7 +560,7 @@ class MQLLMEngineClient(EngineClient):
560
from the LLMEngine to the caller.
561
562
Args:
563
- prompt: The prompt to the LLM. See :class:`~vllm.inputs.PromptType`
563
+ prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
564
for more details about the format of each input.
565
pooling_params: The pooling parameters of the request.
566
request_id: The unique id of the request.
vllm/engine/multiprocessing/engine.py
+6
-6
@@ -41,18 +41,18 @@ HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), )
41
42
43
class MQLLMEngine:
44
- """A multiprocessing wrapper for :class:`LLMEngine`.
44
+ """A multiprocessing wrapper for {class}`LLMEngine`.
45
46
- This class is used to wrap the :class:`LLMEngine` class to enable use
46
+ This class is used to wrap the {class}`LLMEngine` class to enable use
47
in concurrnet manner. It runs a background loop and uses zeromq to
48
receive new requests and stream outputs incrementally via ipc.
49
50
- The :class:`LLMEngine` generate or encode process is kicked off when a new
50
+ The {class}`LLMEngine` generate or encode process is kicked off when a new
51
RPCProcessRequest is received by the input_socket.
52
53
The self.engine_loop checks the input_socket for new requests,
54
adds them to the LLMEngine if there are any, calls the internal
55
- :class:`LLMEngine.step()`, and sends the RequestOutputs back over
55
+ {class}`LLMEngine.step()`, and sends the RequestOutputs back over
56
the output_socket.
57
58
If use_async_sockets is set, the logic associated with reading new
@@ -64,8 +64,8 @@ class MQLLMEngine:
64
ipc_path: Base path for zeromq interprocess messaging
65
use_async_sockets: Whether to make send/recv async with GPU
66
log_requests: Whether to log the requests.
67
- *args: Arguments for :class:`LLMEngine`.
68
- **kwargs: Arguments for :class:`LLMEngine`.
67
+ *args: Arguments for {class}`LLMEngine`.
68
+ **kwargs: Arguments for {class}`LLMEngine`.
69
"""
70
71
def __init__(self,
vllm/engine/output_processor/multi_step.py
+2
-2
@@ -56,8 +56,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
56
scheduled computation.
57
58
Args:
59
- seq_group: the outputs are associated with this :class:`SequenceGroup`
60
- outputs: the :class:`SequenceGroupOutput`s for all scheduler steps
59
+ seq_group: the outputs are associated with this {class}`SequenceGroup`
60
+ outputs: the {class}`SequenceGroupOutput`s for all scheduler steps
61
"""
62
for output in outputs:
63
# Concatenate single-step prompt logprob processing results.
vllm/engine/output_processor/single_step.py
+6
-6
@@ -19,7 +19,7 @@ logger = init_logger(__name__)
19
def single_step_process_prompt_logprob(
20
sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
21
output: CompletionSequenceGroupOutput) -> None:
22
- """Process prompt logprobs associated with the :class:`SequenceGroupOutput`
22
+ """Process prompt logprobs associated with the {class}`SequenceGroupOutput`
23
for a given step.
24
25
Do nothing if the output has no prompt logprobs.
@@ -27,9 +27,9 @@ def single_step_process_prompt_logprob(
27
Account for the fact that transformers do not compute first-token logprobs.
28
29
Args:
30
- sg_output_proc: :class:`SequenceGroupOutputProcessor` instance
31
- seq_group: the output is associated with this :class:`SequenceGroup`
32
- output: the :class:`SequenceGroupOutput` for a single scheduler step
30
+ sg_output_proc: {class}`SequenceGroupOutputProcessor` instance
31
+ seq_group: the output is associated with this {class}`SequenceGroup`
32
+ output: the {class}`SequenceGroupOutput` for a single scheduler step
33
"""
34
prompt_logprobs = output.prompt_logprobs
35
@@ -103,8 +103,8 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
103
scheduled computation.
104
105
Args:
106
- seq_group: the output is associated with this :class:`SequenceGroup`
107
- outputs: the :class:`SequenceGroupOutput` for a single scheduler step
106
+ seq_group: the output is associated with this {class}`SequenceGroup`
107
+ outputs: the {class}`SequenceGroupOutput` for a single scheduler step
108
"""
109
assert len(outputs) == 1, "Single step should only have 1 output."
110
output = outputs[0]
vllm/entrypoints/llm.py
+33
-29
@@ -115,7 +115,7 @@ class LLM:
115
to eager mode. Additionally for encoder-decoder models, if the
116
sequence length of the encoder input is larger than this, we fall
117
back to the eager mode.
118
- disable_custom_all_reduce: See :class:`~vllm.config.ParallelConfig`
118
+ disable_custom_all_reduce: See {class}`~vllm.config.ParallelConfig`
119
disable_async_output_proc: Disable async output processing.
120
This may result in lower performance.
121
hf_token: The token to use as HTTP bearer authorization for remote files
@@ -127,12 +127,13 @@ class LLM:
127
compilation_config: Either an integer or a dictionary. If it is an
128
integer, it is used as the level of compilation optimization. If it
129
is a dictionary, it can specify the full compilation configuration.
130
- **kwargs: Arguments for :class:`~vllm.EngineArgs`. (See
131
- :ref:`engine-args`)
130
+ **kwargs: Arguments for {class}`~vllm.EngineArgs`. (See
131
+ {ref}`engine-args`)
132
133
- Note:
134
- This class is intended to be used for offline inference. For online
135
- serving, use the :class:`~vllm.AsyncLLMEngine` class instead.
133
+ :::{note}
134
+ This class is intended to be used for offline inference. For online
135
+ serving, use the {class}`~vllm.AsyncLLMEngine` class instead.
136
+ :::
137
"""
138
139
DEPRECATE_LEGACY: ClassVar[bool] = True
@@ -141,7 +142,7 @@ class LLM:
142
DEPRECATE_INIT_POSARGS: ClassVar[bool] = True
143
"""
144
A flag to toggle whether to deprecate positional arguments in
144
- :meth:`LLM.__init__`.
145
+ {meth}`LLM.__init__`.
146
"""
147
148
@classmethod
@@ -398,7 +399,7 @@ class LLM:
399
400
Args:
401
prompts: The prompts to the LLM. You may pass a sequence of prompts
401
- for batch inference. See :class:`~vllm.inputs.PromptType`
402
+ for batch inference. See {class}`~vllm.inputs.PromptType`
403
for more details about the format of each prompts.
404
sampling_params: The sampling parameters for text generation. If
405
None, we use the default sampling parameters.
@@ -413,13 +414,14 @@ class LLM:
414
Only applicable when priority scheduling policy is enabled.
415
416
Returns:
416
- A list of ``RequestOutput`` objects containing the
417
+ A list of `RequestOutput` objects containing the
418
generated completions in the same order as the input prompts.
419
419
- Note:
420
- Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
421
- considered legacy and may be deprecated in the future. You should
422
- instead pass them via the ``inputs`` parameter.
420
+ :::{note}
421
+ Using `prompts` and `prompt_token_ids` as keyword parameters is
422
+ considered legacy and may be deprecated in the future. You should
423
+ instead pass them via the `inputs` parameter.
424
+ :::
425
"""
426
runner_type = self.llm_engine.model_config.runner_type
427
if runner_type not in ["generate", "transcription"]:
@@ -488,16 +490,17 @@ class LLM:
490
`self` argument, in addition to the arguments passed in `args`
491
and `kwargs`. The `self` argument will be the worker object.
492
timeout: Maximum time in seconds to wait for execution. Raises a
491
- :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
493
+ {exc}`TimeoutError` on timeout. `None` means wait indefinitely.
494
args: Positional arguments to pass to the worker method.
495
kwargs: Keyword arguments to pass to the worker method.
496
497
Returns:
498
A list containing the results from each worker.
497
-
498
- Note:
499
- It is recommended to use this API to only pass control messages,
500
- and set up data-plane communication to pass data.
499
+
500
+ :::{note}
501
+ It is recommended to use this API to only pass control messages,
502
+ and set up data-plane communication to pass data.
503
+ :::
504
"""
505
506
return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
@@ -664,7 +667,7 @@ class LLM:
667
Generate responses for a chat conversation.
668
669
The chat conversation is converted into a text prompt using the
667
- tokenizer and calls the :meth:`generate` method to generate the
670
+ tokenizer and calls the {meth}`generate` method to generate the
671
responses.
672
673
Multi-modal inputs can be passed in the same way you would pass them
@@ -903,7 +906,7 @@ class LLM:
906
907
Args:
908
prompts: The prompts to the LLM. You may pass a sequence of prompts
906
- for batch inference. See :class:`~vllm.inputs.PromptType`
909
+ for batch inference. See {class}`~vllm.inputs.PromptType`
910
for more details about the format of each prompts.
911
pooling_params: The pooling parameters for pooling. If None, we
912
use the default pooling parameters.
@@ -913,13 +916,14 @@ class LLM:
916
generation, if any.
917
918
Returns:
916
- A list of ``PoolingRequestOutput`` objects containing the
919
+ A list of `PoolingRequestOutput` objects containing the
920
pooled hidden states in the same order as the input prompts.
921
919
- Note:
920
- Using ``prompts`` and ``prompt_token_ids`` as keyword parameters is
921
- considered legacy and may be deprecated in the future. You should
922
- instead pass them via the ``inputs`` parameter.
922
+ :::{note}
923
+ Using `prompts` and `prompt_token_ids` as keyword parameters is
924
+ considered legacy and may be deprecated in the future. You should
925
+ instead pass them via the `inputs` parameter.
926
+ :::
927
"""
928
runner_type = self.llm_engine.model_config.runner_type
929
if runner_type != "pooling":
@@ -992,7 +996,7 @@ class LLM:
996
997
Args:
998
prompts: The prompts to the LLM. You may pass a sequence of prompts
995
- for batch inference. See :class:`~vllm.inputs.PromptType`
999
+ for batch inference. See {class}`~vllm.inputs.PromptType`
1000
for more details about the format of each prompts.
1001
pooling_params: The pooling parameters for pooling. If None, we
1002
use the default pooling parameters.
@@ -1036,7 +1040,7 @@ class LLM:
1040
1041
Args:
1042
prompts: The prompts to the LLM. You may pass a sequence of prompts
1039
- for batch inference. See :class:`~vllm.inputs.PromptType`
1043
+ for batch inference. See {class}`~vllm.inputs.PromptType`
1044
for more details about the format of each prompts.
1045
use_tqdm: Whether to use tqdm to display the progress bar.
1046
lora_request: LoRA request to use for generation, if any.
@@ -1168,7 +1172,7 @@ class LLM:
1172
text_1: can be a single prompt or a list of prompts, in which
1173
case it has to have the same length as the ``text_2`` list
1174
text_2: The texts to pair with the query to form the input
1171
- to the LLM. See :class:`~vllm.inputs.PromptType` for
1175
+ to the LLM. See {class}`~vllm.inputs.PromptType` for
1176
more details about the format of each prompts.
1177
use_tqdm: Whether to use tqdm to display the progress bar.
1178
lora_request: LoRA request to use for generation, if any.
@@ -1277,7 +1281,7 @@ class LLM:
1281
1282
def wake_up(self, tags: Optional[list[str]] = None):
1283
"""
1280
- Wake up the engine from sleep mode. See the :meth:`sleep` method
1284
+ Wake up the engine from sleep mode. See the {meth}`sleep` method
1285
for more details.
1286
1287
Args:
vllm/entrypoints/openai/protocol.py
+1
-18
@@ -5,7 +5,6 @@
5
import json
6
import re
7
import time
8
-from argparse import Namespace
8
from typing import Annotated, Any, ClassVar, Literal, Optional, Union
9
10
import torch
@@ -25,23 +24,7 @@ from vllm.utils import random_uuid, resolve_obj_by_qualname
24
25
logger = init_logger(__name__)
26
28
-# torch is mocked during docs generation,
29
-# so we have to provide the values as literals
30
-_MOCK_LONG_INFO = Namespace(min=-9223372036854775808, max=9223372036854775807)
31
-_LONG_INFO: Union["torch.iinfo", Namespace]
32
-
33
-try:
34
- from sphinx.ext.autodoc.mock import _MockModule
35
-
36
- if isinstance(torch, _MockModule):
37
- _LONG_INFO = _MOCK_LONG_INFO
38
- else:
39
- _LONG_INFO = torch.iinfo(torch.long)
40
-except ModuleNotFoundError:
41
- _LONG_INFO = torch.iinfo(torch.long)
42
-
43
-assert _LONG_INFO.min == _MOCK_LONG_INFO.min
44
-assert _LONG_INFO.max == _MOCK_LONG_INFO.max
27
+_LONG_INFO = torch.iinfo(torch.long)
28
29
30
class OpenAIBaseModel(BaseModel):
vllm/entrypoints/openai/serving_engine.py
+2
-2
@@ -275,7 +275,7 @@ class OpenAIServing:
275
add_special_tokens: bool = True,
276
) -> TextTokensPrompt:
277
"""
278
- A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
278
+ A simpler implementation of {meth}`_tokenize_prompt_input_or_inputs`
279
that assumes single input.
280
"""
281
return next(
@@ -296,7 +296,7 @@ class OpenAIServing:
296
add_special_tokens: bool = True,
297
) -> Iterator[TextTokensPrompt]:
298
"""
299
- A simpler implementation of :meth:`_tokenize_prompt_input_or_inputs`
299
+ A simpler implementation of {meth}`_tokenize_prompt_input_or_inputs`
300
that assumes multiple inputs.
301
"""
302
for text in prompt_inputs:
vllm/executor/executor_base.py
+1
-1
@@ -74,7 +74,7 @@ class ExecutorBase(ABC):
74
`self` argument, in addition to the arguments passed in `args`
75
and `kwargs`. The `self` argument will be the worker object.
76
timeout: Maximum time in seconds to wait for execution. Raises a
77
- :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
77
+ {exc}`TimeoutError` on timeout. `None` means wait indefinitely.
78
args: Positional arguments to pass to the worker method.
79
kwargs: Keyword arguments to pass to the worker method.
80
vllm/inputs/__init__.py
+1
-1
@@ -10,7 +10,7 @@ from .registry import (DummyData, InputContext, InputProcessingContext,
10
11
INPUT_REGISTRY = InputRegistry()
12
"""
13
-The global :class:`~InputRegistry` which is used by :class:`~vllm.LLMEngine`
13
+The global {class}`~InputRegistry` which is used by {class}`~vllm.LLMEngine`
14
to dispatch data processing according to the target model.
15
"""
16
vllm/inputs/data.py
+21
-21
@@ -80,22 +80,22 @@ SingletonPrompt = Union[str, TextPrompt, TokensPrompt, EmbedsPrompt]
80
"""
81
Set of possible schemas for a single prompt:
82
83
-- A text prompt (:class:`str` or :class:`TextPrompt`)
84
-- A tokenized prompt (:class:`TokensPrompt`)
85
-- An embeddings prompt (:class:`EmbedsPrompt`)
83
+- A text prompt ({class}`str` or {class}`TextPrompt`)
84
+- A tokenized prompt ({class}`TokensPrompt`)
85
+- An embeddings prompt ({class}`EmbedsPrompt`)
86
87
Note that "singleton" is as opposed to a data structure
88
which encapsulates multiple prompts, i.e. of the sort
89
which may be utilized for encoder/decoder models when
90
the user desires to express both the encoder & decoder
91
-prompts explicitly, i.e. :class:`ExplicitEncoderDecoderPrompt`
91
+prompts explicitly, i.e. {class}`ExplicitEncoderDecoderPrompt`
92
93
-A prompt of type :class:`SingletonPrompt` may be employed
93
+A prompt of type {class}`SingletonPrompt` may be employed
94
as (1) input to a decoder-only model, (2) input to
95
the encoder of an encoder/decoder model, in the scenario
96
where the decoder-prompt is not specified explicitly, or
97
(3) as a member of a larger data structure encapsulating
98
-more than one prompt, i.e. :class:`ExplicitEncoderDecoderPrompt`
98
+more than one prompt, i.e. {class}`ExplicitEncoderDecoderPrompt`
99
"""
100
101
_T1_co = TypeVar("_T1_co",
@@ -115,18 +115,18 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
115
comprising an explicit encoder prompt and a decoder prompt.
116
117
The encoder and decoder prompts, respectively, may be formatted
118
- according to any of the :class:`SingletonPrompt` schemas,
118
+ according to any of the {class}`SingletonPrompt` schemas,
119
and are not required to have the same schema.
120
121
Only the encoder prompt may have multi-modal data. mm_processor_kwargs
122
should be at the top-level, and should not be set in the encoder/decoder
123
prompts, since they are agnostic to the encoder/decoder.
124
125
- Note that an :class:`ExplicitEncoderDecoderPrompt` may not
125
+ Note that an {class}`ExplicitEncoderDecoderPrompt` may not
126
be used as an input to a decoder-only model,
127
- and that the :code:`encoder_prompt` and :code:`decoder_prompt`
127
+ and that the `encoder_prompt` and `decoder_prompt`
128
fields of this data structure themselves must be
129
- :class:`SingletonPrompt` instances.
129
+ {class}`SingletonPrompt` instances.
130
"""
131
132
encoder_prompt: _T1_co
@@ -141,11 +141,11 @@ PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
141
Set of possible schemas for an LLM input, including
142
both decoder-only and encoder/decoder input types:
143
144
-- A text prompt (:class:`str` or :class:`TextPrompt`)
145
-- A tokenized prompt (:class:`TokensPrompt`)
146
-- An embeddings prompt (:class:`EmbedsPrompt`)
144
+- A text prompt ({class}`str` or {class}`TextPrompt`)
145
+- A tokenized prompt ({class}`TokensPrompt`)
146
+- An embeddings prompt ({class}`EmbedsPrompt`)
147
- A single data structure containing both an encoder and a decoder prompt
148
- (:class:`ExplicitEncoderDecoderPrompt`)
148
+ ({class}`ExplicitEncoderDecoderPrompt`)
149
"""
150
151
@@ -178,7 +178,7 @@ def token_inputs(
178
prompt: Optional[str] = None,
179
cache_salt: Optional[str] = None,
180
) -> TokenInputs:
181
- """Construct :class:`TokenInputs` from optional values."""
181
+ """Construct {class}`TokenInputs` from optional values."""
182
inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
183
184
if prompt is not None:
@@ -221,7 +221,7 @@ def embeds_inputs(
221
222
DecoderOnlyInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
223
"""
224
-The inputs in :class:`~vllm.LLMEngine` before they are
224
+The inputs in {class}`~vllm.LLMEngine` before they are
225
passed to the model executor.
226
This specifies the data required for decoder-only models.
227
"""
@@ -229,7 +229,7 @@ This specifies the data required for decoder-only models.
229
230
class EncoderDecoderInputs(TypedDict):
231
"""
232
- The inputs in :class:`~vllm.LLMEngine` before they are
232
+ The inputs in {class}`~vllm.LLMEngine` before they are
233
passed to the model executor.
234
235
This specifies the required data for encoder-decoder models.
@@ -243,13 +243,13 @@ class EncoderDecoderInputs(TypedDict):
243
244
SingletonInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
245
"""
246
-A processed :class:`SingletonPrompt` which can be passed to
247
-:class:`vllm.sequence.Sequence`.
246
+A processed {class}`SingletonPrompt` which can be passed to
247
+{class}`vllm.sequence.Sequence`.
248
"""
249
250
ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
251
"""
252
-The inputs to :data:`vllm.inputs.InputProcessor`.
252
+The inputs to {data}`vllm.inputs.InputProcessor`.
253
"""
254
255
_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
@@ -277,7 +277,7 @@ def zip_enc_dec_prompts(
277
) -> list[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
278
"""
279
Zip encoder and decoder prompts together into a list of
280
- :class:`ExplicitEncoderDecoderPrompt` instances.
280
+ {class}`ExplicitEncoderDecoderPrompt` instances.
281
282
``mm_processor_kwargs`` may also be provided; if a dict is passed, the same
283
dictionary will be used for every encoder/decoder prompt. If an iterable is
vllm/inputs/preprocess.py
+11
-11
@@ -224,7 +224,7 @@ class InputPreprocessor:
224
lora_request: Optional[LoRARequest],
225
tokenization_kwargs: Optional[dict[str, Any]] = None,
226
) -> list[int]:
227
- """Async version of :meth:`_tokenize_prompt`."""
227
+ """Async version of {meth}`_tokenize_prompt`."""
228
tokenizer = self.get_tokenizer_group()
229
tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
230
@@ -287,7 +287,7 @@ class InputPreprocessor:
287
lora_request: Optional[LoRARequest],
288
return_mm_hashes: bool = False,
289
) -> MultiModalInputs:
290
- """Async version of :meth:`_process_multimodal`."""
290
+ """Async version of {meth}`_process_multimodal`."""
291
tokenizer = await self._get_mm_tokenizer_async(lora_request)
292
293
mm_processor = self.mm_registry.create_processor(self.model_config,
@@ -472,7 +472,7 @@ class InputPreprocessor:
472
473
Returns:
474
475
- * :class:`SingletonInputs` instance
475
+ * {class}`SingletonInputs` instance
476
"""
477
parsed = parse_singleton_prompt(prompt)
478
@@ -508,7 +508,7 @@ class InputPreprocessor:
508
lora_request: Optional[LoRARequest] = None,
509
return_mm_hashes: bool = False,
510
) -> SingletonInputs:
511
- """Async version of :meth:`_prompt_to_llm_inputs`."""
511
+ """Async version of {meth}`_prompt_to_llm_inputs`."""
512
parsed = parse_singleton_prompt(prompt)
513
514
if parsed["type"] == "embeds":
@@ -644,7 +644,7 @@ class InputPreprocessor:
644
) -> EncoderDecoderInputs:
645
"""
646
For encoder/decoder models only:
647
- Process an input prompt into an :class:`EncoderDecoderInputs` instance.
647
+ Process an input prompt into an {class}`EncoderDecoderInputs` instance.
648
649
There are two types of input prompts:
650
singleton prompts which carry only the
@@ -670,7 +670,7 @@ class InputPreprocessor:
670
671
Returns:
672
673
- * :class:`EncoderDecoderInputs` instance
673
+ * {class}`EncoderDecoderInputs` instance
674
"""
675
encoder_inputs: SingletonInputs
676
decoder_inputs: Optional[SingletonInputs]
@@ -710,7 +710,7 @@ class InputPreprocessor:
710
prompt: PromptType,
711
tokenization_kwargs: Optional[dict[str, Any]] = None,
712
) -> EncoderDecoderInputs:
713
- """Async version of :meth:`_process_encoder_decoder_prompt`."""
713
+ """Async version of {meth}`_process_encoder_decoder_prompt`."""
714
encoder_inputs: SingletonInputs
715
decoder_inputs: Optional[SingletonInputs]
716
@@ -778,7 +778,7 @@ class InputPreprocessor:
778
) -> DecoderOnlyInputs:
779
"""
780
For decoder-only models:
781
- Process an input prompt into an :class:`DecoderOnlyInputs` instance.
781
+ Process an input prompt into an {class}`DecoderOnlyInputs` instance.
782
783
Arguments:
784
@@ -789,7 +789,7 @@ class InputPreprocessor:
789
790
Returns:
791
792
- * :class:`DecoderOnlyInputs` instance
792
+ * {class}`DecoderOnlyInputs` instance
793
"""
794
795
prompt_comps = self._prompt_to_llm_inputs(
@@ -812,7 +812,7 @@ class InputPreprocessor:
812
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
813
return_mm_hashes: bool = False,
814
) -> DecoderOnlyInputs:
815
- """Async version of :meth:`_process_decoder_only_prompt`."""
815
+ """Async version of {meth}`_process_decoder_only_prompt`."""
816
prompt_comps = await self._prompt_to_llm_inputs_async(
817
prompt,
818
tokenization_kwargs=tokenization_kwargs,
@@ -863,7 +863,7 @@ class InputPreprocessor:
863
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
864
return_mm_hashes: bool = False,
865
) -> ProcessorInputs:
866
- """Async version of :meth:`preprocess`."""
866
+ """Async version of {meth}`preprocess`."""
867
if self.model_config.is_encoder_decoder:
868
assert not return_mm_hashes, (
869
"Multimodal hashes for encoder-decoder models should not be ",
vllm/inputs/registry.py
+4
-4
@@ -38,7 +38,7 @@ class InputContext:
38
) -> _C:
39
"""
40
Get the HuggingFace configuration
41
- (:class:`transformers.PretrainedConfig`) of the model,
41
+ ({class}`transformers.PretrainedConfig`) of the model,
42
additionally checking its type.
43
44
Raises:
@@ -79,7 +79,7 @@ class InputContext:
79
) -> _P:
80
"""
81
Get the HuggingFace processor
82
- (:class:`transformers.ProcessorMixin`) of the model,
82
+ ({class}`transformers.ProcessorMixin`) of the model,
83
additionally checking its type.
84
85
Raises:
@@ -135,8 +135,8 @@ class InputProcessingContext(InputContext):
135
kwargs: Mapping[str, object] = {},
136
) -> BatchFeature:
137
"""
138
- Call :code:`hf_processor` on the prompt :code:`data`
139
- (text, image, audio...) with configurable options :code:`kwargs`.
138
+ Call `hf_processor` on the prompt `data`
139
+ (text, image, audio...) with configurable options `kwargs`.
140
"""
141
assert callable(hf_processor)
142
vllm/logger.py
+3
-3
@@ -68,21 +68,21 @@ class _VllmLogger(Logger):
68
"""
69
Note:
70
This class is just to provide type information.
71
- We actually patch the methods directly on the :class:`logging.Logger`
71
+ We actually patch the methods directly on the {class}`logging.Logger`
72
instance to avoid conflicting with other libraries such as
73
`intel_extension_for_pytorch.utils._logger`.
74
"""
75
76
def info_once(self, msg: str, *args: Hashable) -> None:
77
"""
78
- As :meth:`info`, but subsequent calls with the same message
78
+ As {meth}`info`, but subsequent calls with the same message
79
are silently dropped.
80
"""
81
_print_info_once(self, msg, *args)
82
83
def warning_once(self, msg: str, *args: Hashable) -> None:
84
"""
85
- As :meth:`warning`, but subsequent calls with the same message
85
+ As {meth}`warning`, but subsequent calls with the same message
86
are silently dropped.
87
"""
88
_print_warning_once(self, msg, *args)
vllm/lora/ops/triton_ops/__init__.py
+2
-2
@@ -1,8 +1,8 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from vllm.lora.ops.triton_ops.lora_expand import lora_expand
3
+from vllm.lora.ops.triton_ops.lora_expand_op import lora_expand
4
from vllm.lora.ops.triton_ops.lora_kernel_metadata import LoRAKernelMeta
5
-from vllm.lora.ops.triton_ops.lora_shrink import lora_shrink
5
+from vllm.lora.ops.triton_ops.lora_shrink_op import lora_shrink
6
7
__all__ = [
8
"lora_expand",
vllm/lora/ops/triton_ops/lora_expand_op.py
renamed
vllm/lora/ops/triton_ops/lora_shrink_op.py
renamed
vllm/model_executor/layers/rejection_sampler.py
+17
-14
@@ -261,15 +261,16 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
261
True, then a token can be accepted, else it should be
262
rejected.
263
264
- Given :math:`q(\hat{x}_{n+1}|x_1, \dots, x_n)`, the probability of
265
- :math:`\hat{x}_{n+1}` given context :math:`x_1, \dots, x_n` according
266
- to the target model, and :math:`p(\hat{x}_{n+1}|x_1, \dots, x_n)`, the
264
+ Given {math}`q(\hat{x}_{n+1}|x_1, \dots, x_n)`, the probability of
265
+ {math}`\hat{x}_{n+1}` given context {math}`x_1, \dots, x_n` according
266
+ to the target model, and {math}`p(\hat{x}_{n+1}|x_1, \dots, x_n)`, the
267
same conditional probability according to the draft model, the token
268
is accepted with probability:
269
270
- .. math::
271
- \min\left(1, \frac{q(\hat{x}_{n+1}|x_1, \dots, x_n)}
272
- {p(\hat{x}_{n+1}|x_1, \dots, x_n)}\right)
270
+ :::{math}
271
+ \min\left(1, \frac{q(\hat{x}_{n+1}|x_1, \dots, x_n)}
272
+ {p(\hat{x}_{n+1}|x_1, \dots, x_n)}\right)
273
+ :::
274
275
This implementation does not apply causality. When using the output,
276
if a token is rejected, subsequent tokens should not be used.
@@ -312,18 +313,20 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
313
target model is recovered (within hardware numerics).
314
315
The probability distribution used in this rejection case is constructed
315
- as follows. Given :math:`q(x|x_1, \dots, x_n)`, the probability of
316
- :math:`x` given context :math:`x_1, \dots, x_n` according to the target
317
- model and :math:`p(x|x_1, \dots, x_n)`, the same conditional probability
316
+ as follows. Given {math}`q(x|x_1, \dots, x_n)`, the probability of
317
+ {math}`x` given context {math}`x_1, \dots, x_n` according to the target
318
+ model and {math}`p(x|x_1, \dots, x_n)`, the same conditional probability
319
according to the draft model:
320
320
- .. math::
321
- x_{n+1} \sim (q(x|x_1, \dots, x_n) - p(x|x_1, \dots, x_n))_+
321
+ :::{math}
322
+ x_{n+1} \sim (q(x|x_1, \dots, x_n) - p(x|x_1, \dots, x_n))_+
323
+ :::
324
323
- where :math:`(f(x))_+` is defined as:
325
+ where {math}`(f(x))_+` is defined as:
326
325
- .. math::
326
- (f(x))_+ = \frac{\max(0, f(x))}{\sum_x \max(0, f(x))}
327
+ :::{math}
328
+ (f(x))_+ = \frac{\max(0, f(x))}{\sum_x \max(0, f(x))}
329
+ :::
330
331
See https://github.com/vllm-project/vllm/pull/2336 for a visualization
332
of the draft, target, and recovered probability distributions.
vllm/model_executor/layers/sampler.py
+1
-1
@@ -235,7 +235,7 @@ class Sampler(nn.Module):
235
* Defer Pythonization of sampling result & logprobs
236
tensor
237
* Encapsulate arguments required for deferred Pythonization
238
- in the :class:`SamplerOutput` structure
238
+ in the {class}`SamplerOutput` structure
239
240
Args:
241
logits: (num_tokens, vocab_size).
vllm/model_executor/layers/typical_acceptance_sampler.py
+8
-7
@@ -107,14 +107,15 @@ class TypicalAcceptanceSampler(SpecDecodeDeterministicBaseSampler):
107
A draft token_id x_{n+k} is accepted if it satisfies the
108
following condition
109
110
- .. math::
111
- p_{\text{original}}(x_{n+k} | x_1, x_2, \dots, x_{n+k-1}) >
112
- \min \left( \epsilon, \delta * \exp \left(
113
- -H(p_{\text{original}}(
114
- \cdot | x_1, x_2, \ldots, x_{n+k-1})) \right) \right)
110
+ :::{math}
111
+ p_{\text{original}}(x_{n+k} | x_1, x_2, \dots, x_{n+k-1}) >
112
+ \min \left( \epsilon, \delta * \exp \left(
113
+ -H(p_{\text{original}}(
114
+ \cdot | x_1, x_2, \ldots, x_{n+k-1})) \right) \right)
115
+ :::
116
116
- where :math:`p_{\text{original}}` corresponds to target_probs
117
- and :math:`\epsilon` and :math:`\delta` correspond to hyperparameters
117
+ where {math}`p_{\text{original}}` corresponds to target_probs
118
+ and {math}`\epsilon` and {math}`\delta` correspond to hyperparameters
119
specified using self._posterior_threshold and self._posterior_alpha
120
121
This method computes the posterior probabilities for the given
vllm/model_executor/models/blip2.py
+3
-2
@@ -681,8 +681,9 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,
681
batch.
682
pixel_values: The pixels in each input image.
683
684
- See also:
685
- :class:`Blip2ImageInputs`
684
+ :::{seealso}
685
+ {class}`Blip2ImageInputs`
686
+ :::
687
"""
688
689
if intermediate_tensors is not None:
vllm/model_executor/models/interfaces.py
+2
-2
@@ -226,9 +226,9 @@ class SupportsPP(Protocol):
226
intermediate_tensors: Optional["IntermediateTensors"],
227
) -> Union[Tensor, "IntermediateTensors"]:
228
"""
229
- Accept :class:`IntermediateTensors` when PP rank > 0.
229
+ Accept {class}`IntermediateTensors` when PP rank > 0.
230
231
- Return :class:`IntermediateTensors` only for the last PP rank.
231
+ Return {class}`IntermediateTensors` only for the last PP rank.
232
"""
233
...
234
vllm/model_executor/models/llava.py
+3
-2
@@ -721,8 +721,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
721
batch.
722
pixel_values: The pixels in each input image.
723
724
- See also:
725
- :class:`LlavaImageInputs`
724
+ :::{seealso}
725
+ {class}`LlavaImageInputs`
726
+ :::
727
"""
728
if intermediate_tensors is not None:
729
inputs_embeds = None
vllm/model_executor/models/llava_next.py
+4
-3
@@ -537,7 +537,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
537
Unlike in LLaVA-1.5, the number of image tokens inputted to the language
538
model depends on the original size of the input image. Including the
539
original image token in the input, the required number of image tokens
540
- is given by :func:`get_llava_next_image_feature_size`.
540
+ is given by {func}`get_llava_next_image_feature_size`.
541
542
This way, the `positions` and `attn_metadata` are consistent
543
with the `input_ids`.
@@ -548,8 +548,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal,
548
pixel_values: The pixels in each grid patch for each input image.
549
image_sizes: The original `(height, width)` for each input image.
550
551
- See also:
552
- :class:`LlavaNextImageInputs`
551
+ :::{seealso}
552
+ {class}`LlavaNextImageInputs`
553
+ :::
554
"""
555
if intermediate_tensors is not None:
556
inputs_embeds = None
vllm/model_executor/models/mistral3.py
+3
-2
@@ -559,8 +559,9 @@ class Mistral3ForConditionalGeneration(nn.Module, SupportsLoRA,
559
batch.
560
pixel_values: The pixels in each input image.
561
562
- See also:
563
- :class:`Mistral3ImagePixelInputs`
562
+ :::{seealso}
563
+ {class}`Mistral3ImagePixelInputs`
564
+ :::
565
"""
566
if intermediate_tensors is not None:
567
inputs_embeds = None
vllm/model_executor/models/molmo.py
+1
-1
@@ -965,7 +965,7 @@ def select_tiling(
965
966
class MolmoProcessorWrapper:
967
"""
968
- Wraps :class:`MolmoProcessor` so that it can be called directly.
968
+ Wraps {class}`MolmoProcessor` so that it can be called directly.
969
970
The original definition can be found here:
971
https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/preprocessing_molmo.py
vllm/model_executor/models/phi4mm_utils.py
+2
-2
@@ -12,7 +12,7 @@ import torch.nn.functional as F
12
from torch import Tensor, nn
13
14
15
-class Block(nn.Module):
15
+class BlockBase(nn.Module):
16
"""Block abstract module"""
17
18
def __init__(self, input_size, output_size):
@@ -1602,7 +1602,7 @@ class AttModule(nn.Module):
1602
return x, memory, pos_emb, att_mask
1603
1604
1605
-class AttBlock(Block, AttModule):
1605
+class AttBlock(BlockBase, AttModule):
1606
"""Attention Block module to support both Attention and Block module."""
1607
1608
def memory_dims(self, max_len=False):
vllm/model_executor/models/pixtral.py
+2
-2
@@ -65,14 +65,14 @@ class PixtralImagePixelInputs(TypedDict):
65
"""
66
Shape: `(batch_size * num_images, num_channels, image_width, image_height)`
67
68
- The result of stacking :attr:`ImageEncoding.tokens` from each prompt.
68
+ The result of stacking {attr}`ImageEncoding.tokens` from each prompt.
69
"""
70
71
72
class PixtralProcessorAdapter:
73
"""
74
Provide a HF-compatible interface for
75
- :class:`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
75
+ {class}`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
76
"""
77
78
def __init__(self, tokenizer: MistralTokenizer) -> None:
vllm/model_executor/models/qwen_vl.py
+1
-1
@@ -383,7 +383,7 @@ def _get_tokenizer_without_image_pad(
383
tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
384
"""
385
The logic of adding image pad tokens should only be applied in
386
- :class:`QwenVLProcessor`, so they are patched out here.
386
+ {class}`QwenVLProcessor`, so they are patched out here.
387
388
The definition of the wrapped tokenizer can be found here:
389
https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
vllm/model_executor/models/registry.py
+5
-7
@@ -19,7 +19,6 @@ import cloudpickle
19
import torch.nn as nn
20
21
from vllm.logger import init_logger
22
-from vllm.utils import is_in_doc_build
22
23
from .interfaces import (has_inner_state, has_noops, is_attention_free,
24
is_hybrid, supports_cross_encoding,
@@ -375,13 +374,13 @@ class _ModelRegistry:
374
"""
375
Register an external model to be used in vLLM.
376
378
- :code:`model_cls` can be either:
377
+ `model_cls` can be either:
378
380
- - A :class:`torch.nn.Module` class directly referencing the model.
381
- - A string in the format :code:`<module>:<class>` which can be used to
379
+ - A {class}`torch.nn.Module` class directly referencing the model.
380
+ - A string in the format `<module>:<class>` which can be used to
381
lazily import the model. This is useful to avoid initializing CUDA
382
when importing the model and thus the related error
384
- :code:`RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
383
+ `RuntimeError: Cannot re-initialize CUDA in forked subprocess`.
384
"""
385
if not isinstance(model_arch, str):
386
msg = f"`model_arch` should be a string, not a {type(model_arch)}"
@@ -400,8 +399,7 @@ class _ModelRegistry:
399
raise ValueError(msg)
400
401
model = _LazyRegisteredModel(*split_str)
403
- elif isinstance(model_cls, type) and (is_in_doc_build() or issubclass(
404
- model_cls, nn.Module)):
402
+ elif isinstance(model_cls, type) and issubclass(model_cls, nn.Module):
403
model = _RegisteredModel.from_model_cls(model_cls)
404
else:
405
msg = ("`model_cls` should be a string or PyTorch model class, "
vllm/model_executor/models/utils.py
+1
-1
@@ -66,7 +66,7 @@ class WeightsMapper:
66
67
class AutoWeightsLoader:
68
"""
69
- Helper class to load weights into a :class:`torch.nn.Module`. It is able
69
+ Helper class to load weights into a {class}`torch.nn.Module`. It is able
70
to automatically detect child modules and parameters while iterating over
71
the weights only once.
72
vllm/multimodal/__init__.py
+4
-3
@@ -8,11 +8,12 @@ from .registry import MultiModalRegistry
8
9
MULTIMODAL_REGISTRY = MultiModalRegistry()
10
"""
11
-The global :class:`~MultiModalRegistry` is used by model runners to
11
+The global {class}`~MultiModalRegistry` is used by model runners to
12
dispatch data processing according to the target model.
13
14
-See also:
15
- :ref:`mm-processing`
14
+:::{seealso}
15
+{ref}`mm-processing`
16
+:::
17
"""
18
19
__all__ = [
vllm/multimodal/base.py
+22
-22
@@ -64,35 +64,35 @@ class MultiModalPlaceholderMap:
64
65
Examples:
66
67
- .. code-block::
67
+ ```
68
+ Prompt: |AAAA BBBB What's in these images?|
69
+ Positions: |.................................|
70
69
- Prompt: |AAAA BBBB What's in these images?|
70
- Positions: |.................................|
71
+ images = [A, B]
72
+ src_ranges = [(0, 4), (4, 8)]
73
+ dest_ranges = [(0, 4), (5, 9)]
74
72
- images = [A, B]
73
- src_ranges = [(0, 4), (4, 8)]
74
- dest_ranges = [(0, 4), (5, 9)]
75
+ Prompt: |AAAA BBBB What's in these images?|
76
+ Positions: | ..... |
77
76
- Prompt: |AAAA BBBB What's in these images?|
77
- Positions: | ..... |
78
+ images = [A, B]
79
+ src_ranges = [(2, 4), (4, 6)]
80
+ dest_ranges = [(0, 2), (3, 5)]
81
79
- images = [A, B]
80
- src_ranges = [(2, 4), (4, 6)]
81
- dest_ranges = [(0, 2), (3, 5)]
82
+ Prompt: |AAAA BBBB What's in these images?|
83
+ Positions: | ......... |
84
83
- Prompt: |AAAA BBBB What's in these images?|
84
- Positions: | ......... |
85
+ images = [B]
86
+ src_ranges = [(0, 4)]
87
+ dest_ranges = [(0, 4)]
88
86
- images = [B]
87
- src_ranges = [(0, 4)]
88
- dest_ranges = [(0, 4)]
89
+ Prompt: |AAAA BBBB What's in these images?|
90
+ Positions: | .......................|
91
90
- Prompt: |AAAA BBBB What's in these images?|
91
- Positions: | .......................|
92
-
93
- images = []
94
- src_ranges = []
95
- dest_ranges = []
92
+ images = []
93
+ src_ranges = []
94
+ dest_ranges = []
95
+ ```
96
"""
97
seq_mm_data = seq_group.multi_modal_data
98
seq_mm_placeholders = seq_group.multi_modal_placeholders
vllm/multimodal/inputs.py
+129
-126
@@ -26,27 +26,27 @@ _T = TypeVar("_T")
26
27
HfImageItem: TypeAlias = Union[Image, np.ndarray, torch.Tensor]
28
"""
29
-A :class:`transformers.image_utils.ImageInput` representing a single image
30
-item, which can be passed to a HuggingFace :code:`ImageProcessor`.
29
+A {class}`transformers.image_utils.ImageInput` representing a single image
30
+item, which can be passed to a HuggingFace `ImageProcessor`.
31
"""
32
33
HfVideoItem: TypeAlias = Union[list[Image], np.ndarray, torch.Tensor,
34
list[np.ndarray], list[torch.Tensor]]
35
"""
36
-A :class:`transformers.image_utils.VideoInput` representing a single video
37
-item, which can be passed to a HuggingFace :code:`VideoProcessor`.
36
+A {class}`transformers.image_utils.VideoInput` representing a single video
37
+item, which can be passed to a HuggingFace `VideoProcessor`.
38
"""
39
40
HfAudioItem: TypeAlias = Union[list[float], np.ndarray, torch.Tensor]
41
"""
42
Represents a single audio
43
-item, which can be passed to a HuggingFace :code:`AudioProcessor`.
43
+item, which can be passed to a HuggingFace `AudioProcessor`.
44
"""
45
46
ImageItem: TypeAlias = Union[HfImageItem, torch.Tensor]
47
"""
48
-A :class:`transformers.image_utils.ImageInput` representing a single image
49
-item, which can be passed to a HuggingFace :code:`ImageProcessor`.
48
+A {class}`transformers.image_utils.ImageInput` representing a single image
49
+item, which can be passed to a HuggingFace `ImageProcessor`.
50
51
Alternatively, a 3-D tensor or batch of 2-D tensors,
52
which are treated as image embeddings;
@@ -55,8 +55,8 @@ these are directly passed to the model without HF processing.
55
56
VideoItem: TypeAlias = Union[HfVideoItem, torch.Tensor]
57
"""
58
-A :class:`transformers.image_utils.VideoInput` representing a single video
59
-item, which can be passed to a HuggingFace :code:`VideoProcessor`.
58
+A {class}`transformers.image_utils.VideoInput` representing a single video
59
+item, which can be passed to a HuggingFace `VideoProcessor`.
60
61
Alternatively, a 3-D tensor or batch of 2-D tensors,
62
which are treated as video embeddings;
@@ -67,7 +67,7 @@ AudioItem: TypeAlias = Union[HfAudioItem, tuple[np.ndarray, float],
67
torch.Tensor]
68
"""
69
Represents a single audio
70
-item, which can be passed to a HuggingFace :code:`AudioProcessor`.
70
+item, which can be passed to a HuggingFace `AudioProcessor`.
71
72
Alternatively, a tuple `(audio, sampling_rate)`, where the sampling rate
73
is different from that expected by the model;
@@ -83,7 +83,7 @@ ModalityData: TypeAlias = Union[_T, list[_T]]
83
Either a single data item, or a list of data items.
84
85
The number of data items allowed per modality is restricted by
86
-:code:`--limit-mm-per-prompt`.
86
+`--limit-mm-per-prompt`.
87
"""
88
89
@@ -105,7 +105,7 @@ MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
105
"""
106
A dictionary containing an entry for each modality type to input.
107
108
-The built-in modalities are defined by :class:`MultiModalDataBuiltins`.
108
+The built-in modalities are defined by {class}`MultiModalDataBuiltins`.
109
"""
110
111
@@ -116,14 +116,14 @@ class PlaceholderRange:
116
117
Example:
118
119
- Prompt: :code:`AAAA BBBB What is in these images?`
119
+ Prompt: `AAAA BBBB What is in these images?`
120
121
- Images A and B will have:
121
+ Images A and B will have:
122
123
- .. code-block::
124
-
125
- A: PlaceholderRange(offset=0, length=4)
126
- B: PlaceholderRange(offset=5, length=4)
123
+ ```
124
+ A: PlaceholderRange(offset=0, length=4)
125
+ B: PlaceholderRange(offset=5, length=4)
126
+ ```
127
"""
128
129
offset: int
@@ -166,7 +166,7 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
166
167
168
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
169
- """Equality check between :data:`NestedTensors` objects."""
169
+ """Equality check between {data}`NestedTensors` objects."""
170
if isinstance(a, torch.Tensor):
171
return isinstance(b, torch.Tensor) and torch.equal(a, b)
172
elif isinstance(b, torch.Tensor):
@@ -186,7 +186,7 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
186
BatchedTensorInputs: TypeAlias = Mapping[str, NestedTensors]
187
"""
188
A dictionary containing nested tensors which have been batched via
189
-:meth:`MultiModalKwargs.batch`.
189
+{meth}`MultiModalKwargs.batch`.
190
"""
191
192
@@ -194,7 +194,7 @@ A dictionary containing nested tensors which have been batched via
194
class MultiModalFieldElem:
195
"""
196
Represents a keyword argument corresponding to a multi-modal item
197
- in :class:`MultiModalKwargs`.
197
+ in {class}`MultiModalKwargs`.
198
"""
199
200
modality: str
@@ -205,13 +205,13 @@ class MultiModalFieldElem:
205
206
key: str
207
"""
208
- The key of this field in :class:`MultiModalKwargs`,
208
+ The key of this field in {class}`MultiModalKwargs`,
209
i.e. the name of the keyword argument to be passed to the model.
210
"""
211
212
data: NestedTensors
213
"""
214
- The tensor data of this field in :class:`MultiModalKwargs`,
214
+ The tensor data of this field in {class}`MultiModalKwargs`,
215
i.e. the value of the keyword argument to be passed to the model.
216
"""
217
@@ -234,7 +234,7 @@ class MultiModalFieldElem:
234
class BaseMultiModalField(ABC):
235
"""
236
Defines how to interpret tensor data belonging to a keyword argument in
237
- :class:`MultiModalKwargs` for multiple multi-modal items, and vice versa.
237
+ {class}`MultiModalKwargs` for multiple multi-modal items, and vice versa.
238
"""
239
240
def _field_factory(self, *, modality: str, key: str):
@@ -259,10 +259,10 @@ class BaseMultiModalField(ABC):
259
data: NestedTensors,
260
) -> Sequence[MultiModalFieldElem]:
261
"""
262
- Construct :class:`MultiModalFieldElem` instances to represent
262
+ Construct {class}`MultiModalFieldElem` instances to represent
263
the provided data.
264
265
- This is the inverse of :meth:`reduce_data`.
265
+ This is the inverse of {meth}`reduce_data`.
266
"""
267
raise NotImplementedError
268
@@ -272,9 +272,9 @@ class BaseMultiModalField(ABC):
272
273
def reduce_data(self, elems: list[MultiModalFieldElem]) -> NestedTensors:
274
"""
275
- Merge the data from multiple instances of :class:`MultiModalFieldElem`.
275
+ Merge the data from multiple instances of {class}`MultiModalFieldElem`.
276
277
- This is the inverse of :meth:`build_elems`.
277
+ This is the inverse of {meth}`build_elems`.
278
"""
279
field_types = [type(item.field) for item in elems]
280
if len(set(field_types)) > 1:
@@ -286,8 +286,9 @@ class BaseMultiModalField(ABC):
286
@dataclass(frozen=True)
287
class MultiModalBatchedField(BaseMultiModalField):
288
"""
289
- See also:
290
- :func:`MultiModalFieldConfig.batched`
289
+ :::{seealso}
290
+ {func}`MultiModalFieldConfig.batched`
291
+ :::
292
"""
293
294
def build_elems(
@@ -316,9 +317,10 @@ class MultiModalBatchedField(BaseMultiModalField):
317
@dataclass(frozen=True)
318
class MultiModalFlatField(BaseMultiModalField):
319
"""
319
- See also:
320
- :func:`MultiModalFieldConfig.flat`
321
- :func:`MultiModalFieldConfig.flat_from_sizes`
320
+ :::{seealso}
321
+ {func}`MultiModalFieldConfig.flat`
322
+ {func}`MultiModalFieldConfig.flat_from_sizes`
323
+ :::
324
"""
325
slices: Union[Sequence[slice], Sequence[Sequence[slice]]]
326
dim: int = 0
@@ -358,8 +360,9 @@ class MultiModalFlatField(BaseMultiModalField):
360
@dataclass(frozen=True)
361
class MultiModalSharedField(BaseMultiModalField):
362
"""
361
- See also:
362
- :func:`MultiModalFieldConfig.shared`
363
+ :::{seealso}
364
+ {func}`MultiModalFieldConfig.shared`
365
+ :::
366
"""
367
batch_size: int
368
@@ -390,17 +393,17 @@ class MultiModalFieldConfig:
393
394
Example:
395
393
- .. code-block::
394
-
395
- Input:
396
- Data: [[AAAA]
397
- [BBBB]
398
- [CCCC]]
399
-
400
- Output:
401
- Element 1: [AAAA]
402
- Element 2: [BBBB]
403
- Element 3: [CCCC]
396
+ ```
397
+ Input:
398
+ Data: [[AAAA]
399
+ [BBBB]
400
+ [CCCC]]
401
+
402
+ Output:
403
+ Element 1: [AAAA]
404
+ Element 2: [BBBB]
405
+ Element 3: [CCCC]
406
+ ```
407
"""
408
return MultiModalFieldConfig(
409
field=MultiModalBatchedField(),
@@ -425,35 +428,35 @@ class MultiModalFieldConfig:
428
429
Example:
430
428
- .. code-block::
429
-
430
- Given:
431
- slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
432
-
433
- Input:
434
- Data: [AAABBBBCC]
435
-
436
- Output:
437
- Element 1: [AAA]
438
- Element 2: [BBBB]
439
- Element 3: [CC]
440
-
441
- .. code-block::
442
-
443
- Given:
444
- slices: [
445
- (slice(None), slice(0, 3)),
446
- (slice(None), slice(3, 7)),
447
- (slice(None), slice(7, 9))]
448
- dim: 1
449
-
450
- Input:
451
- Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
452
-
453
- Output:
454
- Element 1: [[A],[A],[A]]
455
- Element 2: [[B],[B],[B],[B]]
456
- Element 3: [[C],[C]]
431
+ ```
432
+ Given:
433
+ slices: [slice(0, 3), slice(3, 7), slice(7, 9)]
434
+
435
+ Input:
436
+ Data: [AAABBBBCC]
437
+
438
+ Output:
439
+ Element 1: [AAA]
440
+ Element 2: [BBBB]
441
+ Element 3: [CC]
442
+ ```
443
+
444
+ ```
445
+ Given:
446
+ slices: [
447
+ (slice(None), slice(0, 3)),
448
+ (slice(None), slice(3, 7)),
449
+ (slice(None), slice(7, 9))]
450
+ dim: 1
451
+
452
+ Input:
453
+ Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
454
+
455
+ Output:
456
+ Element 1: [[A],[A],[A]]
457
+ Element 2: [[B],[B],[B],[B]]
458
+ Element 3: [[C],[C]]
459
+ ```
460
"""
461
return MultiModalFieldConfig(
462
field=MultiModalFlatField(slices=slices, dim=dim),
@@ -477,36 +480,36 @@ class MultiModalFieldConfig:
480
481
Example:
482
480
- .. code-block::
481
-
482
- Given:
483
- size_per_item: [3, 4, 2]
484
-
485
- Input:
486
- Data: [AAABBBBCC]
487
-
488
- Output:
489
- Element 1: [AAA]
490
- Element 2: [BBBB]
491
- Element 3: [CC]
492
-
493
-
494
- .. code-block::
495
-
496
- Given:
497
- slices: [3, 4, 2]
498
- dim: 1
499
-
500
- Input:
501
- Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
502
-
503
- Output:
504
- Element 1: [[A],[A],[A]]
505
- Element 2: [[B],[B],[B],[B]]
506
- Element 3: [[C],[C]]
507
-
508
- See also:
509
- :func:`MultiModalFieldConfig.flat`
483
+ ```
484
+ Given:
485
+ size_per_item: [3, 4, 2]
486
+
487
+ Input:
488
+ Data: [AAABBBBCC]
489
+
490
+ Output:
491
+ Element 1: [AAA]
492
+ Element 2: [BBBB]
493
+ Element 3: [CC]
494
+ ```
495
+
496
+ ```
497
+ Given:
498
+ slices: [3, 4, 2]
499
+ dim: 1
500
+
501
+ Input:
502
+ Data: [[A],[A],[A],[B],[B],[B],[B],[C],[C]]
503
+
504
+ Output:
505
+ Element 1: [[A],[A],[A]]
506
+ Element 2: [[B],[B],[B],[B]]
507
+ Element 3: [[C],[C]]
508
+ ```
509
+
510
+ :::{seealso}
511
+ {func}`MultiModalFieldConfig.flat`
512
+ :::
513
"""
514
515
if size_per_item.ndim != 1:
@@ -535,19 +538,19 @@ class MultiModalFieldConfig:
538
539
Example:
540
538
- .. code-block::
539
-
540
- Given:
541
- batch_size: 4
541
+ ```
542
+ Given:
543
+ batch_size: 4
544
543
- Input:
544
- Data: [XYZ]
545
+ Input:
546
+ Data: [XYZ]
547
546
- Output:
547
- Element 1: [XYZ]
548
- Element 2: [XYZ]
549
- Element 3: [XYZ]
550
- Element 4: [XYZ]
548
+ Output:
549
+ Element 1: [XYZ]
550
+ Element 2: [XYZ]
551
+ Element 3: [XYZ]
552
+ Element 4: [XYZ]
553
+ ```
554
"""
555
return MultiModalFieldConfig(
556
field=MultiModalSharedField(batch_size),
@@ -570,8 +573,8 @@ class MultiModalFieldConfig:
573
574
class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
575
"""
573
- A collection of :class:`MultiModalFieldElem`
574
- corresponding to a data item in :class:`MultiModalDataItems`.
576
+ A collection of {class}`MultiModalFieldElem`
577
+ corresponding to a data item in {class}`MultiModalDataItems`.
578
"""
579
580
@staticmethod
@@ -590,11 +593,11 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
593
class MultiModalKwargs(UserDict[str, NestedTensors]):
594
"""
595
A dictionary that represents the keyword arguments to
593
- :meth:`~torch.nn.Module.forward`.
596
+ {meth}`~torch.nn.Module.forward`.
597
595
- The metadata :code:`items` enables us to obtain the keyword arguments
596
- corresponding to each data item in :class:`MultiModalDataItems`, via
597
- :meth:`get_item` and :meth:`get_items`.
598
+ The metadata `items` enables us to obtain the keyword arguments
599
+ corresponding to each data item in {class}`MultiModalDataItems`, via
600
+ {meth}`get_item` and {meth}`get_items`.
601
"""
602
603
@staticmethod
@@ -633,7 +636,7 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
636
637
@staticmethod
638
def from_items(items: Sequence[MultiModalKwargsItem]):
636
- """Construct a new :class:`MultiModalKwargs` from multiple items."""
639
+ """Construct a new {class}`MultiModalKwargs` from multiple items."""
640
elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
641
for item in items:
642
for key, elem in item.items():
@@ -798,7 +801,7 @@ A dictionary containing placeholder ranges for each modality.
801
class MultiModalInputs(TypedDict):
802
"""
803
Represents the outputs of
801
- :class:`vllm.multimodal.processing.BaseMultiModalProcessor`,
804
+ {class}`vllm.multimodal.processing.BaseMultiModalProcessor`,
805
ready to be passed to vLLM internals.
806
"""
807
@@ -823,7 +826,7 @@ class MultiModalInputs(TypedDict):
826
mm_placeholders: MultiModalPlaceholderDict
827
"""
828
For each modality, information about the placeholder tokens in
826
- :code:`prompt_token_ids`.
829
+ `prompt_token_ids`.
830
"""
831
832
cache_salt: NotRequired[str]
@@ -834,7 +837,7 @@ class MultiModalInputs(TypedDict):
837
838
class MultiModalEncDecInputs(MultiModalInputs):
839
"""
837
- Represents the outputs of :class:`vllm.multimodal.EncDecMultiModalProcessor`
840
+ Represents the outputs of {class}`vllm.multimodal.EncDecMultiModalProcessor`
841
ready to be passed to vLLM internals.
842
"""
843
vllm/multimodal/parse.py
+5
-5
@@ -25,7 +25,7 @@ _I = TypeVar("_I")
25
26
class ModalityDataItems(ABC, Generic[_T, _I]):
27
"""
28
- Represents data items for a modality in :class:`MultiModalDataItems`.
28
+ Represents data items for a modality in {class}`MultiModalDataItems`.
29
"""
30
31
def __init__(self, data: _T, modality: str) -> None:
@@ -246,7 +246,7 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
246
247
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
248
"""
249
- As :data:`~vllm.multimodal.inputs.MultiModalDataDict`, but normalized
249
+ As {data}`~vllm.multimodal.inputs.MultiModalDataDict`, but normalized
250
such that each entry corresponds to a list.
251
"""
252
@@ -254,7 +254,7 @@ class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
254
"""
255
Get the number of data items belonging to a modality.
256
257
- If `strict=False`, return `0` instead of raising :exc:`KeyError`
257
+ If `strict=False`, return `0` instead of raising {exc}`KeyError`
258
even if the modality is not found.
259
"""
260
if modality not in self:
@@ -300,8 +300,8 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
300
301
class MultiModalDataParser:
302
"""
303
- Parses :data:`~vllm.multimodal.inputs.MultiModalDataDict` into
304
- :class:`MultiModalDataItems`.
303
+ Parses {data}`~vllm.multimodal.inputs.MultiModalDataDict` into
304
+ {class}`MultiModalDataItems`.
305
306
Args:
307
target_sr (float, optional): Enables automatic resampling of audio
vllm/multimodal/processing.py
+122
-122
@@ -111,13 +111,13 @@ class PromptUpdateDetails(Generic[_S]):
111
112
is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None
113
"""
114
- Given :attr:`full`, return a boolean mask of shape `(len(full),)`
114
+ Given {attr}`full`, return a boolean mask of shape `(len(full),)`
115
indicating which positions of `full` to assign embeddings to.
116
117
`None` (default) means to assign embeddings to all positions of `full`.
118
119
The embeddings are obtained by calling
120
- :class:`SupportsMultiModal.get_multimodal_embeddings`.
120
+ {class}`SupportsMultiModal.get_multimodal_embeddings`.
121
"""
122
123
@staticmethod
@@ -156,13 +156,13 @@ PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
156
The token sequence or text that are part of the update.
157
158
If only part of the content corresponds to feature placeholders, you can
159
-use :class:`PromptUpdateDetails` to specify which part.
159
+use {class}`PromptUpdateDetails` to specify which part.
160
"""
161
162
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
163
PromptUpdateInfo]
164
"""
165
-Given the index of the processed item within :attr:`modality`,
165
+Given the index of the processed item within {attr}`modality`,
166
output the corresponding token sequence (or text).
167
168
For convenience, you can directly pass in the token sequence (or text)
@@ -213,52 +213,52 @@ class PromptInsertion(PromptUpdate):
213
214
Example:
215
216
- For each image, insert a number of ``<image>`` feature placeholders
217
- equal to the feature size of the vision encoder after the ``<s>`` token:
218
-
219
- .. code-block:: python
220
-
221
- PromptInsertion(
222
- modality="image",
223
- target="<s>",
224
- insertion="<image>" * image_feature_size,
225
- )
226
-
227
- Insert these tokens at the start of the prompt:
228
-
229
- .. code-block:: python
230
-
231
- PromptInsertion(
232
- modality="image",
233
- target=PromptIndexTargets.start(),
234
- insertion="<image>" * image_feature_size,
235
- )
236
-
237
- Insert these tokens after a prefix ``Images:``:
238
-
239
- .. code-block:: python
240
-
241
- PromptInsertion(
242
- modality="image",
243
- target=PromptIndexTargets.prefix("Images:"),
244
- insertion="<image>" * image_feature_size,
245
- )
246
-
247
- Insert these tokens at the end of the prompt:
248
-
249
- .. code-block:: python
250
-
251
- PromptInsertion(
252
- modality="image",
253
- target=PromptIndexTargets.end(),
254
- insertion="<image>" * image_feature_size,
255
- )
216
+ For each image, insert a number of ``<image>`` feature placeholders
217
+ equal to the feature size of the vision encoder after the ``<s>`` token:
218
+
219
+ ```python
220
+ PromptInsertion(
221
+ modality="image",
222
+ target="<s>",
223
+ insertion="<image>" * image_feature_size,
224
+ )
225
+ ```
226
+
227
+ Insert these tokens at the start of the prompt:
228
+
229
+ ```python
230
+ PromptInsertion(
231
+ modality="image",
232
+ target=PromptIndexTargets.start(),
233
+ insertion="<image>" * image_feature_size,
234
+ )
235
+ ```
236
+
237
+ Insert these tokens after a prefix ``Images:``:
238
+
239
+ ```python
240
+ PromptInsertion(
241
+ modality="image",
242
+ target=PromptIndexTargets.prefix("Images:"),
243
+ insertion="<image>" * image_feature_size,
244
+ )
245
+ ```
246
+
247
+ Insert these tokens at the end of the prompt:
248
+
249
+ ```python
250
+ PromptInsertion(
251
+ modality="image",
252
+ target=PromptIndexTargets.end(),
253
+ insertion="<image>" * image_feature_size,
254
+ )
255
+ ```
256
"""
257
258
insertion: PromptUpdateContent = field(repr=False)
259
"""
260
- Given the index of the processed item within :attr:`modality`,
261
- output the token sequence (or text) to insert right after :attr:`target`.
260
+ Given the index of the processed item within {attr}`modality`,
261
+ output the token sequence (or text) to insert right after {attr}`target`.
262
263
For convenience, you can directly pass in the token sequence (or text)
264
instead of a function if it does not depend on the input.
@@ -280,57 +280,57 @@ class PromptReplacement(PromptUpdate):
280
281
Example:
282
283
- For each image, replace one ``<image>`` input placeholder in the prompt
284
- with a number of ``<image>`` feature placeholders
285
- equal to the feature size of the vision encoder:
286
-
287
- .. code-block:: python
288
-
289
- PromptReplacement(
290
- modality="image",
291
- target="<image>",
292
- replacement="<image>" * image_feature_size,
293
- )
294
-
295
- As above, but further pad the feature placeholders with ``<image_bos>``
296
- and `<image_eos>``, which are not supposed to be passed to the vision
297
- encoder:
298
-
299
- .. code-block:: python
300
-
301
- PromptReplacement(
302
- modality="image",
303
- target="<image>",
304
- replacement=PromptUpdateDetails(
305
- full="".join([
306
- "<image_bos>",
307
- "<image>" * image_feature_size,
308
- "<image_eos>",
309
- ]),
310
- features="<image>" * image_feature_size,
311
- ),
312
- )
313
-
314
- To avoid unnecessary tokenization during prompt replacement,
315
- we recommended passing token sequences instead of text:
316
-
317
- .. code-block:: python
318
-
319
- PromptReplacement(
320
- modality="image",
321
- target=[image_token_id],
322
- replacement=PromptUpdateDetails(
323
- full=([image_bos_id] + [image_token_id] * image_feature_size
324
- + [image_eos_id]),
325
- features=[image_token_id] * image_feature_size,
326
- ),
327
- )
283
+ For each image, replace one ``<image>`` input placeholder in the prompt
284
+ with a number of ``<image>`` feature placeholders
285
+ equal to the feature size of the vision encoder:
286
+
287
+ ```python
288
+ PromptReplacement(
289
+ modality="image",
290
+ target="<image>",
291
+ replacement="<image>" * image_feature_size,
292
+ )
293
+ ```
294
+
295
+ As above, but further pad the feature placeholders with ``<image_bos>``
296
+ and `<image_eos>``, which are not supposed to be passed to the vision
297
+ encoder:
298
+
299
+ ```python
300
+ PromptReplacement(
301
+ modality="image",
302
+ target="<image>",
303
+ replacement=PromptUpdateDetails(
304
+ full="".join([
305
+ "<image_bos>",
306
+ "<image>" * image_feature_size,
307
+ "<image_eos>",
308
+ ]),
309
+ features="<image>" * image_feature_size,
310
+ ),
311
+ )
312
+ ```
313
+
314
+ To avoid unnecessary tokenization during prompt replacement,
315
+ we recommended passing token sequences instead of text:
316
+
317
+ ```python
318
+ PromptReplacement(
319
+ modality="image",
320
+ target=[image_token_id],
321
+ replacement=PromptUpdateDetails(
322
+ full=([image_bos_id] + [image_token_id] * image_feature_size
323
+ + [image_eos_id]),
324
+ features=[image_token_id] * image_feature_size,
325
+ ),
326
+ )
327
+ ```
328
"""
329
330
replacement: PromptUpdateContent = field(repr=False)
331
"""
332
- Given the index of the processed item within :attr:`modality`,
333
- output the token sequence (or text) to replace :attr:`target`.
332
+ Given the index of the processed item within {attr}`modality`,
333
+ output the token sequence (or text) to replace {attr}`target`.
334
335
For convenience, you can directly pass in the token sequence (or text)
336
instead of a function if it does not depend on the input.
@@ -384,14 +384,14 @@ _M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
384
385
386
def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
387
- """Convenience function to apply :func:`full_groupby` based on modality."""
387
+ """Convenience function to apply {func}`full_groupby` based on modality."""
388
return full_groupby(values, key=lambda x: x.modality)
389
390
391
@dataclass
392
class _BoundPromptSequence:
393
"""
394
- A :data:`_PromptSeq` bound to a tokenizer to automatically
394
+ A {data}`_PromptSeq` bound to a tokenizer to automatically
395
convert between token sequence and text representations.
396
"""
397
tokenizer: AnyTokenizer = field(repr=False)
@@ -443,8 +443,8 @@ class _BoundPromptContent:
443
@dataclass
444
class BoundPromptUpdate:
445
"""
446
- A :class:`PromptUpdate` bound to a tokenizer to automatically convert
447
- :attr:`target` and the result of :meth:`get_content` between
446
+ A {class}`PromptUpdate` bound to a tokenizer to automatically convert
447
+ {attr}`target` and the result of {meth}`get_content` between
448
token sequence and text representations.
449
"""
450
_origin: PromptUpdate
@@ -479,7 +479,7 @@ class BoundPromptUpdate:
479
480
def get_content(self, item_idx: int) -> _BoundPromptContent:
481
"""
482
- Given the index of the processed item within :attr:`modality`,
482
+ Given the index of the processed item within {attr}`modality`,
483
output the token sequence (or text) to update.
484
"""
485
content = self.content
@@ -516,7 +516,7 @@ def iter_token_matches(
516
match_ids: list[int],
517
) -> Generator[_TokenMatch]:
518
"""
519
- Yield each occurrence of :code:`match_ids` in :code:`token_ids`.
519
+ Yield each occurrence of `match_ids` in `token_ids`.
520
521
Note that empty matches are ignored.
522
"""
@@ -545,8 +545,8 @@ def replace_token_matches(
545
new_ids: list[int],
546
) -> list[int]:
547
"""
548
- Replace each occurrence of :code:`match_ids` in :code:`token_ids`
549
- with :code:`new_ids`.
548
+ Replace each occurrence of `match_ids` in `token_ids`
549
+ with `new_ids`.
550
551
Note that empty matches are ignored.
552
"""
@@ -654,7 +654,7 @@ def find_token_matches(
654
prompt: list[int],
655
prompt_updates: Sequence[BoundPromptUpdate],
656
) -> Sequence[PromptTargetMatch]:
657
- """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
657
+ """Return each target of `prompt_updates` found in `prompt`."""
658
659
def get_matches(update: BoundPromptUpdate):
660
target = update.target
@@ -680,7 +680,7 @@ def find_text_matches(
680
prompt: str,
681
prompt_updates: Sequence[BoundPromptUpdate],
682
) -> Sequence[PromptTargetMatch]:
683
- """Return each target of :code:`prompt_updates` found in :code:`prompt`."""
683
+ """Return each target of `prompt_updates` found in `prompt`."""
684
685
def get_matches(update: BoundPromptUpdate):
686
target = update.target
@@ -707,7 +707,7 @@ def _resolve_matches(
707
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
708
) -> list[PromptTargetMatch]:
709
"""
710
- Resolve :code:`mm_matches` to ensure that there are no overlapping matches,
710
+ Resolve `mm_matches` to ensure that there are no overlapping matches,
711
and sort them such that earlier matches take priority over later ones.
712
"""
713
matches = [m for matches in mm_matches.values() for m in matches]
@@ -731,7 +731,7 @@ def _apply_matches(
731
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
732
mm_item_counts: Mapping[str, int],
733
) -> list[_S]:
734
- """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
734
+ """Apply the updates in `mm_matches` to `prompt`."""
735
out_seqs = list[Union[str, list[int]]]()
736
prev_end_idx = 0
737
next_idx_by_modality = defaultdict[str, int](lambda: 0)
@@ -780,7 +780,7 @@ def apply_token_matches(
780
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
781
mm_item_counts: Mapping[str, int],
782
) -> list[int]:
783
- """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
783
+ """Apply the updates in `mm_matches` to `prompt`."""
784
if not mm_matches:
785
return prompt
786
@@ -794,7 +794,7 @@ def apply_text_matches(
794
mm_matches: Mapping[str, Sequence[PromptTargetMatch]],
795
mm_item_counts: Mapping[str, int],
796
) -> str:
797
- """Apply the updates in :code:`mm_matches` to :code:`prompt`."""
797
+ """Apply the updates in `mm_matches` to `prompt`."""
798
if not mm_matches:
799
return prompt
800
@@ -809,7 +809,7 @@ def _iter_placeholders(
809
mm_item_counts: Mapping[str, int],
810
) -> Iterable[PlaceholderFeaturesInfo]:
811
"""
812
- Yield each set of placeholder tokens found in :code:`prompt`.
812
+ Yield each set of placeholder tokens found in `prompt`.
813
814
Matches are exclusive even when multiple modalities share
815
the same placeholder tokens. In that case, the modality that
@@ -1016,7 +1016,7 @@ class ProcessingCache:
1016
) -> None:
1017
"""
1018
Put a processed multi-modal item into the cache
1019
- according to its dependencies (see :meth:`get`).
1019
+ according to its dependencies (see {meth}`get`).
1020
"""
1021
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
1022
**{modality: input_item},
@@ -1083,7 +1083,7 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
1083
1084
MultiModalHashes = dict[str, list[str]]
1085
"""
1086
-A collection of hashes with a similar structure as :class:`MultiModalKwargs`.
1086
+A collection of hashes with a similar structure as {class}`MultiModalKwargs`.
1087
"""
1088
1089
@@ -1091,7 +1091,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1091
"""
1092
Abstract base class to process multi-modal inputs to be used in vLLM.
1093
1094
- Not to be confused with :class:`transformers.ProcessorMixin`.
1094
+ Not to be confused with {class}`transformers.ProcessorMixin`.
1095
"""
1096
1097
def __init__(self,
@@ -1118,10 +1118,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1118
def _get_data_parser(self) -> MultiModalDataParser:
1119
"""
1120
Construct a parser to preprocess multi-modal data items
1121
- before passing them to :meth:`_get_hf_mm_data`.
1121
+ before passing them to {meth}`_get_hf_mm_data`.
1122
1123
You can support additional modalities by creating a subclass
1124
- of :class:`MultiModalDataParser` that has additional subparsers.
1124
+ of {class}`MultiModalDataParser` that has additional subparsers.
1125
"""
1126
return MultiModalDataParser()
1127
@@ -1130,8 +1130,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1130
mm_data: MultiModalDataDict,
1131
) -> MultiModalDataItems:
1132
"""
1133
- Normalize :class:`MultiModalDataDict` to :class:`MultiModalDataItems`
1134
- before passing them to :meth:`_get_hf_mm_data`.
1133
+ Normalize {class}`MultiModalDataDict` to {class}`MultiModalDataItems`
1134
+ before passing them to {meth}`_get_hf_mm_data`.
1135
"""
1136
mm_items = self.data_parser.parse_mm_data(mm_data)
1137
supported_mm_limits = self.info.get_supported_mm_limits()
@@ -1183,7 +1183,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1183
inputs.
1184
1185
Moreover, this information is critical to determine the token positions
1186
- in order to construct :class:`~vllm-multimodal.input.PlaceholderRange`
1186
+ in order to construct {class}`~vllm-multimodal.input.PlaceholderRange`
1187
for each multi-modal item.
1188
"""
1189
raise NotImplementedError
@@ -1237,8 +1237,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1237
"""
1238
Return whether the HF processor applies prompt updates.
1239
1240
- For most HF processors, this should be :code:`True` when multi-modal
1241
- data items are passed, but :code:`False` when multi-modal embeddings
1240
+ For most HF processors, this should be `True` when multi-modal
1241
+ data items are passed, but `False` when multi-modal embeddings
1242
are passed.
1243
"""
1244
return not any(
@@ -1307,7 +1307,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1307
Most HF processors accept prompt text but not prompt tokens.
1308
If the HF processor adds or removes tokens that are not related to
1309
multi-modal data, you should override this method so it is consistent
1310
- with the output of :meth:`_apply_hf_processor_text_only` on the
1310
+ with the output of {meth}`_apply_hf_processor_text_only` on the
1311
corresponding text.
1312
"""
1313
return prompt_tokens
@@ -1322,7 +1322,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1322
1323
Since HF processor requires that text and multi-modal items
1324
correspond to each other, we generate dummy text using
1325
- :class:`DummyInputsBuilder` to go along with the multi-modal data.
1325
+ {class}`DummyInputsBuilder` to go along with the multi-modal data.
1326
"""
1327
mm_counts = mm_items.get_all_counts()
1328
@@ -1346,10 +1346,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1346
Apply the HF processor on the prompt text and multi-modal data.
1347
1348
In addition, return whether prompt updates have been applied
1349
- (for most HF processors, this should be :code:`True`).
1349
+ (for most HF processors, this should be `True`).
1350
1351
Note:
1352
- If :code:`enable_hf_prompt_update=False`, we use HF processor
1352
+ If `enable_hf_prompt_update=False`, we use HF processor
1353
to perform prompt updates if available; HF processor requires
1354
that the prompt corresponds to multi-modal items.
1355
"""
vllm/multimodal/profiling.py
+2
-2
@@ -25,7 +25,7 @@ logger = init_logger(__name__)
25
class ProcessorInputs:
26
"""
27
Represents the keyword arguments to
28
- :meth:`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
28
+ {meth}`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
29
"""
30
prompt_text: str
31
mm_data: MultiModalDataDict
@@ -63,7 +63,7 @@ class BaseDummyInputsBuilder(ABC, Generic[_I]):
63
# TODO: @abstractmethod after transition
64
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
65
"""
66
- Build the text input corresponding to :code:`mm_counts`.
66
+ Build the text input corresponding to `mm_counts`.
67
"""
68
if (type(self).get_dummy_processor_inputs ==
69
BaseDummyInputsBuilder.get_dummy_processor_inputs):
vllm/multimodal/registry.py
+11
-9
@@ -29,7 +29,7 @@ _I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
29
30
31
class ProcessingInfoFactory(Protocol[_I_co]):
32
- """Constructs a :class:`MultiModalProcessor` instance from the context."""
32
+ """Constructs a {class}`MultiModalProcessor` instance from the context."""
33
34
def __call__(
35
self,
@@ -40,7 +40,7 @@ class ProcessingInfoFactory(Protocol[_I_co]):
40
41
class DummyInputsBuilderFactory(Protocol[_I]):
42
"""
43
- Constructs a :class:`BaseDummyInputsBuilder` instance from the context.
43
+ Constructs a {class}`BaseDummyInputsBuilder` instance from the context.
44
"""
45
46
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
@@ -48,7 +48,7 @@ class DummyInputsBuilderFactory(Protocol[_I]):
48
49
50
class MultiModalProcessorFactory(Protocol[_I]):
51
- """Constructs a :class:`MultiModalProcessor` instance from the context."""
51
+ """Constructs a {class}`MultiModalProcessor` instance from the context."""
52
53
def __call__(
54
self,
@@ -150,7 +150,7 @@ class MultiModalRegistry:
150
Get the maximum number of tokens from each modality
151
for profiling the memory usage of a model.
152
153
- See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
153
+ See {meth}`MultiModalPlugin.get_max_multimodal_tokens` for more details.
154
"""
155
mm_limits = self.get_mm_limits_per_prompt(model_config)
156
@@ -165,7 +165,7 @@ class MultiModalRegistry:
165
Get the maximum number of multi-modal tokens
166
for profiling the memory usage of a model.
167
168
- See :meth:`MultiModalPlugin.get_max_multimodal_tokens` for more details.
168
+ See {meth}`MultiModalPlugin.get_max_multimodal_tokens` for more details.
169
"""
170
return sum(self.get_max_tokens_by_modality(model_config).values())
171
@@ -208,8 +208,9 @@ class MultiModalRegistry:
208
When the model receives multi-modal data, the provided function is
209
invoked to transform the data into a dictionary of model inputs.
210
211
- See also:
212
- :ref:`mm-processing`
211
+ :::{seealso}
212
+ {ref}`mm-processing`
213
+ :::
214
"""
215
216
def wrapper(model_cls: N) -> N:
@@ -253,8 +254,9 @@ class MultiModalRegistry:
254
"""
255
Create a multi-modal processor for a specific model and tokenizer.
256
256
- See also:
257
- :ref:`mm-processing`
257
+ :::{seealso}
258
+ {ref}`mm-processing`
259
+ :::
260
"""
261
if not model_config.is_multimodal_model:
262
raise ValueError(f"{model_config.model} is not a multimodal model")
vllm/multimodal/utils.py
+23
-18
@@ -2,7 +2,7 @@
2
3
from itertools import groupby
4
from pathlib import Path
5
-from typing import TYPE_CHECKING, Optional, TypeVar, Union
5
+from typing import TYPE_CHECKING, Any, Optional, TypeVar, Union
6
from urllib.parse import ParseResult, urlparse
7
8
import numpy as np
@@ -24,6 +24,10 @@ _M = TypeVar("_M")
24
if TYPE_CHECKING:
25
from .hasher import MultiModalHashDict
26
from .inputs import MultiModalKwargs, MultiModalPlaceholderDict
27
+else:
28
+ MultiModalHashDict = Any
29
+ MultiModalKwargs = Any
30
+ MultiModalPlaceholderDict = Any
31
32
33
class MediaConnector:
@@ -255,7 +259,7 @@ class MediaConnector:
259
260
261
global_media_connector = MediaConnector()
258
-"""The global :class:`MediaConnector` instance used by vLLM."""
262
+"""The global {class}`MediaConnector` instance used by vLLM."""
263
264
fetch_audio = global_media_connector.fetch_audio
265
fetch_image = global_media_connector.fetch_image
@@ -293,24 +297,24 @@ def encode_video_base64(frames: npt.NDArray) -> str:
297
298
299
def merge_and_sort_multimodal_metadata(
296
- mm_positions: "MultiModalPlaceholderDict",
297
- mm_hashes: Optional["MultiModalHashDict"],
300
+ mm_positions: MultiModalPlaceholderDict,
301
+ mm_hashes: Optional[MultiModalHashDict],
302
) -> tuple[list[str], list[PlaceholderRange], Optional[list[str]]]:
303
"""Given a MultiModalPlaceholderDict, merge all PlaceholderRange
304
objects from all available modalities into a single list of
301
- PlaceholderRange, sorted by their offset (starting index in the input
305
+ PlaceholderRange, sorted by their offset (starting index in the input
306
sequence) in the ascending order.
307
304
- Optionally if a MultiModalHashDict is given, same operation will be
308
+ Optionally if a `MultiModalHashDict` is given, same operation will be
309
applied to the object and the sorted list of hashes will be returned.
310
311
Returns:
308
- list[str]: List of item modalities in order of their positions in
309
- the input sequence.
310
- list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
311
- mm_positions.
312
- Optional[list[str]]: Sorted list of all hashes from mm_hashes if
313
- given, None otherwise.
312
+ list[str]: List of item modalities in order of their positions in the
313
+ input sequence.
314
+ list[PlaceholderRange]: Sorted list of all PlaceholdeRanges from
315
+ mm_positions.
316
+ Optional[list[str]]: Sorted list of all hashes from mm_hashes if given,
317
+ None otherwise.
318
"""
319
320
modalities = list(mm_positions.keys())
@@ -352,22 +356,23 @@ def merge_and_sort_multimodal_metadata(
356
357
358
def group_mm_inputs_by_modality(
355
- mm_inputs: list["MultiModalKwargs"]) -> list[list["MultiModalKwargs"]]:
356
- """Group consecutive MultiModalKwargs from mm_inputs with the same modality
357
- together into the same list for batching purpose. For MultiModalKwargs with
359
+ mm_inputs: list[MultiModalKwargs]) -> list[list[MultiModalKwargs]]:
360
+ """Group consecutive MultiModalKwargs from mm_inputs with the same modality
361
+ together into the same list for batching purpose. For MultiModalKwargs with
362
multiple modalities, put them into their own list.
363
364
Args:
365
mm_inputs: List of MultiModalKwargs.
366
367
Returns:
364
- list[list[MultiModalKwargs]]: List of list of MultiModalKwargs, each
365
- inner list contains consecutive MultiModalKwargs with same modality.
368
+ list[list[vllm.multimodal.MultiModalKwargs]]: List of list of
369
+ `MultiModalKwargs`, each inner list contains consecutive
370
+ `MultiModalKwargs` with same modality.
371
"""
372
if not mm_inputs:
373
return []
374
370
- def modality_group_func(mm_input: "MultiModalKwargs") -> Union[str, int]:
375
+ def modality_group_func(mm_input: MultiModalKwargs) -> Union[str, int]:
376
# If the input has multiple modalities, return a id as the unique key
377
# for the mm_input input.
378
if len(mm_input.modalities) > 1:
vllm/platforms/cpu.py
-2
@@ -19,8 +19,6 @@ if TYPE_CHECKING:
19
else:
20
VllmConfig = None
21
22
-logger = init_logger(__name__)
23
-
22
23
class CpuPlatform(Platform):
24
_enum = PlatformEnum.CPU
vllm/platforms/cuda.py
+1
-7
@@ -454,10 +454,4 @@ finally:
454
455
CudaPlatform = NvmlCudaPlatform if nvml_available else NonNvmlCudaPlatform
456
457
-try:
458
- from sphinx.ext.autodoc.mock import _MockModule
459
-
460
- if not isinstance(pynvml, _MockModule):
461
- CudaPlatform.log_warnings()
462
-except ModuleNotFoundError:
463
- CudaPlatform.log_warnings()
457
+CudaPlatform.log_warnings()
vllm/platforms/interface.py
+3
-3
@@ -146,7 +146,7 @@ class Platform:
146
return self._enum == PlatformEnum.OOT
147
148
def is_cuda_alike(self) -> bool:
149
- """Stateless version of :func:`torch.cuda.is_available`."""
149
+ """Stateless version of {func}`torch.cuda.is_available`."""
150
return self._enum in (PlatformEnum.CUDA, PlatformEnum.ROCM)
151
152
def is_sleep_mode_available(self) -> bool:
@@ -165,7 +165,7 @@ class Platform:
165
cls,
166
device_id: int = 0,
167
) -> Optional[DeviceCapability]:
168
- """Stateless version of :func:`torch.cuda.get_device_capability`."""
168
+ """Stateless version of {func}`torch.cuda.get_device_capability`."""
169
return None
170
171
@classmethod
@@ -180,7 +180,7 @@ class Platform:
180
The ``capability`` argument can either be:
181
182
- A tuple ``(major, minor)``.
183
- - An integer ``<major><minor>``. (See :meth:`DeviceCapability.to_int`)
183
+ - An integer ``<major><minor>``. (See {meth}`DeviceCapability.to_int`)
184
"""
185
current_capability = cls.get_device_capability(device_id=device_id)
186
if current_capability is None:
vllm/profiler/__init__.py
-7
@@ -1,7 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-
3
-from .layerwise_profile import layerwise_profile
4
-
5
-__all__ = [
6
- "layerwise_profile",
7
-]
vllm/sequence.py
+7
-7
@@ -27,7 +27,7 @@ VLLM_INVALID_TOKEN_ID = -1
27
28
29
def array_full(token_id: int, count: int):
30
- """:class:`array` equivalent of :func:`numpy.full`."""
30
+ """{class}`array` equivalent of {func}`numpy.full`."""
31
return array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
32
33
@@ -192,11 +192,11 @@ class SequenceData(msgspec.Struct,
192
def from_prompt_token_counts(
193
*token_counts: tuple[int, int]) -> "SequenceData":
194
"""
195
- Construct a :class:`SequenceData` instance by concatenating
195
+ Construct a {class}`SequenceData` instance by concatenating
196
prompt token sequences.
197
198
Each tuple represents one token sequence, expressed in the form
199
- :code:`(token_id, count)`.
199
+ `(token_id, count)`.
200
"""
201
if len(token_counts) == 0:
202
return SequenceData.from_seqs([])
@@ -216,7 +216,7 @@ class SequenceData(msgspec.Struct,
216
prompt_embeds: Optional[torch.Tensor] = None,
217
) -> "SequenceData":
218
"""
219
- Construct a :class:`SequenceData` instance from prompt and output
219
+ Construct a {class}`SequenceData` instance from prompt and output
220
token sequences.
221
"""
222
prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
@@ -452,9 +452,9 @@ class SequenceData(msgspec.Struct,
452
class Sequence:
453
"""Stores the data, status, and block information of a sequence.
454
455
- The sequence is constructed from the :data:`DecoderOnlyInputs`
456
- (for decoder-only) or :data:`EncoderDecoderInputs` (for encoder-decoder)
457
- instance passed in through the :code:`inputs` constructor argument.
455
+ The sequence is constructed from the {data}`DecoderOnlyInputs`
456
+ (for decoder-only) or {data}`EncoderDecoderInputs` (for encoder-decoder)
457
+ instance passed in through the `inputs` constructor argument.
458
459
Args:
460
seq_id: The ID of the sequence.
vllm/spec_decode/smaller_tp_proposer_worker.py
+2
-1
@@ -52,7 +52,8 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
52
"""Create a SmallerTpProposerWorker.
53
54
Args:
55
- worker (MultiStepWorker): an actual worker wrapped with this class
55
+ worker (~vllm.spec_decode.multi_step_worker.MultiStepWorker): an
56
+ actual worker wrapped with this class
57
draft_ranks (List[int]): if this value is given, only the GPU ranks
58
written in this value participate in draft generation
59
"""
vllm/transformers_utils/configs/dbrx.py
+1
-2
@@ -196,8 +196,7 @@ class DbrxConfig(PretrainedConfig):
196
initializer_range (`float`, *optional*, defaults to 0.02):
197
The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
198
output_router_logits (`bool`, *optional*, defaults to `False`):
199
- Whether or not the router logits should be returned by the model. Enabling this will also
200
- allow the model to output the auxiliary loss. See [here]() for more details
199
+ Whether or not the router logits should be returned by the model. Enabling this will also allow the model to output the auxiliary loss.
200
router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
201
The aux loss factor for the total loss.
202
vllm/transformers_utils/configs/exaone.py
+22
-22
@@ -35,22 +35,22 @@ class ExaoneConfig(PretrainedConfig):
35
Instantiating a configuration with the defaults will yield a similar
36
configuration to that of the Exaone
37
38
- Configuration objects inherit from :class:`~transformers.PretrainedConfig`
38
+ Configuration objects inherit from {class}`~transformers.PretrainedConfig`
39
and can be used to control the model outputs. Read the documentation from :
40
class:`~transformers.PretrainedConfig` for more information.
41
42
Args:
43
- vocab_size (:obj:`int`, `optional`, defaults to 50257):
43
+ vocab_size ({obj}`int`, `optional`, defaults to 50257):
44
Vocabulary size of the GPT Lingvo model. Defines the number of
45
- different tokens that can be represented by the :obj:`inputs_ids`
46
- passed when calling :class:`~transformers.ExaoneModel`. Vocabulary
45
+ different tokens that can be represented by the {obj}`inputs_ids`
46
+ passed when calling {class}`~transformers.ExaoneModel`. Vocabulary
47
size of the model.
48
Defines the different tokens that can be represented by the
49
`inputs_ids` passed to the forward method of :class:
50
`~transformers.EXAONEModel`.
51
- hidden_size (:obj:`int`, `optional`, defaults to 2048):
51
+ hidden_size ({obj}`int`, `optional`, defaults to 2048):
52
Dimensionality of the encoder layers and the pooler layer.
53
- num_layers (:obj:`int`, `optional`, defaults to 24):
53
+ num_layers ({obj}`int`, `optional`, defaults to 24):
54
Number of hidden layers in the Transformer encoder.
55
num_attention_heads (`int`, *optional*, defaults to 32):
56
Number of attention heads for each attention layer in the
@@ -68,37 +68,37 @@ class ExaoneConfig(PretrainedConfig):
68
specified, will default to `num_attention_heads`.
69
rotary_pct (`float`, *optional*, defaults to 0.25):
70
percentage of hidden dimensions to allocate to rotary embeddings
71
- intermediate_size (:obj:`int`, `optional`, defaults to 8192):
71
+ intermediate_size ({obj}`int`, `optional`, defaults to 8192):
72
Dimensionality of the "intermediate" (i.e., feed-forward) layer in
73
the Transformer encoder.
74
- activation_function (:obj:`str` or :obj:`function`, `optional`,
75
- defaults to :obj:`"gelu_new"`):
74
+ activation_function ({obj}`str` or {obj}`function`, `optional`,
75
+ defaults to {obj}`"gelu_new"`):
76
The non-linear activation function (function or string) in the
77
- encoder and pooler. If string, :obj:`"gelu"`, :obj:`"relu"`,
78
- :obj:`"selu"` and :obj:`"gelu_new"` are supported.
79
- embed_dropout (:obj:`float`, `optional`, defaults to 0.0):
77
+ encoder and pooler. If string, {obj}`"gelu"`, {obj}`"relu"`,
78
+ {obj}`"selu"` and {obj}`"gelu_new"` are supported.
79
+ embed_dropout ({obj}`float`, `optional`, defaults to 0.0):
80
The dropout probabilitiy for all fully connected layers in the
81
embeddings, encoder, and pooler.
82
- attention_dropout (:obj:`float`, `optional`, defaults to 0.0):
82
+ attention_dropout ({obj}`float`, `optional`, defaults to 0.0):
83
The dropout ratio for the attention probabilities.
84
- max_position_embeddings (:obj:`int`, `optional`, defaults to 2048):
84
+ max_position_embeddings ({obj}`int`, `optional`, defaults to 2048):
85
The maximum sequence length that this model might ever be used with.
86
Typically set this to something large just in case
87
(e.g., 512 or 1024 or 2048).
88
- type_vocab_size (:obj:`int`, `optional`, defaults to 2):
89
- The vocabulary size of the :obj:`token_type_ids` passed when calling
90
- :class:`~transformers.EXAONEModel`.
91
- initializer_range (:obj:`float`, `optional`, defaults to 0.02):
88
+ type_vocab_size ({obj}`int`, `optional`, defaults to 2):
89
+ The vocabulary size of the {obj}`token_type_ids` passed when calling
90
+ {class}`~transformers.EXAONEModel`.
91
+ initializer_range ({obj}`float`, `optional`, defaults to 0.02):
92
The standard deviation of the truncated_normal_initializer for
93
initializing all weight matrices.
94
- layer_norm_epsilon (:obj:`float`, `optional`, defaults to 1e-5):
94
+ layer_norm_epsilon ({obj}`float`, `optional`, defaults to 1e-5):
95
The epsilon used by the layer normalization layers.
96
- use_cache (:obj:`bool`, `optional`, defaults to :obj:`True`):
96
+ use_cache ({obj}`bool`, `optional`, defaults to {obj}`True`):
97
Whether or not the model should return the last key/values
98
attentions (not used by all models).
99
Only relevant if ``config.is_decoder=True``.
100
- gradient_checkpointing (:obj:`bool`, `optional`,
101
- defaults to :obj:`False`):
100
+ gradient_checkpointing ({obj}`bool`, `optional`,
101
+ defaults to {obj}`False`):
102
If True, use gradient checkpointing to save memory at the expense
103
of slower backward pass.
104
Example::
vllm/transformers_utils/tokenizer.py
+4
-4
@@ -39,9 +39,9 @@ def decode_tokens(
39
) -> str:
40
"""
41
Backend-agnostic equivalent of HF's
42
- :code:`tokenizer.decode(token_ids, ...)`.
42
+ `tokenizer.decode(token_ids, ...)`.
43
44
- :code:`skip_special_tokens=None` means to use the backend's default
44
+ `skip_special_tokens=None` means to use the backend's default
45
settings.
46
"""
47
if skip_special_tokens is not None:
@@ -61,9 +61,9 @@ def encode_tokens(
61
) -> list[int]:
62
"""
63
Backend-agnostic equivalent of HF's
64
- :code:`tokenizer.encode(text, ...)`.
64
+ `tokenizer.encode(text, ...)`.
65
66
- :code:`add_special_tokens=None` means to use the backend's default
66
+ `add_special_tokens=None` means to use the backend's default
67
settings.
68
"""
69
vllm/utils.py
+7
-17
@@ -309,8 +309,8 @@ class LRUCache(cachetools.LRUCache[_K, _V], Generic[_K, _V]):
309
"""
310
Gets the cumulative number of hits and queries against this cache.
311
312
- If :code:`delta=True`, instead gets these statistics
313
- since the last call that also passed :code:`delta=True`.
312
+ If `delta=True`, instead gets these statistics
313
+ since the last call that also passed `delta=True`.
314
"""
315
info = CacheInfo(hits=self._hits, total=self._total)
316
@@ -983,7 +983,7 @@ def flatten_2d_lists(lists: Iterable[Iterable[T]]) -> list[T]:
983
984
def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
985
"""
986
- Unlike :class:`itertools.groupby`, groups are not broken by
986
+ Unlike {class}`itertools.groupby`, groups are not broken by
987
non-contiguous data.
988
"""
989
groups = defaultdict[_K, list[_V]](list)
@@ -1773,14 +1773,6 @@ def get_cuda_view_from_cpu_tensor(cpu_tensor: torch.Tensor) -> torch.Tensor:
1773
return torch.ops._C.get_cuda_view_from_cpu_tensor(cpu_tensor)
1774
1775
1776
-def is_in_doc_build() -> bool:
1777
- try:
1778
- from sphinx.ext.autodoc.mock import _MockModule
1779
- return isinstance(torch, _MockModule)
1780
- except ModuleNotFoundError:
1781
- return False
1782
-
1783
-
1776
def import_from_path(module_name: str, file_path: Union[str, os.PathLike]):
1777
"""
1778
Import a Python file according to its file path.
@@ -1820,10 +1812,11 @@ class _PlaceholderBase:
1812
Disallows downstream usage of placeholder modules.
1813
1814
We need to explicitly override each dunder method because
1823
- :meth:`__getattr__` is not called when they are accessed.
1815
+ {meth}`__getattr__` is not called when they are accessed.
1816
1825
- See also:
1826
- [Special method lookup](https://docs.python.org/3/reference/datamodel.html#special-lookup)
1817
+ :::{seealso}
1818
+ [Special method lookup](https://docs.python.org/3/reference/datamodel.html#special-lookup)
1819
+ :::
1820
"""
1821
1822
def __getattr__(self, key: str) -> Never:
@@ -2052,9 +2045,6 @@ def direct_register_custom_op(
2045
library object. If you want to bind the operator to a different library,
2046
make sure the library object is alive when the operator is used.
2047
"""
2055
- if is_in_doc_build():
2056
- return
2057
-
2048
if not supports_custom_op():
2049
from vllm.platforms import current_platform
2050
assert not current_platform.is_cuda_alike(), (
vllm/v1/attention/backends/mla/common.py
+2
@@ -1,5 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""
3
+# MLA Common Components
4
+
5
This file implements common components for MLA implementations.
6
7
First we define:
vllm/v1/core/kv_cache_manager.py
+2
@@ -180,6 +180,7 @@ class KVCacheManager:
180
as eagle.
181
182
Blocks layout:
183
+ ```
184
-----------------------------------------------------------------------
185
| < computed > | < new computed > | < new > | < pre-allocated > |
186
-----------------------------------------------------------------------
@@ -189,6 +190,7 @@ class KVCacheManager:
190
------------------------------------------------
191
| <new full> |
192
--------------
193
+ ```
194
The following *_blocks are illustrated in this layout.
195
196
Returns:
vllm/v1/engine/output_processor.py
+1
-3
@@ -308,7 +308,7 @@ class OutputProcessor:
308
* If there is no queue (for usage with LLMEngine),
309
return a list of RequestOutput objects.
310
311
- ****************** NOTE FOR DEVELOPERS ******************
311
+ NOTE FOR DEVELOPERS
312
313
vLLM V1 minimizes the number of python loops over the full
314
batch to ensure system overheads are minimized. This is the
@@ -316,8 +316,6 @@ class OutputProcessor:
316
317
If you need to touch every element of the batch, do it from
318
within the loop below.
319
-
320
- **********************************************************
319
"""
320
321
request_outputs: list[RequestOutput] = []
vllm/v1/sample/rejection_sampler.py
+1
-1
@@ -75,7 +75,7 @@ class RejectionSampler(nn.Module):
75
outside of the rejection sampler with the default sampling
76
strategy. It allows for more flexibility in the sampling
77
process such as top_p, top_k sampling.
78
- sampling_metadata (SamplingMetadata):
78
+ sampling_metadata (vllm.v1.sample.metadata.SamplingMetadata):
79
Additional metadata needed for sampling, such as temperature,
80
top-k/top-p parameters, or other relevant information.
81
Returns:
vllm/v1/worker/gpu_worker.py
+4
-3
@@ -170,9 +170,10 @@ class Worker(WorkerBase):
170
Then, it calculate the free memory that can be used for KV cache in
171
bytes.
172
173
- .. tip::
174
- You may limit the usage of GPU memory
175
- by adjusting the `gpu_memory_utilization` parameter.
173
+ :::{tip}
174
+ You may limit the usage of GPU memory
175
+ by adjusting the `gpu_memory_utilization` parameter.
176
+ :::
177
"""
178
torch.cuda.empty_cache()
179
torch.cuda.reset_peak_memory_stats()
vllm/v1/worker/utils.py
+3
-3
@@ -10,7 +10,7 @@ def sanity_check_mm_encoder_outputs(
10
) -> None:
11
"""
12
Perform sanity checks for the result of
13
- :meth:`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.
13
+ {meth}`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.
14
"""
15
assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (
16
"Expected multimodal embeddings to be a list/tuple of 2D tensors, "
@@ -39,7 +39,7 @@ def scatter_mm_placeholders(
39
Scatter the multimodal embeddings into a contiguous tensor that represents
40
the placeholder tokens.
41
42
- :class:`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
42
+ {class}`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
43
44
Args:
45
embeds: The multimodal embeddings.
@@ -66,7 +66,7 @@ def gather_mm_placeholders(
66
"""
67
Reconstructs the embeddings from the placeholder tokens.
68
69
- This is the operation of :func:`scatter_mm_placeholders`.
69
+ This is the operation of {func}`scatter_mm_placeholders`.
70
"""
71
if is_embed is None:
72
return placeholders
vllm/worker/hpu_worker.py
+4
-3
@@ -201,9 +201,10 @@ class HPUWorker(LocalOrDistributedWorkerBase):
201
Then, it calculate the maximum possible number of GPU and CPU blocks
202
that can be allocated with the remaining free memory.
203
204
- .. tip::
205
- You may limit the usage of GPU memory
206
- by adjusting the `gpu_memory_utilization` parameter.
204
+ :::{tip}
205
+ You may limit the usage of GPU memory
206
+ by adjusting the `gpu_memory_utilization` parameter.
207
+ :::
208
"""
209
# Profile the memory usage of the model and get the maximum number of
210
# cache blocks that can be allocated with the remaining free memory.
vllm/worker/multi_step_model_runner.py
+2
-2
@@ -734,11 +734,11 @@ def _pythonize_sampler_output(
734
cache: Optional[PythonizationCache],
735
) -> None:
736
""" This function is only called when the output tensors are ready.
737
- See :class:`ModelOutput`.
737
+ See {class}`ModelOutput`.
738
739
Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place,
740
adding a Pythonized output data structure
741
- (:class:`CompletionSequenceGroupOutput`) for each :class:`SequenceGroup`.
741
+ ({class}`CompletionSequenceGroupOutput`) for each {class}`SequenceGroup`.
742
743
Args:
744
model_input
vllm/worker/worker.py
+4
-3
@@ -230,9 +230,10 @@ class Worker(LocalOrDistributedWorkerBase):
230
Then, it calculate the maximum possible number of GPU and CPU blocks
231
that can be allocated with the remaining free memory.
232
233
- .. tip::
234
- You may limit the usage of GPU memory
235
- by adjusting the `gpu_memory_utilization` parameter.
233
+ :::{tip}
234
+ You may limit the usage of GPU memory
235
+ by adjusting the `gpu_memory_utilization` parameter.
236
+ :::
237
"""
238
# Profile the memory usage of the model and get the maximum number of
239
# cache blocks that can be allocated with the remaining free memory.
vllm/worker/xpu_worker.py
+5
-4
@@ -26,7 +26,7 @@ logger = init_logger(__name__)
26
27
class XPUWorker(LoRANotSupportedWorkerBase, Worker):
28
"""A worker class that executes (a partition of) the model on a GPU.
29
-
29
+
30
Each worker is associated with a single XPU device. The worker is
31
responsible for maintaining the KV cache and executing the model on the
32
XPU. In case of distributed inference, each worker is assigned a partition
@@ -93,9 +93,10 @@ class XPUWorker(LoRANotSupportedWorkerBase, Worker):
93
Then, it calculate the maximum possible number of GPU and CPU blocks
94
that can be allocated with the remaining free memory.
95
96
- .. tip::
97
- You may limit the usage of GPU memory
98
- by adjusting the `gpu_memory_utilization` parameter.
96
+ :::{tip}
97
+ You may limit the usage of GPU memory
98
+ by adjusting the `gpu_memory_utilization` parameter.
99
+ :::
100
"""
101
# Profile the memory usage of the model and get the maximum number of
102
# cache blocks that can be allocated with the remaining free memory.