A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Doc] Convert Sphinx directives ( `{class}`, `{meth}`, `{attr}`, ...) to MkDocs format for better documentation linking (#18663)
Signed-off-by: Zerohertz <ohg3417@gmail.com>
Hyogeun Oh (오효근) committed
May 27, 2025 at 17:44 UTC
a68e293cb9dd40d415a9b25391b13ae370adc62d
37 files changed
+360
-247
vllm/compilation/compiler_interface.py
+2
-1
@@ -39,7 +39,8 @@ class CompilerInterface:
39
Gather all the relevant information from the vLLM config,
40
to compute a hash so that we can cache the compiled model.
41
42
- See {meth}`VllmConfig.compute_hash` to check what information
42
+ See [`VllmConfig.compute_hash`][vllm.config.VllmConfig.compute_hash]
43
+ to check what information
44
is already considered by default. This function should only
45
consider the information that is specific to the compiler.
46
"""
vllm/config.py
+20
-16
@@ -2986,7 +2986,7 @@ class PoolerConfig:
2986
pooling_type: Optional[str] = None
2987
"""
2988
The pooling method of the pooling model. This should be a key in
2989
- {class}`vllm.model_executor.layers.pooler.PoolingType`.
2989
+ [`vllm.model_executor.layers.pooler.PoolingType`][].
2990
"""
2991
2992
normalize: Optional[bool] = None
@@ -3697,23 +3697,27 @@ class CompilationConfig:
3697
"""Configuration for compilation. It has three parts:
3698
3699
- Top-level Compilation control:
3700
- - {attr}`level`
3701
- - {attr}`debug_dump_path`
3702
- - {attr}`cache_dir`
3703
- - {attr}`backend`
3704
- - {attr}`custom_ops`
3705
- - {attr}`splitting_ops`
3700
+ - [`level`][vllm.config.CompilationConfig.level]
3701
+ - [`debug_dump_path`][vllm.config.CompilationConfig.debug_dump_path]
3702
+ - [`cache_dir`][vllm.config.CompilationConfig.cache_dir]
3703
+ - [`backend`][vllm.config.CompilationConfig.backend]
3704
+ - [`custom_ops`][vllm.config.CompilationConfig.custom_ops]
3705
+ - [`splitting_ops`][vllm.config.CompilationConfig.splitting_ops]
3706
- CudaGraph capture:
3707
- - {attr}`use_cudagraph`
3708
- - {attr}`cudagraph_capture_sizes`
3709
- - {attr}`cudagraph_num_of_warmups`
3710
- - {attr}`cudagraph_copy_inputs`
3711
- - {attr}`full_cuda_graph`
3707
+ - [`use_cudagraph`][vllm.config.CompilationConfig.use_cudagraph]
3708
+ - [`cudagraph_capture_sizes`]
3709
+ [vllm.config.CompilationConfig.cudagraph_capture_sizes]
3710
+ - [`cudagraph_num_of_warmups`]
3711
+ [vllm.config.CompilationConfig.cudagraph_num_of_warmups]
3712
+ - [`cudagraph_copy_inputs`]
3713
+ [vllm.config.CompilationConfig.cudagraph_copy_inputs]
3714
+ - [`full_cuda_graph`][vllm.config.CompilationConfig.full_cuda_graph]
3715
- Inductor compilation:
3713
- - {attr}`use_inductor`
3714
- - {attr}`compile_sizes`
3715
- - {attr}`inductor_compile_config`
3716
- - {attr}`inductor_passes`
3716
+ - [`use_inductor`][vllm.config.CompilationConfig.use_inductor]
3717
+ - [`compile_sizes`][vllm.config.CompilationConfig.compile_sizes]
3718
+ - [`inductor_compile_config`]
3719
+ [vllm.config.CompilationConfig.inductor_compile_config]
3720
+ - [`inductor_passes`][vllm.config.CompilationConfig.inductor_passes]
3721
- custom inductor passes
3722
3723
Why we have different sizes for cudagraph and inductor:
vllm/connections.py
+4
-1
@@ -167,4 +167,7 @@ class HTTPConnection:
167
168
169
global_http_connection = HTTPConnection()
170
-"""The global {class}`HTTPConnection` instance used by vLLM."""
170
+"""
171
+The global [`HTTPConnection`][vllm.connections.HTTPConnection] instance used
172
+by vLLM.
173
+"""
vllm/engine/async_llm_engine.py
+27
-23
@@ -475,7 +475,8 @@ class _AsyncLLMEngine(LLMEngine):
475
*,
476
inputs: Optional[PromptType] = None, # DEPRECATED
477
) -> None:
478
- """Async version of {meth}`add_request`."""
478
+ """Async version of
479
+ [`add_request`][vllm.engine.llm_engine.LLMEngine.add_request]."""
480
if inputs is not None:
481
prompt = inputs
482
assert prompt is not None and params is not None
@@ -582,20 +583,21 @@ async def build_guided_decoding_logits_processor_async(
583
584
585
class AsyncLLMEngine(EngineClient):
585
- """An asynchronous wrapper for {class}`LLMEngine`.
586
+ """An asynchronous wrapper for [`LLMEngine`][vllm.LLMEngine].
587
587
- This class is used to wrap the {class}`LLMEngine` class to make it
588
- asynchronous. It uses asyncio to create a background loop that keeps
589
- processing incoming requests. The {class}`LLMEngine` is kicked by the
590
- generate method when there are requests in the waiting queue. The generate
591
- method yields the outputs from the {class}`LLMEngine` to the caller.
588
+ This class is used to wrap the [`LLMEngine`][vllm.LLMEngine] class to
589
+ make it asynchronous. It uses asyncio to create a background loop that keeps
590
+ processing incoming requests. The [`LLMEngine`][vllm.LLMEngine] is kicked
591
+ by the generate method when there are requests in the waiting queue. The
592
+ generate method yields the outputs from the [`LLMEngine`][vllm.LLMEngine]
593
+ to the caller.
594
595
Args:
596
log_requests: Whether to log the requests.
597
start_engine_loop: If True, the background task to run the engine
598
will be automatically started in the generate call.
597
- *args: Arguments for {class}`LLMEngine`.
598
- **kwargs: Arguments for {class}`LLMEngine`.
599
+ *args: Arguments for [`LLMEngine`][vllm.LLMEngine].
600
+ **kwargs: Arguments for [`LLMEngine`][vllm.LLMEngine].
601
"""
602
603
_engine_class: Type[_AsyncLLMEngine] = _AsyncLLMEngine
@@ -985,8 +987,9 @@ class AsyncLLMEngine(EngineClient):
987
from the LLMEngine to the caller.
988
989
Args:
988
- prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
989
- for more details about the format of each input.
990
+ prompt: The prompt to the LLM. See
991
+ [`PromptType`][vllm.inputs.PromptType] for more details about
992
+ the format of each input.
993
sampling_params: The sampling parameters of the request.
994
request_id: The unique id of the request.
995
lora_request: LoRA request to use for generation, if any.
@@ -1003,7 +1006,7 @@ class AsyncLLMEngine(EngineClient):
1006
Details:
1007
- If the engine is not running, start the background loop,
1008
which iteratively invokes
1006
- {meth}`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1009
+ [`engine_step`][vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step]
1010
to process the waiting requests.
1011
- Add the request to the engine's `RequestTracker`.
1012
On the next background loop, this request will be sent to
@@ -1075,8 +1078,9 @@ class AsyncLLMEngine(EngineClient):
1078
from the LLMEngine to the caller.
1079
1080
Args:
1078
- prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
1079
- for more details about the format of each input.
1081
+ prompt: The prompt to the LLM. See
1082
+ [`PromptType`][vllm.inputs.PromptType] for more details about
1083
+ the format of each input.
1084
pooling_params: The pooling parameters of the request.
1085
request_id: The unique id of the request.
1086
lora_request: LoRA request to use for generation, if any.
@@ -1089,15 +1093,15 @@ class AsyncLLMEngine(EngineClient):
1093
for the request.
1094
1095
Details:
1092
- - If the engine is not running, start the background loop,
1093
- which iteratively invokes
1094
- {meth}`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
1095
- to process the waiting requests.
1096
- - Add the request to the engine's `RequestTracker`.
1097
- On the next background loop, this request will be sent to
1098
- the underlying engine.
1099
- Also, a corresponding `AsyncStream` will be created.
1100
- - Wait for the request outputs from `AsyncStream` and yield them.
1096
+ - If the engine is not running, start the background loop,
1097
+ which iteratively invokes
1098
+ [`vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`][]
1099
+ to process the waiting requests.
1100
+ - Add the request to the engine's `RequestTracker`.
1101
+ On the next background loop, this request will be sent to
1102
+ the underlying engine.
1103
+ Also, a corresponding `AsyncStream` will be created.
1104
+ - Wait for the request outputs from `AsyncStream` and yield them.
1105
1106
Example:
1107
```
vllm/engine/llm_engine.py
+4
-4
@@ -130,11 +130,11 @@ class LLMEngine:
130
iteration-level scheduling and efficient memory management to maximize the
131
serving throughput.
132
133
- The [LLM][vllm.LLM] class wraps this class for offline batched inference
134
- and the [AsyncLLMEngine][] class wraps this class for online serving.
133
+ The [`LLM`][vllm.LLM] class wraps this class for offline batched inference
134
+ and the [`AsyncLLMEngine`][vllm.engine.async_llm_engine.AsyncLLMEngine]
135
+ class wraps this class for online serving.
136
136
- The config arguments are derived from [EngineArgs][vllm.EngineArgs]. (See
137
- [engine-args][])
137
+ The config arguments are derived from [`EngineArgs`][vllm.EngineArgs].
138
139
Args:
140
vllm_config: The configuration for initializing and running vLLM.
vllm/engine/multiprocessing/client.py
+6
-4
@@ -492,8 +492,9 @@ class MQLLMEngineClient(EngineClient):
492
from the LLMEngine to the caller.
493
494
Args:
495
- prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
496
- for more details about the format of each input.
495
+ prompt: The prompt to the LLM. See
496
+ [`PromptType`][vllm.inputs.PromptType] for more details about
497
+ the format of each input.
498
sampling_params: The sampling parameters of the request.
499
request_id: The unique id of the request.
500
lora_request: LoRA request to use for generation, if any.
@@ -561,8 +562,9 @@ class MQLLMEngineClient(EngineClient):
562
from the LLMEngine to the caller.
563
564
Args:
564
- prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`
565
- for more details about the format of each input.
565
+ prompt: The prompt to the LLM. See
566
+ [`PromptType`][vllm.inputs.PromptType] for more details about
567
+ the format of each input.
568
pooling_params: The pooling parameters of the request.
569
request_id: The unique id of the request.
570
lora_request: LoRA request to use for generation, if any.
vllm/engine/multiprocessing/engine.py
+11
-8
@@ -42,19 +42,22 @@ HEALTHY_RESPONSE = (pickle.dumps(VLLM_RPC_SUCCESS_STR), )
42
43
44
class MQLLMEngine:
45
- """A multiprocessing wrapper for {class}`LLMEngine`.
45
+ """A multiprocessing wrapper for
46
+ [`LLMEngine`][vllm.engine.llm_engine.LLMEngine].
47
47
- This class is used to wrap the {class}`LLMEngine` class to enable use
48
+ This class is used to wrap the
49
+ [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] class to enable use
50
in concurrnet manner. It runs a background loop and uses zeromq to
51
receive new requests and stream outputs incrementally via ipc.
52
51
- The {class}`LLMEngine` generate or encode process is kicked off when a new
52
- RPCProcessRequest is received by the input_socket.
53
+ The [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] generate or encode
54
+ process is kicked off when a new RPCProcessRequest is received by the
55
+ input_socket.
56
57
The self.engine_loop checks the input_socket for new requests,
58
adds them to the LLMEngine if there are any, calls the internal
56
- {class}`LLMEngine.step()`, and sends the RequestOutputs back over
57
- the output_socket.
59
+ [`LLMEngine.step()`][vllm.engine.llm_engine.LLMEngine.step], and sends
60
+ the RequestOutputs back over the output_socket.
61
62
If use_async_sockets is set, the logic associated with reading new
63
requests from the socket and sending data to the socket is passed
@@ -65,8 +68,8 @@ class MQLLMEngine:
68
ipc_path: Base path for zeromq interprocess messaging
69
use_async_sockets: Whether to make send/recv async with GPU
70
log_requests: Whether to log the requests.
68
- *args: Arguments for {class}`LLMEngine`.
69
- **kwargs: Arguments for {class}`LLMEngine`.
71
+ *args: Arguments for [`LLMEngine`][vllm.engine.llm_engine.LLMEngine].
72
+ **kwargs: Arguments for [`LLMEngine`][vllm.engine.llm_engine.LLMEngine].
73
"""
74
75
def __init__(self,
vllm/engine/output_processor/multi_step.py
+5
-2
@@ -56,8 +56,11 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
56
scheduled computation.
57
58
Args:
59
- seq_group: the outputs are associated with this {class}`SequenceGroup`
60
- outputs: the {class}`SequenceGroupOutput`s for all scheduler steps
59
+ seq_group: the outputs are associated with this
60
+ [`SequenceGroup`][vllm.sequence.SequenceGroup]
61
+ outputs: the
62
+ [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput]s
63
+ for all scheduler steps
64
"""
65
for output in outputs:
66
# Concatenate single-step prompt logprob processing results.
vllm/engine/output_processor/single_step.py
+14
-7
@@ -19,17 +19,21 @@ logger = init_logger(__name__)
19
def single_step_process_prompt_logprob(
20
sg_output_proc: SequenceGroupOutputProcessor, seq_group: SequenceGroup,
21
output: CompletionSequenceGroupOutput) -> None:
22
- """Process prompt logprobs associated with the {class}`SequenceGroupOutput`
23
- for a given step.
22
+ """Process prompt logprobs associated with the
23
+ [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput] for a given step.
24
25
Do nothing if the output has no prompt logprobs.
26
27
Account for the fact that transformers do not compute first-token logprobs.
28
29
Args:
30
- sg_output_proc: {class}`SequenceGroupOutputProcessor` instance
31
- seq_group: the output is associated with this {class}`SequenceGroup`
32
- output: the {class}`SequenceGroupOutput` for a single scheduler step
30
+ sg_output_proc:
31
+ [`SequenceGroupOutputProcessor`][vllm.engine.output_processor.interfaces.SequenceGroupOutputProcessor]
32
+ instance
33
+ seq_group: the output is associated with this
34
+ [`SequenceGroup`][vllm.sequence.SequenceGroup]
35
+ output: the [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput]
36
+ for a single scheduler step
37
"""
38
prompt_logprobs = output.prompt_logprobs
39
@@ -103,8 +107,11 @@ class SingleStepOutputProcessor(SequenceGroupOutputProcessor):
107
scheduled computation.
108
109
Args:
106
- seq_group: the output is associated with this {class}`SequenceGroup`
107
- outputs: the {class}`SequenceGroupOutput` for a single scheduler step
110
+ seq_group: the output is associated with this
111
+ [`SequenceGroup`][vllm.sequence.SequenceGroup]
112
+ outputs: the
113
+ [`SequenceGroupOutput`][vllm.sequence.SequenceGroupOutput]
114
+ for a single scheduler step
115
"""
116
assert len(outputs) == 1, "Single step should only have 1 output."
117
output = outputs[0]
vllm/entrypoints/llm.py
+2
-3
@@ -129,8 +129,7 @@ class LLM:
129
compilation_config: Either an integer or a dictionary. If it is an
130
integer, it is used as the level of compilation optimization. If it
131
is a dictionary, it can specify the full compilation configuration.
132
- **kwargs: Arguments for [EngineArgs][vllm.EngineArgs]. (See
133
- [engine-args][])
132
+ **kwargs: Arguments for [`EngineArgs`][vllm.EngineArgs].
133
134
Note:
135
This class is intended to be used for offline inference. For online
@@ -494,7 +493,7 @@ class LLM:
493
`self` argument, in addition to the arguments passed in `args`
494
and `kwargs`. The `self` argument will be the worker object.
495
timeout: Maximum time in seconds to wait for execution. Raises a
497
- {exc}`TimeoutError` on timeout. `None` means wait indefinitely.
496
+ [`TimeoutError`][] on timeout. `None` means wait indefinitely.
497
args: Positional arguments to pass to the worker method.
498
kwargs: Keyword arguments to pass to the worker method.
499
vllm/entrypoints/openai/serving_engine.py
+4
-2
@@ -582,7 +582,8 @@ class OpenAIServing:
582
add_special_tokens: bool = True,
583
) -> TextTokensPrompt:
584
"""
585
- A simpler implementation of {meth}`_tokenize_prompt_input_or_inputs`
585
+ A simpler implementation of
586
+ [`_tokenize_prompt_input_or_inputs`][vllm.entrypoints.openai.serving_engine.OpenAIServing._tokenize_prompt_input_or_inputs]
587
that assumes single input.
588
"""
589
return next(
@@ -603,7 +604,8 @@ class OpenAIServing:
604
add_special_tokens: bool = True,
605
) -> Iterator[TextTokensPrompt]:
606
"""
606
- A simpler implementation of {meth}`_tokenize_prompt_input_or_inputs`
607
+ A simpler implementation of
608
+ [`_tokenize_prompt_input_or_inputs`][vllm.entrypoints.openai.serving_engine.OpenAIServing._tokenize_prompt_input_or_inputs]
609
that assumes multiple inputs.
610
"""
611
for text in prompt_inputs:
vllm/executor/executor_base.py
+1
-1
@@ -74,7 +74,7 @@ class ExecutorBase(ABC):
74
`self` argument, in addition to the arguments passed in `args`
75
and `kwargs`. The `self` argument will be the worker object.
76
timeout: Maximum time in seconds to wait for execution. Raises a
77
- {exc}`TimeoutError` on timeout. `None` means wait indefinitely.
77
+ [`TimeoutError`][] on timeout. `None` means wait indefinitely.
78
args: Positional arguments to pass to the worker method.
79
kwargs: Keyword arguments to pass to the worker method.
80
vllm/inputs/__init__.py
+3
-2
@@ -10,8 +10,9 @@ from .registry import (DummyData, InputContext, InputProcessingContext,
10
11
INPUT_REGISTRY = InputRegistry()
12
"""
13
-The global {class}`~InputRegistry` which is used by {class}`~vllm.LLMEngine`
14
-to dispatch data processing according to the target model.
13
+The global [`InputRegistry`][vllm.inputs.registry.InputRegistry] which is used
14
+by [`LLMEngine`][vllm.LLMEngine] to dispatch data processing according to the
15
+target model.
16
"""
17
18
__all__ = [
vllm/inputs/data.py
+39
-29
@@ -80,22 +80,24 @@ SingletonPrompt = Union[str, TextPrompt, TokensPrompt, EmbedsPrompt]
80
"""
81
Set of possible schemas for a single prompt:
82
83
-- A text prompt ({class}`str` or {class}`TextPrompt`)
84
-- A tokenized prompt ({class}`TokensPrompt`)
85
-- An embeddings prompt ({class}`EmbedsPrompt`)
83
+- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])
84
+- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])
85
+- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])
86
87
Note that "singleton" is as opposed to a data structure
88
which encapsulates multiple prompts, i.e. of the sort
89
which may be utilized for encoder/decoder models when
90
the user desires to express both the encoder & decoder
91
-prompts explicitly, i.e. {class}`ExplicitEncoderDecoderPrompt`
91
+prompts explicitly, i.e.
92
+[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
93
93
-A prompt of type {class}`SingletonPrompt` may be employed
94
-as (1) input to a decoder-only model, (2) input to
94
+A prompt of type [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] may be
95
+employed as (1) input to a decoder-only model, (2) input to
96
the encoder of an encoder/decoder model, in the scenario
97
where the decoder-prompt is not specified explicitly, or
98
(3) as a member of a larger data structure encapsulating
98
-more than one prompt, i.e. {class}`ExplicitEncoderDecoderPrompt`
99
+more than one prompt, i.e.
100
+[`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
101
"""
102
103
@@ -126,18 +128,20 @@ class ExplicitEncoderDecoderPrompt(TypedDict, Generic[_T1_co, _T2_co]):
128
comprising an explicit encoder prompt and a decoder prompt.
129
130
The encoder and decoder prompts, respectively, may be formatted
129
- according to any of the {class}`SingletonPrompt` schemas,
131
+ according to any of the
132
+ [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] schemas,
133
and are not required to have the same schema.
134
135
Only the encoder prompt may have multi-modal data. mm_processor_kwargs
136
should be at the top-level, and should not be set in the encoder/decoder
137
prompts, since they are agnostic to the encoder/decoder.
138
136
- Note that an {class}`ExplicitEncoderDecoderPrompt` may not
137
- be used as an input to a decoder-only model,
139
+ Note that an
140
+ [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
141
+ may not be used as an input to a decoder-only model,
142
and that the `encoder_prompt` and `decoder_prompt`
143
fields of this data structure themselves must be
140
- {class}`SingletonPrompt` instances.
144
+ [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] instances.
145
"""
146
147
encoder_prompt: _T1_co
@@ -152,11 +156,11 @@ PromptType = Union[SingletonPrompt, ExplicitEncoderDecoderPrompt]
156
Set of possible schemas for an LLM input, including
157
both decoder-only and encoder/decoder input types:
158
155
-- A text prompt ({class}`str` or {class}`TextPrompt`)
156
-- A tokenized prompt ({class}`TokensPrompt`)
157
-- An embeddings prompt ({class}`EmbedsPrompt`)
159
+- A text prompt ([`str`][] or [`TextPrompt`][vllm.inputs.data.TextPrompt])
160
+- A tokenized prompt ([`TokensPrompt`][vllm.inputs.data.TokensPrompt])
161
+- An embeddings prompt ([`EmbedsPrompt`][vllm.inputs.data.EmbedsPrompt])
162
- A single data structure containing both an encoder and a decoder prompt
159
- ({class}`ExplicitEncoderDecoderPrompt`)
163
+ ([`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt])
164
"""
165
166
@@ -189,7 +193,8 @@ def token_inputs(
193
prompt: Optional[str] = None,
194
cache_salt: Optional[str] = None,
195
) -> TokenInputs:
192
- """Construct {class}`TokenInputs` from optional values."""
196
+ """Construct [`TokenInputs`][vllm.inputs.data.TokenInputs] from optional
197
+ values."""
198
inputs = TokenInputs(type="token", prompt_token_ids=prompt_token_ids)
199
200
if prompt is not None:
@@ -221,7 +226,8 @@ def embeds_inputs(
226
prompt_embeds: torch.Tensor,
227
cache_salt: Optional[str] = None,
228
) -> EmbedsInputs:
224
- """Construct :class:`EmbedsInputs` from optional values."""
229
+ """Construct [`EmbedsInputs`][vllm.inputs.data.EmbedsInputs] from optional
230
+ values."""
231
inputs = EmbedsInputs(type="embeds", prompt_embeds=prompt_embeds)
232
233
if cache_salt is not None:
@@ -232,7 +238,7 @@ def embeds_inputs(
238
239
DecoderOnlyInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
240
"""
235
-The inputs in {class}`~vllm.LLMEngine` before they are
241
+The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they are
242
passed to the model executor.
243
This specifies the data required for decoder-only models.
244
"""
@@ -240,11 +246,12 @@ This specifies the data required for decoder-only models.
246
247
class EncoderDecoderInputs(TypedDict):
248
"""
243
- The inputs in {class}`~vllm.LLMEngine` before they are
244
- passed to the model executor.
249
+ The inputs in [`LLMEngine`][vllm.engine.llm_engine.LLMEngine] before they
250
+ are passed to the model executor.
251
252
This specifies the required data for encoder-decoder models.
253
"""
254
+
255
encoder: Union[TokenInputs, "MultiModalInputs"]
256
"""The inputs for the encoder portion."""
257
@@ -254,13 +261,13 @@ class EncoderDecoderInputs(TypedDict):
261
262
SingletonInputs = Union[TokenInputs, EmbedsInputs, "MultiModalInputs"]
263
"""
257
-A processed {class}`SingletonPrompt` which can be passed to
258
-{class}`vllm.sequence.Sequence`.
264
+A processed [`SingletonPrompt`][vllm.inputs.data.SingletonPrompt] which can be
265
+passed to [`vllm.sequence.Sequence`][].
266
"""
267
268
ProcessorInputs = Union[DecoderOnlyInputs, EncoderDecoderInputs]
269
"""
263
-The inputs to {data}`vllm.inputs.InputProcessor`.
270
+The outputs from [`vllm.inputs.preprocess.InputPreprocessor`][].
271
"""
272
273
_T1 = TypeVar("_T1", bound=SingletonPrompt, default=SingletonPrompt)
@@ -277,7 +284,8 @@ def build_explicit_enc_dec_prompt(
284
return ExplicitEncoderDecoderPrompt(
285
encoder_prompt=encoder_prompt,
286
decoder_prompt=decoder_prompt,
280
- mm_processor_kwargs=mm_processor_kwargs)
287
+ mm_processor_kwargs=mm_processor_kwargs,
288
+ )
289
290
291
def zip_enc_dec_prompts(
@@ -288,7 +296,8 @@ def zip_enc_dec_prompts(
296
) -> list[ExplicitEncoderDecoderPrompt[_T1, _T2]]:
297
"""
298
Zip encoder and decoder prompts together into a list of
291
- {class}`ExplicitEncoderDecoderPrompt` instances.
299
+ [`ExplicitEncoderDecoderPrompt`][vllm.inputs.data.ExplicitEncoderDecoderPrompt]
300
+ instances.
301
302
``mm_processor_kwargs`` may also be provided; if a dict is passed, the same
303
dictionary will be used for every encoder/decoder prompt. If an iterable is
@@ -299,10 +308,11 @@ def zip_enc_dec_prompts(
308
if isinstance(mm_processor_kwargs, dict):
309
return [
310
build_explicit_enc_dec_prompt(
302
- encoder_prompt, decoder_prompt,
303
- cast(dict[str, Any], mm_processor_kwargs))
304
- for (encoder_prompt,
305
- decoder_prompt) in zip(enc_prompts, dec_prompts)
311
+ encoder_prompt,
312
+ decoder_prompt,
313
+ cast(dict[str, Any], mm_processor_kwargs),
314
+ ) for (encoder_prompt,
315
+ decoder_prompt) in zip(enc_prompts, dec_prompts)
316
]
317
return [
318
build_explicit_enc_dec_prompt(encoder_prompt, decoder_prompt,
vllm/inputs/parse.py
+4
-4
@@ -23,13 +23,13 @@ class ParsedTokens(TypedDict):
23
24
@overload
25
def parse_and_batch_prompt(
26
- prompt: Union[str, list[str]]) -> Sequence[ParsedText]:
26
+ prompt: Union[str, list[str]], ) -> Sequence[ParsedText]:
27
...
28
29
30
@overload
31
def parse_and_batch_prompt(
32
- prompt: Union[list[int], list[list[int]]]) -> Sequence[ParsedTokens]:
32
+ prompt: Union[list[int], list[list[int]]], ) -> Sequence[ParsedTokens]:
33
...
34
35
@@ -86,7 +86,7 @@ class ParsedTokensPrompt(TypedDict):
86
87
88
class ParsedEmbedsPrompt(TypedDict):
89
- type: Literal['embeds']
89
+ type: Literal["embeds"]
90
content: EmbedsPrompt
91
92
@@ -133,7 +133,7 @@ def parse_singleton_prompt(prompt: SingletonPrompt) -> ParsedSingletonPrompt:
133
134
135
def is_explicit_encoder_decoder_prompt(
136
- prompt: PromptType) -> TypeIs[ExplicitEncoderDecoderPrompt]:
136
+ prompt: PromptType, ) -> TypeIs[ExplicitEncoderDecoderPrompt]:
137
return isinstance(prompt, dict) and "encoder_prompt" in prompt
138
139
vllm/inputs/preprocess.py
+39
-17
@@ -67,11 +67,11 @@ class InputPreprocessor:
67
return self.tokenizer.get_lora_tokenizer(lora_request).eos_token_id
68
69
def get_decoder_start_token_id(self) -> Optional[int]:
70
- '''
70
+ """
71
Obtain the decoder start token id employed by an encoder/decoder
72
model. Returns None for non-encoder/decoder models or if the
73
model config is unavailable.
74
- '''
74
+ """
75
76
if not self.model_config.is_encoder_decoder:
77
logger.warning_once(
@@ -79,14 +79,14 @@ class InputPreprocessor:
79
"this is not an encoder/decoder model.")
80
return None
81
82
- if (self.model_config is None or self.model_config.hf_config is None):
82
+ if self.model_config is None or self.model_config.hf_config is None:
83
logger.warning_once(
84
"Using None for decoder start token id because "
85
"model config is not available.")
86
return None
87
88
dec_start_token_id = getattr(self.model_config.hf_config,
89
- 'decoder_start_token_id', None)
89
+ "decoder_start_token_id", None)
90
if dec_start_token_id is None:
91
logger.warning_once(
92
"Falling back on <BOS> for decoder start token "
@@ -97,7 +97,7 @@ class InputPreprocessor:
97
return dec_start_token_id
98
99
def _get_default_enc_dec_decoder_prompt(self) -> list[int]:
100
- '''
100
+ """
101
Specifically for encoder/decoder models:
102
generate a default decoder prompt for when
103
the user specifies only the encoder prompt.
@@ -126,7 +126,7 @@ class InputPreprocessor:
126
Returns:
127
128
* prompt_token_ids
129
- '''
129
+ """
130
131
bos_token_id = self.get_bos_token_id()
132
assert bos_token_id is not None
@@ -224,7 +224,10 @@ class InputPreprocessor:
224
lora_request: Optional[LoRARequest],
225
tokenization_kwargs: Optional[dict[str, Any]] = None,
226
) -> list[int]:
227
- """Async version of {meth}`_tokenize_prompt`."""
227
+ """
228
+ Async version of
229
+ [`_tokenize_prompt`][vllm.inputs.preprocess.InputPreprocessor._tokenize_prompt].
230
+ """
231
tokenizer = self.get_tokenizer_group()
232
tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs)
233
@@ -287,7 +290,10 @@ class InputPreprocessor:
290
lora_request: Optional[LoRARequest],
291
return_mm_hashes: bool = False,
292
) -> MultiModalInputs:
290
- """Async version of {meth}`_process_multimodal`."""
293
+ """
294
+ Async version of
295
+ [`_process_multimodal`][vllm.inputs.preprocess.InputPreprocessor._process_multimodal].
296
+ """
297
tokenizer = await self._get_mm_tokenizer_async(lora_request)
298
299
mm_processor = self.mm_registry.create_processor(self.model_config,
@@ -472,7 +478,7 @@ class InputPreprocessor:
478
479
Returns:
480
475
- * {class}`SingletonInputs` instance
481
+ * [`SingletonInputs`][vllm.inputs.data.SingletonInputs] instance
482
"""
483
parsed = parse_singleton_prompt(prompt)
484
@@ -508,7 +514,10 @@ class InputPreprocessor:
514
lora_request: Optional[LoRARequest] = None,
515
return_mm_hashes: bool = False,
516
) -> SingletonInputs:
511
- """Async version of {meth}`_prompt_to_llm_inputs`."""
517
+ """
518
+ Async version of
519
+ [`_prompt_to_llm_inputs`][vllm.inputs.preprocess.InputPreprocessor._prompt_to_llm_inputs].
520
+ """
521
parsed = parse_singleton_prompt(prompt)
522
523
if parsed["type"] == "embeds":
@@ -644,7 +653,9 @@ class InputPreprocessor:
653
) -> EncoderDecoderInputs:
654
"""
655
For encoder/decoder models only:
647
- Process an input prompt into an {class}`EncoderDecoderInputs` instance.
656
+ Process an input prompt into an
657
+ [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
658
+ instance.
659
660
There are two types of input prompts:
661
singleton prompts which carry only the
@@ -670,7 +681,8 @@ class InputPreprocessor:
681
682
Returns:
683
673
- * {class}`EncoderDecoderInputs` instance
684
+ * [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
685
+ instance
686
"""
687
encoder_inputs: SingletonInputs
688
decoder_inputs: Optional[SingletonInputs]
@@ -710,7 +722,10 @@ class InputPreprocessor:
722
prompt: PromptType,
723
tokenization_kwargs: Optional[dict[str, Any]] = None,
724
) -> EncoderDecoderInputs:
713
- """Async version of {meth}`_process_encoder_decoder_prompt`."""
725
+ """
726
+ Async version of
727
+ [`_process_encoder_decoder_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_encoder_decoder_prompt].
728
+ """
729
encoder_inputs: SingletonInputs
730
decoder_inputs: Optional[SingletonInputs]
731
@@ -778,7 +793,8 @@ class InputPreprocessor:
793
) -> DecoderOnlyInputs:
794
"""
795
For decoder-only models:
781
- Process an input prompt into an {class}`DecoderOnlyInputs` instance.
796
+ Process an input prompt into a
797
+ [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance.
798
799
Arguments:
800
@@ -789,7 +805,7 @@ class InputPreprocessor:
805
806
Returns:
807
792
- * {class}`DecoderOnlyInputs` instance
808
+ * [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] instance
809
"""
810
811
prompt_comps = self._prompt_to_llm_inputs(
@@ -812,7 +828,10 @@ class InputPreprocessor:
828
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
829
return_mm_hashes: bool = False,
830
) -> DecoderOnlyInputs:
815
- """Async version of {meth}`_process_decoder_only_prompt`."""
831
+ """
832
+ Async version of
833
+ [`_process_decoder_only_prompt`][vllm.inputs.preprocess.InputPreprocessor._process_decoder_only_prompt].
834
+ """
835
prompt_comps = await self._prompt_to_llm_inputs_async(
836
prompt,
837
tokenization_kwargs=tokenization_kwargs,
@@ -863,7 +882,10 @@ class InputPreprocessor:
882
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
883
return_mm_hashes: bool = False,
884
) -> ProcessorInputs:
866
- """Async version of {meth}`preprocess`."""
885
+ """
886
+ Async version of
887
+ [`preprocess`][vllm.inputs.preprocess.InputPreprocessor.preprocess].
888
+ """
889
if self.model_config.is_encoder_decoder:
890
assert not return_mm_hashes, (
891
"Multimodal hashes for encoder-decoder models should not be ",
vllm/inputs/registry.py
+2
-2
@@ -38,7 +38,7 @@ class InputContext:
38
) -> _C:
39
"""
40
Get the HuggingFace configuration
41
- ({class}`transformers.PretrainedConfig`) of the model,
41
+ (`transformers.PretrainedConfig`) of the model,
42
additionally checking its type.
43
44
Raises:
@@ -79,7 +79,7 @@ class InputContext:
79
) -> _P:
80
"""
81
Get the HuggingFace processor
82
- ({class}`transformers.ProcessorMixin`) of the model,
82
+ (`transformers.ProcessorMixin`) of the model,
83
additionally checking its type.
84
85
Raises:
vllm/logger.py
+5
-5
@@ -68,22 +68,22 @@ class _VllmLogger(Logger):
68
"""
69
Note:
70
This class is just to provide type information.
71
- We actually patch the methods directly on the {class}`logging.Logger`
71
+ We actually patch the methods directly on the [`logging.Logger`][]
72
instance to avoid conflicting with other libraries such as
73
`intel_extension_for_pytorch.utils._logger`.
74
"""
75
76
def info_once(self, msg: str, *args: Hashable) -> None:
77
"""
78
- As {meth}`info`, but subsequent calls with the same message
79
- are silently dropped.
78
+ As [`info`][logging.Logger.info], but subsequent calls with
79
+ the same message are silently dropped.
80
"""
81
_print_info_once(self, msg, *args)
82
83
def warning_once(self, msg: str, *args: Hashable) -> None:
84
"""
85
- As {meth}`warning`, but subsequent calls with the same message
86
- are silently dropped.
85
+ As [`warning`][logging.Logger.warning], but subsequent calls with
86
+ the same message are silently dropped.
87
"""
88
_print_warning_once(self, msg, *args)
89
vllm/model_executor/layers/sampler.py
+11
-9
@@ -228,17 +228,19 @@ class Sampler(nn.Module):
228
) -> Optional[SamplerOutput]:
229
"""
230
Single-step scheduling:
231
- * Perform GPU-side sampling computation & compute
232
- GPU-side logprobs tensor
233
- * Pythonize sampling result & logprobs tensor
231
+ * Perform GPU-side sampling computation & compute
232
+ GPU-side logprobs tensor
233
+ * Pythonize sampling result & logprobs tensor
234
235
Multi-step scheduling:
236
- * Perform GPU-side sampling computation & compute
237
- GPU-side logprobs tensor
238
- * Defer Pythonization of sampling result & logprobs
239
- tensor
240
- * Encapsulate arguments required for deferred Pythonization
241
- in the {class}`SamplerOutput` structure
236
+ * Perform GPU-side sampling computation & compute
237
+ GPU-side logprobs tensor
238
+ * Defer Pythonization of sampling result & logprobs
239
+ tensor
240
+ * Encapsulate arguments required for deferred Pythonization
241
+ in the
242
+ [`SamplerOutput`][vllm.model_executor.layers.sampler.SamplerOutput]
243
+ structure
244
245
Args:
246
logits: (num_tokens, vocab_size).
vllm/model_executor/models/interfaces.py
+4
-2
@@ -226,9 +226,11 @@ class SupportsPP(Protocol):
226
intermediate_tensors: Optional["IntermediateTensors"],
227
) -> Union[Tensor, "IntermediateTensors"]:
228
"""
229
- Accept {class}`IntermediateTensors` when PP rank > 0.
229
+ Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
230
+ PP rank > 0.
231
231
- Return {class}`IntermediateTensors` only for the last PP rank.
232
+ Return [`IntermediateTensors`][vllm.sequence.IntermediateTensors] only
233
+ for the last PP rank.
234
"""
235
...
236
vllm/model_executor/models/molmo.py
+1
-1
@@ -965,7 +965,7 @@ def select_tiling(
965
966
class MolmoProcessorWrapper:
967
"""
968
- Wraps {class}`MolmoProcessor` so that it can be called directly.
968
+ Wraps `MolmoProcessor` so that it can be called directly.
969
970
The original definition can be found here:
971
https://huggingface.co/allenai/Molmo-7B-D-0924/blob/main/preprocessing_molmo.py
vllm/model_executor/models/pixtral.py
+2
-2
@@ -67,14 +67,14 @@ class PixtralImagePixelInputs(TypedDict):
67
"""
68
Shape: `(batch_size * num_images, num_channels, image_width, image_height)`
69
70
- The result of stacking {attr}`ImageEncoding.tokens` from each prompt.
70
+ The result of stacking `ImageEncoding.tokens` from each prompt.
71
"""
72
73
74
class PixtralProcessorAdapter:
75
"""
76
Provide a HF-compatible interface for
77
- {class}`mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
77
+ `mistral_common.tokens.tokenizers.multimodal.ImageEncoder`.
78
"""
79
80
def __init__(self, tokenizer: MistralTokenizer) -> None:
vllm/model_executor/models/qwen_vl.py
+2
-1
@@ -382,7 +382,8 @@ def _get_tokenizer_without_image_pad(
382
tokenizer: PreTrainedTokenizer) -> PreTrainedTokenizer:
383
"""
384
The logic of adding image pad tokens should only be applied in
385
- {class}`QwenVLProcessor`, so they are patched out here.
385
+ [`QwenVLProcessor`][vllm.model_executor.models.qwen_vl.QwenVLProcessor],
386
+ so they are patched out here.
387
388
The definition of the wrapped tokenizer can be found here:
389
https://huggingface.co/Qwen/Qwen-VL/blob/main/tokenization_qwen.py
vllm/model_executor/models/registry.py
+1
-1
@@ -383,7 +383,7 @@ class _ModelRegistry:
383
384
`model_cls` can be either:
385
386
- - A {class}`torch.nn.Module` class directly referencing the model.
386
+ - A [`torch.nn.Module`][] class directly referencing the model.
387
- A string in the format `<module>:<class>` which can be used to
388
lazily import the model. This is useful to avoid initializing CUDA
389
when importing the model and thus the related error
vllm/model_executor/models/utils.py
+1
-1
@@ -66,7 +66,7 @@ class WeightsMapper:
66
67
class AutoWeightsLoader:
68
"""
69
- Helper class to load weights into a {class}`torch.nn.Module`. It is able
69
+ Helper class to load weights into a [`torch.nn.Module`][]. It is able
70
to automatically detect child modules and parameters while iterating over
71
the weights only once.
72
vllm/multimodal/__init__.py
+4
-3
@@ -8,11 +8,12 @@ from .registry import MultiModalRegistry
8
9
MULTIMODAL_REGISTRY = MultiModalRegistry()
10
"""
11
-The global {class}`~MultiModalRegistry` is used by model runners to
12
-dispatch data processing according to the target model.
11
+The global [`MultiModalRegistry`][vllm.multimodal.registry.MultiModalRegistry]
12
+is used by model runners to dispatch data processing according to the target
13
+model.
14
15
Info:
15
- [mm-processing][]
16
+ [mm_processing](../../../design/mm_processing.html)
17
"""
18
19
__all__ = [
vllm/multimodal/inputs.py
+45
-29
@@ -29,14 +29,14 @@ _T = TypeVar("_T")
29
30
HfImageItem: TypeAlias = Union["Image", np.ndarray, "torch.Tensor"]
31
"""
32
-A {class}`transformers.image_utils.ImageInput` representing a single image
32
+A `transformers.image_utils.ImageInput` representing a single image
33
item, which can be passed to a HuggingFace `ImageProcessor`.
34
"""
35
36
HfVideoItem: TypeAlias = Union[list["Image"], np.ndarray, "torch.Tensor",
37
list[np.ndarray], list["torch.Tensor"]]
38
"""
39
-A {class}`transformers.image_utils.VideoInput` representing a single video
39
+A `transformers.image_utils.VideoInput` representing a single video
40
item, which can be passed to a HuggingFace `VideoProcessor`.
41
"""
42
@@ -48,7 +48,7 @@ item, which can be passed to a HuggingFace `AudioProcessor`.
48
49
ImageItem: TypeAlias = Union[HfImageItem, "torch.Tensor"]
50
"""
51
-A {class}`transformers.image_utils.ImageInput` representing a single image
51
+A `transformers.image_utils.ImageInput` representing a single image
52
item, which can be passed to a HuggingFace `ImageProcessor`.
53
54
Alternatively, a 3-D tensor or batch of 2-D tensors,
@@ -58,7 +58,7 @@ these are directly passed to the model without HF processing.
58
59
VideoItem: TypeAlias = Union[HfVideoItem, "torch.Tensor"]
60
"""
61
-A {class}`transformers.image_utils.VideoInput` representing a single video
61
+A `transformers.image_utils.VideoInput` representing a single video
62
item, which can be passed to a HuggingFace `VideoProcessor`.
63
64
Alternatively, a 3-D tensor or batch of 2-D tensors,
@@ -108,7 +108,8 @@ MultiModalDataDict: TypeAlias = Mapping[str, ModalityData[Any]]
108
"""
109
A dictionary containing an entry for each modality type to input.
110
111
-The built-in modalities are defined by {class}`MultiModalDataBuiltins`.
111
+The built-in modalities are defined by
112
+[`MultiModalDataBuiltins`][vllm.multimodal.inputs.MultiModalDataBuiltins].
113
"""
114
115
@@ -169,7 +170,8 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
170
171
172
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
172
- """Equality check between {data}`NestedTensors` objects."""
173
+ """Equality check between
174
+ [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects."""
175
if isinstance(a, torch.Tensor):
176
return isinstance(b, torch.Tensor) and torch.equal(a, b)
177
elif isinstance(b, torch.Tensor):
@@ -189,7 +191,7 @@ def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
191
BatchedTensorInputs: TypeAlias = Mapping[str, NestedTensors]
192
"""
193
A dictionary containing nested tensors which have been batched via
192
-{meth}`MultiModalKwargs.batch`.
194
+[`MultiModalKwargs.batch`][vllm.multimodal.inputs.MultiModalKwargs.batch].
195
"""
196
197
@@ -197,7 +199,7 @@ A dictionary containing nested tensors which have been batched via
199
class MultiModalFieldElem:
200
"""
201
Represents a keyword argument corresponding to a multi-modal item
200
- in {class}`MultiModalKwargs`.
202
+ in [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs].
203
"""
204
205
modality: str
@@ -208,13 +210,15 @@ class MultiModalFieldElem:
210
211
key: str
212
"""
211
- The key of this field in {class}`MultiModalKwargs`,
213
+ The key of this field in
214
+ [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs],
215
i.e. the name of the keyword argument to be passed to the model.
216
"""
217
218
data: NestedTensors
219
"""
217
- The tensor data of this field in {class}`MultiModalKwargs`,
220
+ The tensor data of this field in
221
+ [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs],
222
i.e. the value of the keyword argument to be passed to the model.
223
"""
224
@@ -237,7 +241,8 @@ class MultiModalFieldElem:
241
class BaseMultiModalField(ABC):
242
"""
243
Defines how to interpret tensor data belonging to a keyword argument in
240
- {class}`MultiModalKwargs` for multiple multi-modal items, and vice versa.
244
+ [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs] for multiple
245
+ multi-modal items, and vice versa.
246
"""
247
248
def _field_factory(self, *, modality: str, key: str):
@@ -262,10 +267,12 @@ class BaseMultiModalField(ABC):
267
data: NestedTensors,
268
) -> Sequence[MultiModalFieldElem]:
269
"""
265
- Construct {class}`MultiModalFieldElem` instances to represent
266
- the provided data.
270
+ Construct
271
+ [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]
272
+ instances to represent the provided data.
273
268
- This is the inverse of {meth}`reduce_data`.
274
+ This is the inverse of
275
+ [`reduce_data`][vllm.multimodal.inputs.BaseMultiModalField.reduce_data].
276
"""
277
raise NotImplementedError
278
@@ -275,9 +282,11 @@ class BaseMultiModalField(ABC):
282
283
def reduce_data(self, elems: list[MultiModalFieldElem]) -> NestedTensors:
284
"""
278
- Merge the data from multiple instances of {class}`MultiModalFieldElem`.
285
+ Merge the data from multiple instances of
286
+ [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem].
287
280
- This is the inverse of {meth}`build_elems`.
288
+ This is the inverse of
289
+ [`build_elems`][vllm.multimodal.inputs.BaseMultiModalField.build_elems].
290
"""
291
field_types = [type(item.field) for item in elems]
292
if len(set(field_types)) > 1:
@@ -290,7 +299,7 @@ class BaseMultiModalField(ABC):
299
class MultiModalBatchedField(BaseMultiModalField):
300
"""
301
Info:
293
- [MultiModalFieldConfig.batched][]
302
+ [`MultiModalFieldConfig.batched`][vllm.multimodal.inputs.MultiModalFieldConfig.batched]
303
"""
304
305
def build_elems(
@@ -320,8 +329,8 @@ class MultiModalBatchedField(BaseMultiModalField):
329
class MultiModalFlatField(BaseMultiModalField):
330
"""
331
Info:
323
- [MultiModalFieldConfig.flat][]
324
- [MultiModalFieldConfig.flat_from_sizes][]
332
+ [`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
333
+ [`MultiModalFieldConfig.flat_from_sizes`][vllm.multimodal.inputs.MultiModalFieldConfig.flat_from_sizes]
334
"""
335
slices: Union[Sequence[slice], Sequence[Sequence[slice]]]
336
dim: int = 0
@@ -362,7 +371,7 @@ class MultiModalFlatField(BaseMultiModalField):
371
class MultiModalSharedField(BaseMultiModalField):
372
"""
373
Info:
365
- [MultiModalFieldConfig.shared][]
374
+ [`MultiModalFieldConfig.shared`][vllm.multimodal.inputs.MultiModalFieldConfig.shared]
375
"""
376
batch_size: int
377
@@ -508,7 +517,7 @@ class MultiModalFieldConfig:
517
```
518
519
Info:
511
- [MultiModalFieldConfig.flat][]
520
+ [`MultiModalFieldConfig.flat`][vllm.multimodal.inputs.MultiModalFieldConfig.flat]
521
"""
522
523
if size_per_item.ndim != 1:
@@ -572,8 +581,10 @@ class MultiModalFieldConfig:
581
582
class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
583
"""
575
- A collection of {class}`MultiModalFieldElem`
576
- corresponding to a data item in {class}`MultiModalDataItems`.
584
+ A collection of
585
+ [`MultiModalFieldElem`][vllm.multimodal.inputs.MultiModalFieldElem]
586
+ corresponding to a data item in
587
+ [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
588
"""
589
590
@staticmethod
@@ -592,11 +603,13 @@ class MultiModalKwargsItem(UserDict[str, MultiModalFieldElem]):
603
class MultiModalKwargs(UserDict[str, NestedTensors]):
604
"""
605
A dictionary that represents the keyword arguments to
595
- {meth}`~torch.nn.Module.forward`.
606
+ [`torch.nn.Module.forward`][].
607
608
The metadata `items` enables us to obtain the keyword arguments
598
- corresponding to each data item in {class}`MultiModalDataItems`, via
599
- {meth}`get_item` and {meth}`get_items`.
609
+ corresponding to each data item in
610
+ [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems], via
611
+ [`get_item`][vllm.multimodal.inputs.MultiModalKwargs.get_item] and
612
+ [`get_items`][vllm.multimodal.inputs.MultiModalKwargs.get_items].
613
"""
614
615
@staticmethod
@@ -635,7 +648,9 @@ class MultiModalKwargs(UserDict[str, NestedTensors]):
648
649
@staticmethod
650
def from_items(items: Sequence[MultiModalKwargsItem]):
638
- """Construct a new {class}`MultiModalKwargs` from multiple items."""
651
+ """Construct a new
652
+ [`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs]
653
+ from multiple items."""
654
elems_by_key = defaultdict[str, list[MultiModalFieldElem]](list)
655
for item in items:
656
for key, elem in item.items():
@@ -800,7 +815,7 @@ A dictionary containing placeholder ranges for each modality.
815
class MultiModalInputs(TypedDict):
816
"""
817
Represents the outputs of
803
- {class}`vllm.multimodal.processing.BaseMultiModalProcessor`,
818
+ [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor],
819
ready to be passed to vLLM internals.
820
"""
821
@@ -836,7 +851,8 @@ class MultiModalInputs(TypedDict):
851
852
class MultiModalEncDecInputs(MultiModalInputs):
853
"""
839
- Represents the outputs of {class}`vllm.multimodal.EncDecMultiModalProcessor`
854
+ Represents the outputs of
855
+ [`EncDecMultiModalProcessor`][vllm.multimodal.processing.EncDecMultiModalProcessor]
856
ready to be passed to vLLM internals.
857
"""
858
vllm/multimodal/parse.py
+7
-6
@@ -28,7 +28,8 @@ else:
28
29
class ModalityDataItems(ABC, Generic[_T, _I]):
30
"""
31
- Represents data items for a modality in {class}`MultiModalDataItems`.
31
+ Represents data items for a modality in
32
+ [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
33
"""
34
35
def __init__(self, data: _T, modality: str) -> None:
@@ -251,15 +252,15 @@ _D = TypeVar("_D", bound=ModalityDataItems[Any, Any])
252
253
class MultiModalDataItems(UserDict[str, ModalityDataItems[Any, Any]]):
254
"""
254
- As {data}`~vllm.multimodal.inputs.MultiModalDataDict`, but normalized
255
- such that each entry corresponds to a list.
255
+ As [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict], but
256
+ normalized such that each entry corresponds to a list.
257
"""
258
259
def get_count(self, modality: str, *, strict: bool = True) -> int:
260
"""
261
Get the number of data items belonging to a modality.
262
262
- If `strict=False`, return `0` instead of raising {exc}`KeyError`
263
+ If `strict=False`, return `0` instead of raising [`KeyError`][]
264
even if the modality is not found.
265
"""
266
if modality not in self:
@@ -305,8 +306,8 @@ ModalityDataParser: TypeAlias = Callable[[ModalityData[Any]],
306
307
class MultiModalDataParser:
308
"""
308
- Parses {data}`~vllm.multimodal.inputs.MultiModalDataDict` into
309
- {class}`MultiModalDataItems`.
309
+ Parses [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
310
+ into [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems].
311
312
Args:
313
target_sr (float, optional): Enables automatic resampling of audio
vllm/multimodal/processing.py
+48
-25
@@ -114,13 +114,14 @@ class PromptUpdateDetails(Generic[_S]):
114
115
is_embed: Optional[Callable[["_BoundPromptSequence"], torch.Tensor]] = None
116
"""
117
- Given {attr}`full`, return a boolean mask of shape `(len(full),)`
118
- indicating which positions of `full` to assign embeddings to.
117
+ Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full],
118
+ return a boolean mask of shape `(len(full),)` indicating which positions
119
+ of `full` to assign embeddings to.
120
121
`None` (default) means to assign embeddings to all positions of `full`.
122
123
The embeddings are obtained by calling
123
- {class}`SupportsMultiModal.get_multimodal_embeddings`.
124
+ [`SupportsMultiModal.get_multimodal_embeddings`][vllm.model_executor.models.interfaces.SupportsMultiModal.get_multimodal_embeddings].
125
"""
126
127
@staticmethod
@@ -159,13 +160,15 @@ PromptUpdateInfo = Union[PromptSeq, PromptUpdateDetails]
160
The token sequence or text that are part of the update.
161
162
If only part of the content corresponds to feature placeholders, you can
162
-use {class}`PromptUpdateDetails` to specify which part.
163
+use [`PromptUpdateDetails`][vllm.multimodal.processing.PromptUpdateDetails] to
164
+specify which part.
165
"""
166
167
PromptUpdateContent = Union[Callable[[int], PromptUpdateInfo],
168
PromptUpdateInfo]
169
"""
168
-Given the index of the processed item within {attr}`modality`,
170
+Given the index of the processed item within
171
+[`modality`][vllm.multimodal.processing.PromptUpdate.modality],
172
output the corresponding token sequence (or text).
173
174
For convenience, you can directly pass in the token sequence (or text)
@@ -260,8 +263,10 @@ class PromptInsertion(PromptUpdate):
263
264
insertion: PromptUpdateContent = field(repr=False)
265
"""
263
- Given the index of the processed item within {attr}`modality`,
264
- output the token sequence (or text) to insert right after {attr}`target`.
266
+ Given the index of the processed item within
267
+ [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
268
+ output the token sequence (or text) to insert right after
269
+ [`target`][vllm.multimodal.processing.PromptUpdate.target].
270
271
For convenience, you can directly pass in the token sequence (or text)
272
instead of a function if it does not depend on the input.
@@ -332,8 +337,10 @@ class PromptReplacement(PromptUpdate):
337
338
replacement: PromptUpdateContent = field(repr=False)
339
"""
335
- Given the index of the processed item within {attr}`modality`,
336
- output the token sequence (or text) to replace {attr}`target`.
340
+ Given the index of the processed item within
341
+ [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
342
+ output the token sequence (or text) to replace
343
+ [`target`][vllm.multimodal.processing.PromptUpdate.target].
344
345
For convenience, you can directly pass in the token sequence (or text)
346
instead of a function if it does not depend on the input.
@@ -387,14 +394,16 @@ _M = TypeVar("_M", bound=Union[_HasModalityAttr, _HasModalityProp])
394
395
396
def full_groupby_modality(values: Iterable[_M]) -> ItemsView[str, list[_M]]:
390
- """Convenience function to apply [full_groupby][] based on modality."""
397
+ """Convenience function to apply [`full_groupby`][vllm.utils.full_groupby]
398
+ based on modality."""
399
return full_groupby(values, key=lambda x: x.modality)
400
401
402
@dataclass
403
class _BoundPromptSequence:
404
"""
397
- A {data}`_PromptSeq` bound to a tokenizer to automatically
405
+ A [`_PromptSeq`][vllm.multimodal.processing.PromptSeq] bound
406
+ to a tokenizer to automatically
407
convert between token sequence and text representations.
408
"""
409
tokenizer: AnyTokenizer = field(repr=False)
@@ -446,9 +455,11 @@ class _BoundPromptContent:
455
@dataclass
456
class BoundPromptUpdate:
457
"""
449
- A {class}`PromptUpdate` bound to a tokenizer to automatically convert
450
- {attr}`target` and the result of {meth}`get_content` between
451
- token sequence and text representations.
458
+ A [`PromptUpdate`][vllm.multimodal.processing.PromptUpdate] bound
459
+ to a tokenizer to automatically convert
460
+ [`target`][vllm.multimodal.processing.PromptUpdate.target] and the result of
461
+ [`get_content`][vllm.multimodal.processing.BoundPromptUpdate.get_content]
462
+ between token sequence and text representations.
463
"""
464
_origin: PromptUpdate
465
tokenizer: AnyTokenizer = field(repr=False)
@@ -482,7 +493,8 @@ class BoundPromptUpdate:
493
494
def get_content(self, item_idx: int) -> _BoundPromptContent:
495
"""
485
- Given the index of the processed item within {attr}`modality`,
496
+ Given the index of the processed item within
497
+ [`modality`][vllm.multimodal.processing.PromptUpdate.modality],
498
output the token sequence (or text) to update.
499
"""
500
content = self.content
@@ -1019,7 +1031,8 @@ class ProcessingCache:
1031
) -> None:
1032
"""
1033
Put a processed multi-modal item into the cache
1022
- according to its dependencies (see {meth}`get`).
1034
+ according to its dependencies
1035
+ (see [`get`][vllm.multimodal.processing.ProcessingCache.get]).
1036
"""
1037
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
1038
**{modality: input_item},
@@ -1091,7 +1104,8 @@ _I = TypeVar("_I", bound=BaseProcessingInfo)
1104
1105
MultiModalHashes = dict[str, list[str]]
1106
"""
1094
-A collection of hashes with a similar structure as {class}`MultiModalKwargs`.
1107
+A collection of hashes with a similar structure as
1108
+[`MultiModalKwargs`][vllm.multimodal.inputs.MultiModalKwargs].
1109
"""
1110
1111
@@ -1099,7 +1113,7 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1113
"""
1114
Abstract base class to process multi-modal inputs to be used in vLLM.
1115
1102
- Not to be confused with {class}`transformers.ProcessorMixin`.
1116
+ Not to be confused with `transformers.ProcessorMixin`.
1117
"""
1118
1119
def __init__(self,
@@ -1126,10 +1140,12 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1140
def _get_data_parser(self) -> MultiModalDataParser:
1141
"""
1142
Construct a parser to preprocess multi-modal data items
1129
- before passing them to {meth}`_get_hf_mm_data`.
1143
+ before passing them to
1144
+ [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1145
1146
You can support additional modalities by creating a subclass
1132
- of {class}`MultiModalDataParser` that has additional subparsers.
1147
+ of [`MultiModalDataParser`][vllm.multimodal.parse.MultiModalDataParser]
1148
+ that has additional subparsers.
1149
"""
1150
return MultiModalDataParser()
1151
@@ -1138,8 +1154,11 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1154
mm_data: MultiModalDataDict,
1155
) -> MultiModalDataItems:
1156
"""
1141
- Normalize {class}`MultiModalDataDict` to {class}`MultiModalDataItems`
1142
- before passing them to {meth}`_get_hf_mm_data`.
1157
+ Normalize
1158
+ [`MultiModalDataDict`][vllm.multimodal.inputs.MultiModalDataDict]
1159
+ to [`MultiModalDataItems`][vllm.multimodal.parse.MultiModalDataItems]
1160
+ before passing them to
1161
+ [`_get_hf_mm_data`][vllm.multimodal.processing.BaseMultiModalProcessor._get_hf_mm_data].
1162
"""
1163
mm_items = self.data_parser.parse_mm_data(mm_data)
1164
supported_mm_limits = self.info.get_supported_mm_limits()
@@ -1191,7 +1210,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1210
inputs.
1211
1212
Moreover, this information is critical to determine the token positions
1194
- in order to construct {class}`~vllm-multimodal.input.PlaceholderRange`
1213
+ in order to construct
1214
+ [`PlaceholderRange`][vllm.multimodal.inputs.PlaceholderRange]
1215
for each multi-modal item.
1216
"""
1217
raise NotImplementedError
@@ -1315,7 +1335,9 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1335
Most HF processors accept prompt text but not prompt tokens.
1336
If the HF processor adds or removes tokens that are not related to
1337
multi-modal data, you should override this method so it is consistent
1318
- with the output of {meth}`_apply_hf_processor_text_only` on the
1338
+ with the output of
1339
+ [`_apply_hf_processor_text_only`][vllm.multimodal.processing.BaseMultiModalProcessor._apply_hf_processor_text_only]
1340
+ on the
1341
corresponding text.
1342
"""
1343
return prompt_tokens
@@ -1330,7 +1352,8 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
1352
1353
Since HF processor requires that text and multi-modal items
1354
correspond to each other, we generate dummy text using
1333
- {class}`DummyInputsBuilder` to go along with the multi-modal data.
1355
+ [`DummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
1356
+ to go along with the multi-modal data.
1357
"""
1358
mm_counts = mm_items.get_all_counts()
1359
vllm/multimodal/profiling.py
+1
-1
@@ -25,7 +25,7 @@ logger = init_logger(__name__)
25
class ProcessorInputs:
26
"""
27
Represents the keyword arguments to
28
- {meth}`vllm.multimodal.processing.BaseMultiModalProcessor.apply`.
28
+ [`vllm.multimodal.processing.BaseMultiModalProcessor.apply`][].
29
"""
30
prompt: Union[str, list[int]]
31
mm_data: MultiModalDataDict
vllm/multimodal/registry.py
+13
-13
@@ -29,7 +29,11 @@ _I_co = TypeVar("_I_co", bound=BaseProcessingInfo, covariant=True)
29
30
31
class ProcessingInfoFactory(Protocol[_I_co]):
32
- """Constructs a {class}`MultiModalProcessor` instance from the context."""
32
+ """
33
+ Constructs a
34
+ [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
35
+ instance from the context.
36
+ """
37
38
def __call__(
39
self,
@@ -40,7 +44,9 @@ class ProcessingInfoFactory(Protocol[_I_co]):
44
45
class DummyInputsBuilderFactory(Protocol[_I]):
46
"""
43
- Constructs a {class}`BaseDummyInputsBuilder` instance from the context.
47
+ Constructs a
48
+ [`BaseDummyInputsBuilder`][vllm.multimodal.profiling.BaseDummyInputsBuilder]
49
+ instance from the context.
50
"""
51
52
def __call__(self, info: _I) -> BaseDummyInputsBuilder[_I]:
@@ -48,7 +54,11 @@ class DummyInputsBuilderFactory(Protocol[_I]):
54
55
56
class MultiModalProcessorFactory(Protocol[_I]):
51
- """Constructs a {class}`MultiModalProcessor` instance from the context."""
57
+ """
58
+ Constructs a
59
+ [`BaseMultiModalProcessor`][vllm.multimodal.processing.BaseMultiModalProcessor]
60
+ instance from the context.
61
+ """
62
63
def __call__(
64
self,
@@ -155,8 +165,6 @@ class MultiModalRegistry:
165
"""
166
Get the maximum number of tokens from each modality
167
for profiling the memory usage of a model.
158
-
159
- See {meth}`MultiModalPlugin.get_max_multimodal_tokens` for more details.
168
"""
169
mm_limits = self.get_mm_limits_per_prompt(model_config)
170
@@ -170,8 +178,6 @@ class MultiModalRegistry:
178
"""
179
Get the maximum number of multi-modal tokens
180
for profiling the memory usage of a model.
173
-
174
- See {meth}`MultiModalPlugin.get_max_multimodal_tokens` for more details.
181
"""
182
return sum(self.get_max_tokens_by_modality(model_config).values())
183
@@ -213,9 +219,6 @@ class MultiModalRegistry:
219
220
When the model receives multi-modal data, the provided function is
221
invoked to transform the data into a dictionary of model inputs.
216
-
217
- Info:
218
- [mm-processing][]
222
"""
223
224
def wrapper(model_cls: N) -> N:
@@ -258,9 +261,6 @@ class MultiModalRegistry:
261
) -> BaseMultiModalProcessor[BaseProcessingInfo]:
262
"""
263
Create a multi-modal processor for a specific model and tokenizer.
261
-
262
- Info:
263
- [mm-processing][]
264
"""
265
if not model_config.is_multimodal_model:
266
raise ValueError(f"{model_config.model} is not a multimodal model")
vllm/multimodal/utils.py
+2
-1
@@ -259,7 +259,8 @@ class MediaConnector:
259
260
261
global_media_connector = MediaConnector()
262
-"""The global {class}`MediaConnector` instance used by vLLM."""
262
+"""The global [`MediaConnector`][vllm.multimodal.utils.MediaConnector]
263
+instance used by vLLM."""
264
265
fetch_audio = global_media_connector.fetch_audio
266
fetch_image = global_media_connector.fetch_image
vllm/platforms/interface.py
+5
-4
@@ -84,7 +84,7 @@ class DeviceCapability(NamedTuple):
84
85
def to_int(self) -> int:
86
"""
87
- Express device capability as an integer ``<major><minor>``.
87
+ Express device capability as an integer `<major><minor>`.
88
89
It is assumed that the minor version is always a single digit.
90
"""
@@ -206,10 +206,11 @@ class Platform:
206
"""
207
Test whether this platform is compatible with a device capability.
208
209
- The ``capability`` argument can either be:
209
+ The `capability` argument can either be:
210
211
- - A tuple ``(major, minor)``.
212
- - An integer ``<major><minor>``. (See {meth}`DeviceCapability.to_int`)
211
+ - A tuple `(major, minor)`.
212
+ - An integer `<major><minor>`. (See
213
+ [`DeviceCapability.to_int`][vllm.platforms.interface.DeviceCapability.to_int])
214
"""
215
current_capability = cls.get_device_capability(device_id=device_id)
216
if current_capability is None:
vllm/sequence.py
+10
-8
@@ -27,7 +27,7 @@ VLLM_INVALID_TOKEN_ID = -1
27
28
29
def array_full(token_id: int, count: int):
30
- """{class}`array` equivalent of [numpy.full][]."""
30
+ """[`array`][] equivalent of [numpy.full][]."""
31
return array(VLLM_TOKEN_ID_ARRAY_TYPE, [token_id]) * count
32
33
@@ -192,8 +192,8 @@ class SequenceData(msgspec.Struct,
192
def from_prompt_token_counts(
193
*token_counts: tuple[int, int]) -> "SequenceData":
194
"""
195
- Construct a {class}`SequenceData` instance by concatenating
196
- prompt token sequences.
195
+ Construct a [`SequenceData`][vllm.sequence.SequenceData] instance
196
+ by concatenating prompt token sequences.
197
198
Each tuple represents one token sequence, expressed in the form
199
`(token_id, count)`.
@@ -216,8 +216,8 @@ class SequenceData(msgspec.Struct,
216
prompt_embeds: Optional[torch.Tensor] = None,
217
) -> "SequenceData":
218
"""
219
- Construct a {class}`SequenceData` instance from prompt and output
220
- token sequences.
219
+ Construct a [`SequenceData`][vllm.sequence.SequenceData] instance
220
+ from prompt and output token sequences.
221
"""
222
prompt_token_ids_arr = array(VLLM_TOKEN_ID_ARRAY_TYPE,
223
prompt_token_ids)
@@ -452,9 +452,11 @@ class SequenceData(msgspec.Struct,
452
class Sequence:
453
"""Stores the data, status, and block information of a sequence.
454
455
- The sequence is constructed from the {data}`DecoderOnlyInputs`
456
- (for decoder-only) or {data}`EncoderDecoderInputs` (for encoder-decoder)
457
- instance passed in through the `inputs` constructor argument.
455
+ The sequence is constructed from the
456
+ [`DecoderOnlyInputs`][vllm.inputs.data.DecoderOnlyInputs] (for decoder-only)
457
+ or [`EncoderDecoderInputs`][vllm.inputs.data.EncoderDecoderInputs]
458
+ (for encoder-decoder) instance passed in through the `inputs`
459
+ constructor argument.
460
461
Args:
462
seq_id: The ID of the sequence.
vllm/utils.py
+3
-2
@@ -1005,7 +1005,7 @@ def flatten_2d_lists(lists: Iterable[Iterable[T]]) -> list[T]:
1005
1006
def full_groupby(values: Iterable[_V], *, key: Callable[[_V], _K]):
1007
"""
1008
- Unlike {class}`itertools.groupby`, groups are not broken by
1008
+ Unlike [`itertools.groupby`][], groups are not broken by
1009
non-contiguous data.
1010
"""
1011
groups = defaultdict[_K, list[_V]](list)
@@ -1926,7 +1926,8 @@ class _PlaceholderBase:
1926
Disallows downstream usage of placeholder modules.
1927
1928
We need to explicitly override each dunder method because
1929
- {meth}`__getattr__` is not called when they are accessed.
1929
+ [`__getattr__`][vllm.utils._PlaceholderBase.__getattr__]
1930
+ is not called when they are accessed.
1931
1932
Info:
1933
[Special method lookup](https://docs.python.org/3/reference/datamodel.html#special-lookup)
vllm/v1/worker/utils.py
+2
-2
@@ -10,7 +10,7 @@ def sanity_check_mm_encoder_outputs(
10
) -> None:
11
"""
12
Perform sanity checks for the result of
13
- {meth}`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`.
13
+ [`vllm.model_executor.models.SupportsMultiModal.get_multimodal_embeddings`][].
14
"""
15
assert isinstance(mm_embeddings, (list, tuple, torch.Tensor)), (
16
"Expected multimodal embeddings to be a list/tuple of 2D tensors, "
@@ -39,7 +39,7 @@ def scatter_mm_placeholders(
39
Scatter the multimodal embeddings into a contiguous tensor that represents
40
the placeholder tokens.
41
42
- {class}`vllm.multimodal.processing.PromptUpdateDetails.is_embed`.
42
+ [`vllm.multimodal.processing.PromptUpdateDetails.is_embed`][].
43
44
Args:
45
embeds: The multimodal embeddings.
vllm/worker/multi_step_model_runner.py
+6
-5
@@ -733,12 +733,13 @@ def _pythonize_sampler_output(
733
logprobs_tensor: Optional[torch.Tensor],
734
cache: Optional[PythonizationCache],
735
) -> None:
736
- """ This function is only called when the output tensors are ready.
737
- See {class}`ModelOutput`.
738
-
739
- Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place,
736
+ """ This function is only called when the output tensors are ready.
737
+ See [`ModelOutput`][vllm.worker.multi_step_model_runner.ModelOutput].
738
+
739
+ Modifies `output.outputs` and `pinned_sampled_token_buffer` in-place,
740
adding a Pythonized output data structure
741
- ({class}`CompletionSequenceGroupOutput`) for each {class}`SequenceGroup`.
741
+ ([`CompletionSequenceGroupOutput`][vllm.sequence.CompletionSequenceGroupOutput])
742
+ for each [`SequenceGroup`][vllm.sequence.SequenceGroup].
743
744
Args:
745
model_input