@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 273cb3b4d9

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 08:30 UTC

[Doc] Fix top-level API links/docs (#18621)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed May 24, 2025 at 00:46 UTC 273cb3b4d9fb3d8454e8662ad42ded2f096bbf99

9 files changed +88 -103

vllm/benchmarks/datasets.py

+12 -10

@@ -129,16 +129,17 @@ class BenchmarkDataset(ABC):

129

130 Args:

131 tokenizer (PreTrainedTokenizerBase): The base tokenizer to use if no

132 - LoRA is selected. max_loras (Optional[int]): The maximum number of

133 - LoRAs available. If None, LoRA is not used. lora_path

134 - (Optional[str]): Path to the LoRA parameters on disk. If None, LoRA

135 - is not used.

132 + LoRA is selected.

133 + max_loras (Optional[int]): The maximum number of LoRAs available.

134 + If `None`, LoRA is not used.

135 + lora_path (Optional[str]): Path to the LoRA parameters on disk.

136 + If `None`, LoRA is not used.

137

138 Returns:

138 - tuple[Optional[LoRARequest], AnyTokenizer]: A tuple where the first

139 - element is a LoRARequest (or None if not applicable) and the second

140 - element is the tokenizer associated with the LoRA request (or the

141 - base tokenizer).

139 + A tuple with the following elements:

140 + - A new [LoRARequest][] (or `None` if not applicable).

141 + - The tokenizer associated with the LoRA request

142 + (or the base tokenizer).

143 """

144 if max_loras is None or lora_path is None:

145 return None, tokenizer

@@ -167,7 +168,7 @@ class BenchmarkDataset(ABC):

168

169 Args:

170 tokenizer (PreTrainedTokenizerBase): The tokenizer to be used

170 - for processing the dataset's text.

171 + for processing the dataset's text.

172 num_requests (int): The number of sample requests to generate.

173

174 Returns:

@@ -184,7 +185,8 @@ class BenchmarkDataset(ABC):

185

186 Args:

187 requests (List[SampleRequest]): The current list of sampled

187 - requests. num_requests (int): The target number of requests.

188 + requests.

189 + num_requests (int): The target number of requests.

190 """

191 if len(requests) < num_requests:

192 random.seed(self.random_seed)

vllm/config.py

+1 -1

@@ -4552,7 +4552,7 @@ def contains_object_print(text):

4552 text (str): The text to check

4553

4554 Returns:

4555 - bool: True if a match is found, False otherwise

4555 + result (bool): `True` if a match is found, `False` otherwise.

4556 """

4557 pattern = r'at 0x[a-fA-F0-9]{2,16}>'

4558 match = re.search(pattern, text)

vllm/distributed/kv_transfer/kv_connector/v1/base.py

+5 -4

@@ -210,10 +210,11 @@ class KVConnectorBase_V1(ABC):

210 computed tokens for this request

211

212 Returns:

213 - * the number of tokens that can be loaded from the

214 - external KV cache beyond what is already computed.

215 - * true if external KV cache tokens will be loaded

216 - asynchronously (between scheduler steps).

213 + A tuple with the following elements:

214 + - The number of tokens that can be loaded from the

215 + external KV cache beyond what is already computed.

216 + - `True` if external KV cache tokens will be loaded

217 + asynchronously (between scheduler steps).

218 """

219 pass

220

vllm/distributed/kv_transfer/kv_pipe/pynccl_pipe.py

+20 -20

@@ -118,11 +118,11 @@ class PyNcclPipe(KVPipeBase):

118 """

119 Create the metadata as a dictionary based on the input tensor.

120

121 - Parameters:

122 - - tensor: The input tensor or None if no tensor is provided.

121 + Args:

122 + tensor: The input tensor or None if no tensor is provided.

123

124 Returns:

125 - - metadata: A dictionary with the following keys:

125 + metadata: A dictionary with the following keys:

126 - "dtype": The data type of the tensor or None.

127 - "shape": The shape of the tensor or None.

128 """

@@ -135,13 +135,13 @@ class PyNcclPipe(KVPipeBase):

135 """

136 Create a buffer to receive the tensor based on the provided metadata.

137

138 - Parameters:

139 - - metadata: A dictionary with keys "dtype" and "shape", describing

140 - the tensor's data type and shape.

138 + Args:

139 + metadata: A dictionary with keys "dtype" and "shape",

140 + describing the tensor's data type and shape.

141

142 Returns:

143 - - buffer: A tensor of the specified type and shape, allocated on

144 - self.device.

143 + buffer: A tensor of the specified type and shape,

144 + allocated on `self.device`.

145 """

146 return torch.empty(metadata["shape"],

147 dtype=metadata["dtype"],

@@ -151,8 +151,8 @@ class PyNcclPipe(KVPipeBase):

151 """

152 Send the metadata dictionary to the target rank.

153

154 - Parameters:

155 - - metadata: A dictionary with keys "dtype" and "shape".

154 + Args:

155 + metadata: A dictionary with keys "dtype" and "shape".

156 """

157 self.group.send_obj(metadata, self.target_rank_for_send)

158

@@ -161,8 +161,8 @@ class PyNcclPipe(KVPipeBase):

161 Receive the metadata dictionary from the target rank.

162

163 Returns:

164 - - metadata: A dictionary with keys "dtype" and "shape" describing

165 - the tensor.

164 + metadata: A dictionary with keys "dtype" and "shape"

165 + describing the tensor.

166 """

167 return self.group.recv_obj(self.target_rank_for_recv)

168

@@ -171,9 +171,9 @@ class PyNcclPipe(KVPipeBase):

171 The actual implementation of sending the tensor and its metadata to the

172 target rank.

173

174 - Parameters:

175 - - tensor: The input tensor to be sent, or None if no tensor is

176 - being sent.

174 + Args:

175 + tensor: The input tensor to be sent, or `None` if no tensor is

176 + being sent.

177 """

178 metadata = self._make_metadata(tensor)

179 self._send_metadata(metadata)

@@ -187,7 +187,7 @@ class PyNcclPipe(KVPipeBase):

187 the target rank.

188

189 Returns:

190 - - buffer: The received tensor, or None if no tensor is received.

190 + buffer: The received tensor, or `None` if no tensor is received.

191 """

192 metadata = self._recv_metadata()

193 if metadata["dtype"] is None:

@@ -227,8 +227,8 @@ class PyNcclPipe(KVPipeBase):

227 Sends a tensor and its metadata to the destination rank in a

228 non-blocking way.

229

230 - Parameters:

231 - - tensor: The tensor to send, or None if no tensor is being sent.

230 + Args:

231 + tensor: The tensor to send, or `None` if no tensor is being sent.

232 """

233 if self.transport_thread is None:

234 self.transport_thread = ThreadPoolExecutor(max_workers=1)

@@ -250,8 +250,8 @@ class PyNcclPipe(KVPipeBase):

250 """

251 Receives a tensor and its metadata from the source rank. Blocking call.

252

253 - Returns:

254 - - tensor: The received tensor, or None if no tensor is received.

253 + Args:

254 + tensor: The received tensor, or `None` if no tensor is received.

255 """

256 if self.transport_thread is None:

257 self.transport_thread = ThreadPoolExecutor(max_workers=1)

vllm/engine/llm_engine.py

+15 -25

@@ -130,26 +130,16 @@ class LLMEngine:

130 iteration-level scheduling and efficient memory management to maximize the

131 serving throughput.

132

133 - The {class}`~vllm.LLM` class wraps this class for offline batched inference

134 - and the {class}`AsyncLLMEngine` class wraps this class for online serving.

133 + The [LLM][vllm.LLM] class wraps this class for offline batched inference

134 + and the [AsyncLLMEngine][] class wraps this class for online serving.

135

136 - The config arguments are derived from {class}`~vllm.EngineArgs`. (See

137 - {ref}`engine-args`)

136 + The config arguments are derived from [EngineArgs][vllm.EngineArgs]. (See

137 + [engine-args][])

138

139 Args:

140 - model_config: The configuration related to the LLM model.

141 - cache_config: The configuration related to the KV cache memory

142 - management.

143 - parallel_config: The configuration related to distributed execution.

144 - scheduler_config: The configuration related to the request scheduler.

145 - device_config: The configuration related to the device.

146 - lora_config (Optional): The configuration related to serving multi-LoRA.

147 - speculative_config (Optional): The configuration related to speculative

148 - decoding.

140 + vllm_config: The configuration for initializing and running vLLM.

141 executor_class: The model executor class for managing distributed

142 execution.

151 - prompt_adapter_config (Optional): The configuration related to serving

152 - prompt adapters.

143 log_stats: Whether to log statistics.

144 usage_context: Specified entry point, used for usage info collection.

145 """

@@ -695,11 +685,12 @@ class LLMEngine:

685

686 Args:

687 request_id: The unique ID of the request.

698 - prompt: The prompt to the LLM. See {class}`~vllm.inputs.PromptType`

688 + prompt: The prompt to the LLM. See

689 + [PromptType][vllm.inputs.PromptType]

690 for more details about the format of each input.

691 params: Parameters for sampling or pooling.

701 - {class}`~vllm.SamplingParams` for text generation.

702 - {class}`~vllm.PoolingParams` for pooling.

692 + [SamplingParams][vllm.SamplingParams] for text generation.

693 + [PoolingParams][vllm.PoolingParams] for pooling.

694 arrival_time: The arrival time of the request. If None, we use

695 the current monotonic time.

696 lora_request: The LoRA request to add.

@@ -711,10 +702,11 @@ class LLMEngine:

702 Details:

703 - Set arrival_time to the current time if it is None.

704 - Set prompt_token_ids to the encoded prompt if it is None.

714 - - Create `n` number of {class}`~vllm.Sequence` objects.

715 - - Create a {class}`~vllm.SequenceGroup` object

716 - from the list of {class}`~vllm.Sequence`.

717 - - Add the {class}`~vllm.SequenceGroup` object to the scheduler.

705 + - Create `n` number of [Sequence][vllm.Sequence] objects.

706 + - Create a [SequenceGroup][vllm.SequenceGroup] object

707 + from the list of [Sequence][vllm.Sequence].

708 + - Add the [SequenceGroup][vllm.SequenceGroup] object to the

709 + scheduler.

710

711 Example:

712 >>> # initialize engine

@@ -861,9 +853,7 @@ class LLMEngine:

853 request_id: The ID(s) of the request to abort.

854

855 Details:

864 - - Refer to the

865 - {meth}`~vllm.core.scheduler.Scheduler.abort_seq_group`

866 - from class {class}`~vllm.core.scheduler.Scheduler`.

856 + - Refer to [vllm.core.scheduler.Scheduler.abort_seq_group][].

857

858 Example:

859 >>> # initialize engine and add a request with request_id

vllm/entrypoints/llm.py

+32 -31

@@ -116,7 +116,8 @@ class LLM:

116 to eager mode. Additionally for encoder-decoder models, if the

117 sequence length of the encoder input is larger than this, we fall

118 back to the eager mode.

119 - disable_custom_all_reduce: See {class}`~vllm.config.ParallelConfig`

119 + disable_custom_all_reduce: See

120 + [ParallelConfig][vllm.config.ParallelConfig].

121 disable_async_output_proc: Disable async output processing.

122 This may result in lower performance.

123 hf_token: The token to use as HTTP bearer authorization for remote files

@@ -128,12 +129,12 @@ class LLM:

129 compilation_config: Either an integer or a dictionary. If it is an

130 integer, it is used as the level of compilation optimization. If it

131 is a dictionary, it can specify the full compilation configuration.

131 - **kwargs: Arguments for {class}`~vllm.EngineArgs`. (See

132 - {ref}`engine-args`)

132 + **kwargs: Arguments for [EngineArgs][vllm.EngineArgs]. (See

133 + [engine-args][])

134

135 Note:

136 This class is intended to be used for offline inference. For online

136 - serving, use the {class}`~vllm.AsyncLLMEngine` class instead.

137 + serving, use the [AsyncLLMEngine][vllm.AsyncLLMEngine] class instead.

138 """

139

140 DEPRECATE_LEGACY: ClassVar[bool] = True

@@ -142,7 +143,7 @@ class LLM:

143 DEPRECATE_INIT_POSARGS: ClassVar[bool] = True

144 """

145 A flag to toggle whether to deprecate positional arguments in

145 - {meth}`LLM.__init__`.

146 + [LLM.__init__][].

147 """

148

149 @classmethod

@@ -403,7 +404,7 @@ class LLM:

404

405 Args:

406 prompts: The prompts to the LLM. You may pass a sequence of prompts

406 - for batch inference. See {class}`~vllm.inputs.PromptType`

407 + for batch inference. See [PromptType][vllm.inputs.PromptType]

408 for more details about the format of each prompts.

409 sampling_params: The sampling parameters for text generation. If

410 None, we use the default sampling parameters.

@@ -669,7 +670,7 @@ class LLM:

670 Generate responses for a chat conversation.

671

672 The chat conversation is converted into a text prompt using the

672 - tokenizer and calls the {meth}`generate` method to generate the

673 + tokenizer and calls the [generate][] method to generate the

674 responses.

675

676 Multi-modal inputs can be passed in the same way you would pass them

@@ -678,8 +679,8 @@ class LLM:

679 Args:

680 messages: A list of conversations or a single conversation.

681

681 - - Each conversation is represented as a list of messages.

682 - - Each message is a dictionary with 'role' and 'content' keys.

682 + - Each conversation is represented as a list of messages.

683 + - Each message is a dictionary with 'role' and 'content' keys.

684

685 sampling_params: The sampling parameters for text generation.

686 If None, we use the default sampling parameters. When it

@@ -689,27 +690,27 @@ class LLM:

690 use_tqdm: Whether to use tqdm to display the progress bar.

691 lora_request: LoRA request to use for generation, if any.

692 chat_template: The template to use for structuring the chat.

692 - If not provided, the model's default chat template will be used.

693 + If not provided, the model's default chat template will be used.

694 chat_template_content_format: The format to render message content.

695

695 - - "string" will render the content as a string.

696 - Example: ``"Who are you?"``

697 - - "openai" will render the content as a list of dictionaries,

698 - similar to OpenAI schema.

699 - Example: ``[{"type": "text", "text": "Who are you?"}]``

696 + - "string" will render the content as a string.

697 + Example: `"Who are you?"`

698 + - "openai" will render the content as a list of dictionaries,

699 + similar to OpenAI schema.

700 + Example: `[{"type": "text", "text": "Who are you?"}]`

701

702 add_generation_prompt: If True, adds a generation template

703 to each message.

704 continue_final_message: If True, continues the final message in

705 the conversation instead of starting a new one. Cannot be

705 - ``True`` if ``add_generation_prompt`` is also ``True``.

706 + `True` if `add_generation_prompt` is also `True`.

707 chat_template_kwargs: Additional kwargs to pass to the chat

708 template.

709 mm_processor_kwargs: Multimodal processor kwarg overrides for this

710 chat request. Only used for offline requests.

711

712 Returns:

712 - A list of ``RequestOutput`` objects containing the generated

713 + A list of `RequestOutput` objects containing the generated

714 responses in the same order as the input messages.

715 """

716 list_of_messages: list[list[ChatCompletionMessageParam]]

@@ -908,7 +909,7 @@ class LLM:

909

910 Args:

911 prompts: The prompts to the LLM. You may pass a sequence of prompts

911 - for batch inference. See {class}`~vllm.inputs.PromptType`

912 + for batch inference. See [PromptType][vllm.inputs.PromptType]

913 for more details about the format of each prompts.

914 pooling_params: The pooling parameters for pooling. If None, we

915 use the default pooling parameters.

@@ -997,7 +998,7 @@ class LLM:

998

999 Args:

1000 prompts: The prompts to the LLM. You may pass a sequence of prompts

1000 - for batch inference. See {class}`~vllm.inputs.PromptType`

1001 + for batch inference. See [PromptType][vllm.inputs.PromptType]

1002 for more details about the format of each prompts.

1003 pooling_params: The pooling parameters for pooling. If None, we

1004 use the default pooling parameters.

@@ -1007,7 +1008,7 @@ class LLM:

1008 generation, if any.

1009

1010 Returns:

1010 - A list of ``EmbeddingRequestOutput`` objects containing the

1011 + A list of `EmbeddingRequestOutput` objects containing the

1012 embedding vectors in the same order as the input prompts.

1013 """

1014 if self.llm_engine.model_config.task != "embed":

@@ -1041,7 +1042,7 @@ class LLM:

1042

1043 Args:

1044 prompts: The prompts to the LLM. You may pass a sequence of prompts

1044 - for batch inference. See {class}`~vllm.inputs.PromptType`

1045 + for batch inference. See [PromptType][vllm.inputs.PromptType]

1046 for more details about the format of each prompts.

1047 use_tqdm: Whether to use tqdm to display the progress bar.

1048 lora_request: LoRA request to use for generation, if any.

@@ -1049,7 +1050,7 @@ class LLM:

1050 generation, if any.

1051

1052 Returns:

1052 - A list of ``ClassificationRequestOutput`` objects containing the

1053 + A list of `ClassificationRequestOutput` objects containing the

1054 embedding vectors in the same order as the input prompts.

1055 """

1056 if self.llm_engine.model_config.task != "classify":

@@ -1159,11 +1160,11 @@ class LLM:

1160 lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,

1161 prompt_adapter_request: Optional[PromptAdapterRequest] = None,

1162 ) -> list[ScoringRequestOutput]:

1162 - """Generate similarity scores for all pairs ``<text,text_pair>``.

1163 + """Generate similarity scores for all pairs `<text,text_pair>`.

1164

1164 - The inputs can be ``1 -> 1``, ``1 -> N`` or ``N -> N``.

1165 - In the ``1 - N`` case the ``text_1`` sentence will be replicated ``N``

1166 - times to pair with the ``text_2`` sentences.

1165 + The inputs can be `1 -> 1`, `1 -> N` or `N -> N`.

1166 + In the `1 - N` case the `text_1` sentence will be replicated `N`

1167 + times to pair with the `text_2` sentences.

1168 The input pairs are used to build a list of prompts for the

1169 cross encoder model. This class automatically batches the prompts,

1170 considering the memory constraint. For the best performance, put all

@@ -1171,9 +1172,9 @@ class LLM:

1172

1173 Args:

1174 text_1: can be a single prompt or a list of prompts, in which

1174 - case it has to have the same length as the ``text_2`` list

1175 + case it has to have the same length as the `text_2` list

1176 text_2: The texts to pair with the query to form the input

1176 - to the LLM. See {class}`~vllm.inputs.PromptType` for

1177 + to the LLM. See [PromptType][vllm.inputs.PromptType] for

1178 more details about the format of each prompts.

1179 use_tqdm: Whether to use tqdm to display the progress bar.

1180 lora_request: LoRA request to use for generation, if any.

@@ -1181,7 +1182,7 @@ class LLM:

1182 generation, if any.

1183

1184 Returns:

1184 - A list of ``ScoringRequestOutput`` objects containing the

1185 + A list of `ScoringRequestOutput` objects containing the

1186 generated scores in the same order as the input prompts.

1187 """

1188 runner_type = self.llm_engine.model_config.runner_type

@@ -1282,13 +1283,13 @@ class LLM:

1283

1284 def wake_up(self, tags: Optional[list[str]] = None):

1285 """

1285 - Wake up the engine from sleep mode. See the {meth}`sleep` method

1286 + Wake up the engine from sleep mode. See the [sleep][] method

1287 for more details.

1288

1289 Args:

1290 tags: An optional list of tags to reallocate the engine memory

1291 for specific memory allocations. Values must be in

1291 - ("weights", "kv_cache",). If None, all memory is reallocated.

1292 + `("weights", "kv_cache")`. If None, all memory is reallocated.

1293 wake_up should be called with all tags (or None) before the

1294 engine is used again.

1295 """

vllm/multimodal/__init__.py

+1 -1

@@ -12,7 +12,7 @@ The global {class}`~MultiModalRegistry` is used by model runners to

12 dispatch data processing according to the target model.

14 Info:

15 - {ref}`mm-processing`

15 + [mm-processing][]

16 """

18 __all__ = [

vllm/multimodal/registry.py

+2 -2

@@ -215,7 +215,7 @@ class MultiModalRegistry:

215 invoked to transform the data into a dictionary of model inputs.

216

217 Info:

218 - {ref}`mm-processing`

218 + [mm-processing][]

219 """

220

221 def wrapper(model_cls: N) -> N:

@@ -260,7 +260,7 @@ class MultiModalRegistry:

260 Create a multi-modal processor for a specific model and tokenizer.

261

262 Info:

263 - {ref}`mm-processing`

263 + [mm-processing][]

264 """

265 if not model_config.is_multimodal_model:

266 raise ValueError(f"{model_config.model} is not a multimodal model")

vllm/outputs.py

-9

@@ -391,15 +391,6 @@ class PoolingRequestOutput(Generic[_O]):

391 prompt_token_ids, finished)

392

393 def __repr__(self):

394 - """

395 - Returns a string representation of an PoolingRequestOutput instance.

396 -

397 - The representation includes the request_id and the number of outputs,

398 - providing a quick overview of the pooling request's results.

399 -

400 - Returns:

401 - str: A string representation of the PoolingRequestOutput instance.

402 - """

394 return (f"{type(self).__name__}(request_id={self.request_id!r}, "

395 f"outputs={self.outputs!r}, "

396 f"prompt_token_ids={self.prompt_token_ids}, "