@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 41f17bf290

Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 14, 05:30 UTC

[Docs] Fix warnings in mkdocs build (continued) (#24740)

Signed-off-by: Zerohertz <ohg3417@gmail.com>

Hyogeun Oh (오효근) committed Sep 12, 2025 at 22:43 UTC 41f17bf2900611211f9a2145e11008f0c8cb45b8

10 files changed +121 -176

vllm/model_executor/layers/quantization/torchao.py

+5 -5

@@ -144,9 +144,9 @@ def torchao_quantize_param_data(param: torch.Tensor,

144 """Quantize a Tensor with torchao quantization specified by torchao_config

145

146 Args:

147 - `param`: weight parameter of the linear module

148 - `torchao_config`: type of quantization and their arguments we want to

149 - use to quantize the Tensor

147 + param: weight parameter of the linear module

148 + torchao_config: type of quantization and their arguments we want to

149 + use to quantize the Tensor

150 """

151 from torchao.core.config import AOBaseConfig

152 from torchao.quantization import quantize_

@@ -172,8 +172,8 @@ class TorchAOLinearMethod(LinearMethodBase):

172 """Linear method for torchao.

173

174 Args:

175 - torchao_config: The torchao quantization config, a string

176 - that encodes the type of quantization and all relevant arguments.

175 + quant_config: The torchao quantization config, a string that encodes

176 + the type of quantization and all relevant arguments.

177 """

178

179 def __init__(self, quant_config: TorchAOConfig):

vllm/model_executor/layers/quantization/utils/int8_utils.py

+1 -1

@@ -423,7 +423,7 @@ def w8a8_block_int8_matmul(

423 Bs: The per-block quantization scale for `B`.

424 block_size: The block size for per-block quantization. It should be

425 2-dim, e.g., [128, 128].

426 - output_dytpe: The dtype of the returned tensor.

426 + output_dtype: The dtype of the returned tensor.

427

428 Returns:

429 torch.Tensor: The result of matmul.

vllm/model_executor/layers/rotary_embedding/mrope.py

+2 -2

@@ -135,8 +135,8 @@ def triton_mrope(

135 """Qwen2VL mrope kernel.

136

137 Args:

138 - query: [num_tokens, num_heads * head_size]

139 - key: [num_tokens, num_kv_heads * head_size]

138 + q: [num_tokens, num_heads * head_size]

139 + k: [num_tokens, num_kv_heads * head_size]

140 cos: [3, num_tokens, head_size //2 ]

141 (T/H/W positions with multimodal inputs)

142 sin: [3, num_tokens, head_size //2 ]

vllm/model_executor/model_loader/tensorizer.py

+45 -44

@@ -171,51 +171,52 @@ class TensorizerConfig(MutableMapping):

171 _is_sharded: bool = field(init=False, default=False)

172 _fields: ClassVar[tuple[str, ...]]

173 _keys: ClassVar[frozenset[str]]

174 - """

175 - Args for the TensorizerConfig class. These are used to configure the

176 - behavior of model serialization and deserialization using Tensorizer.

174 + """Configuration class for Tensorizer settings.

175

178 - Args:

179 - tensorizer_uri: Path to serialized model tensors. Can be a local file

180 - path or a S3 URI. This is a required field unless lora_dir is

181 - provided and the config is meant to be used for the

182 - `tensorize_lora_adapter` function. Unless a `tensorizer_dir` or

183 - `lora_dir` is passed to this object's initializer, this is a required

184 - argument.

185 - tensorizer_dir: Path to a directory containing serialized model tensors,

186 - and all other potential model artifacts to load the model, such as

187 - configs and tokenizer files. Can be passed instead of `tensorizer_uri`

188 - where the `model.tensors` file will be assumed to be in this

189 - directory.

190 - vllm_tensorized: If True, indicates that the serialized model is a

191 - vLLM model. This is used to determine the behavior of the

192 - TensorDeserializer when loading tensors from a serialized model.

193 - It is far faster to deserialize a vLLM model as it utilizes

194 - tensorizer's optimized GPU loading. Note that this is now

195 - deprecated, as serialized vLLM models are now automatically

196 - inferred as vLLM models.

197 - verify_hash: If True, the hashes of each tensor will be verified against

198 - the hashes stored in the metadata. A `HashMismatchError` will be

199 - raised if any of the hashes do not match.

200 - num_readers: Controls how many threads are allowed to read concurrently

201 - from the source file. Default is `None`, which will dynamically set

202 - the number of readers based on the number of available

203 - resources and model size. This greatly increases performance.

204 - encryption_keyfile: File path to a binary file containing a

205 - binary key to use for decryption. `None` (the default) means

206 - no decryption. See the example script in

207 - examples/others/tensorize_vllm_model.py.

208 - s3_access_key_id: The access key for the S3 bucket. Can also be set via

209 - the S3_ACCESS_KEY_ID environment variable.

210 - s3_secret_access_key: The secret access key for the S3 bucket. Can also

211 - be set via the S3_SECRET_ACCESS_KEY environment variable.

212 - s3_endpoint: The endpoint for the S3 bucket. Can also be set via the

213 - S3_ENDPOINT_URL environment variable.

214 - lora_dir: Path to a directory containing LoRA adapter artifacts for

215 - serialization or deserialization. When serializing LoRA adapters

216 - this is the only necessary parameter to pass to this object's

217 - initializer.

218 - """

176 + These settings configure the behavior of model serialization and

177 + deserialization using Tensorizer.

178 +

179 + Attributes:

180 + tensorizer_uri: Path to serialized model tensors. Can be a local file

181 + path or a S3 URI. This is a required field unless lora_dir is

182 + provided and the config is meant to be used for the

183 + `tensorize_lora_adapter` function. Unless a `tensorizer_dir` or

184 + `lora_dir` is passed to this object's initializer, this is

185 + a required argument.

186 + tensorizer_dir: Path to a directory containing serialized model tensors,

187 + and all other potential model artifacts to load the model, such as

188 + configs and tokenizer files. Can be passed instead of

189 + `tensorizer_uri` where the `model.tensors` file will be assumed

190 + to be in this directory.

191 + vllm_tensorized: If True, indicates that the serialized model is a

192 + vLLM model. This is used to determine the behavior of the

193 + TensorDeserializer when loading tensors from a serialized model.

194 + It is far faster to deserialize a vLLM model as it utilizes

195 + tensorizer's optimized GPU loading. Note that this is now

196 + deprecated, as serialized vLLM models are now automatically

197 + inferred as vLLM models.

198 + verify_hash: If True, the hashes of each tensor will be verified

199 + against the hashes stored in the metadata. A `HashMismatchError`

200 + will be raised if any of the hashes do not match.

201 + num_readers: Controls how many threads are allowed to read concurrently

202 + from the source file. Default is `None`, which will dynamically set

203 + the number of readers based on the number of available

204 + resources and model size. This greatly increases performance.

205 + encryption_keyfile: File path to a binary file containing a

206 + binary key to use for decryption. `None` (the default) means

207 + no decryption. See the example script in

208 + examples/others/tensorize_vllm_model.py.

209 + s3_access_key_id: The access key for the S3 bucket. Can also be set via

210 + the S3_ACCESS_KEY_ID environment variable.

211 + s3_secret_access_key: The secret access key for the S3 bucket. Can also

212 + be set via the S3_SECRET_ACCESS_KEY environment variable.

213 + s3_endpoint: The endpoint for the S3 bucket. Can also be set via the

214 + S3_ENDPOINT_URL environment variable.

215 + lora_dir: Path to a directory containing LoRA adapter artifacts for

216 + serialization or deserialization. When serializing LoRA adapters

217 + this is the only necessary parameter to pass to this object's

218 + initializer.

219 + """

220

221 def __post_init__(self):

222 # check if the configuration is for a sharded vLLM model

vllm/model_executor/models/aria.py

+4 -12

@@ -143,16 +143,8 @@ class AriaProjector(nn.Module):

143 projects ViT's outputs into MoE's inputs.

144

145 Args:

146 - patch_to_query_dict (dict): Maps patch numbers to their corresponding

147 - query numbers,

148 - e.g., {1225: 128, 4900: 256}. This allows for different query sizes

149 - based on image resolution.

150 - embed_dim (int): Embedding dimension.

151 - num_heads (int): Number of attention heads.

152 - kv_dim (int): Dimension of key and value.

153 - ff_dim (int): Hidden dimension of the feed-forward network.

154 - output_dim (int): Output dimension.

155 - norm_layer (nn.Module): Normalization layer. Default is nn.LayerNorm.

146 + config: [AriaConfig](https://huggingface.co/docs/transformers/main/model_doc/aria#transformers.AriaConfig)

147 + containing projector configuration parameters.

148

149 Outputs:

150 A tensor with the shape of (batch_size, query_number, output_dim)

@@ -282,8 +274,8 @@ class AriaTextMoELayer(nn.Module):

274 Forward pass of the MoE Layer.

275

276 Args:

285 - hidden_states (torch.Tensor): Input tensor of shape (batch_size,

286 - sequence_length, hidden_size).

277 + hidden_states: Input tensor of shape

278 + (batch_size, sequence_length, hidden_size).

279

280 Returns:

281 torch.Tensor: Output tensor after passing through the MoE layer.

vllm/model_executor/models/bart.py

+40 -64

@@ -401,8 +401,7 @@ class BartEncoderLayer(nn.Module):

401 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

402 r"""

403 Args:

404 - hidden_states

405 - torch.Tensor of *encoder* input embeddings.

404 + hidden_states: torch.Tensor of *encoder* input embeddings.

405 Returns:

406 Encoder layer output torch.Tensor

407 """

@@ -490,10 +489,8 @@ class BartDecoderLayer(nn.Module):

489 ) -> torch.Tensor:

490 r"""

491 Args:

493 - decoder_hidden_states

494 - torch.Tensor of *decoder* input embeddings.

495 - encoder_hidden_states

496 - torch.Tensor of *encoder* input embeddings.

492 + decoder_hidden_states: torch.Tensor of *decoder* input embeddings.

493 + encoder_hidden_states: torch.Tensor of *encoder* input embeddings.

494 Returns:

495 Decoder layer output torch.Tensor

496 """

@@ -584,12 +581,10 @@ class BartEncoder(nn.Module):

581 ) -> torch.Tensor:

582 r"""

583 Args:

587 - input_ids

588 - Indices of *encoder* input sequence tokens in the vocabulary.

589 - Padding will be ignored by default should you

590 - provide it.

591 - positions

592 - Positions of *encoder* input sequence tokens.

584 + input_ids: Indices of *encoder* input sequence tokens in the

585 + vocabulary.

586 + Padding will be ignored by default should you provide it.

587 + positions: Positions of *encoder* input sequence tokens.

588 Returns:

589 Decoder output torch.Tensor

590 """

@@ -663,14 +658,11 @@ class BartDecoder(nn.Module):

658 ) -> torch.Tensor:

659 r"""

660 Args:

666 - decoder_input_ids

667 - Indices of *decoder* input sequence tokens in the vocabulary.

668 - Padding will be ignored by default should you

669 - provide it.

670 - decoder_positions

671 - Positions of *decoder* input sequence tokens.

672 - encoder_hidden_states:

673 - Tensor of encoder output embeddings

661 + decoder_input_ids: Indices of *decoder* input sequence tokens

662 + in the vocabulary.

663 + Padding will be ignored by default should you provide it.

664 + decoder_positions: Positions of *decoder* input sequence tokens.

665 + encoder_hidden_states: Tensor of encoder output embeddings.

666 Returns:

667 Decoder output torch.Tensor

668 """

@@ -732,16 +724,13 @@ class BartModel(nn.Module, SupportsQuant):

724 encoder_positions: torch.Tensor) -> torch.Tensor:

725 r"""

726 Args:

735 - input_ids

736 - Indices of *decoder* input sequence tokens in the vocabulary.

737 - Padding will be ignored by default should you

738 - provide it.

739 - positions

740 - Positions of *decoder* input sequence tokens.

741 - encoder_input_ids

742 - Indices of *encoder* input sequence tokens in the vocabulary.

743 - encoder_positions:

744 - Positions of *encoder* input sequence tokens.

727 + input_ids: Indices of *decoder* input sequence tokens

728 + in the vocabulary.

729 + Padding will be ignored by default should you provide it.

730 + positions: Positions of *decoder* input sequence tokens.

731 + encoder_input_ids: Indices of *encoder* input sequence tokens

732 + in the vocabulary.

733 + encoder_positions: Positions of *encoder* input sequence tokens.

734 Returns:

735 Model output torch.Tensor

736 """

@@ -848,14 +837,10 @@ class BartForConditionalGeneration(nn.Module, SupportsV0Only, SupportsQuant):

837 ) -> torch.Tensor:

838 r"""

839 Args:

851 - input_ids

852 - torch.Tensor of *decoder* input token ids.

853 - positions

854 - torch.Tensor of *decoder* position indices.

855 - encoder_input_ids

856 - torch.Tensor of *encoder* input token ids.

857 - encoder_positions

858 - torch.Tensor of *encoder* position indices

840 + input_ids: torch.Tensor of *decoder* input token ids.

841 + positions: torch.Tensor of *decoder* position indices.

842 + encoder_input_ids: torch.Tensor of *encoder* input token ids.

843 + encoder_positions: torch.Tensor of *encoder* position indices.

844 Returns:

845 Output torch.Tensor

846 """

@@ -912,8 +897,7 @@ class MBartEncoderLayer(BartEncoderLayer):

897 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:

898 r"""

899 Args:

915 - hidden_states

916 - torch.Tensor of *encoder* input embeddings.

900 + hidden_states: torch.Tensor of *encoder* input embeddings.

901 Returns:

902 Encoder layer output torch.Tensor

903 """

@@ -1035,12 +1019,10 @@ class MBartEncoder(nn.Module):

1019 ) -> torch.Tensor:

1020 r"""

1021 Args:

1038 - input_ids

1039 - Indices of *encoder* input sequence tokens in the vocabulary.

1040 - Padding will be ignored by default should you

1041 - provide it.

1042 - positions

1043 - Positions of *encoder* input sequence tokens.

1022 + input_ids: Indices of *encoder* input sequence tokens in the

1023 + vocabulary.

1024 + Padding will be ignored by default should you provide it.

1025 + positions: Positions of *encoder* input sequence tokens.

1026 Returns:

1027 Decoder output torch.Tensor

1028 """

@@ -1116,14 +1098,11 @@ class MBartDecoder(nn.Module):

1098 ) -> torch.Tensor:

1099 r"""

1100 Args:

1119 - decoder_input_ids

1120 - Indices of *decoder* input sequence tokens in the vocabulary.

1121 - Padding will be ignored by default should you

1122 - provide it.

1123 - decoder_positions

1124 - Positions of *decoder* input sequence tokens.

1125 - encoder_hidden_states:

1126 - Tensor of encoder output embeddings

1101 + decoder_input_ids: Indices of *decoder* input sequence tokens

1102 + in the vocabulary.

1103 + Padding will be ignored by default should you provide it.

1104 + decoder_positions: Positions of *decoder* input sequence tokens.

1105 + encoder_hidden_states: Tensor of encoder output embeddings.

1106 Returns:

1107 Decoder output torch.Tensor

1108 """

@@ -1185,16 +1164,13 @@ class MBartModel(nn.Module, SupportsQuant):

1164 encoder_positions: torch.Tensor) -> torch.Tensor:

1165 r"""

1166 Args:

1188 - input_ids

1189 - Indices of *decoder* input sequence tokens in the vocabulary.

1190 - Padding will be ignored by default should you

1191 - provide it.

1192 - positions

1193 - Positions of *decoder* input sequence tokens.

1194 - encoder_input_ids

1195 - Indices of *encoder* input sequence tokens in the vocabulary.

1196 - encoder_positions:

1197 - Positions of *encoder* input sequence tokens.

1167 + input_ids: Indices of *decoder* input sequence tokens

1168 + in the vocabulary.

1169 + Padding will be ignored by default should you provide it.

1170 + positions: Positions of *decoder* input sequence tokens.

1171 + encoder_input_ids: Indices of *encoder* input sequence tokens

1172 + in the vocabulary.

1173 + encoder_positions: Positions of *encoder* input sequence tokens.

1174 Returns:

1175 Model output torch.Tensor

1176 """

vllm/model_executor/models/blip2.py

-1

@@ -678,7 +678,6 @@ class Blip2ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP,

678 Args:

679 input_ids: Flattened (concatenated) input_ids corresponding to a

680 batch.

681 - pixel_values: The pixels in each input image.

681

682 Info:

683 [Blip2ImageInputs][]

vllm/model_executor/models/donut.py

+6 -12

@@ -79,10 +79,8 @@ class DonutLanguageForConditionalGeneration(nn.Module, SupportsV0Only):

79 ) -> torch.Tensor:

80 r"""

81 Args:

82 - input_ids

83 - torch.Tensor of *decoder* input token ids.

84 - positions

85 - torch.Tensor of *decoder* position indices.

82 + input_ids: torch.Tensor of *decoder* input token ids.

83 + positions: torch.Tensor of *decoder* position indices.

84 Returns:

85 Output torch.Tensor

86 """

@@ -351,14 +349,10 @@ class DonutForConditionalGeneration(nn.Module, SupportsMultiModal,

349 ) -> torch.Tensor:

350 r"""

351 Args:

354 - input_ids

355 - torch.Tensor of *decoder* input token ids.

356 - positions

357 - torch.Tensor of *decoder* position indices.

358 - encoder_input_ids

359 - torch.Tensor of *encoder* input token ids.

360 - encoder_positions

361 - torch.Tensor of *encoder* position indices

352 + input_ids: torch.Tensor of *decoder* input token ids.

353 + positions: torch.Tensor of *decoder* position indices.

354 + encoder_input_ids: torch.Tensor of *encoder* input token ids.

355 + encoder_positions: torch.Tensor of *encoder* position indices

356 Returns:

357 Output torch.Tensor

358 """

vllm/model_executor/models/florence2.py

+14 -24

@@ -631,16 +631,14 @@ class Florence2LanguageModel(nn.Module):

631 ) -> torch.Tensor:

632 r"""

633 Args:

634 - input_ids

635 - Indices of *decoder* input sequence tokens in the vocabulary.

634 + input_ids: Indices of *decoder* input sequence tokens

635 + in the vocabulary.

636 Padding will be ignored by default should you

637 provide it.

638 - positions

639 - Positions of *decoder* input sequence tokens.

640 - encoder_input_ids

641 - Indices of *encoder* input sequence tokens in the vocabulary.

642 - encoder_positions:

643 - Positions of *encoder* input sequence tokens.

638 + positions: Positions of *decoder* input sequence tokens.

639 + encoder_input_ids: Indices of *encoder* input sequence tokens

640 + in the vocabulary.

641 + encoder_positions: Positions of *encoder* input sequence tokens.

642 Returns:

643 Model output torch.Tensor

644 """

@@ -699,14 +697,10 @@ class Florence2LanguageForConditionalGeneration(nn.Module, SupportsV0Only):

697 ) -> torch.Tensor:

698 r"""

699 Args:

702 - input_ids

703 - torch.Tensor of *decoder* input token ids.

704 - positions

705 - torch.Tensor of *decoder* position indices.

706 - encoder_input_ids

707 - torch.Tensor of *encoder* input token ids.

708 - encoder_positions

709 - torch.Tensor of *encoder* position indices

700 + input_ids: torch.Tensor of *decoder* input token ids.

701 + positions: torch.Tensor of *decoder* position indices.

702 + encoder_input_ids: torch.Tensor of *encoder* input token ids.

703 + encoder_positions: torch.Tensor of *encoder* position indices

704 Returns:

705 Output torch.Tensor

706 """

@@ -1068,14 +1062,10 @@ class Florence2ForConditionalGeneration(nn.Module, SupportsMultiModal,

1062 ) -> torch.Tensor:

1063 r"""

1064 Args:

1071 - input_ids

1072 - torch.Tensor of *decoder* input token ids.

1073 - positions

1074 - torch.Tensor of *decoder* position indices.

1075 - encoder_input_ids

1076 - torch.Tensor of *encoder* input token ids.

1077 - encoder_positions

1078 - torch.Tensor of *encoder* position indices

1065 + input_ids: torch.Tensor of *decoder* input token ids.

1066 + positions: torch.Tensor of *decoder* position indices.

1067 + encoder_input_ids: torch.Tensor of *encoder* input token ids.

1068 + encoder_positions: torch.Tensor of *encoder* position indices

1069 Returns:

1070 Output torch.Tensor

1071 """

vllm/model_executor/models/glm4_1v.py

+4 -11

@@ -1599,17 +1599,10 @@ class Glm4vForConditionalGeneration(nn.Module, SupportsMultiModal,

1599 **NOTE**: If mrope is enabled (default setting for GLM-4V

1600 opensource models), the shape will be `(3, seq_len)`,

1601 otherwise it will be `(seq_len,).

1602 - pixel_values: Pixel values to be fed to a model.

1603 - `None` if no images are passed.

1604 - image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.

1605 - `None` if no images are passed.

1606 - pixel_values_videos: Pixel values of videos to be fed to a model.

1607 - `None` if no videos are passed.

1608 - video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.

1609 - `None` if no videos are passed.

1610 - second_per_grid_ts: Tensor `(num_videos)` of video time interval (

1611 - in seconds) for each grid along the temporal dimension in the

1612 - 3D position IDs. `None` if no videos are passed.

1602 + intermediate_tensors: Optional intermediate tensors for pipeline

1603 + parallelism.

1604 + inputs_embeds: Optional pre-computed input embeddings.

1605 + **kwargs: Additional keyword arguments.

1606 """

1607 if intermediate_tensors is not None:

1608 inputs_embeds = None