@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c749713f6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[mypy] Forward pass function type hints in lora (#11740)

Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>

Lucas Tucker committed Jan 6, 2025 at 01:59 UTC 9c749713f6990a9f9d12e526d9bfc2669dfa8ee6
3 files changed +14 -5
vllm/lora/layers.py
+9 -3
@@ -405,7 +405,9 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
405 self.output_size = self.base_layer.output_size
406 self.n_slices = 1
407
408 - def forward(self, input_):
408 + def forward(
409 + self, input_: torch.Tensor
410 + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
411 """Forward of ReplicatedLinearWithLoRA
412
413 Args:
@@ -496,7 +498,9 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
498 bias = bias[start_idx:end_idx]
499 return bias
500
499 - def forward(self, input_):
501 + def forward(
502 + self, input_: torch.Tensor
503 + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
504 """Forward of ColumnParallelLinear
505
506 Args:
@@ -833,7 +837,9 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
837 def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
838 return bias
839
836 - def forward(self, input_):
840 + def forward(
841 + self, input_: torch.Tensor
842 + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
843 """Forward of RowParallelLinear
844
845 Args:
vllm/lora/models.py
+2 -1
@@ -4,7 +4,7 @@ import math
4 import os
5 import re
6 from dataclasses import dataclass, field
7 -from typing import Any, Callable, Dict, List, Optional, Sequence, Type
7 +from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union
8
9 import safetensors.torch
10 import torch
@@ -219,6 +219,7 @@ class LoRAModel(AdapterModel):
219
220 config["vllm_max_position_embeddings"] = max_position_embeddings
221 peft_helper = PEFTHelper.from_dict(config)
222 + unexpected_modules: List[Union[list[str], str]]
223 if os.path.isfile(lora_tensor_path):
224 tensors: Dict[str, torch.Tensor] = {}
225 # Find unexpected modules.
vllm/model_executor/layers/linear.py
+3 -1
@@ -238,7 +238,9 @@ class ReplicatedLinear(LinearBase):
238 assert param.size() == loaded_weight.size()
239 param.data.copy_(loaded_weight)
240
241 - def forward(self, x: torch.Tensor) -> torch.Tensor:
241 + def forward(
242 + self, x: torch.Tensor
243 + ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
244 bias = self.bias if not self.skip_bias_add else None
245 assert self.quant_method is not None
246 output = self.quant_method.apply(self, x, bias)