A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[mypy] Forward pass function type hints in lora (#11740)
Signed-off-by: lucast2021 <lucast2021@headroyce.org> Co-authored-by: lucast2021 <lucast2021@headroyce.org>
Lucas Tucker committed
Jan 6, 2025 at 01:59 UTC
9c749713f6990a9f9d12e526d9bfc2669dfa8ee6
3 files changed
+14
-5
vllm/lora/layers.py
+9
-3
@@ -405,7 +405,9 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
405
self.output_size = self.base_layer.output_size
406
self.n_slices = 1
407
408
- def forward(self, input_):
408
+ def forward(
409
+ self, input_: torch.Tensor
410
+ ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
411
"""Forward of ReplicatedLinearWithLoRA
412
413
Args:
@@ -496,7 +498,9 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
498
bias = bias[start_idx:end_idx]
499
return bias
500
499
- def forward(self, input_):
501
+ def forward(
502
+ self, input_: torch.Tensor
503
+ ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
504
"""Forward of ColumnParallelLinear
505
506
Args:
@@ -833,7 +837,9 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
837
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
838
return bias
839
836
- def forward(self, input_):
840
+ def forward(
841
+ self, input_: torch.Tensor
842
+ ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
843
"""Forward of RowParallelLinear
844
845
Args:
vllm/lora/models.py
+2
-1
@@ -4,7 +4,7 @@ import math
4
import os
5
import re
6
from dataclasses import dataclass, field
7
-from typing import Any, Callable, Dict, List, Optional, Sequence, Type
7
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union
8
9
import safetensors.torch
10
import torch
@@ -219,6 +219,7 @@ class LoRAModel(AdapterModel):
219
220
config["vllm_max_position_embeddings"] = max_position_embeddings
221
peft_helper = PEFTHelper.from_dict(config)
222
+ unexpected_modules: List[Union[list[str], str]]
223
if os.path.isfile(lora_tensor_path):
224
tensors: Dict[str, torch.Tensor] = {}
225
# Find unexpected modules.
vllm/model_executor/layers/linear.py
+3
-1
@@ -238,7 +238,9 @@ class ReplicatedLinear(LinearBase):
238
assert param.size() == loaded_weight.size()
239
param.data.copy_(loaded_weight)
240
241
- def forward(self, x: torch.Tensor) -> torch.Tensor:
241
+ def forward(
242
+ self, x: torch.Tensor
243
+ ) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor]]:
244
bias = self.bias if not self.skip_bias_add else None
245
assert self.quant_method is not None
246
output = self.quant_method.apply(self, x, bias)