A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Bugfix] Fix LoRA bug (#4032)
Jee Li committed
Apr 13, 2024 at 07:56 UTC
b8aacac31a4e2e03381fdaef6f1e4bbb895f3b64
1 file changed
+9
-6
vllm/lora/layers.py
+9
-6
@@ -32,14 +32,17 @@ if TYPE_CHECKING:
32
def _get_lora_device(base_layer: nn.Module) -> torch.device:
33
# code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34
34
"""Returns the device for where to place the LoRA tensors."""
35
+ # unquantizedLinear
36
if hasattr(base_layer, "weight"):
37
return base_layer.weight.device
37
- if hasattr(base_layer, "linear_weights") and isinstance(
38
- base_layer.linear_weights, dict):
39
- values = list(base_layer.linear_weights.values())
40
- if len(values) and isinstance(values[0], torch.Tensor):
41
- return values[0].device
42
- raise ValueError(f"Unsupported base layer: {base_layer}")
38
+ # GPTQ/AWQ/SqueezeLLM
39
+ elif hasattr(base_layer, "qweight"):
40
+ return base_layer.qweight.device
41
+ # marlin
42
+ elif hasattr(base_layer, "B"):
43
+ return base_layer.B.device
44
+ else:
45
+ raise ValueError(f"Unsupported base layer: {base_layer}")
46
47
48
def _apply_lora(