@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b8aacac31a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Bugfix] Fix LoRA bug (#4032)

Jee Li committed Apr 13, 2024 at 07:56 UTC b8aacac31a4e2e03381fdaef6f1e4bbb895f3b64
1 file changed +9 -6
vllm/lora/layers.py
+9 -6
@@ -32,14 +32,17 @@ if TYPE_CHECKING:
32 def _get_lora_device(base_layer: nn.Module) -> torch.device:
33 # code borrowed from https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/vllm/lora/layers.py#L34
34 """Returns the device for where to place the LoRA tensors."""
35 + # unquantizedLinear
36 if hasattr(base_layer, "weight"):
37 return base_layer.weight.device
37 - if hasattr(base_layer, "linear_weights") and isinstance(
38 - base_layer.linear_weights, dict):
39 - values = list(base_layer.linear_weights.values())
40 - if len(values) and isinstance(values[0], torch.Tensor):
41 - return values[0].device
42 - raise ValueError(f"Unsupported base layer: {base_layer}")
38 + # GPTQ/AWQ/SqueezeLLM
39 + elif hasattr(base_layer, "qweight"):
40 + return base_layer.qweight.device
41 + # marlin
42 + elif hasattr(base_layer, "B"):
43 + return base_layer.B.device
44 + else:
45 + raise ValueError(f"Unsupported base layer: {base_layer}")
46
47
48 def _apply_lora(