@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21906a6f50
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] Fix lora loading for Compressed Tensors in #9120 (#9179)

Ahmad Fahadh Ilyas committed Oct 9, 2024 at 05:10 UTC 21906a6f50ee0edf49ede856a82e8840bab41471
1 file changed +3
vllm/lora/layers.py
+3
@@ -39,6 +39,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
39 # unquantizedLinear
40 if hasattr(base_layer, "weight"):
41 return base_layer.weight.device
42 + # Compressed Tensor
43 + elif hasattr(base_layer, "weight_packed"):
44 + return base_layer.weight_packed.device
45 # GPTQ/AWQ
46 elif hasattr(base_layer, "qweight"):
47 return base_layer.qweight.device