A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix lora loading for Compressed Tensors in #9120 (#9179)
Ahmad Fahadh Ilyas committed
Oct 9, 2024 at 05:10 UTC
21906a6f50ee0edf49ede856a82e8840bab41471
1 file changed
+3
vllm/lora/layers.py
+3
@@ -39,6 +39,9 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
39
# unquantizedLinear
40
if hasattr(base_layer, "weight"):
41
return base_layer.weight.device
42
+ # Compressed Tensor
43
+ elif hasattr(base_layer, "weight_packed"):
44
+ return base_layer.weight_packed.device
45
# GPTQ/AWQ
46
elif hasattr(base_layer, "qweight"):
47
return base_layer.qweight.device