A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Specify device when loading LoRA and embedding tensors (#7129)
Co-authored-by: Jacob Schein <jacobschein@Jacobs-MacBook-Pro-2.local>
Jacob Schein committed
Aug 5, 2024 at 16:35 UTC
89b8db6bb2ce2948073c21231f103c76456844da
1 file changed
+3
-2
vllm/lora/models.py
+3
-2
@@ -248,7 +248,7 @@ class LoRAModel(AdapterModel):
248
f" target modules in {expected_lora_modules}"
249
f" but received {unexpected_modules}."
250
f" Please verify that the loaded LoRA module is correct")
251
- tensors = torch.load(lora_bin_file_path)
251
+ tensors = torch.load(lora_bin_file_path, map_location=device)
252
else:
253
raise ValueError(f"{lora_dir} doesn't contain tensors")
254
@@ -257,7 +257,8 @@ class LoRAModel(AdapterModel):
257
embeddings = safetensors.torch.load_file(
258
new_embeddings_tensor_path)
259
elif os.path.isfile(new_embeddings_bin_file_path):
260
- embeddings = torch.load(new_embeddings_bin_file_path)
260
+ embeddings = torch.load(new_embeddings_bin_file_path,
261
+ map_location=device)
262
263
rank = config["r"]
264
lora_alpha = config["lora_alpha"]