@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89b8db6bb2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Specify device when loading LoRA and embedding tensors (#7129)

Co-authored-by: Jacob Schein <jacobschein@Jacobs-MacBook-Pro-2.local>

Jacob Schein committed Aug 5, 2024 at 16:35 UTC 89b8db6bb2ce2948073c21231f103c76456844da
1 file changed +3 -2
vllm/lora/models.py
+3 -2
@@ -248,7 +248,7 @@ class LoRAModel(AdapterModel):
248 f" target modules in {expected_lora_modules}"
249 f" but received {unexpected_modules}."
250 f" Please verify that the loaded LoRA module is correct")
251 - tensors = torch.load(lora_bin_file_path)
251 + tensors = torch.load(lora_bin_file_path, map_location=device)
252 else:
253 raise ValueError(f"{lora_dir} doesn't contain tensors")
254
@@ -257,7 +257,8 @@ class LoRAModel(AdapterModel):
257 embeddings = safetensors.torch.load_file(
258 new_embeddings_tensor_path)
259 elif os.path.isfile(new_embeddings_bin_file_path):
260 - embeddings = torch.load(new_embeddings_bin_file_path)
260 + embeddings = torch.load(new_embeddings_bin_file_path,
261 + map_location=device)
262
263 rank = config["r"]
264 lora_alpha = config["lora_alpha"]