A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Misc] Delete unused LoRA modules (#13151)
Jee Jee Li committed
Feb 13, 2025 at 00:58 UTC
82cabf53a32be91ec08f214e97de06b99d0eef18
3 files changed
+20
-8
tests/lora/test_lora_manager.py
+12
-6
@@ -606,20 +606,26 @@ def test_packed_loras(dist_init, dummy_model_gate_up, device):
606
607
assert isinstance(model.get_submodule("gate_up_proj"),
608
MergedColumnParallelLinearWithLoRA)
609
+ # Verify packed lora is correct
610
+ model_lora_clone = model_lora.clone(1)
611
+ model_lora_clone1 = model_lora1.clone(1)
612
assert manager.add_adapter(model_lora)
613
assert manager.add_adapter(model_lora1)
614
615
+ assert model_lora.get_lora("gate_proj") is None
616
+ assert model_lora.get_lora("up_proj") is None
617
+ assert model_lora1.get_lora("up_proj") is None
618
packed_lora = model_lora.get_lora("gate_up_proj")
619
assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
620
621
torch.testing.assert_close(packed_lora.lora_a[0],
616
- model_lora.get_lora("gate_proj").lora_a)
622
+ model_lora_clone.get_lora("gate_proj").lora_a)
623
torch.testing.assert_close(packed_lora.lora_b[0],
618
- model_lora.get_lora("gate_proj").lora_b)
624
+ model_lora_clone.get_lora("gate_proj").lora_b)
625
torch.testing.assert_close(packed_lora.lora_a[1],
620
- model_lora.get_lora("up_proj").lora_a)
626
+ model_lora_clone.get_lora("up_proj").lora_a)
627
torch.testing.assert_close(packed_lora.lora_b[1],
622
- model_lora.get_lora("up_proj").lora_b)
628
+ model_lora_clone.get_lora("up_proj").lora_b)
629
630
packed_lora1 = model_lora1.get_lora("gate_up_proj")
631
assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
@@ -627,6 +633,6 @@ def test_packed_loras(dist_init, dummy_model_gate_up, device):
633
assert packed_lora1.lora_a[0] is None
634
assert packed_lora1.lora_b[0] is None
635
torch.testing.assert_close(packed_lora1.lora_a[1],
630
- model_lora1.get_lora("up_proj").lora_a)
636
+ model_lora_clone1.get_lora("up_proj").lora_a)
637
torch.testing.assert_close(packed_lora1.lora_b[1],
632
- model_lora1.get_lora("up_proj").lora_b)
638
+ model_lora_clone1.get_lora("up_proj").lora_b)
vllm/lora/models.py
+7
-1
@@ -5,7 +5,8 @@ import math
5
import os
6
import re
7
from dataclasses import dataclass, field
8
-from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union
8
+from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Type,
9
+ Union)
10
11
import safetensors.torch
12
import torch
@@ -619,12 +620,14 @@ class LoRAModelManager(AdapterModelManager):
620
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
621
for module_name, new_module_names in self.packed_modules.items():
622
replacement_loras: List[Optional[LoRALayerWeights]] = []
623
+ replaced_module: Set[str] = set()
624
has_replacement = False
625
for r in new_module_names:
626
lora = lora_model.get_lora(r)
627
replacement_loras.append(lora)
628
if lora:
629
has_replacement = True
630
+ replaced_module.add(r)
631
if not has_replacement:
632
continue
633
for i in range(len(replacement_loras)):
@@ -633,6 +636,9 @@ class LoRAModelManager(AdapterModelManager):
636
replacement_loras[i] = None
637
lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
638
replacement_loras)
639
+ # Remove the modules that have been replaced.
640
+ for module in replaced_module:
641
+ lora_model.loras.pop(module, None)
642
643
def deactivate_adapter(self, adapter_id: int) -> bool:
644
return deactivate_adapter(adapter_id, self._active_adapters,
vllm/lora/punica_wrapper/punica_base.py
+1
-1
@@ -147,7 +147,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
147
dtype=torch.long,
148
device=device)
149
150
- # 5 is the number of indicies tensors.
150
+ # 5 is the number of indices tensors.
151
# base_indices, sampler_indices, sampler_indices_padded,
152
# embeddings_indices,long_lora_indices
153
self.indices_len: List[Optional[int]] = [None] * 5