@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82cabf53a3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Misc] Delete unused LoRA modules (#13151)

Jee Jee Li committed Feb 13, 2025 at 00:58 UTC 82cabf53a32be91ec08f214e97de06b99d0eef18
3 files changed +20 -8
tests/lora/test_lora_manager.py
+12 -6
@@ -606,20 +606,26 @@ def test_packed_loras(dist_init, dummy_model_gate_up, device):
606
607 assert isinstance(model.get_submodule("gate_up_proj"),
608 MergedColumnParallelLinearWithLoRA)
609 + # Verify packed lora is correct
610 + model_lora_clone = model_lora.clone(1)
611 + model_lora_clone1 = model_lora1.clone(1)
612 assert manager.add_adapter(model_lora)
613 assert manager.add_adapter(model_lora1)
614
615 + assert model_lora.get_lora("gate_proj") is None
616 + assert model_lora.get_lora("up_proj") is None
617 + assert model_lora1.get_lora("up_proj") is None
618 packed_lora = model_lora.get_lora("gate_up_proj")
619 assert packed_lora and isinstance(packed_lora, PackedLoRALayerWeights)
620
621 torch.testing.assert_close(packed_lora.lora_a[0],
616 - model_lora.get_lora("gate_proj").lora_a)
622 + model_lora_clone.get_lora("gate_proj").lora_a)
623 torch.testing.assert_close(packed_lora.lora_b[0],
618 - model_lora.get_lora("gate_proj").lora_b)
624 + model_lora_clone.get_lora("gate_proj").lora_b)
625 torch.testing.assert_close(packed_lora.lora_a[1],
620 - model_lora.get_lora("up_proj").lora_a)
626 + model_lora_clone.get_lora("up_proj").lora_a)
627 torch.testing.assert_close(packed_lora.lora_b[1],
622 - model_lora.get_lora("up_proj").lora_b)
628 + model_lora_clone.get_lora("up_proj").lora_b)
629
630 packed_lora1 = model_lora1.get_lora("gate_up_proj")
631 assert packed_lora1 and isinstance(packed_lora1, PackedLoRALayerWeights)
@@ -627,6 +633,6 @@ def test_packed_loras(dist_init, dummy_model_gate_up, device):
633 assert packed_lora1.lora_a[0] is None
634 assert packed_lora1.lora_b[0] is None
635 torch.testing.assert_close(packed_lora1.lora_a[1],
630 - model_lora1.get_lora("up_proj").lora_a)
636 + model_lora_clone1.get_lora("up_proj").lora_a)
637 torch.testing.assert_close(packed_lora1.lora_b[1],
632 - model_lora1.get_lora("up_proj").lora_b)
638 + model_lora_clone1.get_lora("up_proj").lora_b)
vllm/lora/models.py
+7 -1
@@ -5,7 +5,8 @@ import math
5 import os
6 import re
7 from dataclasses import dataclass, field
8 -from typing import Any, Callable, Dict, List, Optional, Sequence, Type, Union
8 +from typing import (Any, Callable, Dict, List, Optional, Sequence, Set, Type,
9 + Union)
10
11 import safetensors.torch
12 import torch
@@ -619,12 +620,14 @@ class LoRAModelManager(AdapterModelManager):
620 def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
621 for module_name, new_module_names in self.packed_modules.items():
622 replacement_loras: List[Optional[LoRALayerWeights]] = []
623 + replaced_module: Set[str] = set()
624 has_replacement = False
625 for r in new_module_names:
626 lora = lora_model.get_lora(r)
627 replacement_loras.append(lora)
628 if lora:
629 has_replacement = True
630 + replaced_module.add(r)
631 if not has_replacement:
632 continue
633 for i in range(len(replacement_loras)):
@@ -633,6 +636,9 @@ class LoRAModelManager(AdapterModelManager):
636 replacement_loras[i] = None
637 lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
638 replacement_loras)
639 + # Remove the modules that have been replaced.
640 + for module in replaced_module:
641 + lora_model.loras.pop(module, None)
642
643 def deactivate_adapter(self, adapter_id: int) -> bool:
644 return deactivate_adapter(adapter_id, self._active_adapters,
vllm/lora/punica_wrapper/punica_base.py
+1 -1
@@ -147,7 +147,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
147 dtype=torch.long,
148 device=device)
149
150 - # 5 is the number of indicies tensors.
150 + # 5 is the number of indices tensors.
151 # base_indices, sampler_indices, sampler_indices_padded,
152 # embeddings_indices,long_lora_indices
153 self.indices_len: List[Optional[int]] = [None] * 5