@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 82c49d3260
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Misc][LoRA] Support Rank Stabilized LoRA (RSLoRA) (#6909)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>

John Giorgi committed Dec 31, 2024 at 01:15 UTC 82c49d3260f1fb9fcd686736e8439dc69cd2f1c4
4 files changed +30 -22
tests/lora/test_lora_manager.py
+13 -7
@@ -1,4 +1,5 @@
1 import json
2 +import math
3 import os
4 from typing import Dict, List
5
@@ -50,6 +51,18 @@ def test_peft_helper(sql_lora_files):
51 "embed_tokens",
52 "lm_head",
53 ]
54 + scaling = peft_helper.lora_alpha / peft_helper.r
55 + assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
56 +
57 + # test RSLoRA
58 + config = dict(r=8,
59 + lora_alpha=16,
60 + target_modules=["gate_proj"],
61 + use_rslora=True)
62 + peft_helper = PEFTHelper.from_dict(config)
63 +
64 + scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
65 + assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
66
67 expected_error = "vLLM only supports modules_to_save being None."
68 with pytest.raises(ValueError, match=expected_error):
@@ -60,13 +73,6 @@ def test_peft_helper(sql_lora_files):
73 modules_to_save=["lm_head"],
74 )
75 PEFTHelper.from_dict(config)
63 - expected_error = "vLLM does not yet support RSLoRA."
64 - with pytest.raises(ValueError, match=expected_error):
65 - config = dict(r=8,
66 - lora_alpha=16,
67 - target_modules=["gate_proj"],
68 - use_rslora=True)
69 - PEFTHelper.from_dict(config)
76
77 expected_error = "vLLM does not yet support DoRA."
78 with pytest.raises(ValueError, match=expected_error):
vllm/lora/lora.py
+3 -9
@@ -67,15 +67,9 @@ class LoRALayerWeights:
67 peft_helper: PEFTHelper,
68 embeddings_tensor: Optional[torch.Tensor] = None,
69 ) -> "LoRALayerWeights":
70 - return cls(
71 - module_name,
72 - peft_helper.r,
73 - peft_helper.lora_alpha,
74 - None,
75 - None,
76 - None,
77 - embeddings_tensor,
78 - )
70 + return cls(module_name, peft_helper.r, peft_helper.lora_alpha, None,
71 + None, None, embeddings_tensor,
72 + peft_helper.vllm_lora_scaling_factor)
73
74 @classmethod
75 def create_dummy_lora_weights(
vllm/lora/models.py
+1 -1
@@ -173,7 +173,7 @@ class LoRAModel(AdapterModel):
173 return cls(lora_model_id,
174 peft_helper.r,
175 loras,
176 - scaling_factor=peft_helper.vllm_scaling_factor)
176 + scaling_factor=peft_helper.vllm_long_context_scaling_factor)
177
178 @classmethod
179 def from_local_checkpoint(
vllm/lora/peft_helper.py
+13 -5
@@ -4,6 +4,8 @@ import math
4 from dataclasses import MISSING, dataclass, field, fields
5 from typing import Literal, Optional, Union
6
7 +from vllm.utils import print_info_once
8 +
9
10 @dataclass
11 class PEFTHelper:
@@ -14,21 +16,22 @@ class PEFTHelper:
16
17 bias: Literal["none", "all", "lora_only"] = field(default="none")
18 modules_to_save: Optional[list[str]] = field(default=None)
19 + # True to use Rank-Stabilized LoRA (rsLoRA, see: https://arxiv.org/abs/2312.03732)
20 use_rslora: bool = field(default=False)
21 + # True to use Weight-Decomposed Low-Rank Adaptation (DoRA, see: https://arxiv.org/abs/2402.09353)
22 use_dora: bool = field(default=False)
19 - # long lora field
23 + # long context lora field
24 context_length: int = field(default=0)
25 # Extra vllm field, start with 'vllm_' to avoid conflict
26 + vllm_lora_scaling_factor: float = field(default=1.0)
27 vllm_max_position_embeddings: Optional[int] = field(default=False)
23 - vllm_scaling_factor: Optional[float] = field(default=None)
28 + vllm_long_context_scaling_factor: Optional[float] = field(default=None)
29
30 def _validate_features(self):
31 error_msg = []
32
33 if self.modules_to_save:
34 error_msg.append("vLLM only supports modules_to_save being None.")
30 - if self.use_rslora:
31 - error_msg.append("vLLM does not yet support RSLoRA.")
35
36 if self.use_dora:
37 error_msg.append("vLLM does not yet support DoRA.")
@@ -38,10 +41,15 @@ class PEFTHelper:
41
42 def __post_init__(self):
43 self._validate_features()
44 + if self.use_rslora:
45 + print_info_once("Loading LoRA weights trained with rsLoRA.")
46 + self.vllm_lora_scaling_factor = self.lora_alpha / math.sqrt(self.r)
47 + else:
48 + self.vllm_lora_scaling_factor = self.lora_alpha / self.r
49 if self.context_length:
50 if self.vllm_max_position_embeddings is None:
51 self.vllm_max_position_embeddings = self.context_length
44 - self.vllm_scaling_factor = float(
52 + self.vllm_long_context_scaling_factor = float(
53 math.ceil(self.context_length /
54 self.vllm_max_position_embeddings))
55