A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Misc][LoRA] Support Rank Stabilized LoRA (RSLoRA) (#6909)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
John Giorgi committed
Dec 31, 2024 at 01:15 UTC
82c49d3260f1fb9fcd686736e8439dc69cd2f1c4
4 files changed
+30
-22
tests/lora/test_lora_manager.py
+13
-7
@@ -1,4 +1,5 @@
1
import json
2
+import math
3
import os
4
from typing import Dict, List
5
@@ -50,6 +51,18 @@ def test_peft_helper(sql_lora_files):
51
"embed_tokens",
52
"lm_head",
53
]
54
+ scaling = peft_helper.lora_alpha / peft_helper.r
55
+ assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
56
+
57
+ # test RSLoRA
58
+ config = dict(r=8,
59
+ lora_alpha=16,
60
+ target_modules=["gate_proj"],
61
+ use_rslora=True)
62
+ peft_helper = PEFTHelper.from_dict(config)
63
+
64
+ scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
65
+ assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
66
67
expected_error = "vLLM only supports modules_to_save being None."
68
with pytest.raises(ValueError, match=expected_error):
@@ -60,13 +73,6 @@ def test_peft_helper(sql_lora_files):
73
modules_to_save=["lm_head"],
74
)
75
PEFTHelper.from_dict(config)
63
- expected_error = "vLLM does not yet support RSLoRA."
64
- with pytest.raises(ValueError, match=expected_error):
65
- config = dict(r=8,
66
- lora_alpha=16,
67
- target_modules=["gate_proj"],
68
- use_rslora=True)
69
- PEFTHelper.from_dict(config)
76
77
expected_error = "vLLM does not yet support DoRA."
78
with pytest.raises(ValueError, match=expected_error):
vllm/lora/lora.py
+3
-9
@@ -67,15 +67,9 @@ class LoRALayerWeights:
67
peft_helper: PEFTHelper,
68
embeddings_tensor: Optional[torch.Tensor] = None,
69
) -> "LoRALayerWeights":
70
- return cls(
71
- module_name,
72
- peft_helper.r,
73
- peft_helper.lora_alpha,
74
- None,
75
- None,
76
- None,
77
- embeddings_tensor,
78
- )
70
+ return cls(module_name, peft_helper.r, peft_helper.lora_alpha, None,
71
+ None, None, embeddings_tensor,
72
+ peft_helper.vllm_lora_scaling_factor)
73
74
@classmethod
75
def create_dummy_lora_weights(
vllm/lora/models.py
+1
-1
@@ -173,7 +173,7 @@ class LoRAModel(AdapterModel):
173
return cls(lora_model_id,
174
peft_helper.r,
175
loras,
176
- scaling_factor=peft_helper.vllm_scaling_factor)
176
+ scaling_factor=peft_helper.vllm_long_context_scaling_factor)
177
178
@classmethod
179
def from_local_checkpoint(
vllm/lora/peft_helper.py
+13
-5
@@ -4,6 +4,8 @@ import math
4
from dataclasses import MISSING, dataclass, field, fields
5
from typing import Literal, Optional, Union
6
7
+from vllm.utils import print_info_once
8
+
9
10
@dataclass
11
class PEFTHelper:
@@ -14,21 +16,22 @@ class PEFTHelper:
16
17
bias: Literal["none", "all", "lora_only"] = field(default="none")
18
modules_to_save: Optional[list[str]] = field(default=None)
19
+ # True to use Rank-Stabilized LoRA (rsLoRA, see: https://arxiv.org/abs/2312.03732)
20
use_rslora: bool = field(default=False)
21
+ # True to use Weight-Decomposed Low-Rank Adaptation (DoRA, see: https://arxiv.org/abs/2402.09353)
22
use_dora: bool = field(default=False)
19
- # long lora field
23
+ # long context lora field
24
context_length: int = field(default=0)
25
# Extra vllm field, start with 'vllm_' to avoid conflict
26
+ vllm_lora_scaling_factor: float = field(default=1.0)
27
vllm_max_position_embeddings: Optional[int] = field(default=False)
23
- vllm_scaling_factor: Optional[float] = field(default=None)
28
+ vllm_long_context_scaling_factor: Optional[float] = field(default=None)
29
30
def _validate_features(self):
31
error_msg = []
32
33
if self.modules_to_save:
34
error_msg.append("vLLM only supports modules_to_save being None.")
30
- if self.use_rslora:
31
- error_msg.append("vLLM does not yet support RSLoRA.")
35
36
if self.use_dora:
37
error_msg.append("vLLM does not yet support DoRA.")
@@ -38,10 +41,15 @@ class PEFTHelper:
41
42
def __post_init__(self):
43
self._validate_features()
44
+ if self.use_rslora:
45
+ print_info_once("Loading LoRA weights trained with rsLoRA.")
46
+ self.vllm_lora_scaling_factor = self.lora_alpha / math.sqrt(self.r)
47
+ else:
48
+ self.vllm_lora_scaling_factor = self.lora_alpha / self.r
49
if self.context_length:
50
if self.vllm_max_position_embeddings is None:
51
self.vllm_max_position_embeddings = self.context_length
44
- self.vllm_scaling_factor = float(
52
+ self.vllm_long_context_scaling_factor = float(
53
math.ceil(self.context_length /
54
self.vllm_max_position_embeddings))
55