A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Misc][LoRA] Add PEFTHelper for LoRA (#11003)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 10, 2024 at 19:12 UTC
d05f88679bedd73939251a17c3d785a354b2946c
4 files changed
+160
-28
tests/lora/test_lora_manager.py
+55
-3
@@ -1,3 +1,4 @@
1
+import json
2
import os
3
from typing import Dict, List
4
@@ -13,6 +14,7 @@ from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
14
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
15
from vllm.lora.models import (LoRAMapping, LoRAModel, LoRAModelManager,
16
LRUCacheLoRAModelManager)
17
+from vllm.lora.peft_helper import PEFTHelper
18
from vllm.lora.request import LoRARequest
19
from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
20
WorkerLoRAManager)
@@ -30,18 +32,68 @@ CUDA_DEVICES = [
32
]
33
34
35
+def test_peft_helper(sql_lora_files):
36
+ lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
37
+ with open(lora_config_path) as f:
38
+ config = json.load(f)
39
+ peft_helper = PEFTHelper.from_dict(config)
40
+ assert peft_helper.r == 8
41
+ assert peft_helper.lora_alpha == 16
42
+ assert peft_helper.target_modules == [
43
+ "q_proj",
44
+ "v_proj",
45
+ "k_proj",
46
+ "o_proj",
47
+ "gate_proj",
48
+ "up_proj",
49
+ "down_proj",
50
+ "embed_tokens",
51
+ "lm_head",
52
+ ]
53
+
54
+ expected_error = "vLLM only supports modules_to_save being None."
55
+ with pytest.raises(ValueError, match=expected_error):
56
+ config = dict(
57
+ r=8,
58
+ lora_alpha=16,
59
+ target_modules=["gate_proj"],
60
+ modules_to_save=["lm_head"],
61
+ )
62
+ PEFTHelper.from_dict(config)
63
+ expected_error = "vLLM does not yet support RSLoRA."
64
+ with pytest.raises(ValueError, match=expected_error):
65
+ config = dict(r=8,
66
+ lora_alpha=16,
67
+ target_modules=["gate_proj"],
68
+ use_rslora=True)
69
+ PEFTHelper.from_dict(config)
70
+
71
+ expected_error = "vLLM does not yet support DoRA."
72
+ with pytest.raises(ValueError, match=expected_error):
73
+ config = dict(r=8,
74
+ lora_alpha=16,
75
+ target_modules=["gate_proj"],
76
+ use_dora=True)
77
+ PEFTHelper.from_dict(config)
78
+
79
+
80
@pytest.mark.parametrize("device", CUDA_DEVICES)
81
def test_from_lora_tensors(sql_lora_files, device):
82
tensors = load_file(
83
os.path.join(sql_lora_files, "adapter_model.safetensors"))
84
new_embeddings = load_file(
85
os.path.join(sql_lora_files, "new_embeddings.safetensors"))
86
+
87
+ lora_config_path = os.path.join(sql_lora_files, "adapter_config.json")
88
+ with open(lora_config_path) as f:
89
+ config = json.load(f)
90
+
91
+ peft_helper = PEFTHelper.from_dict(config)
92
lora_model = LoRAModel.from_lora_tensors(
93
1,
41
- 8,
42
- 16,
94
tensors,
44
- device,
95
+ peft_helper=peft_helper,
96
+ device=device,
97
embeddings=new_embeddings,
98
embedding_modules=EMBEDDING_MODULES,
99
embedding_padding_modules=EMBEDDING_PADDING_MODULES)
vllm/lora/lora.py
+18
@@ -4,6 +4,7 @@ from typing import Sequence as GenericSequence
4
import torch
5
import torch.types
6
7
+from vllm.lora.peft_helper import PEFTHelper
8
from vllm.utils import is_pin_memory_available
9
10
@@ -59,6 +60,23 @@ class LoRALayerWeights:
60
return self.embeddings_tensor.shape[
61
0] if self.embeddings_tensor is not None else 0
62
63
+ @classmethod
64
+ def from_config(
65
+ cls,
66
+ module_name: str,
67
+ peft_helper: PEFTHelper,
68
+ embeddings_tensor: Optional[torch.Tensor] = None,
69
+ ) -> "LoRALayerWeights":
70
+ return cls(
71
+ module_name,
72
+ peft_helper.r,
73
+ peft_helper.lora_alpha,
74
+ None,
75
+ None,
76
+ None,
77
+ embeddings_tensor,
78
+ )
79
+
80
@classmethod
81
def create_dummy_lora_weights(
82
cls,
vllm/lora/models.py
+17
-25
@@ -21,6 +21,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA,
21
LinearScalingRotaryEmbeddingWithLora,
22
LoRAMapping)
23
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
24
+from vllm.lora.peft_helper import PEFTHelper
25
from vllm.lora.punica_wrapper import get_punica_wrapper
26
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
27
is_regex_target_modules,
@@ -104,14 +105,12 @@ class LoRAModel(AdapterModel):
105
def from_lora_tensors(
106
cls,
107
lora_model_id: int,
107
- rank: int,
108
- lora_alpha: int,
108
tensors: Dict[str, torch.Tensor],
109
+ peft_helper: PEFTHelper,
110
device: str = "cuda",
111
dtype: Optional[torch.dtype] = None,
112
embeddings: Optional[Dict[str, torch.Tensor]] = None,
113
target_embedding_padding: Optional[int] = None,
114
- scaling_factor: Optional[float] = None,
114
embedding_modules: Optional[Dict[str, str]] = None,
115
embedding_padding_modules: Optional[List[str]] = None,
116
) -> "LoRAModel":
@@ -135,10 +134,9 @@ class LoRAModel(AdapterModel):
134
if pin_memory:
135
lora_embeddings_tensor = (
136
lora_embeddings_tensor.pin_memory())
138
- loras[module_name] = LoRALayerWeights(module_name, rank,
139
- lora_alpha, None, None,
140
- None,
141
- lora_embeddings_tensor)
137
+ loras[module_name] = LoRALayerWeights.from_config(
138
+ module_name, peft_helper, lora_embeddings_tensor)
139
+
140
if is_bias:
141
loras[module_name].bias = tensor.to(device=device,
142
dtype=dtype).t()
@@ -170,7 +168,11 @@ class LoRAModel(AdapterModel):
168
169
for lora in loras.values():
170
lora.optimize()
173
- return cls(lora_model_id, rank, loras, scaling_factor=scaling_factor)
171
+
172
+ return cls(lora_model_id,
173
+ peft_helper.r,
174
+ loras,
175
+ scaling_factor=peft_helper.vllm_scaling_factor)
176
177
@classmethod
178
def from_local_checkpoint(
@@ -212,6 +214,9 @@ class LoRAModel(AdapterModel):
214
"new_embeddings.bin")
215
with open(lora_config_path) as f:
216
config = json.load(f)
217
+
218
+ config["vllm_max_position_embeddings"] = max_position_embeddings
219
+ peft_helper = PEFTHelper.from_dict(config)
220
if os.path.isfile(lora_tensor_path):
221
tensors: Dict[str, torch.Tensor] = {}
222
# Find unexpected modules.
@@ -242,7 +247,7 @@ class LoRAModel(AdapterModel):
247
# When a bin file is provided, we rely on config to find unexpected
248
# modules.
249
unexpected_modules = []
245
- target_modules = config["target_modules"]
250
+ target_modules = peft_helper.target_modules
251
if not isinstance(target_modules, list):
252
target_modules = [target_modules]
253
for module in target_modules:
@@ -256,7 +261,7 @@ class LoRAModel(AdapterModel):
261
# https://github.com/vllm-project/vllm/pull/5909. But there's no
262
# other better mechanism.
263
if unexpected_modules and not is_regex_target_modules(
259
- config["target_modules"], expected_lora_modules):
264
+ peft_helper.target_modules, expected_lora_modules):
265
raise ValueError(
266
f"While loading {lora_dir}, expected"
267
f" target modules in {expected_lora_modules}"
@@ -274,30 +279,17 @@ class LoRAModel(AdapterModel):
279
embeddings = torch.load(new_embeddings_bin_file_path,
280
map_location=device)
281
277
- rank = config["r"]
278
- lora_alpha = config["lora_alpha"]
279
- context_length = config.get("context_length", None)
280
- scaling_factor = None
281
- if context_length:
282
- if max_position_embeddings is None:
283
- max_position_embeddings = context_length
284
- scaling_factor = float(
285
- math.ceil(context_length / max_position_embeddings))
286
-
282
return cls.from_lora_tensors(
283
lora_model_id=get_lora_id()
284
if lora_model_id is None else lora_model_id,
290
- rank=rank,
291
- lora_alpha=lora_alpha,
285
tensors=tensors,
286
+ peft_helper=peft_helper,
287
device=device,
288
dtype=dtype,
289
embeddings=embeddings,
290
target_embedding_padding=target_embedding_padding,
297
- scaling_factor=scaling_factor,
291
embedding_modules=embedding_modules,
299
- embedding_padding_modules=embedding_padding_modules,
300
- )
292
+ embedding_padding_modules=embedding_padding_modules)
293
294
295
class LoRAModelManager(AdapterModelManager):
vllm/lora/peft_helper.py
new
+70
@@ -0,0 +1,70 @@
1
+# Adapted from: https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora/config.py
2
+
3
+import math
4
+from dataclasses import MISSING, dataclass, field, fields
5
+from typing import Literal, Optional, Union
6
+
7
+
8
+@dataclass
9
+class PEFTHelper:
10
+ # Required fields
11
+ r: int
12
+ lora_alpha: int
13
+ target_modules: Union[list[str], str]
14
+
15
+ bias: Literal["none", "all", "lora_only"] = field(default="none")
16
+ modules_to_save: Optional[list[str]] = field(default=None)
17
+ use_rslora: bool = field(default=False)
18
+ use_dora: bool = field(default=False)
19
+ # long lora field
20
+ context_length: int = field(default=0)
21
+ # Extra vllm field, start with 'vllm_' to avoid conflict
22
+ vllm_max_position_embeddings: Optional[int] = field(default=False)
23
+ vllm_scaling_factor: Optional[float] = field(default=None)
24
+
25
+ def _validate_features(self):
26
+ error_msg = []
27
+
28
+ if self.modules_to_save:
29
+ error_msg.append("vLLM only supports modules_to_save being None.")
30
+ if self.use_rslora:
31
+ error_msg.append("vLLM does not yet support RSLoRA.")
32
+
33
+ if self.use_dora:
34
+ error_msg.append("vLLM does not yet support DoRA.")
35
+
36
+ if error_msg:
37
+ raise ValueError(f"{', '.join(error_msg)}")
38
+
39
+ def __post_init__(self):
40
+ self._validate_features()
41
+ if self.context_length:
42
+ if self.vllm_max_position_embeddings is None:
43
+ self.vllm_max_position_embeddings = self.context_length
44
+ self.vllm_scaling_factor = float(
45
+ math.ceil(self.context_length /
46
+ self.vllm_max_position_embeddings))
47
+
48
+ @classmethod
49
+ def from_dict(cls, config_dict: dict) -> "PEFTHelper":
50
+ # Get all field information from the class
51
+ class_fields = {f.name: f for f in fields(cls)}
52
+ # Check for required fields
53
+ required_fields = {
54
+ name
55
+ for name, f in class_fields.items()
56
+ if f.default is MISSING and f.default_factory is MISSING
57
+ }
58
+
59
+ # Identify any missing required fields
60
+ missing_fields = required_fields - set(config_dict.keys())
61
+ if missing_fields:
62
+ raise ValueError(
63
+ f"Missing required configuration fields: {missing_fields}")
64
+
65
+ # Filter out fields that aren't defined in the class
66
+ filtered_dict = {
67
+ k: v
68
+ for k, v in config_dict.items() if k in class_fields
69
+ }
70
+ return cls(**filtered_dict)