A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add LoRA support for Mixtral (#2831)
* add mixtral lora support * formatting * fix incorrectly ported logic * polish tests * minor fixes and refactoring * minor fixes * formatting * rename and remove redundant logic * refactoring * refactoring * minor fix * minor refactoring * fix code smell
Terry committed
Feb 13, 2024 at 15:55 UTC
2a543d6efecc4e0fe391cbccb68d99ab42e37c33
10 files changed
+251
-121
tests/lora/conftest.py
+5
@@ -121,6 +121,11 @@ def sql_lora_files():
121
return snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
122
123
124
+@pytest.fixture(scope="session")
125
+def mixtral_lora_files():
126
+ return snapshot_download(repo_id="terrysun/mixtral-lora-adapter")
127
+
128
+
129
@pytest.fixture
130
def llama_2_7b_engine_extra_embeddings() -> nn.Module:
131
cleanup()
tests/lora/test_lora_manager.py
+47
-35
@@ -11,25 +11,35 @@ from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11
RowParallelLinearWithLoRA,
12
MergedColumnParallelLinearWithLoRA)
13
from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14
-from vllm.lora.models import (EMBEDDING_MODULES, LoRAModel, LoRAModelManager,
14
+from vllm.lora.models import (LoRAModel, LoRAModelManager,
15
LRUCacheLoRAModelManager, LoRAMapping)
16
from vllm.lora.request import LoRARequest
17
from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18
WorkerLoRAManager)
19
from vllm.model_executor.layers.linear import RowParallelLinear
20
21
+EMBEDDING_MODULES = {
22
+ "embed_tokens": "input_embeddings",
23
+ "lm_head": "output_embeddings",
24
+}
25
+
26
+EMBEDDING_PADDING_MODULES = ["lm_head"]
27
+
28
29
def test_from_lora_tensors(sql_lora_files):
30
tensors = load_file(
31
os.path.join(sql_lora_files, "adapter_model.safetensors"))
32
new_embeddings = load_file(
33
os.path.join(sql_lora_files, "new_embeddings.safetensors"))
27
- lora_model = LoRAModel.from_lora_tensors(1,
28
- 8,
29
- 16,
30
- tensors,
31
- "cuda",
32
- embeddings=new_embeddings)
34
+ lora_model = LoRAModel.from_lora_tensors(
35
+ 1,
36
+ 8,
37
+ 16,
38
+ tensors,
39
+ "cuda",
40
+ embeddings=new_embeddings,
41
+ embedding_modules=EMBEDDING_MODULES,
42
+ embedding_padding_modules=EMBEDDING_PADDING_MODULES)
43
for module_name, lora in lora_model.loras.items():
44
assert lora.module_name == module_name
45
assert lora.rank == 8
@@ -90,14 +100,11 @@ def create_packed_lora(
100
101
def test_replace_submodules(dist_init, dummy_model):
102
model = dummy_model
93
- manager = LoRAModelManager(model,
94
- 1,
95
- 1,
96
- 1,
97
- LoRAConfig(max_lora_rank=8,
98
- max_cpu_loras=8,
99
- max_loras=8),
100
- lora_target_modules=["dense1", "layer1.dense2"])
103
+ model.supported_lora_modules = ["dense1", "layer1.dense2"]
104
+ model.packed_modules_mapping = {}
105
+ manager = LoRAModelManager(
106
+ model, 1, 1, 1,
107
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8))
108
model = manager.model
109
110
assert isinstance(model.get_submodule("dense1"),
@@ -111,16 +118,14 @@ def test_replace_submodules(dist_init, dummy_model):
118
119
def test_lora_model_manager(dist_init, dummy_model):
120
model = dummy_model
121
+ model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
122
+ model.packed_modules_mapping = {}
123
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
124
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
125
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
126
manager = LoRAModelManager(
118
- model,
119
- 2,
120
- 2,
121
- 2,
122
- LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
123
- lora_target_modules=["dense1", "dense2", "lm_head"])
127
+ model, 2, 2, 2,
128
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
129
assert all(x is None for x in manager.lora_index_to_id)
130
assert manager.add_lora(model_lora1)
131
assert manager.activate_lora(1)
@@ -159,16 +164,14 @@ def test_lora_model_manager(dist_init, dummy_model):
164
165
def test_lora_lru_cache_model_manager(dist_init, dummy_model):
166
model = dummy_model
167
+ model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
168
+ model.packed_modules_mapping = {}
169
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
170
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
171
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
172
manager = LRUCacheLoRAModelManager(
166
- model,
167
- 2,
168
- 2,
169
- 2,
170
- LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
171
- lora_target_modules=["dense1", "dense2", "lm_head"])
173
+ model, 2, 2, 2,
174
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
175
assert all(x is None for x in manager.lora_index_to_id)
176
assert manager.add_lora(model_lora1)
177
assert manager.activate_lora(1)
@@ -212,14 +215,15 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
215
# This tests just the LRU cache functionality, everything else is
216
# tested in test_lora_model_manager
217
model = dummy_model
218
+ model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
219
+ model.packed_modules_mapping = {}
220
model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
221
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
222
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
223
model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
224
manager = LRUCacheLoRAModelManager(
225
model, 2, 2, 2,
221
- LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
222
- ["dense1", "dense2", "lm_head"])
226
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
227
228
assert all(x is None for x in manager.lora_index_to_id)
229
@@ -289,8 +293,9 @@ def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
293
sql_lora_files):
294
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
295
worker_lora_manager = LRUCacheWorkerLoRAManager(
292
- 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
293
- torch.device("cuda"))
296
+ 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
297
+ lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
298
+ EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
299
worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
300
301
mapping = LoRAMapping([], [])
@@ -362,8 +367,9 @@ def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
367
# Should remove every LoRA not specified in the request.
368
lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
369
worker_lora_manager = WorkerLoRAManager(
365
- 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
366
- torch.device("cuda"))
370
+ 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
371
+ lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
372
+ EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
373
worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
374
375
mapping = LoRAMapping([], [])
@@ -428,6 +434,13 @@ def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
434
435
def test_packed_loras(dist_init, dummy_model_gate_up):
436
model = dummy_model_gate_up
437
+ model.supported_lora_modules = ["gate_up_proj"]
438
+ model.packed_modules_mapping = {
439
+ "gate_up_proj": [
440
+ "gate_proj",
441
+ "up_proj",
442
+ ],
443
+ }
444
model_lora = create_packed_lora(
445
1,
446
model,
@@ -443,8 +456,7 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
456
457
manager = LoRAModelManager(
458
model, 2, 2, 2,
446
- LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
447
- ["gate_up_proj"])
459
+ LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
460
model = manager.model
461
462
assert isinstance(model.get_submodule("gate_up_proj"),
tests/lora/test_mixtral.py
new
+53
@@ -0,0 +1,53 @@
1
+import pytest
2
+import torch
3
+
4
+import vllm
5
+from vllm.lora.request import LoRARequest
6
+
7
+MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
8
+
9
+
10
+def do_sample(llm, lora_path: str, lora_id: int):
11
+ prompts = [
12
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
13
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
14
+ "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
15
+ ]
16
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
17
+ outputs = llm.generate(
18
+ prompts,
19
+ sampling_params,
20
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
21
+ if lora_id else None)
22
+ # Print the outputs.
23
+ generated_texts = []
24
+ for output in outputs:
25
+ prompt = output.prompt
26
+ generated_text = output.outputs[0].text.strip()
27
+ generated_texts.append(generated_text)
28
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
29
+ return generated_texts
30
+
31
+
32
+@pytest.mark.parametrize("tp_size", [4])
33
+def test_mixtral_lora(mixtral_lora_files, tp_size):
34
+ if torch.cuda.device_count() < tp_size:
35
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
36
+
37
+ llm = vllm.LLM(MODEL_PATH,
38
+ enable_lora=True,
39
+ max_num_seqs=16,
40
+ max_loras=4,
41
+ tensor_parallel_size=tp_size,
42
+ worker_use_ray=True)
43
+
44
+ expected_lora_output = [
45
+ "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
46
+ "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
47
+ "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])",
48
+ ]
49
+
50
+ assert do_sample(llm, mixtral_lora_files,
51
+ lora_id=1) == expected_lora_output
52
+ assert do_sample(llm, mixtral_lora_files,
53
+ lora_id=2) == expected_lora_output
vllm/lora/models.py
+31
-65
@@ -4,8 +4,7 @@ import logging
4
import math
5
import os
6
import re
7
-from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type,
8
- Union)
7
+from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type)
8
9
import safetensors.torch
10
import torch
@@ -20,36 +19,6 @@ from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
19
20
logger = logging.getLogger(__name__)
21
23
-# TODO: The mappings below should be moved to individual model classes.
24
-
25
-PACKED_MODULES_CFG = {
26
- "qkv_proj": [
27
- "q_proj",
28
- "k_proj",
29
- "v_proj",
30
- ],
31
- "gate_up_proj": [
32
- "gate_proj",
33
- "up_proj",
34
- ],
35
-}
36
-
37
-TARGET_MODULES_QKV = [
38
- "qkv_proj",
39
- "o_proj",
40
- "gate_up_proj",
41
- "down_proj",
42
- "embed_tokens",
43
- "lm_head",
44
-]
45
-
46
-EMBEDDING_MODULES = {
47
- "embed_tokens": "input_embeddings",
48
- "lm_head": "output_embeddings",
49
-}
50
-
51
-EMBEDDING_PADDING_MODULES = ["lm_head"]
52
-
22
_GLOBAL_LORA_ID = 0
23
24
@@ -169,6 +138,8 @@ class LoRAModel:
138
dtype: Optional[torch.dtype] = None,
139
embeddings: Optional[Dict[str, torch.Tensor]] = None,
140
target_embedding_padding: Optional[int] = None,
141
+ embedding_modules: Optional[Dict[str, str]] = None,
142
+ embedding_padding_modules: Optional[List[str]] = None,
143
) -> "LoRAModel":
144
"""Create a LoRAModel from a dictionary of tensors."""
145
pin_memory = str(device) == "cpu" and not in_wsl()
@@ -179,11 +150,11 @@ class LoRAModel:
150
lora_embeddings_tensor = None
151
if embeddings:
152
embeddings_module = next(
182
- (k for k in EMBEDDING_MODULES if k in module_name),
153
+ (k for k in embedding_modules if k in module_name),
154
None)
155
if embeddings_module:
156
lora_embeddings_tensor = embeddings[
186
- EMBEDDING_MODULES[embeddings_module]].to(
157
+ embedding_modules[embeddings_module]].to(
158
device=device, dtype=dtype)
159
if pin_memory:
160
lora_embeddings_tensor = (
@@ -201,7 +172,7 @@ class LoRAModel:
172
loras[module_name].lora_b = tensor.to(device=device,
173
dtype=dtype).t()
174
if any(name in module_name
204
- for name in EMBEDDING_PADDING_MODULES
175
+ for name in embedding_padding_modules
176
) and target_embedding_padding is not None:
177
lora_b = loras[module_name].lora_b
178
assert target_embedding_padding >= lora_b.shape[1]
@@ -218,12 +189,15 @@ class LoRAModel:
189
190
@classmethod
191
def from_local_checkpoint(
221
- cls,
222
- lora_dir: str,
223
- lora_model_id: Optional[int] = None,
224
- device: str = "cuda",
225
- dtype: Optional[torch.dtype] = None,
226
- target_embedding_padding: Optional[int] = None) -> "LoRAModel":
192
+ cls,
193
+ lora_dir: str,
194
+ lora_model_id: Optional[int] = None,
195
+ device: str = "cuda",
196
+ dtype: Optional[torch.dtype] = None,
197
+ target_embedding_padding: Optional[int] = None,
198
+ embedding_modules: Optional[Dict[str, str]] = None,
199
+ embedding_padding_modules: Optional[List[str]] = None,
200
+ ) -> "LoRAModel":
201
"""Create a LoRAModel from a local checkpoint."""
202
lora_config_path = os.path.join(lora_dir, "adapter_config.json")
203
lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
@@ -260,6 +234,8 @@ class LoRAModel:
234
dtype=dtype,
235
embeddings=embeddings,
236
target_embedding_padding=target_embedding_padding,
237
+ embedding_modules=embedding_modules,
238
+ embedding_padding_modules=embedding_padding_modules,
239
)
240
241
@@ -273,8 +249,6 @@ class LoRAModelManager:
249
max_num_batched_tokens: int,
250
vocab_size: int,
251
lora_config: LoRAConfig,
276
- lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
277
- packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
252
):
253
"""Create a LoRAModelManager and adapter for a given model.
254
@@ -286,13 +260,6 @@ class LoRAModelManager:
260
in a single batch.
261
vocab_size: the vocab size of the model.
262
lora_config: the LoRA configuration.
289
- lora_target_modules: the target modules patterns to be adapted.
290
- Support both single module name and a list of module names.
291
- packed_modules_mapping: the mapping for packed modules. vLLM
292
- packs some modules into one module, e.g., qkv_proj
293
- is packed of q_proj, k_proj, and v_proj. These modules
294
- have a single layer in the original model, but they are split
295
- into multiple layers in the adapted model.
263
"""
264
self.lora_config = lora_config
265
self.max_num_seqs = max_num_seqs
@@ -320,11 +287,11 @@ class LoRAModelManager:
287
self.indices_len = [None] * 4
288
289
self.model: nn.Module = model
323
- self.lora_target_modules: List[str] = ([
324
- lora_target_modules
325
- ] if isinstance(lora_target_modules, str) else lora_target_modules)
326
- self.lora_target_modules = copy.deepcopy(lora_target_modules)
327
- self.packed_modules_mapping = copy.deepcopy(packed_modules_mapping)
290
+ if hasattr(self.model, "supported_lora_modules"):
291
+ self.supported_lora_modules = copy.deepcopy(
292
+ self.model.supported_lora_modules)
293
+ self.packed_modules_mapping = copy.deepcopy(
294
+ self.model.packed_modules_mapping)
295
self.packed_modules: Dict[str, List[str]] = {}
296
self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
297
self._registered_loras: Dict[int, LoRAModel] = {}
@@ -468,7 +435,11 @@ class LoRAModelManager:
435
assert isinstance(module, BaseLayerWithLoRA)
436
self.modules[module_name] = module
437
471
- def create_dummy_lora(self, lora_id: int, rank: int) -> LoRAModel:
438
+ def create_dummy_lora(
439
+ self,
440
+ lora_id: int,
441
+ rank: int,
442
+ embedding_modules: Optional[Dict[str, str]] = None) -> LoRAModel:
443
"""Create zero-initialized LoRAModel for warmup."""
444
model = LoRAModel(lora_id, rank, {})
445
for module_name, module in self.model.named_modules():
@@ -477,7 +448,7 @@ class LoRAModelManager:
448
continue
449
parts = module_name.split(".")
450
if module_name not in self.packed_modules:
480
- if parts[-1] in EMBEDDING_MODULES:
451
+ if parts[-1] in embedding_modules:
452
input_dim = (module.base_layer.org_vocab_size +
453
self.lora_config.lora_extra_vocab_size if
454
hasattr(module.base_layer, "org_vocab_size")
@@ -531,7 +502,7 @@ class LoRAModelManager:
502
re.match(
503
r".*\.{target_module}$".format(target_module=target_module),
504
module_name) or target_module == module_name
534
- for target_module in self.lora_target_modules)
505
+ for target_module in self.supported_lora_modules)
506
507
def _register_packed_modules(self, module_full_name: str) -> None:
508
parts = module_full_name.split(".")
@@ -586,12 +557,9 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
557
max_num_batched_tokens: int,
558
vocab_size: int,
559
lora_config: LoRAConfig,
589
- lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
590
- packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
560
):
561
super().__init__(model, max_num_seqs, max_num_batched_tokens,
593
- vocab_size, lora_config, lora_target_modules,
594
- packed_modules_mapping)
562
+ vocab_size, lora_config)
563
self._registered_loras: LoRALRUCache = LoRALRUCache(
564
self.capacity, self.deactivate_lora)
565
self._active_loras: LoRALRUCache = LoRALRUCache(
@@ -637,11 +605,10 @@ def create_lora_manager(
605
max_num_batched_tokens: int,
606
vocab_size: int,
607
lora_config: LoRAConfig,
640
- target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
608
lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
609
**kwargs) -> LoRAModelManager:
610
"""Create a LoRA adapter for a given model."""
644
- if not getattr(model, "supports_lora", False):
611
+ if not hasattr(model, "supported_lora_modules"):
612
raise ValueError(f"Model {type(model)} is not supported for LoRA.")
613
lora_manager = lora_manager_cls(
614
model=model,
@@ -649,6 +616,5 @@ def create_lora_manager(
616
max_num_batched_tokens=max_num_batched_tokens,
617
vocab_size=vocab_size,
618
lora_config=lora_config,
652
- lora_target_modules=target_modules,
619
**kwargs)
620
return lora_manager
vllm/lora/worker_manager.py
+11
-10
@@ -1,10 +1,10 @@
1
import logging
2
from abc import ABC, abstractmethod, abstractproperty
3
-from typing import Any, List, Optional, Set, Type, Union
3
+from typing import Any, Dict, List, Optional, Set, Type
4
5
import torch
6
7
-from vllm.lora.models import (TARGET_MODULES_QKV, LoRAModel, LoRAModelManager,
7
+from vllm.lora.models import (LoRAModel, LoRAModelManager,
8
LRUCacheLoRAModelManager, create_lora_manager)
9
from vllm.lora.request import LoRARequest
10
from vllm.lora.layers import LoRAMapping
@@ -13,7 +13,7 @@ from vllm.config import LoRAConfig
13
logger = logging.getLogger(__name__)
14
15
16
-class WorkerLoRAManager(ABC):
16
+class AbstractWorkerLoRAManager(ABC):
17
"""Abstract class for managing LoRA models on the worker side."""
18
19
def __init__(self, max_num_seqs: int, max_num_batched_tokens: int,
@@ -33,7 +33,6 @@ class WorkerLoRAManager(ABC):
33
def create_lora_manager(
34
self,
35
model: torch.nn.Module,
36
- target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
36
) -> Any:
37
...
38
@@ -63,7 +62,7 @@ class WorkerLoRAManager(ABC):
62
...
63
64
66
-class WorkerLoRAManager(WorkerLoRAManager):
65
+class WorkerLoRAManager(AbstractWorkerLoRAManager):
66
"""WorkerLoRAManager that manages LoRA models on the worker side.
67
68
Every request, the requested LoRAs will be loaded (unless they are already
@@ -78,10 +77,14 @@ class WorkerLoRAManager(WorkerLoRAManager):
77
vocab_size: int,
78
lora_config: LoRAConfig,
79
device: torch.device,
80
+ embedding_modules: Dict[str, str],
81
+ embedding_padding_modules: List[str],
82
lora_model_cls: Type[LoRAModel] = LoRAModel,
83
):
84
self._lora_manager: Optional[LoRAModelManager] = None
85
self._lora_model_cls = lora_model_cls
86
+ self.embedding_modules = embedding_modules
87
+ self.embedding_padding_modules = embedding_padding_modules
88
super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
89
lora_config, device)
90
@@ -92,13 +95,11 @@ class WorkerLoRAManager(WorkerLoRAManager):
95
def create_lora_manager(
96
self,
97
model: torch.nn.Module,
95
- target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
98
) -> Any:
99
lora_manager = create_lora_manager(
100
model,
101
max_num_seqs=self.max_num_seqs,
102
max_num_batched_tokens=self.max_num_batched_tokens,
101
- target_modules=target_modules,
103
vocab_size=self.vocab_size,
104
lora_config=self.lora_config,
105
lora_manager_cls=self._lora_manager_cls,
@@ -142,6 +143,8 @@ class WorkerLoRAManager(WorkerLoRAManager):
143
dtype=self.lora_config.lora_dtype,
144
target_embedding_padding=self.vocab_size +
145
self.lora_config.lora_extra_vocab_size,
146
+ embedding_modules=self.embedding_modules,
147
+ embedding_padding_modules=self.embedding_padding_modules,
148
)
149
except Exception as e:
150
raise RuntimeError(
@@ -162,7 +165,7 @@ class WorkerLoRAManager(WorkerLoRAManager):
165
return False
166
return self._lora_manager.add_lora(
167
self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
165
- rank))
168
+ rank, self.embedding_modules))
169
170
def add_lora(self, lora_request: LoRARequest) -> bool:
171
if lora_request.lora_int_id in self.list_loras():
@@ -195,11 +198,9 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
198
def create_lora_manager(
199
self,
200
model: torch.nn.Module,
198
- target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
201
) -> Any:
202
lora_manager = create_lora_manager(
203
model,
202
- target_modules=target_modules,
204
lora_manager_cls=self._lora_manager_cls,
205
max_num_seqs=self.max_num_seqs,
206
vocab_size=self.vocab_size,
vllm/model_executor/model_loader.py
+1
-1
@@ -66,7 +66,7 @@ def get_model(model_config: ModelConfig,
66
# Create a model instance.
67
# The weights will be initialized as empty tensors.
68
with torch.device(device_config.device):
69
- if getattr(model_class, "supports_lora", False):
69
+ if hasattr(model_class, "supported_lora_modules"):
70
model = model_class(model_config.hf_config, linear_method,
71
lora_config)
72
elif lora_config:
vllm/model_executor/models/llama.py
+30
-5
@@ -269,7 +269,32 @@ class LlamaModel(nn.Module):
269
270
271
class LlamaForCausalLM(nn.Module):
272
- supports_lora = True
272
+ packed_modules_mapping = {
273
+ "qkv_proj": [
274
+ "q_proj",
275
+ "k_proj",
276
+ "v_proj",
277
+ ],
278
+ "gate_up_proj": [
279
+ "gate_proj",
280
+ "up_proj",
281
+ ],
282
+ }
283
+
284
+ # LoRA specific attributes
285
+ supported_lora_modules = [
286
+ "qkv_proj",
287
+ "o_proj",
288
+ "gate_up_proj",
289
+ "down_proj",
290
+ "embed_tokens",
291
+ "lm_head",
292
+ ]
293
+ embedding_modules = {
294
+ "embed_tokens": "input_embeddings",
295
+ "lm_head": "output_embeddings",
296
+ }
297
+ embedding_padding_modules = ["lm_head"]
298
299
def __init__(
300
self,
@@ -281,11 +306,11 @@ class LlamaForCausalLM(nn.Module):
306
self.config = config
307
self.linear_method = linear_method
308
self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284
- unpadded_vocab_size = config.vocab_size
309
+ self.unpadded_vocab_size = config.vocab_size
310
if lora_config:
286
- unpadded_vocab_size += lora_config.lora_extra_vocab_size
311
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
312
self.lm_head = ParallelLMHead(
288
- unpadded_vocab_size,
313
+ self.unpadded_vocab_size,
314
config.hidden_size,
315
org_num_embeddings=config.vocab_size,
316
padding_size=DEFAULT_VOCAB_PADDING_SIZE
@@ -293,7 +318,7 @@ class LlamaForCausalLM(nn.Module):
318
# compatibility
319
if not lora_config else lora_config.lora_vocab_padding_size,
320
)
296
- self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
321
+ self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
322
323
def forward(
324
self,
vllm/model_executor/models/mistral.py
+26
-1
@@ -265,7 +265,32 @@ class MistralModel(nn.Module):
265
266
267
class MistralForCausalLM(nn.Module):
268
- supports_lora = True
268
+ packed_modules_mapping = {
269
+ "qkv_proj": [
270
+ "q_proj",
271
+ "k_proj",
272
+ "v_proj",
273
+ ],
274
+ "gate_up_proj": [
275
+ "gate_proj",
276
+ "up_proj",
277
+ ],
278
+ }
279
+
280
+ # LoRA specific attributes
281
+ supported_lora_modules = [
282
+ "qkv_proj",
283
+ "o_proj",
284
+ "gate_up_proj",
285
+ "down_proj",
286
+ "embed_tokens",
287
+ "lm_head",
288
+ ]
289
+ embedding_modules = {
290
+ "embed_tokens": "input_embeddings",
291
+ "lm_head": "output_embeddings",
292
+ }
293
+ embedding_padding_modules = ["lm_head"]
294
295
def __init__(
296
self,
vllm/model_executor/models/mixtral.py
+37
-3
@@ -27,6 +27,7 @@ import torch
27
from torch import nn
28
from transformers import MixtralConfig
29
30
+from vllm.config import LoRAConfig
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.fused_moe import fused_moe
@@ -38,7 +39,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
39
from vllm.model_executor.layers.rotary_embedding import get_rope
40
from vllm.model_executor.layers.sampler import Sampler
41
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
- VocabParallelEmbedding, ParallelLMHead)
42
+ VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
43
from vllm.model_executor.parallel_utils.communication_op import (
44
tensor_model_parallel_all_reduce)
45
from vllm.model_executor.parallel_utils.parallel_state import (
@@ -292,6 +293,7 @@ class MixtralModel(nn.Module):
293
self.embed_tokens = VocabParallelEmbedding(
294
config.vocab_size,
295
config.hidden_size,
296
+ org_num_embeddings=self.org_vocab_size,
297
)
298
self.layers = nn.ModuleList([
299
MixtralDecoderLayer(config, linear_method=linear_method)
@@ -318,18 +320,50 @@ class MixtralModel(nn.Module):
320
321
322
class MixtralForCausalLM(nn.Module):
323
+ packed_modules_mapping = {
324
+ "qkv_proj": [
325
+ "q_proj",
326
+ "k_proj",
327
+ "v_proj",
328
+ ],
329
+ }
330
+
331
+ # LoRA specific attributes
332
+ supported_lora_modules = [
333
+ "qkv_proj",
334
+ "o_proj",
335
+ "embed_tokens",
336
+ "lm_head",
337
+ ]
338
+ embedding_modules = {
339
+ "embed_tokens": "input_embeddings",
340
+ "lm_head": "output_embeddings",
341
+ }
342
+ embedding_padding_modules = ["lm_head"]
343
344
def __init__(
345
self,
346
config: MixtralConfig,
347
linear_method: Optional[LinearMethodBase] = None,
348
+ lora_config: Optional[LoRAConfig] = None,
349
) -> None:
350
super().__init__()
351
self.config = config
352
self.linear_method = linear_method
353
self.model = MixtralModel(config, linear_method)
331
- self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
332
- self.sampler = Sampler(config.vocab_size)
354
+ self.unpadded_vocab_size = config.vocab_size
355
+ if lora_config:
356
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
357
+ self.lm_head = ParallelLMHead(
358
+ self.unpadded_vocab_size,
359
+ config.hidden_size,
360
+ org_num_embeddings=config.vocab_size,
361
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
362
+ # We need bigger padding if using lora for kernel
363
+ # compatibility
364
+ if not lora_config else lora_config.lora_vocab_padding_size,
365
+ )
366
+ self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
367
368
def forward(
369
self,
vllm/worker/model_runner.py
+10
-1
@@ -86,11 +86,20 @@ class ModelRunner:
86
vocab_size = self.model.config.vocab_size
87
88
if self.lora_config:
89
+ assert hasattr(
90
+ self.model, "supported_lora_modules"
91
+ ) and self.model.supported_lora_modules, "Model does not support LoRA"
92
+ assert hasattr(
93
+ self.model,
94
+ "embedding_modules"), "Model does not have embedding_modules"
95
+ assert hasattr(self.model, "embedding_padding_modules"
96
+ ), "Model does not have embedding_padding_modules"
97
self.lora_manager = LRUCacheWorkerLoRAManager(
98
self.scheduler_config.max_num_seqs,
99
self.scheduler_config.max_num_batched_tokens +
100
self.scheduler_config.max_paddings, vocab_size,
93
- self.lora_config, self.device)
101
+ self.lora_config, self.device, self.model.embedding_modules,
102
+ self.model.embedding_padding_modules)
103
self.model = self.lora_manager.create_lora_manager(self.model)
104
105
def set_block_size(self, block_size: int) -> None: