@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2a543d6efe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add LoRA support for Mixtral (#2831)

* add mixtral lora support * formatting * fix incorrectly ported logic * polish tests * minor fixes and refactoring * minor fixes * formatting * rename and remove redundant logic * refactoring * refactoring * minor fix * minor refactoring * fix code smell

Terry committed Feb 13, 2024 at 15:55 UTC 2a543d6efecc4e0fe391cbccb68d99ab42e37c33
10 files changed +251 -121
tests/lora/conftest.py
+5
@@ -121,6 +121,11 @@ def sql_lora_files():
121 return snapshot_download(repo_id="yard1/llama-2-7b-sql-lora-test")
122
123
124 +@pytest.fixture(scope="session")
125 +def mixtral_lora_files():
126 + return snapshot_download(repo_id="terrysun/mixtral-lora-adapter")
127 +
128 +
129 @pytest.fixture
130 def llama_2_7b_engine_extra_embeddings() -> nn.Module:
131 cleanup()
tests/lora/test_lora_manager.py
+47 -35
@@ -11,25 +11,35 @@ from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
11 RowParallelLinearWithLoRA,
12 MergedColumnParallelLinearWithLoRA)
13 from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
14 -from vllm.lora.models import (EMBEDDING_MODULES, LoRAModel, LoRAModelManager,
14 +from vllm.lora.models import (LoRAModel, LoRAModelManager,
15 LRUCacheLoRAModelManager, LoRAMapping)
16 from vllm.lora.request import LoRARequest
17 from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
18 WorkerLoRAManager)
19 from vllm.model_executor.layers.linear import RowParallelLinear
20
21 +EMBEDDING_MODULES = {
22 + "embed_tokens": "input_embeddings",
23 + "lm_head": "output_embeddings",
24 +}
25 +
26 +EMBEDDING_PADDING_MODULES = ["lm_head"]
27 +
28
29 def test_from_lora_tensors(sql_lora_files):
30 tensors = load_file(
31 os.path.join(sql_lora_files, "adapter_model.safetensors"))
32 new_embeddings = load_file(
33 os.path.join(sql_lora_files, "new_embeddings.safetensors"))
27 - lora_model = LoRAModel.from_lora_tensors(1,
28 - 8,
29 - 16,
30 - tensors,
31 - "cuda",
32 - embeddings=new_embeddings)
34 + lora_model = LoRAModel.from_lora_tensors(
35 + 1,
36 + 8,
37 + 16,
38 + tensors,
39 + "cuda",
40 + embeddings=new_embeddings,
41 + embedding_modules=EMBEDDING_MODULES,
42 + embedding_padding_modules=EMBEDDING_PADDING_MODULES)
43 for module_name, lora in lora_model.loras.items():
44 assert lora.module_name == module_name
45 assert lora.rank == 8
@@ -90,14 +100,11 @@ def create_packed_lora(
100
101 def test_replace_submodules(dist_init, dummy_model):
102 model = dummy_model
93 - manager = LoRAModelManager(model,
94 - 1,
95 - 1,
96 - 1,
97 - LoRAConfig(max_lora_rank=8,
98 - max_cpu_loras=8,
99 - max_loras=8),
100 - lora_target_modules=["dense1", "layer1.dense2"])
103 + model.supported_lora_modules = ["dense1", "layer1.dense2"]
104 + model.packed_modules_mapping = {}
105 + manager = LoRAModelManager(
106 + model, 1, 1, 1,
107 + LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8))
108 model = manager.model
109
110 assert isinstance(model.get_submodule("dense1"),
@@ -111,16 +118,14 @@ def test_replace_submodules(dist_init, dummy_model):
118
119 def test_lora_model_manager(dist_init, dummy_model):
120 model = dummy_model
121 + model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
122 + model.packed_modules_mapping = {}
123 model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
124 model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
125 model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
126 manager = LoRAModelManager(
118 - model,
119 - 2,
120 - 2,
121 - 2,
122 - LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
123 - lora_target_modules=["dense1", "dense2", "lm_head"])
127 + model, 2, 2, 2,
128 + LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
129 assert all(x is None for x in manager.lora_index_to_id)
130 assert manager.add_lora(model_lora1)
131 assert manager.activate_lora(1)
@@ -159,16 +164,14 @@ def test_lora_model_manager(dist_init, dummy_model):
164
165 def test_lora_lru_cache_model_manager(dist_init, dummy_model):
166 model = dummy_model
167 + model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
168 + model.packed_modules_mapping = {}
169 model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
170 model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
171 model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
172 manager = LRUCacheLoRAModelManager(
166 - model,
167 - 2,
168 - 2,
169 - 2,
170 - LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2),
171 - lora_target_modules=["dense1", "dense2", "lm_head"])
173 + model, 2, 2, 2,
174 + LoRAConfig(max_lora_rank=8, max_cpu_loras=3, max_loras=2))
175 assert all(x is None for x in manager.lora_index_to_id)
176 assert manager.add_lora(model_lora1)
177 assert manager.activate_lora(1)
@@ -212,14 +215,15 @@ def test_lru_lora_model_manager(dist_init, dummy_model):
215 # This tests just the LRU cache functionality, everything else is
216 # tested in test_lora_model_manager
217 model = dummy_model
218 + model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
219 + model.packed_modules_mapping = {}
220 model_lora1 = create_lora(1, model, ["layer1.dense1", "dense2", "lm_head"])
221 model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"])
222 model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"])
223 model_lora4 = create_lora(4, model, ["dense1", "dense2", "lm_head"])
224 manager = LRUCacheLoRAModelManager(
225 model, 2, 2, 2,
221 - LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
222 - ["dense1", "dense2", "lm_head"])
226 + LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
227
228 assert all(x is None for x in manager.lora_index_to_id)
229
@@ -289,8 +293,9 @@ def test_lru_cache_worker_lora_manager(llama_2_7b_model_extra_embeddings,
293 sql_lora_files):
294 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
295 worker_lora_manager = LRUCacheWorkerLoRAManager(
292 - 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
293 - torch.device("cuda"))
296 + 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
297 + lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
298 + EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
299 worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
300
301 mapping = LoRAMapping([], [])
@@ -362,8 +367,9 @@ def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
367 # Should remove every LoRA not specified in the request.
368 lora_config = LoRAConfig(max_lora_rank=8, max_cpu_loras=4, max_loras=4)
369 worker_lora_manager = WorkerLoRAManager(
365 - 4, 2, llama_2_7b_model_extra_embeddings.config.vocab_size, lora_config,
366 - torch.device("cuda"))
370 + 4, 2, llama_2_7b_model_extra_embeddings.unpadded_vocab_size -
371 + lora_config.lora_extra_vocab_size, lora_config, torch.device("cuda"),
372 + EMBEDDING_MODULES, EMBEDDING_PADDING_MODULES)
373 worker_lora_manager.create_lora_manager(llama_2_7b_model_extra_embeddings)
374
375 mapping = LoRAMapping([], [])
@@ -428,6 +434,13 @@ def test_worker_lora_manager(llama_2_7b_model_extra_embeddings,
434
435 def test_packed_loras(dist_init, dummy_model_gate_up):
436 model = dummy_model_gate_up
437 + model.supported_lora_modules = ["gate_up_proj"]
438 + model.packed_modules_mapping = {
439 + "gate_up_proj": [
440 + "gate_proj",
441 + "up_proj",
442 + ],
443 + }
444 model_lora = create_packed_lora(
445 1,
446 model,
@@ -443,8 +456,7 @@ def test_packed_loras(dist_init, dummy_model_gate_up):
456
457 manager = LoRAModelManager(
458 model, 2, 2, 2,
446 - LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2),
447 - ["gate_up_proj"])
459 + LoRAConfig(max_lora_rank=8, max_cpu_loras=2, max_loras=2))
460 model = manager.model
461
462 assert isinstance(model.get_submodule("gate_up_proj"),
tests/lora/test_mixtral.py new
+53
@@ -0,0 +1,53 @@
1 +import pytest
2 +import torch
3 +
4 +import vllm
5 +from vllm.lora.request import LoRARequest
6 +
7 +MODEL_PATH = "mistralai/Mixtral-8x7B-Instruct-v0.1"
8 +
9 +
10 +def do_sample(llm, lora_path: str, lora_id: int):
11 + prompts = [
12 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nSpellForce 3 is a pretty bad game. The developer Grimlore Games is clearly a bunch of no-talent hacks, and 2017 was a terrible year for games anyway. [/user] [assistant]",
13 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nI wanted to like Grimlore Games' 2017 entry, but in SpellForce 3 they just didn't get anything right. [/user] [assistant]",
14 + "[system] Given a target sentence construct the underlying meaning representation\nof the input sentence as a single function with attributes and attribute\nvalues. This function should describe the target string accurately and the\nfunction must be one of the following ['inform', 'request', 'give_opinion',\n'confirm', 'verify_attribute', 'suggest', 'request_explanation',\n'recommend', 'request_attribute'].\n\nThe attributes must be one of the following:\n['name', 'exp_release_date', 'release_year', 'developer', 'esrb', 'rating',\n'genres', 'player_perspective', 'has_multiplayer', 'platforms',\n'available_on_steam', 'has_linux_release', 'has_mac_release', 'specifier'] [/system] [user] Here is the target sentence:\nBioShock is a good role-playing, action-adventure, shooter that released for PlayStation, Xbox, and PC in 2007. It is available on Steam, and it has a Mac release but not a Linux release. [/user] [assistant]",
15 + ]
16 + sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
17 + outputs = llm.generate(
18 + prompts,
19 + sampling_params,
20 + lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
21 + if lora_id else None)
22 + # Print the outputs.
23 + generated_texts = []
24 + for output in outputs:
25 + prompt = output.prompt
26 + generated_text = output.outputs[0].text.strip()
27 + generated_texts.append(generated_text)
28 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
29 + return generated_texts
30 +
31 +
32 +@pytest.mark.parametrize("tp_size", [4])
33 +def test_mixtral_lora(mixtral_lora_files, tp_size):
34 + if torch.cuda.device_count() < tp_size:
35 + pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
36 +
37 + llm = vllm.LLM(MODEL_PATH,
38 + enable_lora=True,
39 + max_num_seqs=16,
40 + max_loras=4,
41 + tensor_parallel_size=tp_size,
42 + worker_use_ray=True)
43 +
44 + expected_lora_output = [
45 + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
46 + "give_opinion(name[SpellForce 3], release_year[2017], developer[Grimlore Games], rating[poor])",
47 + "inform(name[BioShock], release_year[2007], rating[good], genres[action-adventure, role-playing, shooter], platforms[PlayStation, Xbox, PC], available_on_steam[yes], has_linux_release[no], has_mac_release[yes])",
48 + ]
49 +
50 + assert do_sample(llm, mixtral_lora_files,
51 + lora_id=1) == expected_lora_output
52 + assert do_sample(llm, mixtral_lora_files,
53 + lora_id=2) == expected_lora_output
vllm/lora/models.py
+31 -65
@@ -4,8 +4,7 @@ import logging
4 import math
5 import os
6 import re
7 -from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type,
8 - Union)
7 +from typing import (Any, Callable, Dict, Hashable, List, Optional, Tuple, Type)
8
9 import safetensors.torch
10 import torch
@@ -20,36 +19,6 @@ from vllm.lora.utils import parse_fine_tuned_lora_name, replace_submodule
19
20 logger = logging.getLogger(__name__)
21
23 -# TODO: The mappings below should be moved to individual model classes.
24 -
25 -PACKED_MODULES_CFG = {
26 - "qkv_proj": [
27 - "q_proj",
28 - "k_proj",
29 - "v_proj",
30 - ],
31 - "gate_up_proj": [
32 - "gate_proj",
33 - "up_proj",
34 - ],
35 -}
36 -
37 -TARGET_MODULES_QKV = [
38 - "qkv_proj",
39 - "o_proj",
40 - "gate_up_proj",
41 - "down_proj",
42 - "embed_tokens",
43 - "lm_head",
44 -]
45 -
46 -EMBEDDING_MODULES = {
47 - "embed_tokens": "input_embeddings",
48 - "lm_head": "output_embeddings",
49 -}
50 -
51 -EMBEDDING_PADDING_MODULES = ["lm_head"]
52 -
22 _GLOBAL_LORA_ID = 0
23
24
@@ -169,6 +138,8 @@ class LoRAModel:
138 dtype: Optional[torch.dtype] = None,
139 embeddings: Optional[Dict[str, torch.Tensor]] = None,
140 target_embedding_padding: Optional[int] = None,
141 + embedding_modules: Optional[Dict[str, str]] = None,
142 + embedding_padding_modules: Optional[List[str]] = None,
143 ) -> "LoRAModel":
144 """Create a LoRAModel from a dictionary of tensors."""
145 pin_memory = str(device) == "cpu" and not in_wsl()
@@ -179,11 +150,11 @@ class LoRAModel:
150 lora_embeddings_tensor = None
151 if embeddings:
152 embeddings_module = next(
182 - (k for k in EMBEDDING_MODULES if k in module_name),
153 + (k for k in embedding_modules if k in module_name),
154 None)
155 if embeddings_module:
156 lora_embeddings_tensor = embeddings[
186 - EMBEDDING_MODULES[embeddings_module]].to(
157 + embedding_modules[embeddings_module]].to(
158 device=device, dtype=dtype)
159 if pin_memory:
160 lora_embeddings_tensor = (
@@ -201,7 +172,7 @@ class LoRAModel:
172 loras[module_name].lora_b = tensor.to(device=device,
173 dtype=dtype).t()
174 if any(name in module_name
204 - for name in EMBEDDING_PADDING_MODULES
175 + for name in embedding_padding_modules
176 ) and target_embedding_padding is not None:
177 lora_b = loras[module_name].lora_b
178 assert target_embedding_padding >= lora_b.shape[1]
@@ -218,12 +189,15 @@ class LoRAModel:
189
190 @classmethod
191 def from_local_checkpoint(
221 - cls,
222 - lora_dir: str,
223 - lora_model_id: Optional[int] = None,
224 - device: str = "cuda",
225 - dtype: Optional[torch.dtype] = None,
226 - target_embedding_padding: Optional[int] = None) -> "LoRAModel":
192 + cls,
193 + lora_dir: str,
194 + lora_model_id: Optional[int] = None,
195 + device: str = "cuda",
196 + dtype: Optional[torch.dtype] = None,
197 + target_embedding_padding: Optional[int] = None,
198 + embedding_modules: Optional[Dict[str, str]] = None,
199 + embedding_padding_modules: Optional[List[str]] = None,
200 + ) -> "LoRAModel":
201 """Create a LoRAModel from a local checkpoint."""
202 lora_config_path = os.path.join(lora_dir, "adapter_config.json")
203 lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
@@ -260,6 +234,8 @@ class LoRAModel:
234 dtype=dtype,
235 embeddings=embeddings,
236 target_embedding_padding=target_embedding_padding,
237 + embedding_modules=embedding_modules,
238 + embedding_padding_modules=embedding_padding_modules,
239 )
240
241
@@ -273,8 +249,6 @@ class LoRAModelManager:
249 max_num_batched_tokens: int,
250 vocab_size: int,
251 lora_config: LoRAConfig,
276 - lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
277 - packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
252 ):
253 """Create a LoRAModelManager and adapter for a given model.
254
@@ -286,13 +260,6 @@ class LoRAModelManager:
260 in a single batch.
261 vocab_size: the vocab size of the model.
262 lora_config: the LoRA configuration.
289 - lora_target_modules: the target modules patterns to be adapted.
290 - Support both single module name and a list of module names.
291 - packed_modules_mapping: the mapping for packed modules. vLLM
292 - packs some modules into one module, e.g., qkv_proj
293 - is packed of q_proj, k_proj, and v_proj. These modules
294 - have a single layer in the original model, but they are split
295 - into multiple layers in the adapted model.
263 """
264 self.lora_config = lora_config
265 self.max_num_seqs = max_num_seqs
@@ -320,11 +287,11 @@ class LoRAModelManager:
287 self.indices_len = [None] * 4
288
289 self.model: nn.Module = model
323 - self.lora_target_modules: List[str] = ([
324 - lora_target_modules
325 - ] if isinstance(lora_target_modules, str) else lora_target_modules)
326 - self.lora_target_modules = copy.deepcopy(lora_target_modules)
327 - self.packed_modules_mapping = copy.deepcopy(packed_modules_mapping)
290 + if hasattr(self.model, "supported_lora_modules"):
291 + self.supported_lora_modules = copy.deepcopy(
292 + self.model.supported_lora_modules)
293 + self.packed_modules_mapping = copy.deepcopy(
294 + self.model.packed_modules_mapping)
295 self.packed_modules: Dict[str, List[str]] = {}
296 self.modules: Dict[str, "BaseLayerWithLoRA"] = {}
297 self._registered_loras: Dict[int, LoRAModel] = {}
@@ -468,7 +435,11 @@ class LoRAModelManager:
435 assert isinstance(module, BaseLayerWithLoRA)
436 self.modules[module_name] = module
437
471 - def create_dummy_lora(self, lora_id: int, rank: int) -> LoRAModel:
438 + def create_dummy_lora(
439 + self,
440 + lora_id: int,
441 + rank: int,
442 + embedding_modules: Optional[Dict[str, str]] = None) -> LoRAModel:
443 """Create zero-initialized LoRAModel for warmup."""
444 model = LoRAModel(lora_id, rank, {})
445 for module_name, module in self.model.named_modules():
@@ -477,7 +448,7 @@ class LoRAModelManager:
448 continue
449 parts = module_name.split(".")
450 if module_name not in self.packed_modules:
480 - if parts[-1] in EMBEDDING_MODULES:
451 + if parts[-1] in embedding_modules:
452 input_dim = (module.base_layer.org_vocab_size +
453 self.lora_config.lora_extra_vocab_size if
454 hasattr(module.base_layer, "org_vocab_size")
@@ -531,7 +502,7 @@ class LoRAModelManager:
502 re.match(
503 r".*\.{target_module}$".format(target_module=target_module),
504 module_name) or target_module == module_name
534 - for target_module in self.lora_target_modules)
505 + for target_module in self.supported_lora_modules)
506
507 def _register_packed_modules(self, module_full_name: str) -> None:
508 parts = module_full_name.split(".")
@@ -586,12 +557,9 @@ class LRUCacheLoRAModelManager(LoRAModelManager):
557 max_num_batched_tokens: int,
558 vocab_size: int,
559 lora_config: LoRAConfig,
589 - lora_target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
590 - packed_modules_mapping: Dict[str, List[str]] = PACKED_MODULES_CFG,
560 ):
561 super().__init__(model, max_num_seqs, max_num_batched_tokens,
593 - vocab_size, lora_config, lora_target_modules,
594 - packed_modules_mapping)
562 + vocab_size, lora_config)
563 self._registered_loras: LoRALRUCache = LoRALRUCache(
564 self.capacity, self.deactivate_lora)
565 self._active_loras: LoRALRUCache = LoRALRUCache(
@@ -637,11 +605,10 @@ def create_lora_manager(
605 max_num_batched_tokens: int,
606 vocab_size: int,
607 lora_config: LoRAConfig,
640 - target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
608 lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
609 **kwargs) -> LoRAModelManager:
610 """Create a LoRA adapter for a given model."""
644 - if not getattr(model, "supports_lora", False):
611 + if not hasattr(model, "supported_lora_modules"):
612 raise ValueError(f"Model {type(model)} is not supported for LoRA.")
613 lora_manager = lora_manager_cls(
614 model=model,
@@ -649,6 +616,5 @@ def create_lora_manager(
616 max_num_batched_tokens=max_num_batched_tokens,
617 vocab_size=vocab_size,
618 lora_config=lora_config,
652 - lora_target_modules=target_modules,
619 **kwargs)
620 return lora_manager
vllm/lora/worker_manager.py
+11 -10
@@ -1,10 +1,10 @@
1 import logging
2 from abc import ABC, abstractmethod, abstractproperty
3 -from typing import Any, List, Optional, Set, Type, Union
3 +from typing import Any, Dict, List, Optional, Set, Type
4
5 import torch
6
7 -from vllm.lora.models import (TARGET_MODULES_QKV, LoRAModel, LoRAModelManager,
7 +from vllm.lora.models import (LoRAModel, LoRAModelManager,
8 LRUCacheLoRAModelManager, create_lora_manager)
9 from vllm.lora.request import LoRARequest
10 from vllm.lora.layers import LoRAMapping
@@ -13,7 +13,7 @@ from vllm.config import LoRAConfig
13 logger = logging.getLogger(__name__)
14
15
16 -class WorkerLoRAManager(ABC):
16 +class AbstractWorkerLoRAManager(ABC):
17 """Abstract class for managing LoRA models on the worker side."""
18
19 def __init__(self, max_num_seqs: int, max_num_batched_tokens: int,
@@ -33,7 +33,6 @@ class WorkerLoRAManager(ABC):
33 def create_lora_manager(
34 self,
35 model: torch.nn.Module,
36 - target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
36 ) -> Any:
37 ...
38
@@ -63,7 +62,7 @@ class WorkerLoRAManager(ABC):
62 ...
63
64
66 -class WorkerLoRAManager(WorkerLoRAManager):
65 +class WorkerLoRAManager(AbstractWorkerLoRAManager):
66 """WorkerLoRAManager that manages LoRA models on the worker side.
67
68 Every request, the requested LoRAs will be loaded (unless they are already
@@ -78,10 +77,14 @@ class WorkerLoRAManager(WorkerLoRAManager):
77 vocab_size: int,
78 lora_config: LoRAConfig,
79 device: torch.device,
80 + embedding_modules: Dict[str, str],
81 + embedding_padding_modules: List[str],
82 lora_model_cls: Type[LoRAModel] = LoRAModel,
83 ):
84 self._lora_manager: Optional[LoRAModelManager] = None
85 self._lora_model_cls = lora_model_cls
86 + self.embedding_modules = embedding_modules
87 + self.embedding_padding_modules = embedding_padding_modules
88 super().__init__(max_num_seqs, max_num_batched_tokens, vocab_size,
89 lora_config, device)
90
@@ -92,13 +95,11 @@ class WorkerLoRAManager(WorkerLoRAManager):
95 def create_lora_manager(
96 self,
97 model: torch.nn.Module,
95 - target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
98 ) -> Any:
99 lora_manager = create_lora_manager(
100 model,
101 max_num_seqs=self.max_num_seqs,
102 max_num_batched_tokens=self.max_num_batched_tokens,
101 - target_modules=target_modules,
103 vocab_size=self.vocab_size,
104 lora_config=self.lora_config,
105 lora_manager_cls=self._lora_manager_cls,
@@ -142,6 +143,8 @@ class WorkerLoRAManager(WorkerLoRAManager):
143 dtype=self.lora_config.lora_dtype,
144 target_embedding_padding=self.vocab_size +
145 self.lora_config.lora_extra_vocab_size,
146 + embedding_modules=self.embedding_modules,
147 + embedding_padding_modules=self.embedding_padding_modules,
148 )
149 except Exception as e:
150 raise RuntimeError(
@@ -162,7 +165,7 @@ class WorkerLoRAManager(WorkerLoRAManager):
165 return False
166 return self._lora_manager.add_lora(
167 self._lora_manager.create_dummy_lora(lora_request.lora_int_id,
165 - rank))
168 + rank, self.embedding_modules))
169
170 def add_lora(self, lora_request: LoRARequest) -> bool:
171 if lora_request.lora_int_id in self.list_loras():
@@ -195,11 +198,9 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
198 def create_lora_manager(
199 self,
200 model: torch.nn.Module,
198 - target_modules: Union[str, List[str]] = TARGET_MODULES_QKV,
201 ) -> Any:
202 lora_manager = create_lora_manager(
203 model,
202 - target_modules=target_modules,
204 lora_manager_cls=self._lora_manager_cls,
205 max_num_seqs=self.max_num_seqs,
206 vocab_size=self.vocab_size,
vllm/model_executor/model_loader.py
+1 -1
@@ -66,7 +66,7 @@ def get_model(model_config: ModelConfig,
66 # Create a model instance.
67 # The weights will be initialized as empty tensors.
68 with torch.device(device_config.device):
69 - if getattr(model_class, "supports_lora", False):
69 + if hasattr(model_class, "supported_lora_modules"):
70 model = model_class(model_config.hf_config, linear_method,
71 lora_config)
72 elif lora_config:
vllm/model_executor/models/llama.py
+30 -5
@@ -269,7 +269,32 @@ class LlamaModel(nn.Module):
269
270
271 class LlamaForCausalLM(nn.Module):
272 - supports_lora = True
272 + packed_modules_mapping = {
273 + "qkv_proj": [
274 + "q_proj",
275 + "k_proj",
276 + "v_proj",
277 + ],
278 + "gate_up_proj": [
279 + "gate_proj",
280 + "up_proj",
281 + ],
282 + }
283 +
284 + # LoRA specific attributes
285 + supported_lora_modules = [
286 + "qkv_proj",
287 + "o_proj",
288 + "gate_up_proj",
289 + "down_proj",
290 + "embed_tokens",
291 + "lm_head",
292 + ]
293 + embedding_modules = {
294 + "embed_tokens": "input_embeddings",
295 + "lm_head": "output_embeddings",
296 + }
297 + embedding_padding_modules = ["lm_head"]
298
299 def __init__(
300 self,
@@ -281,11 +306,11 @@ class LlamaForCausalLM(nn.Module):
306 self.config = config
307 self.linear_method = linear_method
308 self.model = LlamaModel(config, linear_method, lora_config=lora_config)
284 - unpadded_vocab_size = config.vocab_size
309 + self.unpadded_vocab_size = config.vocab_size
310 if lora_config:
286 - unpadded_vocab_size += lora_config.lora_extra_vocab_size
311 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
312 self.lm_head = ParallelLMHead(
288 - unpadded_vocab_size,
313 + self.unpadded_vocab_size,
314 config.hidden_size,
315 org_num_embeddings=config.vocab_size,
316 padding_size=DEFAULT_VOCAB_PADDING_SIZE
@@ -293,7 +318,7 @@ class LlamaForCausalLM(nn.Module):
318 # compatibility
319 if not lora_config else lora_config.lora_vocab_padding_size,
320 )
296 - self.sampler = Sampler(unpadded_vocab_size, config.vocab_size)
321 + self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
322
323 def forward(
324 self,
vllm/model_executor/models/mistral.py
+26 -1
@@ -265,7 +265,32 @@ class MistralModel(nn.Module):
265
266
267 class MistralForCausalLM(nn.Module):
268 - supports_lora = True
268 + packed_modules_mapping = {
269 + "qkv_proj": [
270 + "q_proj",
271 + "k_proj",
272 + "v_proj",
273 + ],
274 + "gate_up_proj": [
275 + "gate_proj",
276 + "up_proj",
277 + ],
278 + }
279 +
280 + # LoRA specific attributes
281 + supported_lora_modules = [
282 + "qkv_proj",
283 + "o_proj",
284 + "gate_up_proj",
285 + "down_proj",
286 + "embed_tokens",
287 + "lm_head",
288 + ]
289 + embedding_modules = {
290 + "embed_tokens": "input_embeddings",
291 + "lm_head": "output_embeddings",
292 + }
293 + embedding_padding_modules = ["lm_head"]
294
295 def __init__(
296 self,
vllm/model_executor/models/mixtral.py
+37 -3
@@ -27,6 +27,7 @@ import torch
27 from torch import nn
28 from transformers import MixtralConfig
29
30 +from vllm.config import LoRAConfig
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.fused_moe import fused_moe
@@ -38,7 +39,7 @@ from vllm.model_executor.layers.linear import (LinearMethodBase,
39 from vllm.model_executor.layers.rotary_embedding import get_rope
40 from vllm.model_executor.layers.sampler import Sampler
41 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 - VocabParallelEmbedding, ParallelLMHead)
42 + VocabParallelEmbedding, ParallelLMHead, DEFAULT_VOCAB_PADDING_SIZE)
43 from vllm.model_executor.parallel_utils.communication_op import (
44 tensor_model_parallel_all_reduce)
45 from vllm.model_executor.parallel_utils.parallel_state import (
@@ -292,6 +293,7 @@ class MixtralModel(nn.Module):
293 self.embed_tokens = VocabParallelEmbedding(
294 config.vocab_size,
295 config.hidden_size,
296 + org_num_embeddings=self.org_vocab_size,
297 )
298 self.layers = nn.ModuleList([
299 MixtralDecoderLayer(config, linear_method=linear_method)
@@ -318,18 +320,50 @@ class MixtralModel(nn.Module):
320
321
322 class MixtralForCausalLM(nn.Module):
323 + packed_modules_mapping = {
324 + "qkv_proj": [
325 + "q_proj",
326 + "k_proj",
327 + "v_proj",
328 + ],
329 + }
330 +
331 + # LoRA specific attributes
332 + supported_lora_modules = [
333 + "qkv_proj",
334 + "o_proj",
335 + "embed_tokens",
336 + "lm_head",
337 + ]
338 + embedding_modules = {
339 + "embed_tokens": "input_embeddings",
340 + "lm_head": "output_embeddings",
341 + }
342 + embedding_padding_modules = ["lm_head"]
343
344 def __init__(
345 self,
346 config: MixtralConfig,
347 linear_method: Optional[LinearMethodBase] = None,
348 + lora_config: Optional[LoRAConfig] = None,
349 ) -> None:
350 super().__init__()
351 self.config = config
352 self.linear_method = linear_method
353 self.model = MixtralModel(config, linear_method)
331 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
332 - self.sampler = Sampler(config.vocab_size)
354 + self.unpadded_vocab_size = config.vocab_size
355 + if lora_config:
356 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
357 + self.lm_head = ParallelLMHead(
358 + self.unpadded_vocab_size,
359 + config.hidden_size,
360 + org_num_embeddings=config.vocab_size,
361 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
362 + # We need bigger padding if using lora for kernel
363 + # compatibility
364 + if not lora_config else lora_config.lora_vocab_padding_size,
365 + )
366 + self.sampler = Sampler(self.unpadded_vocab_size, config.vocab_size)
367
368 def forward(
369 self,
vllm/worker/model_runner.py
+10 -1
@@ -86,11 +86,20 @@ class ModelRunner:
86 vocab_size = self.model.config.vocab_size
87
88 if self.lora_config:
89 + assert hasattr(
90 + self.model, "supported_lora_modules"
91 + ) and self.model.supported_lora_modules, "Model does not support LoRA"
92 + assert hasattr(
93 + self.model,
94 + "embedding_modules"), "Model does not have embedding_modules"
95 + assert hasattr(self.model, "embedding_padding_modules"
96 + ), "Model does not have embedding_padding_modules"
97 self.lora_manager = LRUCacheWorkerLoRAManager(
98 self.scheduler_config.max_num_seqs,
99 self.scheduler_config.max_num_batched_tokens +
100 self.scheduler_config.max_paddings, vocab_size,
93 - self.lora_config, self.device)
101 + self.lora_config, self.device, self.model.embedding_modules,
102 + self.model.embedding_padding_modules)
103 self.model = self.lora_manager.create_lora_manager(self.model)
104
105 def set_block_size(self, block_size: int) -> None: