A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc] Reduce LoRA-related static variable (#13166)
Jee Jee Li committed
Feb 22, 2025 at 16:21 UTC
105b8ce4c04022d96ca401907303f04752a6ad6b
41 files changed
+120
-395
tests/lora/conftest.py
+15
-2
@@ -23,6 +23,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
23
from vllm.model_executor.layers.sampler import Sampler
24
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
25
from vllm.model_executor.model_loader import get_model
26
+from vllm.model_executor.models.interfaces import SupportsLoRA
27
from vllm.platforms import current_platform
28
29
@@ -98,9 +99,13 @@ def dist_init_torch_only():
99
backend=backend)
100
101
102
+class DummyLoRAModel(nn.Sequential, SupportsLoRA):
103
+ pass
104
+
105
+
106
@pytest.fixture
107
def dummy_model() -> nn.Module:
103
- model = nn.Sequential(
108
+ model = DummyLoRAModel(
109
OrderedDict([
110
("dense1", ColumnParallelLinear(764, 100)),
111
("dense2", RowParallelLinear(100, 50)),
@@ -121,12 +126,13 @@ def dummy_model() -> nn.Module:
126
("sampler", Sampler())
127
]))
128
model.config = MagicMock()
129
+ model.embedding_modules = {"lm_head": "lm_head"}
130
return model
131
132
133
@pytest.fixture
134
def dummy_model_gate_up() -> nn.Module:
129
- model = nn.Sequential(
135
+ model = DummyLoRAModel(
136
OrderedDict([
137
("dense1", ColumnParallelLinear(764, 100)),
138
("dense2", RowParallelLinear(100, 50)),
@@ -147,6 +153,13 @@ def dummy_model_gate_up() -> nn.Module:
153
("sampler", Sampler())
154
]))
155
model.config = MagicMock()
156
+ model.packed_modules_mapping = {
157
+ "gate_up_proj": [
158
+ "gate_proj",
159
+ "up_proj",
160
+ ],
161
+ }
162
+ model.embedding_modules = {"lm_head": "lm_head"}
163
return model
164
165
tests/lora/test_lora_checkpoints.py
+9
-4
@@ -12,6 +12,12 @@ from vllm.model_executor.models.utils import WeightsMapper
12
lora_lst = [
13
"baichuan7B", "baichuan7B-zero", "baichuan7B-zero-regex", "chatglm3-6b"
14
]
15
+BAICHUAN_LORA_MODULES = [
16
+ "W_pack",
17
+ "o_proj",
18
+ "gate_up_proj",
19
+ "down_proj",
20
+]
21
22
23
@pytest.mark.parametrize("lora_name", lora_lst)
@@ -22,12 +28,11 @@ def test_load_checkpoints(
28
baichuan_regex_lora_files,
29
chatglm3_lora_files,
30
):
25
- supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
31
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
32
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
33
embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
34
expected_lora_modules: List[str] = []
30
- for module in supported_lora_modules:
35
+ for module in BAICHUAN_LORA_MODULES:
36
if module in packed_modules_mapping:
37
expected_lora_modules.extend(packed_modules_mapping[module])
38
else:
@@ -90,12 +95,12 @@ def test_load_checkpoints(
95
96
97
def test_lora_weights_mapping(baichuan_lora_files):
93
- supported_lora_modules = BaiChuanBaseForCausalLM.supported_lora_modules
98
+
99
packed_modules_mapping = BaiChuanBaseForCausalLM.packed_modules_mapping
100
embedding_modules = BaiChuanBaseForCausalLM.embedding_modules
101
embed_padding_modules = BaiChuanBaseForCausalLM.embedding_padding_modules
102
expected_lora_modules: List[str] = []
98
- for module in supported_lora_modules:
103
+ for module in BAICHUAN_LORA_MODULES:
104
if module in packed_modules_mapping:
105
expected_lora_modules.extend(packed_modules_mapping[module])
106
else:
tests/lora/test_lora_huggingface.py
+5
-2
@@ -11,17 +11,20 @@ from vllm.model_executor.models.llama import LlamaForCausalLM
11
12
# Provide absolute path and huggingface lora ids
13
lora_fixture_name = ["sql_lora_files", "sql_lora_huggingface_id"]
14
+LLAMA_LORA_MODULES = [
15
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
16
+ "lm_head"
17
+]
18
19
20
@pytest.mark.parametrize("lora_fixture_name", lora_fixture_name)
21
def test_load_checkpoints_from_huggingface(lora_fixture_name, request):
22
lora_name = request.getfixturevalue(lora_fixture_name)
19
- supported_lora_modules = LlamaForCausalLM.supported_lora_modules
23
packed_modules_mapping = LlamaForCausalLM.packed_modules_mapping
24
embedding_modules = LlamaForCausalLM.embedding_modules
25
embed_padding_modules = LlamaForCausalLM.embedding_padding_modules
26
expected_lora_modules: List[str] = []
24
- for module in supported_lora_modules:
27
+ for module in LLAMA_LORA_MODULES:
28
if module in packed_modules_mapping:
29
expected_lora_modules.extend(packed_modules_mapping[module])
30
else:
tests/lora/test_lora_manager.py
+8
-18
@@ -19,7 +19,6 @@ from vllm.lora.peft_helper import PEFTHelper
19
from vllm.lora.request import LoRARequest
20
from vllm.lora.worker_manager import (LRUCacheWorkerLoRAManager,
21
WorkerLoRAManager)
22
-from vllm.model_executor.layers.linear import RowParallelLinear
22
from vllm.platforms import current_platform
23
24
EMBEDDING_MODULES = {
@@ -114,19 +113,16 @@ def create_packed_lora(
113
114
def test_replace_submodules(dist_init, dummy_model):
115
model = dummy_model
117
- model.supported_lora_modules = ["dense1", "layer1.dense2"]
118
- model.packed_modules_mapping = {}
116
manager = LoRAModelManager(
117
model, 1, 1, 1,
118
LoRAConfig(max_lora_rank=8, max_cpu_loras=8, max_loras=8),
119
torch.device(DEVICES[0]))
120
model = manager.model
124
-
121
assert isinstance(model.get_submodule("dense1"),
122
ColumnParallelLinearWithLoRA)
123
assert isinstance(model.get_submodule("layer1.dense1"),
124
ColumnParallelLinearWithLoRA)
129
- assert isinstance(model.get_submodule("dense2"), RowParallelLinear)
125
+ assert isinstance(model.get_submodule("dense2"), RowParallelLinearWithLoRA)
126
assert isinstance(model.get_submodule("layer1.dense2"),
127
RowParallelLinearWithLoRA)
128
@@ -134,8 +130,6 @@ def test_replace_submodules(dist_init, dummy_model):
130
@pytest.mark.parametrize("device", DEVICES)
131
def test_lora_model_manager(dist_init, dummy_model, device):
132
model = dummy_model
137
- model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
138
- model.packed_modules_mapping = {}
133
model_lora1 = create_lora(1,
134
model, ["layer1.dense1", "dense2", "lm_head"],
135
device=device)
@@ -190,13 +184,18 @@ def test_lora_model_manager(dist_init, dummy_model, device):
184
185
assert manager.device == device
186
assert manager.punica_wrapper.device == device
187
+ assert hasattr(manager, "supported_lora_modules")
188
+ assert sorted(manager.supported_lora_modules) == [
189
+ "dense1",
190
+ "dense2",
191
+ "lm_head",
192
+ "output",
193
+ ]
194
195
196
@pytest.mark.parametrize("device", DEVICES)
197
def test_lora_lru_cache_model_manager(dist_init, dummy_model, device):
198
model = dummy_model
198
- model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
199
- model.packed_modules_mapping = {}
199
model_lora1 = create_lora(1,
200
model, ["layer1.dense1", "dense2", "lm_head"],
201
device=device)
@@ -289,8 +288,6 @@ def test_lru_lora_model_manager(dist_init, dummy_model, device):
288
# This tests just the LRU cache functionality, everything else is
289
# tested in test_lora_model_manager
290
model = dummy_model
292
- model.supported_lora_modules = ["dense1", "dense2", "lm_head"]
293
- model.packed_modules_mapping = {}
291
model_lora1 = create_lora(1,
292
model, ["layer1.dense1", "dense2", "lm_head"],
293
device=device)
@@ -572,13 +569,6 @@ def test_worker_adapter_manager(llama_2_7b_model_extra_embeddings,
569
@pytest.mark.parametrize("device", DEVICES)
570
def test_packed_loras(dist_init, dummy_model_gate_up, device):
571
model = dummy_model_gate_up
575
- model.supported_lora_modules = ["gate_up_proj"]
576
- model.packed_modules_mapping = {
577
- "gate_up_proj": [
578
- "gate_proj",
579
- "up_proj",
580
- ],
581
- }
572
model_lora = create_packed_lora(
573
1,
574
model,
vllm/lora/models.py
+11
-10
@@ -26,6 +26,7 @@ from vllm.lora.lora import LoRALayerWeights, PackedLoRALayerWeights
26
from vllm.lora.peft_helper import PEFTHelper
27
from vllm.lora.punica_wrapper import get_punica_wrapper
28
from vllm.lora.utils import (from_layer, from_layer_logits_processor,
29
+ get_supported_lora_modules,
30
is_regex_target_modules,
31
parse_fine_tuned_lora_name, replace_submodule)
32
from vllm.model_executor.models import SupportsLoRA, supports_multimodal
@@ -332,15 +333,15 @@ class LoRAModelManager(AdapterModelManager):
333
# Used for long context lora.
334
self.scaling_factor_to_offset: Dict[float, int] = {}
335
super().__init__(model)
335
- if hasattr(self.model, "supported_lora_modules"):
336
- self.supported_lora_modules = copy.deepcopy(
337
- self.model.supported_lora_modules)
338
- if lora_config.long_lora_scaling_factors:
339
- # We need to replace rotary emb layer to do batch computation
340
- # for long lora.
341
- self.supported_lora_modules.append("rotary_emb")
342
- self.packed_modules_mapping = copy.deepcopy(
343
- self.model.packed_modules_mapping)
336
+ self.supported_lora_modules = get_supported_lora_modules(self.model)
337
+ assert self.supported_lora_modules, "No supported LoRA modules found in"
338
+ f"{self.model.__class__.__name__}."
339
+ if lora_config.long_lora_scaling_factors:
340
+ # We need to replace rotary emb layer to do batch computation
341
+ # for long lora.
342
+ self.supported_lora_modules.append("rotary_emb")
343
+ self.packed_modules_mapping = copy.deepcopy(
344
+ self.model.packed_modules_mapping)
345
# Used to indicate whether the model is a multimodal model
346
self.supports_mm: bool = (
347
supports_multimodal(self.model)
@@ -756,7 +757,7 @@ def create_lora_manager(
757
lora_manager_cls: Type[LoRAModelManager] = LoRAModelManager,
758
**kwargs) -> LoRAModelManager:
759
"""Create a LoRA adapter for a given model."""
759
- if not hasattr(model, "supported_lora_modules"):
760
+ if not hasattr(model, "packed_modules_mapping"):
761
raise ValueError(f"Model {type(model)} is not supported for LoRA.")
762
lora_manager = lora_manager_cls(
763
model=model,
vllm/lora/utils.py
+26
@@ -29,6 +29,7 @@ from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
29
ReplicatedLinearWithLoRA,
30
RowParallelLinearWithLoRA,
31
VocabParallelEmbeddingWithLoRA)
32
+from vllm.model_executor.layers.linear import LinearBase
33
# yapf: enable
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
@@ -68,6 +69,14 @@ def from_layer(layer: nn.Module,
69
ret = lora_cls(layer)
70
ret.create_lora_weights(max_loras, lora_config, model_config)
71
return ret
72
+
73
+ # The Case for HFCompatibleLinear
74
+ if (hasattr(layer, "get_lora_class")
75
+ and layer.__class__.__name__ == "HFCompatibleLinear"):
76
+ lora_cls = layer.get_lora_class(lora_config.fully_sharded_loras)
77
+ ret = lora_cls(layer)
78
+ ret.create_lora_weights(max_loras, lora_config, model_config)
79
+ return ret
80
return layer
81
82
@@ -170,6 +179,23 @@ def is_regex_target_modules(load_modules: Union[str, List[str]],
179
return False
180
181
182
+def get_supported_lora_modules(model: nn.Module) -> List[str]:
183
+ """
184
+ In vLLM, all linear layers support LoRA.
185
+ """
186
+ supported_lora_modules: Set[str] = set()
187
+ # step1: traverse the model to get all the linear subfixes.
188
+ for name, module in model.named_modules():
189
+ if isinstance(module, (LinearBase, )):
190
+ supported_lora_modules.add(name.split(".")[-1])
191
+ # step 2: get the embedding modules if the model's mbedding_modules
192
+ # is not empty.
193
+ if model.embedding_modules:
194
+ for name in model.embedding_modules:
195
+ supported_lora_modules.add(name)
196
+ return list(supported_lora_modules)
197
+
198
+
199
def get_adapter_absolute_path(lora_path: str) -> str:
200
"""
201
Resolves the given lora_path to an absolute local path.
vllm/lora/worker_manager.py
+5
-3
@@ -84,9 +84,10 @@ class WorkerLoRAManager(AbstractWorkerManager):
84
85
def _load_adapter(self, lora_request: LoRARequest) -> LoRAModel:
86
try:
87
- model = self._adapter_manager.model
88
- supported_lora_modules = model.supported_lora_modules
89
- packed_modules_mapping = model.packed_modules_mapping
87
+ supported_lora_modules = (
88
+ self._adapter_manager.supported_lora_modules)
89
+ packed_modules_mapping = (
90
+ self._adapter_manager.packed_modules_mapping)
91
expected_lora_modules: List[str] = []
92
for module in supported_lora_modules:
93
if module in packed_modules_mapping:
@@ -107,6 +108,7 @@ class WorkerLoRAManager(AbstractWorkerManager):
108
109
# For some models like Qwen2VL, we need to use hf_to_vllm_mapper
110
# to ensure correct loading of lora weights.
111
+ model = self._adapter_manager.model
112
hf_to_vllm_mapper = None
113
if (hasattr(model, "hf_to_vllm_mapper")
114
and model.hf_to_vllm_mapper is not None):
vllm/model_executor/models/baichuan.py
-9
@@ -342,15 +342,6 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
342
"up_proj",
343
],
344
}
345
- # LoRA specific attributes
346
- supported_lora_modules = [
347
- "W_pack",
348
- "o_proj",
349
- "gate_up_proj",
350
- "down_proj",
351
- ]
352
- embedding_modules = {}
353
- embedding_padding_modules = []
345
346
def __init__(
347
self,
vllm/model_executor/models/bamba.py
-6
@@ -389,12 +389,6 @@ class BambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
389
}
390
391
# LoRA specific attributes
392
- supported_lora_modules = [
393
- "qkv_proj",
394
- "o_proj",
395
- "embed_tokens",
396
- "lm_head",
397
- ]
392
embedding_modules = {
393
"embed_tokens": "input_embeddings",
394
"lm_head": "output_embeddings",
vllm/model_executor/models/chatglm.py
-10
@@ -477,16 +477,6 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP):
477
"query_key_value": ["query_key_value"],
478
"dense_h_to_4h": ["dense_h_to_4h"]
479
}
480
- # LoRA specific attributes
481
- supported_lora_modules = [
482
- "query_key_value",
483
- "dense",
484
- "dense_h_to_4h",
485
- "dense_4h_to_h",
486
- ]
487
-
488
- embedding_modules = {}
489
- embedding_padding_modules = []
480
481
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
482
config = vllm_config.model_config.hf_config
vllm/model_executor/models/commandr.py
-4
@@ -357,11 +357,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
357
],
358
}
359
# LoRA specific attributes
360
- supported_lora_modules = [
361
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
362
- ]
360
embedding_modules = {"embed_tokens": "input_embeddings"}
364
- embedding_padding_modules = []
361
362
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
363
super().__init__()
vllm/model_executor/models/exaone.py
-8
@@ -415,14 +415,6 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
415
}
416
417
# LoRA specific attributes
418
- supported_lora_modules = [
419
- "qkv_proj",
420
- "out_proj",
421
- "gate_up_proj",
422
- "c_proj",
423
- "wte",
424
- "lm_head",
425
- ]
418
embedding_modules = {
419
"wte": "input_embeddings",
420
"lm_head": "output_embeddings",
vllm/model_executor/models/gemma.py
-12
@@ -344,18 +344,6 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
344
],
345
}
346
347
- # LoRA specific attributes
348
- supported_lora_modules = [
349
- "qkv_proj",
350
- "o_proj",
351
- "gate_up_proj",
352
- "down_proj",
353
- ]
354
-
355
- # Gemma does not apply LoRA to the embedding layer.
356
- embedding_modules = {}
357
- embedding_padding_modules = []
358
-
347
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
348
super().__init__()
349
config = vllm_config.model_config.hf_config
vllm/model_executor/models/gemma2.py
-11
@@ -390,17 +390,6 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
390
],
391
}
392
393
- # LoRA specific attributes
394
- supported_lora_modules = [
395
- "qkv_proj",
396
- "o_proj",
397
- "gate_up_proj",
398
- "down_proj",
399
- ]
400
- # Gemma does not apply LoRA to the embedding layer.
401
- embedding_modules = {}
402
- embedding_padding_modules = []
403
-
393
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
394
config = vllm_config.model_config.hf_config
395
quant_config = vllm_config.quant_config
vllm/model_executor/models/glm4v.py
-15
@@ -534,21 +534,6 @@ class GLM4VForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP,
534
"dense_h_to_4h": ["dense_h_to_4h"],
535
"merged_proj": ["gate_proj", "dense_h_to_4h"]
536
}
537
- # LoRA specific attributes
538
- supported_lora_modules = [
539
- "query_key_value",
540
- "dense",
541
- "dense_h_to_4h",
542
- "dense_4h_to_h",
543
- # vision
544
- "fc1",
545
- "fc2",
546
- "merged_proj",
547
- "linear_proj"
548
- ]
549
-
550
- embedding_modules = {}
551
- embedding_padding_modules = []
537
538
def get_mm_mapping(self) -> MultiModelKeys:
539
"""
vllm/model_executor/models/gpt_bigcode.py
+1
-4
@@ -261,15 +261,12 @@ class GPTBigCodeModel(nn.Module):
261
class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
262
packed_modules_mapping = {"c_attn": ["c_attn"]}
263
264
- supported_lora_modules = ["c_fc", "c_proj", "wte", "c_attn"]
265
-
264
+ # LoRA specific attributes
265
embedding_modules = {
266
"wte": "input_embeddings",
267
"lm_head": "output_embeddings",
268
}
269
271
- embedding_padding_modules = []
272
-
270
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
271
super().__init__()
272
config = vllm_config.model_config.hf_config
vllm/model_executor/models/granite.py
-4
@@ -351,10 +351,6 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
351
}
352
353
# LoRA specific attributes
354
- supported_lora_modules = [
355
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
356
- "lm_head"
357
- ]
354
embedding_modules = {
355
"embed_tokens": "input_embeddings",
356
"lm_head": "output_embeddings",
vllm/model_executor/models/granitemoe.py
-7
@@ -329,13 +329,6 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
329
}
330
331
# LoRA specific attributes
332
- supported_lora_modules = [
333
- "qkv_proj",
334
- "o_proj",
335
- "embed_tokens",
336
- "lm_head",
337
- "layer",
338
- ]
332
embedding_modules = {
333
"embed_tokens": "input_embeddings",
334
"lm_head": "output_embeddings",
vllm/model_executor/models/idefics3.py
-15
@@ -597,21 +597,6 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal,
597
"up_proj",
598
],
599
}
600
- # LoRA specific attributes
601
- supported_lora_modules = [
602
- # vision_model
603
- "fc1",
604
- "fc2",
605
- "out_proj",
606
- # text_model
607
- "qkv_proj", # same name with vision encoder
608
- "o_proj",
609
- "gate_up_proj",
610
- "down_proj",
611
- ]
612
-
613
- embedding_modules = {}
614
- embedding_padding_modules = []
600
601
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
602
super().__init__()
vllm/model_executor/models/interfaces.py
+5
-7
@@ -118,11 +118,11 @@ class SupportsLoRA(Protocol):
118
There is no need to redefine this flag if this class is in the
119
MRO of your model class.
120
"""
121
-
122
- packed_modules_mapping: ClassVar[Dict[str, List[str]]]
123
- supported_lora_modules: ClassVar[List[str]]
124
- embedding_modules: ClassVar[Dict[str, str]]
125
- embedding_padding_modules: ClassVar[List[str]]
121
+ # The `embedding_module` and `embedding_padding_modules`
122
+ # are empty by default.
123
+ embedding_modules: ClassVar[Dict[str, str]] = {}
124
+ embedding_padding_modules: ClassVar[List[str]] = []
125
+ packed_modules_mapping: ClassVar[Dict[str, List[str]]] = {}
126
127
128
# We can't use runtime_checkable with ClassVar for issubclass checks
@@ -132,7 +132,6 @@ class _SupportsLoRAType(Protocol):
132
supports_lora: Literal[True]
133
134
packed_modules_mapping: Dict[str, List[str]]
135
- supported_lora_modules: List[str]
135
embedding_modules: Dict[str, str]
136
embedding_padding_modules: List[str]
137
@@ -155,7 +154,6 @@ def supports_lora(
154
if not result:
155
lora_attrs = (
156
"packed_modules_mapping",
158
- "supported_lora_modules",
157
"embedding_modules",
158
"embedding_padding_modules",
159
)
vllm/model_executor/models/internlm2.py
-10
@@ -329,16 +329,6 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
329
"gate_up_proj": ["w1", "w3"],
330
}
331
332
- # LoRA specific attributes
333
- supported_lora_modules = [
334
- "wqkv",
335
- "wo",
336
- "gate_up_proj",
337
- "w2",
338
- ]
339
- embedding_modules = {}
340
- embedding_padding_modules = []
341
-
332
def __init__(self,
333
*,
334
vllm_config: VllmConfig,
vllm/model_executor/models/jamba.py
-4
@@ -380,10 +380,6 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
380
}
381
382
# LoRA specific attributes
383
- supported_lora_modules = [
384
- "qkv_proj", "o_proj", "embed_tokens", "lm_head", "up_proj",
385
- "down_proj", "gate_proj", "out_proj", "in_proj", "x_proj"
386
- ]
383
embedding_modules = {
384
"embed_tokens": "input_embeddings",
385
"lm_head": "output_embeddings",
vllm/model_executor/models/llama.py
-4
@@ -452,10 +452,6 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
452
}
453
454
# LoRA specific attributes
455
- supported_lora_modules = [
456
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens",
457
- "lm_head"
458
- ]
455
embedding_modules = {
456
"embed_tokens": "input_embeddings",
457
"lm_head": "output_embeddings"
vllm/model_executor/models/minicpm.py
-8
@@ -522,14 +522,6 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
522
}
523
524
# LoRA specific attributes
525
- supported_lora_modules = [
526
- "qkv_proj",
527
- "o_proj",
528
- "gate_up_proj",
529
- "down_proj",
530
- "embed_tokens",
531
- "lm_head",
532
- ]
525
embedding_modules = {
526
"embed_tokens": "input_embeddings",
527
"lm_head": "output_embeddings",
vllm/model_executor/models/minicpm3.py
-16
@@ -227,21 +227,5 @@ class MiniCPM3ForCausalLM(MiniCPMForCausalLM):
227
],
228
}
229
230
- # LoRA specific attributes
231
- supported_lora_modules = [
232
- "kv_a_proj_with_mqa",
233
- "q_a_proj",
234
- "q_b_proj",
235
- "kv_b_proj",
236
- "o_proj",
237
- "gate_up_proj",
238
- "down_proj",
239
- "embed_tokens",
240
- "lm_head",
241
- ]
242
-
243
- # `embedding_modules` and `embedding_padding_modules`
244
- # are inherited from MiniCPMForCausalLM
245
-
230
def _init_model(self, *, vllm_config: VllmConfig, prefix: str = ""):
231
return MiniCPM3Model(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/minicpmv.py
-42
@@ -1228,23 +1228,6 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
1228
"up_proj",
1229
],
1230
}
1231
- # LoRA specific attributes
1232
- supported_lora_modules = [
1233
- # vision encoder
1234
- "fc1",
1235
- "fc2",
1236
- "out_proj",
1237
- # language model
1238
- "qkv_proj", # same name with vision encoder
1239
- "o_proj",
1240
- "gate_up_proj",
1241
- "down_proj",
1242
- # resampler
1243
- "kv_proj",
1244
- ]
1245
-
1246
- embedding_modules = {}
1247
- embedding_padding_modules = []
1231
1232
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1233
super().__init__(vllm_config=vllm_config, prefix=prefix)
@@ -1338,23 +1321,6 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1321
"up_proj",
1322
],
1323
}
1341
- # LoRA specific attributes
1342
- supported_lora_modules = [
1343
- # vision encoder
1344
- "fc1",
1345
- "fc2",
1346
- "out_proj",
1347
- # language model
1348
- "qkv_proj", # same name with vision encoder
1349
- "o_proj",
1350
- "gate_up_proj",
1351
- "down_proj",
1352
- # resampler
1353
- "kv_proj",
1354
- ]
1355
-
1356
- embedding_modules = {}
1357
- embedding_padding_modules = []
1324
1325
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1326
super().__init__(vllm_config=vllm_config, prefix=prefix)
@@ -1460,13 +1426,6 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1426
which is not conducive to the current integration logic of LoRA and
1427
bitsandbytes in vLLM. Therefore, it is necessary to separate them.
1428
"""
1463
- # Ensure that the LoRA support check passes when the class is not
1464
- # initialized, but set all these attributes to empty.
1465
- # These will be updated when an instance class is selected
1466
- packed_modules_mapping = {}
1467
- supported_lora_modules = []
1468
- embedding_modules = {}
1469
- embedding_padding_modules = []
1429
1430
def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
1431
config = vllm_config.model_config.hf_config
@@ -1487,7 +1446,6 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
1446
# quant_config references base class members,
1447
# so update values before init is called
1448
cls.packed_modules_mapping.update(instance_cls.packed_modules_mapping)
1490
- cls.supported_lora_modules += instance_cls.supported_lora_modules
1449
cls.embedding_modules.update(instance_cls.embedding_modules)
1450
cls.embedding_padding_modules += instance_cls.embedding_padding_modules
1451
return instance_cls(vllm_config=vllm_config, prefix=prefix)
vllm/model_executor/models/mixtral.py
-4
@@ -332,10 +332,6 @@ class MixtralForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
332
}
333
334
# LoRA specific attributes
335
- supported_lora_modules = [
336
- "qkv_proj", "o_proj", "embed_tokens", "lm_head", "w1", "w2", "w3",
337
- "gate"
338
- ]
335
embedding_modules = {
336
"embed_tokens": "input_embeddings",
337
"lm_head": "output_embeddings",
vllm/model_executor/models/molmo.py
-20
@@ -1440,26 +1440,6 @@ class MolmoForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA,
1440
"merged_linear": ["gate_proj", "up_proj"] # image_projector
1441
}
1442
1443
- # LoRA specific attributes
1444
- supported_lora_modules = [
1445
- # language model
1446
- "qkv_proj",
1447
- "o_proj",
1448
- "gate_up_proj",
1449
- "down_proj", # same name with image_projector
1450
- # vision tower
1451
- "wq",
1452
- "wk",
1453
- "wv",
1454
- "wo",
1455
- "w1",
1456
- "w2",
1457
- # image_projector
1458
- "merged_linear",
1459
- ]
1460
- embedding_modules = {}
1461
- embedding_padding_modules = []
1462
-
1443
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
1444
super().__init__()
1445
config = vllm_config.model_config.hf_config
vllm/model_executor/models/nemotron.py
-3
@@ -389,9 +389,6 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
389
}
390
391
# LoRA specific attributes
392
- supported_lora_modules = [
393
- "qkv_proj", "o_proj", "up_proj", "down_proj", "embed_tokens", "lm_head"
394
- ]
392
embedding_modules = {
393
"embed_tokens": "input_embeddings",
394
"lm_head": "output_embeddings",
vllm/model_executor/models/phi.py
-11
@@ -273,17 +273,6 @@ class PhiForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
273
]
274
}
275
276
- # LoRA specific attributes
277
- supported_lora_modules = [
278
- "qkv_proj",
279
- "dense",
280
- "fc1",
281
- "fc2",
282
- ]
283
-
284
- embedding_modules = {}
285
- embedding_padding_modules = []
286
-
276
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
277
super().__init__()
278
config = vllm_config.model_config.hf_config
vllm/model_executor/models/phimoe.py
-10
@@ -526,16 +526,6 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
526
}
527
528
# LoRA specific attributes
529
- supported_lora_modules = [
530
- "qkv_proj",
531
- "o_proj",
532
- "embed_tokens",
533
- "lm_head",
534
- "w1",
535
- "w2",
536
- "w3",
537
- "gate",
538
- ]
529
embedding_modules = {
530
"embed_tokens": "input_embeddings",
531
"lm_head": "output_embeddings",
vllm/model_executor/models/qwen.py
-9
@@ -354,15 +354,6 @@ class QWenLMHeadModel(QWenBaseModel, SupportsPP, SupportsLoRA):
354
"w1",
355
],
356
}
357
- # LoRA specific attributes
358
- supported_lora_modules = [
359
- "c_attn",
360
- "gate_up_proj",
361
- "c_proj",
362
- ]
363
-
364
- embedding_modules = {}
365
- embedding_padding_modules = []
357
358
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
359
config = vllm_config.model_config.hf_config
vllm/model_executor/models/qwen2.py
-20
@@ -430,16 +430,6 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
430
],
431
}
432
433
- # LoRA specific attributes
434
- supported_lora_modules = [
435
- "qkv_proj",
436
- "o_proj",
437
- "gate_up_proj",
438
- "down_proj",
439
- ]
440
- embedding_modules = {}
441
- embedding_padding_modules = []
442
-
433
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
434
super().__init__()
435
config = vllm_config.model_config.hf_config
@@ -528,16 +518,6 @@ class Qwen2EmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
518
],
519
}
520
531
- # LoRA specific attributes
532
- supported_lora_modules = [
533
- "qkv_proj",
534
- "o_proj",
535
- "gate_up_proj",
536
- "down_proj",
537
- ]
538
- embedding_modules = {}
539
- embedding_padding_modules = []
540
-
521
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"model.": ""})
522
523
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
vllm/model_executor/models/qwen2_5_vl.py
-21
@@ -734,27 +734,6 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
734
"up_proj",
735
],
736
}
737
- # LoRA specific attributes
738
- supported_lora_modules = [
739
- # language model
740
- "qkv_proj",
741
- "o_proj",
742
- "gate_up_proj",
743
- "down_proj", # Same name with vision encoder
744
- # vision tower
745
- "qkv",
746
- "gate_proj",
747
- "up_proj",
748
- "attn.proj", # Distinguish patch_embed.proj
749
- "fc1",
750
- "fc2",
751
- # projector
752
- "mlp.0",
753
- "mlp.2"
754
- ]
755
-
756
- embedding_modules = {}
757
- embedding_padding_modules = []
737
738
# To ensure correct weight loading and mapping.
739
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
vllm/model_executor/models/qwen2_rm.py
-10
@@ -47,16 +47,6 @@ class Qwen2RewardBaseModel(nn.Module, SupportsLoRA, SupportsPP):
47
],
48
}
49
50
- # LoRA specific attributes
51
- supported_lora_modules = [
52
- "qkv_proj",
53
- "o_proj",
54
- "gate_up_proj",
55
- "down_proj",
56
- ]
57
- embedding_modules = {}
58
- embedding_padding_modules = []
59
-
50
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
51
super().__init__()
52
config = vllm_config.model_config.hf_config
vllm/model_executor/models/qwen2_vl.py
-18
@@ -1048,24 +1048,6 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1048
],
1049
}
1050
1051
- # LoRA specific attributes
1052
- supported_lora_modules = [
1053
- "qkv_proj",
1054
- "o_proj",
1055
- "gate_up_proj",
1056
- "down_proj",
1057
- # vision tower
1058
- "qkv",
1059
- "attn.proj", # Distinguish patch_embed.proj
1060
- "fc1",
1061
- "fc2",
1062
- # projector
1063
- "mlp.0",
1064
- "mlp.2"
1065
- ]
1066
- embedding_modules = {}
1067
- embedding_padding_modules = []
1068
-
1051
# To ensure correct weight loading and mapping.
1052
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
1053
"lm_head.": "language_model.lm_head.",
vllm/model_executor/models/qwen_vl.py
-15
@@ -667,21 +667,6 @@ class QwenVLForConditionalGeneration(QWenBaseModel, SupportsPP, SupportsLoRA,
667
"w1",
668
],
669
}
670
- # LoRA specific attributes
671
- supported_lora_modules = [
672
- "c_attn",
673
- "gate_up_proj",
674
- "c_proj",
675
- # visual module
676
- "out_proj",
677
- "in_proj",
678
- "c_fc",
679
- # resampler
680
- "kv_proj",
681
- ]
682
-
683
- embedding_modules = {}
684
- embedding_padding_modules = []
670
671
def get_mm_mapping(self) -> MultiModelKeys:
672
"""
vllm/model_executor/models/solar.py
-8
@@ -386,14 +386,6 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
386
}
387
388
# LoRA specific attributes
389
- supported_lora_modules = [
390
- "qkv_proj",
391
- "o_proj",
392
- "gate_up_proj",
393
- "down_proj",
394
- "embed_tokens",
395
- "lm_head",
396
- ]
389
embedding_modules = {
390
"embed_tokens": "input_embeddings",
391
"lm_head": "output_embeddings",
vllm/model_executor/models/transformers.py
+35
@@ -27,6 +27,11 @@ from vllm.config import VllmConfig
27
from vllm.distributed import get_tensor_model_parallel_world_size
28
from vllm.distributed.utils import divide
29
from vllm.logger import init_logger
30
+from vllm.lora.fully_sharded_layers import (
31
+ ColumnParallelLinearWithShardedLoRA, RowParallelLinearWithShardedLoRA)
32
+from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
33
+ ReplicatedLinearWithLoRA,
34
+ RowParallelLinearWithLoRA)
35
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
36
ReplicatedLinear,
37
RowParallelLinear)
@@ -103,6 +108,23 @@ def replace_linear_class(
108
"rowwise": RowParallelLinear,
109
}.get(style, ReplicatedLinear)
110
111
+ lora_linear_cls = {
112
+ ColumnParallelLinear: {
113
+ True: ColumnParallelLinearWithShardedLoRA, # fully sharded
114
+ False: ColumnParallelLinearWithLoRA # not fully sharded
115
+ },
116
+ RowParallelLinear: {
117
+ True: RowParallelLinearWithShardedLoRA,
118
+ False: RowParallelLinearWithLoRA
119
+ },
120
+ # ReplicatedLinear doesn't support fully sharded LoRA yet,
121
+ # so we use the same class for both cases.
122
+ ReplicatedLinear: {
123
+ True: ReplicatedLinearWithLoRA,
124
+ False: ReplicatedLinearWithLoRA
125
+ }
126
+ }
127
+
128
class HFCompatibleLinear(vllm_linear_cls):
129
"""
130
Wrapper class that removes `output_bias` from returned output.
@@ -111,6 +133,19 @@ def replace_linear_class(
133
def forward(self, input: torch.Tensor) -> torch.Tensor:
134
return super().forward(input)[0]
135
136
+ @classmethod
137
+ def get_lora_class(cls, fully_sharded: bool = False):
138
+ """
139
+ Get the LoRA class corresponding to the current transformer
140
+ linear class.
141
+
142
+ Args:
143
+ fully_sharded (bool): If True, select the LoRA class variant
144
+ that supports fully sharded LoRA. Defaults to False.
145
+
146
+ """
147
+ return lora_linear_cls[vllm_linear_cls][fully_sharded]
148
+
149
return HFCompatibleLinear(
150
input_size=linear.in_features,
151
output_size=linear.out_features,
vllm/model_executor/models/ultravox.py
-8
@@ -360,14 +360,6 @@ class UltravoxModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA):
360
"gate_up_proj": ["gate_proj", "up_proj"]
361
}
362
363
- # LoRA specific attributes
364
- # TODO : Add LoRA to the audio tower and projector.
365
- supported_lora_modules = [
366
- "qkv_proj", "o_proj", "gate_up_proj", "down_proj"
367
- ]
368
- embedding_modules = {}
369
- embedding_padding_modules = []
370
-
363
hf_to_vllm_mapper = WeightsMapper(
364
orig_to_new_prefix={"audio_tower.model.encoder.": "audio_tower."})
365
vllm/worker/hpu_model_runner.py
-3
@@ -650,9 +650,6 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
650
logger.info(msg)
651
652
if self.lora_config:
653
- assert hasattr(self.model, "supported_lora_modules"
654
- ) and self.model.supported_lora_modules, (
655
- "Model does not support LoRA")
653
assert hasattr(self.model, "embedding_modules"
654
), "Model does not have embedding_modules"
655
assert hasattr(