A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Enable more models to inference based on LoRA (#3382)
Co-authored-by: Antoni Baum <antoni.baum@protonmail.com>
Jee Li committed
Mar 26, 2024 at 09:09 UTC
8af890a865bf9f744d7d9bd5515558b42224c744
10 files changed
+402
-45
csrc/punica/bgmv/bgmv_config.h
+6
@@ -16,10 +16,13 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
16
f(in_T, out_T, W_T, narrow, 512) \
17
f(in_T, out_T, W_T, narrow, 768) \
18
f(in_T, out_T, W_T, narrow, 1024) \
19
+ f(in_T, out_T, W_T, narrow, 1152) \
20
f(in_T, out_T, W_T, narrow, 1280) \
21
+ f(in_T, out_T, W_T, narrow, 1536) \
22
f(in_T, out_T, W_T, narrow, 1728) \
23
f(in_T, out_T, W_T, narrow, 1792) \
24
f(in_T, out_T, W_T, narrow, 2048) \
25
+ f(in_T, out_T, W_T, narrow, 2304) \
26
f(in_T, out_T, W_T, narrow, 2560) \
27
f(in_T, out_T, W_T, narrow, 2752) \
28
f(in_T, out_T, W_T, narrow, 2816) \
@@ -27,10 +30,12 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
30
f(in_T, out_T, W_T, narrow, 3456) \
31
f(in_T, out_T, W_T, narrow, 3584) \
32
f(in_T, out_T, W_T, narrow, 4096) \
33
+ f(in_T, out_T, W_T, narrow, 4608) \
34
f(in_T, out_T, W_T, narrow, 5120) \
35
f(in_T, out_T, W_T, narrow, 5504) \
36
f(in_T, out_T, W_T, narrow, 5632) \
37
f(in_T, out_T, W_T, narrow, 6144) \
38
+ f(in_T, out_T, W_T, narrow, 6848) \
39
f(in_T, out_T, W_T, narrow, 6912) \
40
f(in_T, out_T, W_T, narrow, 7168) \
41
f(in_T, out_T, W_T, narrow, 8192) \
@@ -45,6 +50,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
50
f(in_T, out_T, W_T, narrow, 20480) \
51
f(in_T, out_T, W_T, narrow, 22016) \
52
f(in_T, out_T, W_T, narrow, 24576) \
53
+ f(in_T, out_T, W_T, narrow, 27392) \
54
f(in_T, out_T, W_T, narrow, 28672) \
55
f(in_T, out_T, W_T, narrow, 32000) \
56
f(in_T, out_T, W_T, narrow, 32256) \
tests/lora/conftest.py
+10
@@ -134,6 +134,16 @@ def gemma_lora_files():
134
return snapshot_download(repo_id="wskwon/gemma-7b-test-lora")
135
136
137
+@pytest.fixture(scope="session")
138
+def chatglm3_lora_files():
139
+ return snapshot_download(repo_id="jeeejeee/chatglm3-text2sql-spider")
140
+
141
+
142
+@pytest.fixture(scope="session")
143
+def baichuan_lora_files():
144
+ return snapshot_download(repo_id="jeeejeee/baichuan7b-text2sql-spider")
145
+
146
+
147
@pytest.fixture
148
def llama_2_7b_engine_extra_embeddings() -> nn.Module:
149
cleanup()
tests/lora/test_baichuan.py
new
+108
@@ -0,0 +1,108 @@
1
+import pytest
2
+
3
+import vllm
4
+from vllm.lora.request import LoRARequest
5
+
6
+from .conftest import cleanup
7
+
8
+MODEL_PATH = "baichuan-inc/Baichuan-7B"
9
+
10
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
11
+
12
+
13
+def do_sample(llm, lora_path: str, lora_id: int) -> str:
14
+ prompts = [
15
+ PROMPT_TEMPLATE.format(query="How many singers do we have?"),
16
+ PROMPT_TEMPLATE.format(
17
+ query=
18
+ "What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
19
+ ),
20
+ PROMPT_TEMPLATE.format(
21
+ query=
22
+ "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
23
+ ),
24
+ ]
25
+ print(prompts)
26
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=256)
27
+ outputs = llm.generate(
28
+ prompts,
29
+ sampling_params,
30
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
31
+ if lora_id else None)
32
+ # Print the outputs.
33
+ generated_texts = []
34
+ for output in outputs:
35
+ prompt = output.prompt
36
+ generated_text = output.outputs[0].text.strip()
37
+ generated_texts.append(generated_text)
38
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
39
+ return generated_texts
40
+
41
+
42
+def test_baichuan_lora(baichuan_lora_files):
43
+ llm = vllm.LLM(MODEL_PATH,
44
+ max_model_len=1024,
45
+ enable_lora=True,
46
+ max_loras=4,
47
+ max_lora_rank=64,
48
+ trust_remote_code=True)
49
+
50
+ expected_lora_output = [
51
+ "SELECT count(*) FROM singer",
52
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE Country = 'France'", # noqa: E501
53
+ "SELECT name , country , age FROM singer ORDER BY age ASC",
54
+ ]
55
+
56
+ output1 = do_sample(llm, baichuan_lora_files, lora_id=1)
57
+ for i in range(len(expected_lora_output)):
58
+ assert output1[i] == expected_lora_output[i]
59
+ output2 = do_sample(llm, baichuan_lora_files, lora_id=2)
60
+ for i in range(len(expected_lora_output)):
61
+ assert output2[i] == expected_lora_output[i]
62
+
63
+
64
+@pytest.mark.skip("Requires multiple GPUs")
65
+def test_llama_tensor_parallel_equality(baichuan_lora_files):
66
+ # Cannot use as it will initialize torch.cuda too early...
67
+ # if torch.cuda.device_count() < 4:
68
+ # pytest.skip(f"Not enough GPUs for tensor parallelism {4}")
69
+
70
+ llm_tp1 = vllm.LLM(MODEL_PATH,
71
+ enable_lora=True,
72
+ max_num_seqs=16,
73
+ max_loras=4,
74
+ max_lora_rank=64,
75
+ tensor_parallel_size=1,
76
+ trust_remote_code=True)
77
+ output_tp1 = do_sample(llm_tp1, baichuan_lora_files, lora_id=1)
78
+
79
+ del llm_tp1
80
+ cleanup()
81
+
82
+ llm_tp2 = vllm.LLM(MODEL_PATH,
83
+ enable_lora=True,
84
+ max_num_seqs=16,
85
+ max_loras=4,
86
+ max_lora_rank=64,
87
+ tensor_parallel_size=2,
88
+ trust_remote_code=True)
89
+ output_tp2 = do_sample(llm_tp2, baichuan_lora_files, lora_id=2)
90
+
91
+ del llm_tp2
92
+ cleanup()
93
+
94
+ assert output_tp1 == output_tp2
95
+
96
+ llm_tp4 = vllm.LLM(MODEL_PATH,
97
+ enable_lora=True,
98
+ max_num_seqs=16,
99
+ max_loras=4,
100
+ max_lora_rank=64,
101
+ tensor_parallel_size=4,
102
+ trust_remote_code=True)
103
+ output_tp4 = do_sample(llm_tp4, baichuan_lora_files, lora_id=2)
104
+
105
+ del llm_tp4
106
+ cleanup()
107
+
108
+ assert output_tp1 == output_tp4
\ No newline at end of file
tests/lora/test_chatglm3.py
new
+57
@@ -0,0 +1,57 @@
1
+import vllm
2
+from vllm.lora.request import LoRARequest
3
+
4
+MODEL_PATH = "THUDM/chatglm3-6b"
5
+
6
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
7
+
8
+
9
+def do_sample(llm, lora_path: str, lora_id: int) -> str:
10
+ prompts = [
11
+ PROMPT_TEMPLATE.format(query="How many singers do we have?"),
12
+ PROMPT_TEMPLATE.format(
13
+ query=
14
+ "What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
15
+ ),
16
+ PROMPT_TEMPLATE.format(
17
+ query=
18
+ "Show name, country, age for all singers ordered by age from the oldest to the youngest." # noqa: E501
19
+ ),
20
+ ]
21
+ print(prompts)
22
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
23
+ outputs = llm.generate(
24
+ prompts,
25
+ sampling_params,
26
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path)
27
+ if lora_id else None)
28
+ # Print the outputs.
29
+ generated_texts = []
30
+ for output in outputs:
31
+ prompt = output.prompt
32
+ generated_text = output.outputs[0].text.strip()
33
+ generated_texts.append(generated_text)
34
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
35
+ return generated_texts
36
+
37
+
38
+def test_chatglm3_lora(chatglm3_lora_files):
39
+ llm = vllm.LLM(MODEL_PATH,
40
+ max_model_len=1024,
41
+ enable_lora=True,
42
+ max_loras=4,
43
+ max_lora_rank=64,
44
+ trust_remote_code=True)
45
+
46
+ expected_lora_output = [
47
+ "SELECT count(*) FROM singer",
48
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
49
+ "SELECT name , country , age FROM singer ORDER BY age",
50
+ ]
51
+
52
+ output1 = do_sample(llm, chatglm3_lora_files, lora_id=1)
53
+ for i in range(len(expected_lora_output)):
54
+ assert output1[i] == expected_lora_output[i]
55
+ output2 = do_sample(llm, chatglm3_lora_files, lora_id=2)
56
+ for i in range(len(expected_lora_output)):
57
+ assert output2[i] == expected_lora_output[i]
tests/lora/test_layers.py
+10
-3
@@ -8,12 +8,16 @@ import torch
8
import torch.nn.functional as F
9
10
from vllm.config import LoRAConfig
11
+# yapf conflicts with isort for this block
12
+# yapf: disable
13
from vllm.lora.layers import (BaseLayerWithLoRA, ColumnParallelLinearWithLoRA,
14
LogitsProcessorWithLoRA, LoRAMapping,
15
MergedColumnParallelLinearWithLoRA,
16
+ MergedQKVParallelLinearWithLora,
17
QKVParallelLinearWithLora,
18
RowParallelLinearWithLoRA,
19
VocabParallelEmbeddingWithLoRA)
20
+# yapf: enable
21
from vllm.lora.models import (LoRALayerWeights, PackedLoRALayerWeights,
22
convert_mapping)
23
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -93,8 +97,7 @@ def populate_loras(
97
lora_dict: Dict[int, LoRALayerWeights] = dict()
98
99
# Dictionary that maps the lora ID to the
96
- # corresponding subloras. Only useful when
97
- # repeats > 1.
100
+ # corresponding subloras.
101
sublora_dict: Dict[int, List[LoRALayerWeights]] = dict()
102
103
for slot_idx, lora_id in enumerate(id_to_index):
@@ -607,7 +610,7 @@ def test_linear_parallel(dist_init, num_loras, orientation, device) -> None:
610
611
@torch.inference_mode()
612
@pytest.mark.parametrize("num_loras", [1, 2, 4, 8])
610
-@pytest.mark.parametrize("repeats", [2, 3])
613
+@pytest.mark.parametrize("repeats", [1, 2, 3])
614
@pytest.mark.parametrize("device", CUDA_DEVICES)
615
def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
616
@@ -623,6 +626,10 @@ def test_column_parallel_packed(dist_init, num_loras, repeats, device) -> None:
626
bias=False)
627
linear.weight.data = torch.rand_like(linear.weight.data)
628
lora_linear = MergedColumnParallelLinearWithLoRA(linear)
629
+ elif repeats == 3:
630
+ linear = QKVParallelLinear(4096, 64, 32, bias=False)
631
+ linear.weight.data = torch.rand_like(linear.weight.data)
632
+ lora_linear = MergedQKVParallelLinearWithLora(linear)
633
else:
634
linear = QKVParallelLinear(4096, 64, 32, bias=False)
635
linear.weight.data = torch.rand_like(linear.weight.data)
tests/lora/test_punica.py
+4
-3
@@ -43,9 +43,10 @@ def _lora_ref_impl(
43
44
45
H1 = H2 = [
46
- 128, 256, 512, 1024, 1280, 2048, 2560, 2752, 3072, 3456, 3584, 4096, 5120,
47
- 5504, 5632, 6144, 6912, 7168, 8192, 9216, 10240, 11008, 13824, 14336,
48
- 22016, 24576, 32000, 32256, 32512, 32768, 33024
46
+ 128, 256, 512, 1024, 1152, 1280, 1536, 2048, 2304, 2560, 2752, 3072, 3456,
47
+ 3584, 4096, 4608, 5120, 5504, 5632, 6144, 6848, 6912, 7168, 8192, 9216,
48
+ 10240, 11008, 13824, 14336, 22016, 24576, 27392, 32000, 32256, 32512,
49
+ 32768, 33024
50
]
51
SEED = [0xabcdabcd987]
52
vllm/lora/layers.py
+146
-22
@@ -1,7 +1,8 @@
1
# pylint: disable=unused-argument
2
+import inspect
3
import math
4
from dataclasses import dataclass
4
-from typing import TYPE_CHECKING, List, Optional, Tuple
5
+from typing import TYPE_CHECKING, List, Optional, Set, Tuple, Type
6
7
import torch
8
import torch.nn as nn
@@ -114,8 +115,11 @@ class LoRAMapping:
115
116
class BaseLayerWithLoRA(nn.Module):
117
117
- def create_lora_weights(self, max_loras: int, lora_config: LoRAConfig,
118
- model_config: PretrainedConfig) -> None:
118
+ def create_lora_weights(
119
+ self,
120
+ max_loras: int,
121
+ lora_config: LoRAConfig,
122
+ model_config: Optional[PretrainedConfig] = None) -> None:
123
"""Initializes lora matrices."""
124
...
125
@@ -144,6 +148,13 @@ class BaseLayerWithLoRA(nn.Module):
148
"""Sets the mapping indices."""
149
...
150
151
+ @classmethod
152
+ def can_replace_layer(cls, source_layer: nn.Module,
153
+ lora_config: LoRAConfig, packed_modules_list: List,
154
+ model_config: Optional[PretrainedConfig]) -> bool:
155
+ """Returns True if the layer can be replaced by this LoRA layer."""
156
+ raise NotImplementedError
157
+
158
159
class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
160
@@ -278,12 +289,19 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
289
self.indices[:self.indices_len[0]], 0, 1.0)
290
return full_output.view_as(full_output_org)
291
292
+ @classmethod
293
+ def can_replace_layer(cls, source_layer: nn.Module,
294
+ lora_config: LoRAConfig, packed_modules_list: List,
295
+ model_config: Optional[PretrainedConfig]) -> bool:
296
+ return type(source_layer) is VocabParallelEmbedding
297
+
298
299
class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
300
301
def __init__(self, base_layer: ColumnParallelLinear) -> None:
302
super().__init__()
303
self.base_layer = base_layer
304
+ self.tp_size = get_tensor_model_parallel_world_size()
305
306
def create_lora_weights(
307
self,
@@ -309,7 +327,7 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
327
328
self.indices: Optional[torch.Tensor] = None
329
self.indices_len: Optional[List[int]] = None
312
- self.output_dim = self.lora_b_stacked.shape[1]
330
+ self.output_dim = self.lora_b_stacked.shape[2]
331
332
def reset_lora(self, index: int):
333
self.lora_a_stacked[index] = 0
@@ -323,7 +341,12 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
341
embeddings_tensor: Optional[torch.Tensor],
342
):
343
self.reset_lora(index)
326
-
344
+ if self.tp_size > 1:
345
+ tensor_model_parallel_rank = get_tensor_model_parallel_rank()
346
+ shard_size = self.output_dim
347
+ start_idx = tensor_model_parallel_rank * shard_size
348
+ end_idx = (tensor_model_parallel_rank + 1) * shard_size
349
+ lora_b = lora_b[:, start_idx:end_idx]
350
self.lora_a_stacked[index,
351
0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
352
lora_a.T, non_blocking=True)
@@ -383,6 +406,14 @@ class ColumnParallelLinearWithLoRA(BaseLayerWithLoRA):
406
def linear_weights(self):
407
return self.base_layer.linear_weights
408
409
+ @classmethod
410
+ def can_replace_layer(cls, source_layer: nn.Module,
411
+ lora_config: LoRAConfig, packed_modules_list: List,
412
+ model_config: Optional[PretrainedConfig]) -> bool:
413
+ return type(source_layer) is ColumnParallelLinear or (
414
+ type(source_layer) is MergedColumnParallelLinear
415
+ and len(packed_modules_list) == 1)
416
+
417
418
class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
419
"""ColumnParallelLinear layer that is composed of 2 sublayers (slices)
@@ -485,8 +516,80 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
516
)
517
return output
518
519
+ @classmethod
520
+ def can_replace_layer(cls, source_layer: nn.Module,
521
+ lora_config: LoRAConfig, packed_modules_list: List,
522
+ model_config: Optional[PretrainedConfig]) -> bool:
523
+ return type(source_layer) is MergedColumnParallelLinear and len(
524
+ packed_modules_list) == 2
525
+
526
527
class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
528
+ """
529
+ ColumnParallelLinear layer that is specifically designed for
530
+ qkv_proj. Certain models, such as chtglm3 and baichuan-7b,
531
+ only contains a single LoRA within their qkv_proj layer.
532
+
533
+ During inference with Tensor Parallel, the weights of lora_b
534
+ must be accurately partitioned according to the respective ranks.
535
+
536
+ Q slice may have different shape than K and V slices (which both have
537
+ the same shape).
538
+ """
539
+
540
+ def __init__(self, base_layer: QKVParallelLinear) -> None:
541
+ super().__init__(base_layer)
542
+ self.tp_size = get_tensor_model_parallel_world_size()
543
+ self.q_proj_total_size = (self.base_layer.total_num_heads *
544
+ self.base_layer.head_size)
545
+ self.q_proj_shard_size = (self.base_layer.num_heads *
546
+ self.base_layer.head_size)
547
+ self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
548
+ self.base_layer.head_size)
549
+ self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
550
+ self.base_layer.head_size)
551
+
552
+ def set_lora(
553
+ self,
554
+ index: int,
555
+ lora_a: torch.Tensor,
556
+ lora_b: torch.Tensor,
557
+ embeddings_tensor: Optional[torch.Tensor],
558
+ ):
559
+ self.reset_lora(index)
560
+ if self.tp_size > 1:
561
+ tp_rank = get_tensor_model_parallel_rank()
562
+ self.q_shard_id = tp_rank
563
+ self.kv_shard_id = tp_rank // self.base_layer.num_kv_head_replicas
564
+ lora_b_q = lora_b[:, self.q_proj_shard_size *
565
+ self.q_shard_id:self.q_proj_shard_size *
566
+ (self.q_shard_id + 1)]
567
+ k_offset = self.q_proj_total_size
568
+ lora_b_k = lora_b[:, k_offset + self.kv_proj_shard_size *
569
+ self.kv_shard_id:k_offset +
570
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
571
+ v_offset = k_offset + self.kv_proj_total_size
572
+ lora_b_v = lora_b[:, v_offset + self.kv_proj_shard_size *
573
+ self.kv_shard_id:v_offset +
574
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)]
575
+ lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=1)
576
+
577
+ self.lora_a_stacked[index,
578
+ 0, :lora_a.shape[1], :lora_a.shape[0]].copy_(
579
+ lora_a.T, non_blocking=True)
580
+ self.lora_b_stacked[index,
581
+ 0, :lora_b.shape[1], :lora_b.shape[0]].copy_(
582
+ lora_b.T, non_blocking=True)
583
+
584
+ @classmethod
585
+ def can_replace_layer(cls, source_layer: nn.Module,
586
+ lora_config: LoRAConfig, packed_modules_list: List,
587
+ model_config: Optional[PretrainedConfig]) -> bool:
588
+ return type(source_layer) is QKVParallelLinear and len(
589
+ packed_modules_list) == 1
590
+
591
+
592
+class MergedQKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
593
"""ColumnParallelLinear layer that is composed of 3 sublayers (slices)
594
packed together in qkv proj fashion
595
(q_proj + k_proj + v_proj -> qkv_proj).
@@ -654,6 +757,13 @@ class QKVParallelLinearWithLora(ColumnParallelLinearWithLoRA):
757
)
758
return output
759
760
+ @classmethod
761
+ def can_replace_layer(cls, source_layer: nn.Module,
762
+ lora_config: LoRAConfig, packed_modules_list: List,
763
+ model_config: Optional[PretrainedConfig]) -> bool:
764
+ return type(source_layer) is QKVParallelLinear and len(
765
+ packed_modules_list) == 3
766
+
767
768
class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
769
@@ -780,6 +890,12 @@ class RowParallelLinearWithLoRA(BaseLayerWithLoRA):
890
def weight(self):
891
return self.base_layer.weight
892
893
+ @classmethod
894
+ def can_replace_layer(cls, source_layer: nn.Module,
895
+ lora_config: LoRAConfig, packed_modules_list: List,
896
+ model_config: Optional[PretrainedConfig]) -> bool:
897
+ return type(source_layer) is RowParallelLinear
898
+
899
900
class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
901
@@ -900,7 +1016,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1016
hidden_states: torch.Tensor,
1017
embedding: torch.Tensor,
1018
embedding_bias: Optional[torch.Tensor] = None,
903
- ) -> torch.Tensor:
1019
+ ) -> Optional[torch.Tensor]:
1020
# Get the logits for the next tokens.
1021
logits = torch.matmul(hidden_states, embedding.t())
1022
if embedding_bias is not None:
@@ -949,22 +1065,30 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
1065
def forward(self, *args, **kwargs):
1066
return type(self.base_layer).forward(self, *args, **kwargs)
1067
952
-
953
-def from_layer(
954
- layer: nn.Module,
955
- max_loras: int,
956
- lora_config: LoRAConfig,
957
- model_config: Optional[PretrainedConfig] = None) -> BaseLayerWithLoRA:
958
- supported_layer_types = {
959
- VocabParallelEmbedding: VocabParallelEmbeddingWithLoRA,
960
- ColumnParallelLinear: ColumnParallelLinearWithLoRA,
961
- QKVParallelLinear: QKVParallelLinearWithLora,
962
- MergedColumnParallelLinear: MergedColumnParallelLinearWithLoRA,
963
- RowParallelLinear: RowParallelLinearWithLoRA,
964
- }
965
- for src_layer_type, lora_layer_type in supported_layer_types.items():
966
- if type(layer) is src_layer_type: # pylint: disable=unidiomatic-typecheck
967
- ret = lora_layer_type(layer)
1068
+ @classmethod
1069
+ def can_replace_layer(cls, source_layer: nn.Module,
1070
+ lora_config: LoRAConfig, packed_modules_list: List,
1071
+ model_config: Optional[PretrainedConfig]) -> bool:
1072
+ # Special handling for the LogitsProcessor.
1073
+ return False
1074
+
1075
+
1076
+_all_lora_classes: Set[Type[BaseLayerWithLoRA]] = {
1077
+ cls
1078
+ for cls in globals().values() if inspect.isclass(cls)
1079
+ and issubclass(cls, BaseLayerWithLoRA) and cls is not BaseLayerWithLoRA
1080
+}
1081
+
1082
+
1083
+def from_layer(layer: nn.Module,
1084
+ max_loras: int,
1085
+ lora_config: LoRAConfig,
1086
+ packed_modules_list: List,
1087
+ model_config: Optional[PretrainedConfig] = None) -> nn.Module:
1088
+ for lora_cls in _all_lora_classes:
1089
+ if lora_cls.can_replace_layer(layer, lora_config, packed_modules_list,
1090
+ model_config):
1091
+ ret = lora_cls(layer)
1092
ret.create_lora_weights(max_loras, lora_config, model_config)
1093
return ret
1094
return layer
vllm/lora/models.py
+7
-4
@@ -413,11 +413,12 @@ class LoRAModelManager:
413
for module_name, module in self.model.named_modules():
414
if not self._match_target_modules(module_name):
415
continue
416
-
416
+ parts = module_name.split(".")[-1]
417
+ packed_moduled_lst = self.packed_modules_mapping.get(parts, [])
418
new_module = replace_submodule(
419
self.model, module_name,
420
from_layer(module, self.lora_slots, self.lora_config,
420
- self.model.config))
421
+ packed_moduled_lst, self.model.config))
422
# (yard1): TODO make this more robust
423
if "lm_head" in module_name:
424
logits_processor_module = self.model.get_submodule(
@@ -510,8 +511,10 @@ class LoRAModelManager:
511
def _register_packed_modules(self, module_full_name: str) -> None:
512
parts = module_full_name.split(".")
513
module_name = parts[-1]
513
- replacements = self.packed_modules_mapping.get(module_name)
514
- if not replacements:
514
+ replacements = self.packed_modules_mapping.get(module_name, [])
515
+ # When replacements is less than or equal to 1, it indicates that this
516
+ # module is not a packed module.
517
+ if len(replacements) <= 1:
518
return
519
prefix = ".".join(parts[:-1])
520
self.packed_modules[module_full_name] = [
vllm/model_executor/models/baichuan.py
+39
-13
@@ -26,6 +26,7 @@ from torch import nn
26
from transformers import PretrainedConfig
27
28
from vllm.attention import Attention, AttentionMetadata
29
+from vllm.config import LoRAConfig
30
from vllm.model_executor.layers.activation import SiluAndMul
31
from vllm.model_executor.layers.layernorm import RMSNorm
32
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -282,11 +283,30 @@ class BaiChuanModel(nn.Module):
283
284
285
class BaiChuanBaseForCausalLM(nn.Module):
286
+ packed_modules_mapping = {
287
+ "W_pack": ["W_pack"],
288
+ "gate_up_proj": [
289
+ "gate_proj",
290
+ "up_proj",
291
+ ],
292
+ }
293
+ # LoRA specific attributes
294
+ supported_lora_modules = [
295
+ "W_pack",
296
+ "o_proj",
297
+ "gate_up_proj",
298
+ "down_proj",
299
+ ]
300
+ embedding_modules = {}
301
+ embedding_padding_modules = []
302
286
- def __init__(self,
287
- config,
288
- position_embedding: str,
289
- linear_method: Optional[LinearMethodBase] = None):
303
+ def __init__(
304
+ self,
305
+ config,
306
+ position_embedding: str,
307
+ linear_method: Optional[LinearMethodBase] = None,
308
+ lora_config: Optional[LoRAConfig] = None,
309
+ ):
310
super().__init__()
311
self.config = config
312
self.linear_method = linear_method
@@ -371,19 +391,25 @@ class BaiChuanBaseForCausalLM(nn.Module):
391
class BaichuanForCausalLM(BaiChuanBaseForCausalLM):
392
"""Baichuan 13B and Baichuan2 7B/13B."""
393
374
- def __init__(self,
375
- config,
376
- linear_method: Optional[LinearMethodBase] = None):
394
+ def __init__(
395
+ self,
396
+ config,
397
+ linear_method: Optional[LinearMethodBase] = None,
398
+ lora_config: Optional[LoRAConfig] = None,
399
+ ):
400
if config.hidden_size == 4096: # baichuan2 7b
378
- super().__init__(config, "ROPE", linear_method)
401
+ super().__init__(config, "ROPE", linear_method, lora_config)
402
else: # baichuan 13b, baichuan2 13b
380
- super().__init__(config, "ALIBI", linear_method)
403
+ super().__init__(config, "ALIBI", linear_method, lora_config)
404
405
406
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM):
407
"""Baichuan 7B."""
408
386
- def __init__(self,
387
- config,
388
- linear_method: Optional[LinearMethodBase] = None):
389
- super().__init__(config, "ROPE", linear_method)
409
+ def __init__(
410
+ self,
411
+ config,
412
+ linear_method: Optional[LinearMethodBase] = None,
413
+ lora_config: Optional[LoRAConfig] = None,
414
+ ):
415
+ super().__init__(config, "ROPE", linear_method, lora_config)
vllm/model_executor/models/chatglm.py
+15
@@ -9,6 +9,7 @@ from torch import nn
9
from torch.nn import LayerNorm
10
11
from vllm.attention import Attention, AttentionMetadata
12
+from vllm.config import LoRAConfig
13
from vllm.model_executor.layers.activation import SiluAndMul
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -317,11 +318,25 @@ class ChatGLMModel(nn.Module):
318
319
320
class ChatGLMForCausalLM(nn.Module):
321
+ packed_modules_mapping = {
322
+ "query_key_value": ["query_key_value"],
323
+ "dense_h_to_4h": ["dense_h_to_4h"]
324
+ }
325
+ # LoRA specific attributes
326
+ supported_lora_modules = [
327
+ "query_key_value",
328
+ "dense",
329
+ "dense_h_to_4h",
330
+ "dense_4h_to_h",
331
+ ]
332
+ embedding_modules = {}
333
+ embedding_padding_modules = []
334
335
def __init__(
336
self,
337
config: ChatGLMConfig,
338
linear_method: Optional[LinearMethodBase] = None,
339
+ lora_config: Optional[LoRAConfig] = None,
340
):
341
super().__init__()
342
self.config: ChatGLMConfig = config