A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] LoRA gptbigcode implementation (#3949)
raywanb committed
May 23, 2024 at 04:58 UTC
97b030005c7f5cde7c1b97c718a8841db7d6220b
4 files changed
+34
-5
csrc/punica/bgmv/bgmv_config.h
+4
@@ -28,6 +28,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
28
f(in_T, out_T, W_T, narrow, 2752) \
29
f(in_T, out_T, W_T, narrow, 2816) \
30
f(in_T, out_T, W_T, narrow, 3072) \
31
+ f(in_T, out_T, W_T, narrow, 3328) \
32
f(in_T, out_T, W_T, narrow, 3456) \
33
f(in_T, out_T, W_T, narrow, 3584) \
34
f(in_T, out_T, W_T, narrow, 4096) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37
f(in_T, out_T, W_T, narrow, 5504) \
38
f(in_T, out_T, W_T, narrow, 5632) \
39
f(in_T, out_T, W_T, narrow, 6144) \
40
+ f(in_T, out_T, W_T, narrow, 6400) \
41
f(in_T, out_T, W_T, narrow, 6848) \
42
f(in_T, out_T, W_T, narrow, 6912) \
43
f(in_T, out_T, W_T, narrow, 7168) \
@@ -97,6 +99,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
99
f(in_T, out_T, W_T, 2752, narrow) \
100
f(in_T, out_T, W_T, 2816, narrow) \
101
f(in_T, out_T, W_T, 3072, narrow) \
102
+ f(in_T, out_T, W_T, 3328, narrow) \
103
f(in_T, out_T, W_T, 3456, narrow) \
104
f(in_T, out_T, W_T, 3584, narrow) \
105
f(in_T, out_T, W_T, 4096, narrow) \
@@ -105,6 +108,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
108
f(in_T, out_T, W_T, 5504, narrow) \
109
f(in_T, out_T, W_T, 5632, narrow) \
110
f(in_T, out_T, W_T, 6144, narrow) \
111
+ f(in_T, out_T, W_T, 6400, narrow) \
112
f(in_T, out_T, W_T, 6848, narrow) \
113
f(in_T, out_T, W_T, 6912, narrow) \
114
f(in_T, out_T, W_T, 7168, narrow) \
tests/lora/test_punica.py
+2
@@ -58,6 +58,7 @@ H1 = H2 = [
58
2560,
59
2752,
60
3072,
61
+ 3328,
62
3456,
63
3584,
64
4096,
@@ -66,6 +67,7 @@ H1 = H2 = [
67
5504,
68
5632,
69
6144,
70
+ 6400,
71
6848,
72
6912,
73
7168,
vllm/lora/models.py
+2
@@ -310,7 +310,9 @@ class LoRAModel:
310
if part_name not in expected_lora_modules:
311
unexpected_modules.append(module)
312
# loaded lora's target modules must be a subset of expected_lora_modules
313
+
314
if unexpected_modules:
315
+ print(unexpected_modules, "modules")
316
raise ValueError(
317
f"While loading {lora_dir}, expected"
318
f" target modules in {expected_lora_modules}"
vllm/model_executor/models/gpt_bigcode.py
+26
-5
@@ -25,7 +25,7 @@ from torch import nn
25
from transformers import GPTBigCodeConfig
26
27
from vllm.attention import Attention, AttentionMetadata
28
-from vllm.config import CacheConfig
28
+from vllm.config import CacheConfig, LoRAConfig
29
from vllm.distributed import get_tensor_model_parallel_world_size
30
from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -191,14 +191,19 @@ class GPTBigCodeModel(nn.Module):
191
config: GPTBigCodeConfig,
192
cache_config: Optional[CacheConfig] = None,
193
quant_config: Optional[QuantizationConfig] = None,
194
+ lora_config: Optional[LoRAConfig] = None,
195
):
196
super().__init__()
197
self.config = config
198
assert not config.add_cross_attention
199
200
self.embed_dim = config.hidden_size
200
-
201
- self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201
+ lora_vocab = (lora_config.lora_extra_vocab_size *
202
+ (lora_config.max_loras or 1)) if lora_config else 0
203
+ self.vocab_size = config.vocab_size + lora_vocab
204
+ self.wte = VocabParallelEmbedding(self.vocab_size,
205
+ self.embed_dim,
206
+ org_num_embeddings=config.vocab_size)
207
self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
208
self.h = nn.ModuleList([
209
GPTBigCodeBlock(config, cache_config, quant_config)
@@ -226,19 +231,35 @@ class GPTBigCodeModel(nn.Module):
231
232
233
class GPTBigCodeForCausalLM(nn.Module):
234
+ packed_modules_mapping = {"c_attn": ["c_attn"]}
235
+
236
+ supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
237
+
238
+ embedding_modules = {
239
+ "wte": "input_embeddings",
240
+ "lm_head": "output_embeddings",
241
+ }
242
+
243
+ embedding_padding_modules = []
244
245
def __init__(
246
self,
247
config: GPTBigCodeConfig,
248
cache_config: Optional[CacheConfig] = None,
249
quant_config: Optional[QuantizationConfig] = None,
250
+ lora_config: Optional[LoRAConfig] = None,
251
):
252
super().__init__()
253
self.config = config
254
self.quant_config = quant_config
239
- self.transformer = GPTBigCodeModel(config, cache_config, quant_config)
255
+ self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
256
+ lora_config)
257
self.lm_head_weight = self.transformer.wte.weight
241
- self.logits_processor = LogitsProcessor(config.vocab_size)
258
+ self.unpadded_vocab_size = config.vocab_size
259
+ if lora_config:
260
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
261
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
262
+ config.vocab_size)
263
self.sampler = Sampler()
264
265
def forward(