@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97b030005c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] LoRA gptbigcode implementation (#3949)

raywanb committed May 23, 2024 at 04:58 UTC 97b030005c7f5cde7c1b97c718a8841db7d6220b
4 files changed +34 -5
csrc/punica/bgmv/bgmv_config.h
+4
@@ -28,6 +28,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
28 f(in_T, out_T, W_T, narrow, 2752) \
29 f(in_T, out_T, W_T, narrow, 2816) \
30 f(in_T, out_T, W_T, narrow, 3072) \
31 + f(in_T, out_T, W_T, narrow, 3328) \
32 f(in_T, out_T, W_T, narrow, 3456) \
33 f(in_T, out_T, W_T, narrow, 3584) \
34 f(in_T, out_T, W_T, narrow, 4096) \
@@ -36,6 +37,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
37 f(in_T, out_T, W_T, narrow, 5504) \
38 f(in_T, out_T, W_T, narrow, 5632) \
39 f(in_T, out_T, W_T, narrow, 6144) \
40 + f(in_T, out_T, W_T, narrow, 6400) \
41 f(in_T, out_T, W_T, narrow, 6848) \
42 f(in_T, out_T, W_T, narrow, 6912) \
43 f(in_T, out_T, W_T, narrow, 7168) \
@@ -97,6 +99,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
99 f(in_T, out_T, W_T, 2752, narrow) \
100 f(in_T, out_T, W_T, 2816, narrow) \
101 f(in_T, out_T, W_T, 3072, narrow) \
102 + f(in_T, out_T, W_T, 3328, narrow) \
103 f(in_T, out_T, W_T, 3456, narrow) \
104 f(in_T, out_T, W_T, 3584, narrow) \
105 f(in_T, out_T, W_T, 4096, narrow) \
@@ -105,6 +108,7 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
108 f(in_T, out_T, W_T, 5504, narrow) \
109 f(in_T, out_T, W_T, 5632, narrow) \
110 f(in_T, out_T, W_T, 6144, narrow) \
111 + f(in_T, out_T, W_T, 6400, narrow) \
112 f(in_T, out_T, W_T, 6848, narrow) \
113 f(in_T, out_T, W_T, 6912, narrow) \
114 f(in_T, out_T, W_T, 7168, narrow) \
tests/lora/test_punica.py
+2
@@ -58,6 +58,7 @@ H1 = H2 = [
58 2560,
59 2752,
60 3072,
61 + 3328,
62 3456,
63 3584,
64 4096,
@@ -66,6 +67,7 @@ H1 = H2 = [
67 5504,
68 5632,
69 6144,
70 + 6400,
71 6848,
72 6912,
73 7168,
vllm/lora/models.py
+2
@@ -310,7 +310,9 @@ class LoRAModel:
310 if part_name not in expected_lora_modules:
311 unexpected_modules.append(module)
312 # loaded lora's target modules must be a subset of expected_lora_modules
313 +
314 if unexpected_modules:
315 + print(unexpected_modules, "modules")
316 raise ValueError(
317 f"While loading {lora_dir}, expected"
318 f" target modules in {expected_lora_modules}"
vllm/model_executor/models/gpt_bigcode.py
+26 -5
@@ -25,7 +25,7 @@ from torch import nn
25 from transformers import GPTBigCodeConfig
26
27 from vllm.attention import Attention, AttentionMetadata
28 -from vllm.config import CacheConfig
28 +from vllm.config import CacheConfig, LoRAConfig
29 from vllm.distributed import get_tensor_model_parallel_world_size
30 from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -191,14 +191,19 @@ class GPTBigCodeModel(nn.Module):
191 config: GPTBigCodeConfig,
192 cache_config: Optional[CacheConfig] = None,
193 quant_config: Optional[QuantizationConfig] = None,
194 + lora_config: Optional[LoRAConfig] = None,
195 ):
196 super().__init__()
197 self.config = config
198 assert not config.add_cross_attention
199
200 self.embed_dim = config.hidden_size
200 -
201 - self.wte = VocabParallelEmbedding(config.vocab_size, self.embed_dim)
201 + lora_vocab = (lora_config.lora_extra_vocab_size *
202 + (lora_config.max_loras or 1)) if lora_config else 0
203 + self.vocab_size = config.vocab_size + lora_vocab
204 + self.wte = VocabParallelEmbedding(self.vocab_size,
205 + self.embed_dim,
206 + org_num_embeddings=config.vocab_size)
207 self.wpe = nn.Embedding(config.max_position_embeddings, self.embed_dim)
208 self.h = nn.ModuleList([
209 GPTBigCodeBlock(config, cache_config, quant_config)
@@ -226,19 +231,35 @@ class GPTBigCodeModel(nn.Module):
231
232
233 class GPTBigCodeForCausalLM(nn.Module):
234 + packed_modules_mapping = {"c_attn": ["c_attn"]}
235 +
236 + supported_lora_modules = ["c_fc", "c_proj", "wte", "lm_head", "c_attn"]
237 +
238 + embedding_modules = {
239 + "wte": "input_embeddings",
240 + "lm_head": "output_embeddings",
241 + }
242 +
243 + embedding_padding_modules = []
244
245 def __init__(
246 self,
247 config: GPTBigCodeConfig,
248 cache_config: Optional[CacheConfig] = None,
249 quant_config: Optional[QuantizationConfig] = None,
250 + lora_config: Optional[LoRAConfig] = None,
251 ):
252 super().__init__()
253 self.config = config
254 self.quant_config = quant_config
239 - self.transformer = GPTBigCodeModel(config, cache_config, quant_config)
255 + self.transformer = GPTBigCodeModel(config, cache_config, quant_config,
256 + lora_config)
257 self.lm_head_weight = self.transformer.wte.weight
241 - self.logits_processor = LogitsProcessor(config.vocab_size)
258 + self.unpadded_vocab_size = config.vocab_size
259 + if lora_config:
260 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
261 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
262 + config.vocab_size)
263 self.sampler = Sampler()
264
265 def forward(