A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Model] LoRA support added for command-r (#5178)
sergey-tinkoff committed
Jun 18, 2024 at 21:01 UTC
07feecde1a69859d565786a7ad64c0f604f17b28
3 files changed
+50
-6
csrc/punica/bgmv/bgmv_config.h
+6
@@ -69,6 +69,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
69
f(in_T, out_T, W_T, narrow, 36864) \
70
f(in_T, out_T, W_T, narrow, 43264) \
71
f(in_T, out_T, W_T, narrow, 49152) \
72
+ f(in_T, out_T, W_T, narrow, 60544) \
73
+ f(in_T, out_T, W_T, narrow, 60672) \
74
f(in_T, out_T, W_T, narrow, 64000) \
75
f(in_T, out_T, W_T, narrow, 64256) \
76
f(in_T, out_T, W_T, narrow, 64512) \
@@ -78,6 +80,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
80
f(in_T, out_T, W_T, narrow, 128000) \
81
f(in_T, out_T, W_T, narrow, 128256) \
82
f(in_T, out_T, W_T, narrow, 128512) \
83
+
84
+
85
// Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
86
// and vllm/tests/lora/test_punica.py
87
@@ -144,6 +148,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
148
f(in_T, out_T, W_T, 36864, narrow) \
149
f(in_T, out_T, W_T, 43264, narrow) \
150
f(in_T, out_T, W_T, 49152, narrow) \
151
+ f(in_T, out_T, W_T, 60544, narrow) \
152
+ f(in_T, out_T, W_T, 60672, narrow) \
153
f(in_T, out_T, W_T, 64000, narrow) \
154
f(in_T, out_T, W_T, 64256, narrow) \
155
f(in_T, out_T, W_T, 64512, narrow) \
tests/lora/test_punica.py
+2
@@ -94,6 +94,8 @@ H1 = H2 = [
94
36864,
95
43264,
96
49152,
97
+ 60544,
98
+ 60672,
99
64000,
100
64256,
101
102400,
vllm/model_executor/models/commandr.py
+42
-6
@@ -29,7 +29,7 @@ from torch.nn.parameter import Parameter
29
from transformers import CohereConfig
30
31
from vllm.attention import Attention, AttentionMetadata
32
-from vllm.config import CacheConfig
32
+from vllm.config import CacheConfig, LoRAConfig
33
from vllm.distributed import (get_tensor_model_parallel_rank,
34
get_tensor_model_parallel_world_size)
35
from vllm.model_executor.layers.activation import SiluAndMul
@@ -265,10 +265,14 @@ class CohereModel(nn.Module):
265
config: CohereConfig,
266
cache_config: Optional[CacheConfig] = None,
267
quant_config: Optional[QuantizationConfig] = None,
268
+ lora_config: Optional[LoRAConfig] = None,
269
):
270
super().__init__()
271
self.config = config
271
- self.vocab_size = config.vocab_size
272
+ lora_vocab = (lora_config.lora_extra_vocab_size *
273
+ (lora_config.max_loras or 1)) if lora_config else 0
274
+ self.vocab_size = config.vocab_size + lora_vocab
275
+ self.org_vocab_size = config.vocab_size
276
self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
277
config.hidden_size)
278
self.layers = nn.ModuleList([
@@ -302,18 +306,44 @@ class CohereModel(nn.Module):
306
307
class CohereForCausalLM(nn.Module):
308
309
+ packed_modules_mapping = {
310
+ "qkv_proj": [
311
+ "q_proj",
312
+ "k_proj",
313
+ "v_proj",
314
+ ],
315
+ "gate_up_proj": [
316
+ "gate_proj",
317
+ "up_proj",
318
+ ],
319
+ }
320
+ # LoRA specific attributes
321
+ supported_lora_modules = [
322
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
323
+ ]
324
+ embedding_modules = {"embed_tokens": "input_embeddings"}
325
+ embedding_padding_modules = []
326
+
327
def __init__(
328
self,
329
config: CohereConfig,
330
cache_config: Optional[CacheConfig] = None,
331
quant_config: Optional[QuantizationConfig] = None,
332
+ lora_config: Optional[LoRAConfig] = None,
333
) -> None:
334
super().__init__()
335
self.config = config
336
+ self.unpadded_vocab_size = config.vocab_size
337
+ if lora_config:
338
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
339
self.quant_config = quant_config
314
- self.logits_processor = LogitsProcessor(config.vocab_size,
340
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
341
+ config.vocab_size,
342
scale=config.logit_scale)
316
- self.model = CohereModel(config, cache_config, quant_config)
343
+ self.model = CohereModel(config,
344
+ cache_config,
345
+ quant_config,
346
+ lora_config=lora_config)
347
self.sampler = Sampler()
348
349
@torch.no_grad()
@@ -330,8 +360,14 @@ class CohereForCausalLM(nn.Module):
360
361
def compute_logits(self, hidden_states: torch.Tensor,
362
sampling_metadata: SamplingMetadata) -> torch.Tensor:
333
- logits = self.logits_processor(self.model.embed_tokens.weight,
334
- hidden_states, sampling_metadata)
363
+ is_not_lora = hasattr(self.model.embed_tokens, 'weight')
364
+ if is_not_lora:
365
+ embedding_weights = self.model.embed_tokens.weight
366
+ else:
367
+ embedding_weights = self.model.embed_tokens.base_layer.weight
368
+
369
+ logits = self.logits_processor(embedding_weights, hidden_states,
370
+ sampling_metadata)
371
return logits
372
373
def sample(