@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 07feecde1a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Model] LoRA support added for command-r (#5178)

sergey-tinkoff committed Jun 18, 2024 at 21:01 UTC 07feecde1a69859d565786a7ad64c0f604f17b28
3 files changed +50 -6
csrc/punica/bgmv/bgmv_config.h
+6
@@ -69,6 +69,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
69 f(in_T, out_T, W_T, narrow, 36864) \
70 f(in_T, out_T, W_T, narrow, 43264) \
71 f(in_T, out_T, W_T, narrow, 49152) \
72 + f(in_T, out_T, W_T, narrow, 60544) \
73 + f(in_T, out_T, W_T, narrow, 60672) \
74 f(in_T, out_T, W_T, narrow, 64000) \
75 f(in_T, out_T, W_T, narrow, 64256) \
76 f(in_T, out_T, W_T, narrow, 64512) \
@@ -78,6 +80,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
80 f(in_T, out_T, W_T, narrow, 128000) \
81 f(in_T, out_T, W_T, narrow, 128256) \
82 f(in_T, out_T, W_T, narrow, 128512) \
83 +
84 +
85 // Keep above in sync with vllm/lora/layers::LogitsProcessorWithLoRA
86 // and vllm/tests/lora/test_punica.py
87
@@ -144,6 +148,8 @@ void bgmv_kernel(out_T *__restrict__ Y, const in_T *__restrict__ X,
148 f(in_T, out_T, W_T, 36864, narrow) \
149 f(in_T, out_T, W_T, 43264, narrow) \
150 f(in_T, out_T, W_T, 49152, narrow) \
151 + f(in_T, out_T, W_T, 60544, narrow) \
152 + f(in_T, out_T, W_T, 60672, narrow) \
153 f(in_T, out_T, W_T, 64000, narrow) \
154 f(in_T, out_T, W_T, 64256, narrow) \
155 f(in_T, out_T, W_T, 64512, narrow) \
tests/lora/test_punica.py
+2
@@ -94,6 +94,8 @@ H1 = H2 = [
94 36864,
95 43264,
96 49152,
97 + 60544,
98 + 60672,
99 64000,
100 64256,
101 102400,
vllm/model_executor/models/commandr.py
+42 -6
@@ -29,7 +29,7 @@ from torch.nn.parameter import Parameter
29 from transformers import CohereConfig
30
31 from vllm.attention import Attention, AttentionMetadata
32 -from vllm.config import CacheConfig
32 +from vllm.config import CacheConfig, LoRAConfig
33 from vllm.distributed import (get_tensor_model_parallel_rank,
34 get_tensor_model_parallel_world_size)
35 from vllm.model_executor.layers.activation import SiluAndMul
@@ -265,10 +265,14 @@ class CohereModel(nn.Module):
265 config: CohereConfig,
266 cache_config: Optional[CacheConfig] = None,
267 quant_config: Optional[QuantizationConfig] = None,
268 + lora_config: Optional[LoRAConfig] = None,
269 ):
270 super().__init__()
271 self.config = config
271 - self.vocab_size = config.vocab_size
272 + lora_vocab = (lora_config.lora_extra_vocab_size *
273 + (lora_config.max_loras or 1)) if lora_config else 0
274 + self.vocab_size = config.vocab_size + lora_vocab
275 + self.org_vocab_size = config.vocab_size
276 self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
277 config.hidden_size)
278 self.layers = nn.ModuleList([
@@ -302,18 +306,44 @@ class CohereModel(nn.Module):
306
307 class CohereForCausalLM(nn.Module):
308
309 + packed_modules_mapping = {
310 + "qkv_proj": [
311 + "q_proj",
312 + "k_proj",
313 + "v_proj",
314 + ],
315 + "gate_up_proj": [
316 + "gate_proj",
317 + "up_proj",
318 + ],
319 + }
320 + # LoRA specific attributes
321 + supported_lora_modules = [
322 + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
323 + ]
324 + embedding_modules = {"embed_tokens": "input_embeddings"}
325 + embedding_padding_modules = []
326 +
327 def __init__(
328 self,
329 config: CohereConfig,
330 cache_config: Optional[CacheConfig] = None,
331 quant_config: Optional[QuantizationConfig] = None,
332 + lora_config: Optional[LoRAConfig] = None,
333 ) -> None:
334 super().__init__()
335 self.config = config
336 + self.unpadded_vocab_size = config.vocab_size
337 + if lora_config:
338 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
339 self.quant_config = quant_config
314 - self.logits_processor = LogitsProcessor(config.vocab_size,
340 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
341 + config.vocab_size,
342 scale=config.logit_scale)
316 - self.model = CohereModel(config, cache_config, quant_config)
343 + self.model = CohereModel(config,
344 + cache_config,
345 + quant_config,
346 + lora_config=lora_config)
347 self.sampler = Sampler()
348
349 @torch.no_grad()
@@ -330,8 +360,14 @@ class CohereForCausalLM(nn.Module):
360
361 def compute_logits(self, hidden_states: torch.Tensor,
362 sampling_metadata: SamplingMetadata) -> torch.Tensor:
333 - logits = self.logits_processor(self.model.embed_tokens.weight,
334 - hidden_states, sampling_metadata)
363 + is_not_lora = hasattr(self.model.embed_tokens, 'weight')
364 + if is_not_lora:
365 + embedding_weights = self.model.embed_tokens.weight
366 + else:
367 + embedding_weights = self.model.embed_tokens.base_layer.weight
368 +
369 + logits = self.logits_processor(embedding_weights, hidden_states,
370 + sampling_metadata)
371 return logits
372
373 def sample(