@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2003cc3513
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model][LoRA]LoRA support added for LlamaEmbeddingModel (#10071)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Nov 6, 2024 at 17:49 UTC 2003cc35135319b240230e686f26f13524403ee0
2 files changed +20 -2
docs/source/models/supported_models.rst
+1 -1
@@ -333,7 +333,7 @@ Text Embedding
333 * - :code:`MistralModel`
334 - Mistral-based
335 - :code:`intfloat/e5-mistral-7b-instruct`, etc.
336 - -
336 + - ✅︎
337 - ✅︎
338
339 .. important::
vllm/model_executor/models/llama.py
+19 -1
@@ -627,7 +627,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
627 return name, loaded_weight
628
629
630 -class LlamaEmbeddingModel(nn.Module, SupportsPP):
630 +class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
631 """
632 A model that uses Llama with additional embedding functionalities.
633
@@ -638,6 +638,19 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
638 model: An instance of LlamaModel used for forward operations.
639 _pooler: An instance of Pooler used for pooling operations.
640 """
641 + packed_modules_mapping = {
642 + "qkv_proj": ["q_proj", "k_proj", "v_proj"],
643 + "gate_up_proj": ["gate_proj", "up_proj"]
644 + }
645 +
646 + # LoRA specific attributes
647 + supported_lora_modules = [
648 + "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
649 + ]
650 + embedding_modules = {
651 + "embed_tokens": "input_embeddings",
652 + }
653 + embedding_padding_modules = []
654
655 def __init__(
656 self,
@@ -679,3 +692,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
692
693 def load_kv_cache_scales(self, quantization_param_path: str) -> None:
694 self.model.load_kv_cache_scales(quantization_param_path)
695 +
696 + # LRUCacheWorkerLoRAManager instantiation requires model config.
697 + @property
698 + def config(self):
699 + return self.model.config