A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model][LoRA]LoRA support added for LlamaEmbeddingModel (#10071)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Nov 6, 2024 at 17:49 UTC
2003cc35135319b240230e686f26f13524403ee0
2 files changed
+20
-2
docs/source/models/supported_models.rst
+1
-1
@@ -333,7 +333,7 @@ Text Embedding
333
* - :code:`MistralModel`
334
- Mistral-based
335
- :code:`intfloat/e5-mistral-7b-instruct`, etc.
336
- -
336
+ - ✅︎
337
- ✅︎
338
339
.. important::
vllm/model_executor/models/llama.py
+19
-1
@@ -627,7 +627,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
627
return name, loaded_weight
628
629
630
-class LlamaEmbeddingModel(nn.Module, SupportsPP):
630
+class LlamaEmbeddingModel(nn.Module, SupportsLoRA, SupportsPP):
631
"""
632
A model that uses Llama with additional embedding functionalities.
633
@@ -638,6 +638,19 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
638
model: An instance of LlamaModel used for forward operations.
639
_pooler: An instance of Pooler used for pooling operations.
640
"""
641
+ packed_modules_mapping = {
642
+ "qkv_proj": ["q_proj", "k_proj", "v_proj"],
643
+ "gate_up_proj": ["gate_proj", "up_proj"]
644
+ }
645
+
646
+ # LoRA specific attributes
647
+ supported_lora_modules = [
648
+ "qkv_proj", "o_proj", "gate_up_proj", "down_proj", "embed_tokens"
649
+ ]
650
+ embedding_modules = {
651
+ "embed_tokens": "input_embeddings",
652
+ }
653
+ embedding_padding_modules = []
654
655
def __init__(
656
self,
@@ -679,3 +692,8 @@ class LlamaEmbeddingModel(nn.Module, SupportsPP):
692
693
def load_kv_cache_scales(self, quantization_param_path: str) -> None:
694
self.model.load_kv_cache_scales(quantization_param_path)
695
+
696
+ # LRUCacheWorkerLoRAManager instantiation requires model config.
697
+ @property
698
+ def config(self):
699
+ return self.model.config