A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] Add Internlm2 LoRA support (#5064)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Nov 29, 2024 at 01:29 UTC
c83919c7a6bd47bb452321f08017ef5a5cdd553a
2 files changed
+21
-3
docs/source/models/supported_models.rst
+1
-1
@@ -182,7 +182,7 @@ Text Generation
182
* - :code:`InternLM2ForCausalLM`
183
- InternLM2
184
- :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
185
- -
185
+ - ✅︎
186
- ✅︎
187
* - :code:`JAISLMHeadModel`
188
- Jais
vllm/model_executor/models/internlm2.py
+20
-2
@@ -27,7 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.sequence import IntermediateTensors
29
30
-from .interfaces import SupportsPP
30
+from .interfaces import SupportsLoRA, SupportsPP
31
from .utils import (is_pp_missing_parameter,
32
make_empty_intermediate_tensors_factory, make_layers,
33
maybe_prefix)
@@ -319,7 +319,21 @@ class InternLM2Model(nn.Module):
319
return hidden_states
320
321
322
-class InternLM2ForCausalLM(nn.Module, SupportsPP):
322
+class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
323
+ packed_modules_mapping = {
324
+ "wqkv": ["wqkv"],
325
+ "gate_up_proj": ["w1", "w3"],
326
+ }
327
+
328
+ # LoRA specific attributes
329
+ supported_lora_modules = [
330
+ "wqkv",
331
+ "wo",
332
+ "gate_up_proj",
333
+ "w2",
334
+ ]
335
+ embedding_modules = {}
336
+ embedding_padding_modules = []
337
338
def __init__(self,
339
*,
@@ -329,8 +343,12 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
343
super().__init__()
344
config = vllm_config.model_config.hf_config
345
quant_config = vllm_config.quant_config
346
+ lora_config = vllm_config.lora_config
347
+
348
self.config = config
349
self.quant_config = quant_config
350
+ self.lora_config = lora_config
351
+
352
self.model = model_type(vllm_config=vllm_config,
353
prefix=maybe_prefix(prefix, "model"))
354
self.output = ParallelLMHead(config.vocab_size,