@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c83919c7a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Add Internlm2 LoRA support (#5064)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 29, 2024 at 01:29 UTC c83919c7a6bd47bb452321f08017ef5a5cdd553a
2 files changed +21 -3
docs/source/models/supported_models.rst
+1 -1
@@ -182,7 +182,7 @@ Text Generation
182 * - :code:`InternLM2ForCausalLM`
183 - InternLM2
184 - :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
185 - -
185 + - ✅︎
186 - ✅︎
187 * - :code:`JAISLMHeadModel`
188 - Jais
vllm/model_executor/models/internlm2.py
+20 -2
@@ -27,7 +27,7 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.sequence import IntermediateTensors
29
30 -from .interfaces import SupportsPP
30 +from .interfaces import SupportsLoRA, SupportsPP
31 from .utils import (is_pp_missing_parameter,
32 make_empty_intermediate_tensors_factory, make_layers,
33 maybe_prefix)
@@ -319,7 +319,21 @@ class InternLM2Model(nn.Module):
319 return hidden_states
320
321
322 -class InternLM2ForCausalLM(nn.Module, SupportsPP):
322 +class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
323 + packed_modules_mapping = {
324 + "wqkv": ["wqkv"],
325 + "gate_up_proj": ["w1", "w3"],
326 + }
327 +
328 + # LoRA specific attributes
329 + supported_lora_modules = [
330 + "wqkv",
331 + "wo",
332 + "gate_up_proj",
333 + "w2",
334 + ]
335 + embedding_modules = {}
336 + embedding_padding_modules = []
337
338 def __init__(self,
339 *,
@@ -329,8 +343,12 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP):
343 super().__init__()
344 config = vllm_config.model_config.hf_config
345 quant_config = vllm_config.quant_config
346 + lora_config = vllm_config.lora_config
347 +
348 self.config = config
349 self.quant_config = quant_config
350 + self.lora_config = lora_config
351 +
352 self.model = model_type(vllm_config=vllm_config,
353 prefix=maybe_prefix(prefix, "model"))
354 self.output = ParallelLMHead(config.vocab_size,