A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix DeciLM (#2883)
Philipp Moritz committed
Feb 14, 2024 at 22:29 UTC
4f2ad1113553211778640c648e11f5aa2e03dbd4
1 file changed
+5
-1
vllm/model_executor/models/decilm.py
+5
-1
@@ -28,6 +28,7 @@ from typing import Optional
28
import torch
29
from transformers import PretrainedConfig
30
31
+from vllm.config import LoRAConfig
32
from vllm.model_executor.layers.linear import LinearMethodBase
33
from vllm.model_executor.models.llama import LlamaForCausalLM
34
from vllm.model_executor.weight_utils import (default_weight_loader,
@@ -56,10 +57,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
57
self,
58
config: Optional[PretrainedConfig] = None,
59
linear_method: Optional[LinearMethodBase] = None,
60
+ lora_config: Optional[LoRAConfig] = None,
61
) -> None:
62
config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
63
delattr(config, "num_key_value_heads_per_layer")
62
- super().__init__(config=config, linear_method=linear_method)
64
+ super().__init__(config=config,
65
+ linear_method=linear_method,
66
+ lora_config=lora_config)
67
68
def load_weights(self,
69
model_name_or_path: str,