@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4f2ad11135
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix DeciLM (#2883)

Philipp Moritz committed Feb 14, 2024 at 22:29 UTC 4f2ad1113553211778640c648e11f5aa2e03dbd4
1 file changed +5 -1
vllm/model_executor/models/decilm.py
+5 -1
@@ -28,6 +28,7 @@ from typing import Optional
28 import torch
29 from transformers import PretrainedConfig
30
31 +from vllm.config import LoRAConfig
32 from vllm.model_executor.layers.linear import LinearMethodBase
33 from vllm.model_executor.models.llama import LlamaForCausalLM
34 from vllm.model_executor.weight_utils import (default_weight_loader,
@@ -56,10 +57,13 @@ class DeciLMForCausalLM(LlamaForCausalLM):
57 self,
58 config: Optional[PretrainedConfig] = None,
59 linear_method: Optional[LinearMethodBase] = None,
60 + lora_config: Optional[LoRAConfig] = None,
61 ) -> None:
62 config.num_key_value_heads = max(config.num_key_value_heads_per_layer)
63 delattr(config, "num_key_value_heads_per_layer")
62 - super().__init__(config=config, linear_method=linear_method)
64 + super().__init__(config=config,
65 + linear_method=linear_method,
66 + lora_config=lora_config)
67
68 def load_weights(self,
69 model_name_or_path: str,