A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add rope_scaling to Aquila model (#1457)
Qing committed
Oct 29, 2023 at 19:25 UTC
28b47d1e490b2b13ec282ac1cbe0eb51f908bfbd
1 file changed
+5
-1
vllm/model_executor/models/aquila.py
+5
-1
@@ -25,7 +25,7 @@
25
The input of the model is flattened to a 1D tensor of tokens. The model uses
26
InputMetadata to extract the original 2D shape of the input.
27
"""
28
-from typing import List, Optional, Tuple
28
+from typing import Any, Dict, List, Optional, Tuple
29
30
import torch
31
from torch import nn
@@ -110,6 +110,7 @@ class AquilaAttention(nn.Module):
110
num_kv_heads: int,
111
rope_theta: float = 10000,
112
max_position_embeddings: int = 8192,
113
+ rope_scaling: Optional[Dict[str, Any]] = None,
114
):
115
super().__init__()
116
self.hidden_size = hidden_size
@@ -148,6 +149,7 @@ class AquilaAttention(nn.Module):
149
base=self.rope_theta,
150
max_position=self.max_position_embeddings,
151
num_kv_heads=self.num_kv_heads,
152
+ rope_scaling=rope_scaling,
153
)
154
155
def forward(
@@ -173,6 +175,7 @@ class AquilaDecoderLayer(nn.Module):
175
super().__init__()
176
self.hidden_size = config.hidden_size
177
rope_theta = getattr(config, "rope_theta", 10000)
178
+ rope_scaling = getattr(config, "rope_scaling", None)
179
max_position_embeddings = getattr(config, "max_position_embeddings",
180
8192)
181
self.self_attn = AquilaAttention(
@@ -181,6 +184,7 @@ class AquilaDecoderLayer(nn.Module):
184
num_kv_heads=config.num_key_value_heads,
185
rope_theta=rope_theta,
186
max_position_embeddings=max_position_embeddings,
187
+ rope_scaling=rope_scaling,
188
)
189
self.mlp = AquilaMLP(
190
hidden_size=self.hidden_size,