@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 28b47d1e49
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add rope_scaling to Aquila model (#1457)

Qing committed Oct 29, 2023 at 19:25 UTC 28b47d1e490b2b13ec282ac1cbe0eb51f908bfbd
1 file changed +5 -1
vllm/model_executor/models/aquila.py
+5 -1
@@ -25,7 +25,7 @@
25 The input of the model is flattened to a 1D tensor of tokens. The model uses
26 InputMetadata to extract the original 2D shape of the input.
27 """
28 -from typing import List, Optional, Tuple
28 +from typing import Any, Dict, List, Optional, Tuple
29
30 import torch
31 from torch import nn
@@ -110,6 +110,7 @@ class AquilaAttention(nn.Module):
110 num_kv_heads: int,
111 rope_theta: float = 10000,
112 max_position_embeddings: int = 8192,
113 + rope_scaling: Optional[Dict[str, Any]] = None,
114 ):
115 super().__init__()
116 self.hidden_size = hidden_size
@@ -148,6 +149,7 @@ class AquilaAttention(nn.Module):
149 base=self.rope_theta,
150 max_position=self.max_position_embeddings,
151 num_kv_heads=self.num_kv_heads,
152 + rope_scaling=rope_scaling,
153 )
154
155 def forward(
@@ -173,6 +175,7 @@ class AquilaDecoderLayer(nn.Module):
175 super().__init__()
176 self.hidden_size = config.hidden_size
177 rope_theta = getattr(config, "rope_theta", 10000)
178 + rope_scaling = getattr(config, "rope_scaling", None)
179 max_position_embeddings = getattr(config, "max_position_embeddings",
180 8192)
181 self.self_attn = AquilaAttention(
@@ -181,6 +184,7 @@ class AquilaDecoderLayer(nn.Module):
184 num_kv_heads=config.num_key_value_heads,
185 rope_theta=rope_theta,
186 max_position_embeddings=max_position_embeddings,
187 + rope_scaling=rope_scaling,
188 )
189 self.mlp = AquilaMLP(
190 hidden_size=self.hidden_size,