A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Add support for CodeLlama (#854)
Antoni Baum committed
Aug 25, 2023 at 12:44 UTC
4b6f069b6fbb4f2ef7d4c6a62140229be61c5dd3
1 file changed
+6
vllm/model_executor/models/llama.py
+6
@@ -85,6 +85,7 @@ class LlamaAttention(nn.Module):
85
hidden_size: int,
86
num_heads: int,
87
num_kv_heads: int,
88
+ rope_theta: float = 10000,
89
):
90
super().__init__()
91
self.hidden_size = hidden_size
@@ -99,6 +100,7 @@ class LlamaAttention(nn.Module):
100
self.q_size = self.num_heads * self.head_dim
101
self.kv_size = self.num_kv_heads * self.head_dim
102
self.scaling = self.head_dim**-0.5
103
+ self.rope_theta = rope_theta
104
105
self.qkv_proj = ColumnParallelLinear(
106
hidden_size,
@@ -118,6 +120,7 @@ class LlamaAttention(nn.Module):
120
self.attn = PagedAttentionWithRoPE(self.num_heads,
121
self.head_dim,
122
self.scaling,
123
+ base=self.rope_theta,
124
rotary_dim=self.head_dim,
125
num_kv_heads=self.num_kv_heads)
126
@@ -143,10 +146,13 @@ class LlamaDecoderLayer(nn.Module):
146
def __init__(self, config: LlamaConfig):
147
super().__init__()
148
self.hidden_size = config.hidden_size
149
+ # Requires transformers > 4.32.0
150
+ rope_theta = getattr(config, "rope_theta", 10000)
151
self.self_attn = LlamaAttention(
152
hidden_size=self.hidden_size,
153
num_heads=config.num_attention_heads,
154
num_kv_heads=config.num_key_value_heads,
155
+ rope_theta=rope_theta,
156
)
157
self.mlp = LlamaMLP(
158
hidden_size=self.hidden_size,