@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4b6f069b6f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Add support for CodeLlama (#854)

Antoni Baum committed Aug 25, 2023 at 12:44 UTC 4b6f069b6fbb4f2ef7d4c6a62140229be61c5dd3
1 file changed +6
vllm/model_executor/models/llama.py
+6
@@ -85,6 +85,7 @@ class LlamaAttention(nn.Module):
85 hidden_size: int,
86 num_heads: int,
87 num_kv_heads: int,
88 + rope_theta: float = 10000,
89 ):
90 super().__init__()
91 self.hidden_size = hidden_size
@@ -99,6 +100,7 @@ class LlamaAttention(nn.Module):
100 self.q_size = self.num_heads * self.head_dim
101 self.kv_size = self.num_kv_heads * self.head_dim
102 self.scaling = self.head_dim**-0.5
103 + self.rope_theta = rope_theta
104
105 self.qkv_proj = ColumnParallelLinear(
106 hidden_size,
@@ -118,6 +120,7 @@ class LlamaAttention(nn.Module):
120 self.attn = PagedAttentionWithRoPE(self.num_heads,
121 self.head_dim,
122 self.scaling,
123 + base=self.rope_theta,
124 rotary_dim=self.head_dim,
125 num_kv_heads=self.num_kv_heads)
126
@@ -143,10 +146,13 @@ class LlamaDecoderLayer(nn.Module):
146 def __init__(self, config: LlamaConfig):
147 super().__init__()
148 self.hidden_size = config.hidden_size
149 + # Requires transformers > 4.32.0
150 + rope_theta = getattr(config, "rope_theta", 10000)
151 self.self_attn = LlamaAttention(
152 hidden_size=self.hidden_size,
153 num_heads=config.num_attention_heads,
154 num_kv_heads=config.num_key_value_heads,
155 + rope_theta=rope_theta,
156 )
157 self.mlp = LlamaMLP(
158 hidden_size=self.hidden_size,