A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Support YaRN models (#1264)
Signed-off-by: Antoni Baum <antoni.baum@protonmail.com> Co-authored-by: Viktor Ferenczi <viktor@ferenczi.eu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Antoni Baum committed
Nov 3, 2023 at 14:12 UTC
9f669a9a7c2b2d0a7963a6e29253280e57680adb
5 files changed
+128
-8
csrc/activation_kernels.cu
+6
-6
@@ -16,8 +16,8 @@ __global__ void silu_and_mul_kernel(
16
scalar_t* __restrict__ out, // [..., d]
17
const scalar_t* __restrict__ input, // [..., 2, d]
18
const int d) {
19
- const int token_idx = blockIdx.x;
20
- for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
19
+ const int64_t token_idx = blockIdx.x;
20
+ for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
21
const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
22
const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
23
out[token_idx * d + idx] = silu(x) * y;
@@ -30,7 +30,7 @@ void silu_and_mul(
30
torch::Tensor& out, // [..., d]
31
torch::Tensor& input) // [..., 2 * d]
32
{
33
- int num_tokens = input.numel() / input.size(-1);
33
+ int64_t num_tokens = input.numel() / input.size(-1);
34
int d = input.size(-1) / 2;
35
36
dim3 grid(num_tokens);
@@ -55,8 +55,8 @@ __global__ void activation_kernel(
55
scalar_t* __restrict__ out, // [..., d]
56
const scalar_t* __restrict__ input, // [..., d]
57
const int d) {
58
- const int token_idx = blockIdx.x;
59
- for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
58
+ const int64_t token_idx = blockIdx.x;
59
+ for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
60
const scalar_t x = __ldg(&input[token_idx * d + idx]);
61
out[token_idx * d + idx] = ACT_FN(x);
62
}
@@ -67,7 +67,7 @@ __global__ void activation_kernel(
67
// Launch element-wise activation kernel.
68
#define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
69
int d = input.size(-1); \
70
- int num_tokens = input.numel() / d; \
70
+ int64_t num_tokens = input.numel() / d; \
71
dim3 grid(num_tokens); \
72
dim3 block(std::min(d, 1024)); \
73
const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
csrc/pos_encoding_kernels.cu
+1
-1
@@ -84,7 +84,7 @@ void rotary_embedding(
84
int head_size,
85
torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86
bool is_neox) {
87
- int num_tokens = query.numel() / query.size(-1);
87
+ int64_t num_tokens = query.numel() / query.size(-1);
88
int rot_dim = cos_sin_cache.size(1);
89
int num_heads = query.size(-1) / head_size;
90
int num_kv_heads = key.size(-1) / head_size;
vllm/config.py
+3
@@ -390,6 +390,9 @@ def _get_and_verify_max_len(
390
if rope_scaling is not None:
391
assert "factor" in rope_scaling
392
scaling_factor = rope_scaling["factor"]
393
+ if rope_scaling["type"] == "yarn":
394
+ derived_max_model_len = rope_scaling[
395
+ "original_max_position_embeddings"]
396
derived_max_model_len *= scaling_factor
397
398
if max_model_len is None:
vllm/model_executor/layers/attention.py
+14
-1
@@ -12,7 +12,7 @@ from vllm import cache_ops
12
from vllm.model_executor.input_metadata import InputMetadata
13
from vllm.model_executor.layers.rotary_embedding import (
14
DynamicNTKScalingRotaryEmbedding, LinearScalingRotaryEmbedding,
15
- RotaryEmbedding)
15
+ RotaryEmbedding, YaRNScalingRotaryEmbedding)
16
17
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
18
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -334,6 +334,19 @@ class PagedAttentionWithRoPE(PagedAttention):
334
self.rotary_emb = DynamicNTKScalingRotaryEmbedding(
335
head_size, rotary_dim, max_position, base, is_neox_style,
336
scaling_factor)
337
+ elif scaling_type == "yarn":
338
+ original_max_position = rope_scaling[
339
+ "original_max_position_embeddings"]
340
+ assert max_position == original_max_position * scaling_factor
341
+ extra_kwargs = {
342
+ k: v
343
+ for k, v in rope_scaling.items()
344
+ if k in ("extrapolation_factor", "attn_factor",
345
+ "beta_fast", "beta_slow")
346
+ }
347
+ self.rotary_emb = YaRNScalingRotaryEmbedding(
348
+ head_size, rotary_dim, original_max_position, base,
349
+ is_neox_style, scaling_factor, **extra_kwargs)
350
else:
351
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
352
vllm/model_executor/layers/rotary_embedding.py
+104
@@ -21,6 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Rotary Positional Embeddings."""
24
+import math
25
from typing import Tuple, Union
26
27
import torch
@@ -167,3 +168,106 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
168
sin = freqs.sin()
169
cache = torch.cat((cos, sin), dim=-1)
170
return cache
171
+
172
+
173
+# Inverse dim formula to find dim based on number of rotations
174
+def _yarn_find_correction_dim(num_rotations: int,
175
+ dim: int,
176
+ base: float = 10000,
177
+ max_position_embeddings: int = 2048) -> float:
178
+ return (dim * math.log(max_position_embeddings /
179
+ (num_rotations * 2 * math.pi))) / (2 *
180
+ math.log(base))
181
+
182
+
183
+# Find dim range bounds based on rotations
184
+def _yarn_find_correction_range(low_rot: int,
185
+ high_rot: int,
186
+ dim: int,
187
+ base: float = 10000,
188
+ max_position_embeddings: int = 2048) -> int:
189
+ low = math.floor(
190
+ _yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
191
+ high = math.ceil(
192
+ _yarn_find_correction_dim(high_rot, dim, base,
193
+ max_position_embeddings))
194
+ return max(low, 0), min(high, dim - 1) # Clamp values just in case
195
+
196
+
197
+def _yarn_linear_ramp_mask(low: float, high: float, dim: int,
198
+ dtype: torch.dtype,
199
+ device: torch.device) -> torch.Tensor:
200
+ if low == high:
201
+ high += 0.001 # Prevent singularity
202
+
203
+ linear_func = (torch.arange(dim, dtype=dtype, device=device) -
204
+ low) / (high - low)
205
+ ramp_func = torch.clamp(linear_func, 0, 1)
206
+ return ramp_func
207
+
208
+
209
+def _yarn_get_mscale(scale: float = 1) -> float:
210
+ if scale <= 1:
211
+ return 1.0
212
+ return 0.1 * math.log(scale) + 1.0
213
+
214
+
215
+class YaRNScalingRotaryEmbedding(RotaryEmbedding):
216
+ """RotaryEmbedding extended with YaRN method.
217
+
218
+ Credits to Peng et al. github.com/jquesnelle/yarn
219
+ """
220
+
221
+ def __init__(
222
+ self,
223
+ head_size: int,
224
+ rotary_dim: int,
225
+ max_position_embeddings: int,
226
+ base: int,
227
+ is_neox_style: bool,
228
+ scaling_factor: float,
229
+ *,
230
+ extrapolation_factor: float = 1,
231
+ attn_factor: float = 1,
232
+ beta_fast: float = 32,
233
+ beta_slow: float = 1,
234
+ ) -> None:
235
+ self.scaling_factor = scaling_factor
236
+ self.extrapolation_factor = extrapolation_factor
237
+ self.attn_factor = attn_factor
238
+ self.beta_fast = beta_fast
239
+ self.beta_slow = beta_slow
240
+ # Get n-d magnitude scaling corrected for interpolation
241
+ self.mscale = float(
242
+ _yarn_get_mscale(self.scaling_factor) * attn_factor)
243
+ super().__init__(head_size, rotary_dim, max_position_embeddings, base,
244
+ is_neox_style)
245
+
246
+ def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
247
+ pos_freqs = self.base**(torch.arange(
248
+ 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
249
+ self.rotary_dim)
250
+ inv_freq_extrapolation = 1.0 / pos_freqs
251
+ inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
252
+
253
+ low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow,
254
+ self.rotary_dim, self.base,
255
+ self.max_position_embeddings)
256
+ # Get n-d rotational scaling corrected for extrapolation
257
+ inv_freq_mask = (1 - _yarn_linear_ramp_mask(
258
+ low, high, self.rotary_dim // 2, dtype=torch.float,
259
+ device="cuda")) * self.extrapolation_factor
260
+ inv_freq = inv_freq_interpolation * (
261
+ 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
262
+ return inv_freq
263
+
264
+ def _compute_cos_sin_cache(self) -> torch.Tensor:
265
+ inv_freq = self._compute_inv_freq(self.scaling_factor)
266
+ t = torch.arange(self.max_position_embeddings * self.scaling_factor,
267
+ device="cuda",
268
+ dtype=torch.float32)
269
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
270
+ cos = (freqs.cos() * self.mscale)
271
+ sin = (freqs.sin() * self.mscale)
272
+ cache = torch.cat((cos, sin), dim=-1)
273
+ return cache