@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9f669a9a7c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support YaRN models (#1264)

Signed-off-by: Antoni Baum <antoni.baum@protonmail.com> Co-authored-by: Viktor Ferenczi <viktor@ferenczi.eu> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Antoni Baum committed Nov 3, 2023 at 14:12 UTC 9f669a9a7c2b2d0a7963a6e29253280e57680adb
5 files changed +128 -8
csrc/activation_kernels.cu
+6 -6
@@ -16,8 +16,8 @@ __global__ void silu_and_mul_kernel(
16 scalar_t* __restrict__ out, // [..., d]
17 const scalar_t* __restrict__ input, // [..., 2, d]
18 const int d) {
19 - const int token_idx = blockIdx.x;
20 - for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
19 + const int64_t token_idx = blockIdx.x;
20 + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
21 const scalar_t x = __ldg(&input[token_idx * 2 * d + idx]);
22 const scalar_t y = __ldg(&input[token_idx * 2 * d + d + idx]);
23 out[token_idx * d + idx] = silu(x) * y;
@@ -30,7 +30,7 @@ void silu_and_mul(
30 torch::Tensor& out, // [..., d]
31 torch::Tensor& input) // [..., 2 * d]
32 {
33 - int num_tokens = input.numel() / input.size(-1);
33 + int64_t num_tokens = input.numel() / input.size(-1);
34 int d = input.size(-1) / 2;
35
36 dim3 grid(num_tokens);
@@ -55,8 +55,8 @@ __global__ void activation_kernel(
55 scalar_t* __restrict__ out, // [..., d]
56 const scalar_t* __restrict__ input, // [..., d]
57 const int d) {
58 - const int token_idx = blockIdx.x;
59 - for (int idx = threadIdx.x; idx < d; idx += blockDim.x) {
58 + const int64_t token_idx = blockIdx.x;
59 + for (int64_t idx = threadIdx.x; idx < d; idx += blockDim.x) {
60 const scalar_t x = __ldg(&input[token_idx * d + idx]);
61 out[token_idx * d + idx] = ACT_FN(x);
62 }
@@ -67,7 +67,7 @@ __global__ void activation_kernel(
67 // Launch element-wise activation kernel.
68 #define LAUNCH_ACTIVATION_KERNEL(KERNEL) \
69 int d = input.size(-1); \
70 - int num_tokens = input.numel() / d; \
70 + int64_t num_tokens = input.numel() / d; \
71 dim3 grid(num_tokens); \
72 dim3 block(std::min(d, 1024)); \
73 const cudaStream_t stream = at::cuda::getCurrentCUDAStream(); \
csrc/pos_encoding_kernels.cu
+1 -1
@@ -84,7 +84,7 @@ void rotary_embedding(
84 int head_size,
85 torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
86 bool is_neox) {
87 - int num_tokens = query.numel() / query.size(-1);
87 + int64_t num_tokens = query.numel() / query.size(-1);
88 int rot_dim = cos_sin_cache.size(1);
89 int num_heads = query.size(-1) / head_size;
90 int num_kv_heads = key.size(-1) / head_size;
vllm/config.py
+3
@@ -390,6 +390,9 @@ def _get_and_verify_max_len(
390 if rope_scaling is not None:
391 assert "factor" in rope_scaling
392 scaling_factor = rope_scaling["factor"]
393 + if rope_scaling["type"] == "yarn":
394 + derived_max_model_len = rope_scaling[
395 + "original_max_position_embeddings"]
396 derived_max_model_len *= scaling_factor
397
398 if max_model_len is None:
vllm/model_executor/layers/attention.py
+14 -1
@@ -12,7 +12,7 @@ from vllm import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 from vllm.model_executor.layers.rotary_embedding import (
14 DynamicNTKScalingRotaryEmbedding, LinearScalingRotaryEmbedding,
15 - RotaryEmbedding)
15 + RotaryEmbedding, YaRNScalingRotaryEmbedding)
16
17 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
18 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -334,6 +334,19 @@ class PagedAttentionWithRoPE(PagedAttention):
334 self.rotary_emb = DynamicNTKScalingRotaryEmbedding(
335 head_size, rotary_dim, max_position, base, is_neox_style,
336 scaling_factor)
337 + elif scaling_type == "yarn":
338 + original_max_position = rope_scaling[
339 + "original_max_position_embeddings"]
340 + assert max_position == original_max_position * scaling_factor
341 + extra_kwargs = {
342 + k: v
343 + for k, v in rope_scaling.items()
344 + if k in ("extrapolation_factor", "attn_factor",
345 + "beta_fast", "beta_slow")
346 + }
347 + self.rotary_emb = YaRNScalingRotaryEmbedding(
348 + head_size, rotary_dim, original_max_position, base,
349 + is_neox_style, scaling_factor, **extra_kwargs)
350 else:
351 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
352
vllm/model_executor/layers/rotary_embedding.py
+104
@@ -21,6 +21,7 @@
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 """Rotary Positional Embeddings."""
24 +import math
25 from typing import Tuple, Union
26
27 import torch
@@ -167,3 +168,106 @@ class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
168 sin = freqs.sin()
169 cache = torch.cat((cos, sin), dim=-1)
170 return cache
171 +
172 +
173 +# Inverse dim formula to find dim based on number of rotations
174 +def _yarn_find_correction_dim(num_rotations: int,
175 + dim: int,
176 + base: float = 10000,
177 + max_position_embeddings: int = 2048) -> float:
178 + return (dim * math.log(max_position_embeddings /
179 + (num_rotations * 2 * math.pi))) / (2 *
180 + math.log(base))
181 +
182 +
183 +# Find dim range bounds based on rotations
184 +def _yarn_find_correction_range(low_rot: int,
185 + high_rot: int,
186 + dim: int,
187 + base: float = 10000,
188 + max_position_embeddings: int = 2048) -> int:
189 + low = math.floor(
190 + _yarn_find_correction_dim(low_rot, dim, base, max_position_embeddings))
191 + high = math.ceil(
192 + _yarn_find_correction_dim(high_rot, dim, base,
193 + max_position_embeddings))
194 + return max(low, 0), min(high, dim - 1) # Clamp values just in case
195 +
196 +
197 +def _yarn_linear_ramp_mask(low: float, high: float, dim: int,
198 + dtype: torch.dtype,
199 + device: torch.device) -> torch.Tensor:
200 + if low == high:
201 + high += 0.001 # Prevent singularity
202 +
203 + linear_func = (torch.arange(dim, dtype=dtype, device=device) -
204 + low) / (high - low)
205 + ramp_func = torch.clamp(linear_func, 0, 1)
206 + return ramp_func
207 +
208 +
209 +def _yarn_get_mscale(scale: float = 1) -> float:
210 + if scale <= 1:
211 + return 1.0
212 + return 0.1 * math.log(scale) + 1.0
213 +
214 +
215 +class YaRNScalingRotaryEmbedding(RotaryEmbedding):
216 + """RotaryEmbedding extended with YaRN method.
217 +
218 + Credits to Peng et al. github.com/jquesnelle/yarn
219 + """
220 +
221 + def __init__(
222 + self,
223 + head_size: int,
224 + rotary_dim: int,
225 + max_position_embeddings: int,
226 + base: int,
227 + is_neox_style: bool,
228 + scaling_factor: float,
229 + *,
230 + extrapolation_factor: float = 1,
231 + attn_factor: float = 1,
232 + beta_fast: float = 32,
233 + beta_slow: float = 1,
234 + ) -> None:
235 + self.scaling_factor = scaling_factor
236 + self.extrapolation_factor = extrapolation_factor
237 + self.attn_factor = attn_factor
238 + self.beta_fast = beta_fast
239 + self.beta_slow = beta_slow
240 + # Get n-d magnitude scaling corrected for interpolation
241 + self.mscale = float(
242 + _yarn_get_mscale(self.scaling_factor) * attn_factor)
243 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
244 + is_neox_style)
245 +
246 + def _compute_inv_freq(self, scaling_factor: float) -> torch.Tensor:
247 + pos_freqs = self.base**(torch.arange(
248 + 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
249 + self.rotary_dim)
250 + inv_freq_extrapolation = 1.0 / pos_freqs
251 + inv_freq_interpolation = 1.0 / (scaling_factor * pos_freqs)
252 +
253 + low, high = _yarn_find_correction_range(self.beta_fast, self.beta_slow,
254 + self.rotary_dim, self.base,
255 + self.max_position_embeddings)
256 + # Get n-d rotational scaling corrected for extrapolation
257 + inv_freq_mask = (1 - _yarn_linear_ramp_mask(
258 + low, high, self.rotary_dim // 2, dtype=torch.float,
259 + device="cuda")) * self.extrapolation_factor
260 + inv_freq = inv_freq_interpolation * (
261 + 1 - inv_freq_mask) + inv_freq_extrapolation * inv_freq_mask
262 + return inv_freq
263 +
264 + def _compute_cos_sin_cache(self) -> torch.Tensor:
265 + inv_freq = self._compute_inv_freq(self.scaling_factor)
266 + t = torch.arange(self.max_position_embeddings * self.scaling_factor,
267 + device="cuda",
268 + dtype=torch.float32)
269 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
270 + cos = (freqs.cos() * self.mscale)
271 + sin = (freqs.sin() * self.mscale)
272 + cache = torch.cat((cos, sin), dim=-1)
273 + return cache