@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e9bd08f60
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add batched RoPE kernel (#3095)

Terry committed Mar 13, 2024 at 13:45 UTC 7e9bd08f60a4b18e3646ff986caeacde9ffffa53
6 files changed +417 -37
benchmarks/kernels/benchmark_rope.py new
+120
@@ -0,0 +1,120 @@
1 +from typing import Optional
2 +
3 +import argparse
4 +import torch
5 +import nvtx
6 +from itertools import accumulate
7 +from vllm.model_executor.layers.rotary_embedding import get_rope
8 +
9 +
10 +def benchmark_rope_kernels_multi_lora(
11 + is_neox_style: bool,
12 + batch_size: int,
13 + seq_len: int,
14 + num_heads: int,
15 + head_size: int,
16 + rotary_dim: Optional[int],
17 + dtype: torch.dtype,
18 + seed: int,
19 + device: str,
20 + max_position: int = 8192,
21 + base: int = 10000,
22 +) -> None:
23 + torch.random.manual_seed(seed)
24 + if torch.cuda.is_available():
25 + torch.cuda.manual_seed(seed)
26 + torch.set_default_device(device)
27 + if rotary_dim is None:
28 + rotary_dim = head_size
29 + # silulating serving 4 LoRAs
30 + scaling_factors = [1, 2, 4, 8]
31 + # batched RoPE can take multiple scaling factors
32 + batched_rope = get_rope(head_size, rotary_dim, max_position, base,
33 + is_neox_style, {
34 + "type": "linear",
35 + "factor": tuple(scaling_factors)
36 + })
37 + # non-batched RoPE takes only one scaling factor, we create multiple
38 + # instances to simulate the same behavior
39 + non_batched_ropes = []
40 + for scaling_factor in scaling_factors:
41 + non_batched_ropes.append(
42 + get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
43 + {
44 + "type": "linear",
45 + "factor": (scaling_factor, )
46 + }))
47 +
48 + positions = torch.randint(0, max_position, (batch_size, seq_len))
49 + query = torch.randn(batch_size,
50 + seq_len,
51 + num_heads * head_size,
52 + dtype=dtype)
53 + key = torch.randn_like(query)
54 +
55 + # create query offsets for batched RoPE, we concat multiple kv cache
56 + # together and each query needs to find the right kv cache of its type
57 + offset_map = torch.tensor(
58 + list(
59 + accumulate([0] + [
60 + max_position * scaling_factor * 2
61 + for scaling_factor in scaling_factors[:-1]
62 + ])))
63 + query_types = torch.randint(0,
64 + len(scaling_factors), (batch_size, seq_len),
65 + device=device)
66 + # map query types to offsets
67 + query_offsets = offset_map[query_types]
68 + # the kernel takes flattened offsets
69 + flatten_offsets = query_offsets.flatten()
70 +
71 + # batched queries of the same type together for non-batched RoPE
72 + queries = [query[query_types == i] for i in range(len(scaling_factors))]
73 + keys = [key[query_types == i] for i in range(len(scaling_factors))]
74 + packed_qkr = zip(queries, keys, non_batched_ropes)
75 + # synchronize before start timing
76 + torch.cuda.synchronize()
77 + with nvtx.annotate("non-batched", color="yellow"):
78 + for q, k, r in packed_qkr:
79 + r.forward(positions, q, k)
80 + torch.cuda.synchronize()
81 + with nvtx.annotate("batched", color="green"):
82 + batched_rope.forward(positions, query, key, flatten_offsets)
83 + torch.cuda.synchronize()
84 +
85 +
86 +if __name__ == '__main__':
87 + parser = argparse.ArgumentParser(
88 + description="Benchmark the rotary embedding kernels.")
89 + parser.add_argument("--is-neox-style", type=bool, default=True)
90 + parser.add_argument("--batch-size", type=int, default=16)
91 + parser.add_argument("--seq-len", type=int, default=512)
92 + parser.add_argument("--num-heads", type=int, default=8)
93 + parser.add_argument("--head-size",
94 + type=int,
95 + choices=[64, 80, 96, 112, 128, 256],
96 + default=128)
97 + parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
98 + parser.add_argument("--dtype",
99 + type=str,
100 + choices=["bfloat16", "float"],
101 + default="float")
102 + parser.add_argument("--seed", type=int, default=0)
103 + parser.add_argument("--device",
104 + type=str,
105 + choices=["cuda:0", "cuda:1"],
106 + default="cuda:0")
107 + args = parser.parse_args()
108 + print(args)
109 +
110 + benchmark_rope_kernels_multi_lora(
111 + is_neox_style=args.is_neox_style,
112 + batch_size=args.batch_size,
113 + seq_len=args.seq_len,
114 + num_heads=args.num_heads,
115 + head_size=args.head_size,
116 + rotary_dim=args.rotary_dim,
117 + dtype=getattr(torch, args.dtype),
118 + seed=args.seed,
119 + device=args.device,
120 + )
csrc/ops.h
+10
@@ -53,6 +53,16 @@ void rotary_embedding(
53 torch::Tensor& cos_sin_cache,
54 bool is_neox);
55
56 +void batched_rotary_embedding(
57 + torch::Tensor& positions,
58 + torch::Tensor& query,
59 + torch::Tensor& key,
60 + int head_size,
61 + torch::Tensor& cos_sin_cache,
62 + bool is_neox,
63 + int rot_dim,
64 + torch::Tensor& cos_sin_cache_offsets);
65 +
66 void silu_and_mul(
67 torch::Tensor& out,
68 torch::Tensor& input);
csrc/pos_encoding_kernels.cu
+111 -15
@@ -8,7 +8,7 @@
8 namespace vllm {
9
10 template<typename scalar_t, bool IS_NEOX>
11 -inline __device__ void apply_rotary_embedding(
11 +inline __device__ void apply_token_rotary_embedding(
12 scalar_t* __restrict__ arr,
13 const scalar_t* __restrict__ cos_ptr,
14 const scalar_t* __restrict__ sin_ptr,
@@ -38,22 +38,18 @@ inline __device__ void apply_rotary_embedding(
38 }
39
40 template<typename scalar_t, bool IS_NEOX>
41 -__global__ void rotary_embedding_kernel(
42 - const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
41 +inline __device__ void apply_rotary_embedding(
42 scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
43 scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
45 - const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
46 - const int rot_dim,
47 - const int64_t query_stride,
48 - const int64_t key_stride,
44 + const scalar_t* cache_ptr,
45 + const int head_size,
46 const int num_heads,
47 const int num_kv_heads,
51 - const int head_size) {
52 - // Each thread block is responsible for one token.
53 - const int token_idx = blockIdx.x;
54 - int64_t pos = positions[token_idx];
55 - const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
56 -
48 + const int rot_dim,
49 + const int token_idx,
50 + const int64_t query_stride,
51 + const int64_t key_stride)
52 +{
53 const int embed_dim = rot_dim / 2;
54 const scalar_t* cos_ptr = cache_ptr;
55 const scalar_t* sin_ptr = cache_ptr + embed_dim;
@@ -63,7 +59,7 @@ __global__ void rotary_embedding_kernel(
59 const int head_idx = i / embed_dim;
60 const int64_t token_head = token_idx * query_stride + head_idx * head_size;
61 const int rot_offset = i % embed_dim;
66 - apply_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
62 + apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
63 sin_ptr, rot_offset, embed_dim);
64 }
65
@@ -72,11 +68,53 @@ __global__ void rotary_embedding_kernel(
68 const int head_idx = i / embed_dim;
69 const int64_t token_head = token_idx * key_stride + head_idx * head_size;
70 const int rot_offset = i % embed_dim;
75 - apply_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
71 + apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
72 sin_ptr, rot_offset, embed_dim);
73 }
74 }
75
76 +template<typename scalar_t, bool IS_NEOX>
77 +__global__ void rotary_embedding_kernel(
78 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
79 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
80 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
81 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
82 + const int rot_dim,
83 + const int64_t query_stride,
84 + const int64_t key_stride,
85 + const int num_heads,
86 + const int num_kv_heads,
87 + const int head_size) {
88 + // Each thread block is responsible for one token.
89 + const int token_idx = blockIdx.x;
90 + int64_t pos = positions[token_idx];
91 + const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
92 +
93 + apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
94 +}
95 +
96 +template<typename scalar_t, bool IS_NEOX>
97 +__global__ void batched_rotary_embedding_kernel(
98 + const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99 + scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
100 + scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
101 + const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
102 + const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len] or [num_tokens]
103 + const int rot_dim,
104 + const int64_t query_stride,
105 + const int64_t key_stride,
106 + const int num_heads,
107 + const int num_kv_heads,
108 + const int head_size) {
109 + // Each thread block is responsible for one token.
110 + const int token_idx = blockIdx.x;
111 + int64_t pos = positions[token_idx];
112 + int64_t cos_sin_cache_offset = cos_sin_cache_offsets[token_idx];
113 + const scalar_t* cache_ptr = cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
114 +
115 + apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
116 +}
117 +
118 } // namespace vllm
119
120 void rotary_embedding(
@@ -128,3 +166,61 @@ void rotary_embedding(
166 }
167 });
168 }
169 +
170 +/*
171 +Batched version of rotary embedding, pack multiple LoRAs together
172 +and process in batched manner.
173 +*/
174 +void batched_rotary_embedding(
175 + torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
176 + torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
177 + torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
178 + int head_size,
179 + torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
180 + bool is_neox,
181 + int rot_dim,
182 + torch::Tensor& cos_sin_cache_offsets // [num_tokens]
183 +) {
184 + int64_t num_tokens = cos_sin_cache_offsets.size(0);
185 + int num_heads = query.size(-1) / head_size;
186 + int num_kv_heads = key.size(-1) / head_size;
187 + int64_t query_stride = query.stride(-2);
188 + int64_t key_stride = key.stride(-2);
189 +
190 + dim3 grid(num_tokens);
191 + dim3 block(std::min(num_heads * rot_dim / 2, 512));
192 + const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
193 + const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
194 + VLLM_DISPATCH_FLOATING_TYPES(
195 + query.scalar_type(),
196 + "rotary_embedding",
197 + [&] {
198 + if (is_neox) {
199 + vllm::batched_rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
200 + positions.data_ptr<int64_t>(),
201 + query.data_ptr<scalar_t>(),
202 + key.data_ptr<scalar_t>(),
203 + cos_sin_cache.data_ptr<scalar_t>(),
204 + cos_sin_cache_offsets.data_ptr<int64_t>(),
205 + rot_dim,
206 + query_stride,
207 + key_stride,
208 + num_heads,
209 + num_kv_heads,
210 + head_size);
211 + } else {
212 + vllm::batched_rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
213 + positions.data_ptr<int64_t>(),
214 + query.data_ptr<scalar_t>(),
215 + key.data_ptr<scalar_t>(),
216 + cos_sin_cache.data_ptr<scalar_t>(),
217 + cos_sin_cache_offsets.data_ptr<int64_t>(),
218 + rot_dim,
219 + query_stride,
220 + key_stride,
221 + num_heads,
222 + num_kv_heads,
223 + head_size);
224 + }
225 + });
226 +}
csrc/pybind.cpp
+5
@@ -56,6 +56,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
56 &rotary_embedding,
57 "Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
58
59 + ops.def(
60 + "batched_rotary_embedding",
61 + &batched_rotary_embedding,
62 + "Apply GPT-NeoX or GPT-J style rotary embedding to query and key (supports multiple loras)");
63 +
64 // Quantization ops
65 #ifndef USE_ROCM
66 ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
tests/kernels/test_pos_encoding.py
+134 -1
@@ -1,8 +1,9 @@
1 -from typing import Optional
1 +from typing import List, Optional
2
3 import pytest
4 import torch
5 from allclose_default import get_default_atol, get_default_rtol
6 +from itertools import accumulate
7 from vllm.model_executor.layers.rotary_embedding import get_rope
8
9 IS_NEOX_STYLE = [True, False]
@@ -72,3 +73,135 @@ def test_rotary_embedding(
73 ref_key,
74 atol=get_default_atol(out_key),
75 rtol=get_default_rtol(out_key))
76 +
77 +
78 +@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
79 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
80 +@pytest.mark.parametrize("seq_len", SEQ_LENS)
81 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
82 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
83 +@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
84 +@pytest.mark.parametrize("dtype", DTYPES)
85 +@pytest.mark.parametrize("seed", SEEDS)
86 +@pytest.mark.parametrize("device", CUDA_DEVICES)
87 +@torch.inference_mode()
88 +def test_batched_rotary_embedding(
89 + is_neox_style: bool,
90 + batch_size: int,
91 + seq_len: int,
92 + num_heads: int,
93 + head_size: int,
94 + rotary_dim: Optional[int],
95 + dtype: torch.dtype,
96 + seed: int,
97 + device: str,
98 + max_position: int = 8192,
99 + base: int = 10000,
100 +) -> None:
101 + torch.random.manual_seed(seed)
102 + if torch.cuda.is_available():
103 + torch.cuda.manual_seed(seed)
104 + torch.set_default_device(device)
105 + if rotary_dim is None:
106 + rotary_dim = head_size
107 + rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
108 + "type": "linear",
109 + "factor": (1, )
110 + })
111 + rope = rope.to(dtype=dtype)
112 +
113 + positions = torch.randint(0, max_position, (batch_size, seq_len))
114 + query = torch.randn(batch_size,
115 + seq_len,
116 + num_heads * head_size,
117 + dtype=dtype)
118 + key = torch.randn_like(query)
119 +
120 + # NOTE(woosuk): The reference implementation should be executed first
121 + # because the custom kernel is in-place.
122 + ref_query, ref_key = rope._forward(positions, query, key)
123 + out_query, out_key = rope.forward(positions,
124 + query,
125 + key,
126 + offsets=torch.zeros(batch_size * seq_len,
127 + dtype=int,
128 + device=device))
129 + # Compare the results.
130 + assert torch.allclose(out_query,
131 + ref_query,
132 + atol=get_default_atol(out_query),
133 + rtol=get_default_rtol(out_query))
134 + assert torch.allclose(out_key,
135 + ref_key,
136 + atol=get_default_atol(out_key),
137 + rtol=get_default_rtol(out_key))
138 +
139 +
140 +@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
141 +@pytest.mark.parametrize("batch_size", BATCH_SIZES)
142 +@pytest.mark.parametrize("seq_len", SEQ_LENS)
143 +@pytest.mark.parametrize("num_heads", NUM_HEADS)
144 +@pytest.mark.parametrize("head_size", HEAD_SIZES)
145 +@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
146 +@pytest.mark.parametrize("dtype", DTYPES)
147 +@pytest.mark.parametrize("seed", SEEDS)
148 +@pytest.mark.parametrize("device", CUDA_DEVICES)
149 +@torch.inference_mode()
150 +def test_batched_rotary_embedding_multi_lora(
151 + is_neox_style: bool,
152 + batch_size: int,
153 + seq_len: int,
154 + num_heads: int,
155 + head_size: int,
156 + rotary_dim: Optional[int],
157 + dtype: torch.dtype,
158 + seed: int,
159 + device: str,
160 + max_position: int = 8192,
161 + base: int = 10000,
162 +) -> None:
163 + torch.random.manual_seed(seed)
164 + if torch.cuda.is_available():
165 + torch.cuda.manual_seed(seed)
166 + torch.set_default_device(device)
167 + if rotary_dim is None:
168 + rotary_dim = head_size
169 + scaling_factors: List[int] = [1, 2, 4]
170 + rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
171 + "type": "linear",
172 + "factor": tuple(scaling_factors)
173 + })
174 + rope = rope.to(dtype=dtype)
175 +
176 + positions = torch.randint(0, max_position, (batch_size, seq_len))
177 + query = torch.randn(batch_size,
178 + seq_len,
179 + num_heads * head_size,
180 + dtype=dtype)
181 + key = torch.randn_like(query)
182 +
183 + offset_map = torch.tensor(
184 + list(
185 + accumulate([0] + [
186 + max_position * scaling_factor * 2
187 + for scaling_factor in scaling_factors[:-1]
188 + ])))
189 + query_types = torch.randint(0,
190 + len(scaling_factors), (batch_size, seq_len),
191 + device=device)
192 + query_offsets = offset_map[query_types]
193 +
194 + # NOTE(woosuk): The reference implementation should be executed first
195 + # because the custom kernel is in-place.
196 + ref_query, ref_key = rope._forward(positions, query, key, query_offsets)
197 + out_query, out_key = rope.forward(positions, query, key,
198 + query_offsets.flatten())
199 + # Compare the results.
200 + assert torch.allclose(out_query,
201 + ref_query,
202 + atol=get_default_atol(out_query),
203 + rtol=get_default_rtol(out_query))
204 + assert torch.allclose(out_key,
205 + ref_key,
206 + atol=get_default_atol(out_key),
207 + rtol=get_default_rtol(out_key))
vllm/model_executor/layers/rotary_embedding.py
+37 -21
@@ -22,7 +22,7 @@
22 # limitations under the License.
23 """Rotary Positional Embeddings."""
24 import math
25 -from typing import Any, Dict, Optional, Tuple, Union
25 +from typing import Any, Dict, List, Optional, Tuple, Union
26
27 import torch
28 import torch.nn as nn
@@ -96,6 +96,7 @@ class RotaryEmbedding(nn.Module):
96 positions: torch.Tensor,
97 query: torch.Tensor,
98 key: torch.Tensor,
99 + offsets: Optional[torch.Tensor] = None,
100 ) -> Tuple[torch.Tensor, torch.Tensor]:
101 """PyTorch-native implementation equivalent to forward()."""
102 query = query.view(*query.shape[:-1], -1, self.head_size)
@@ -107,7 +108,9 @@ class RotaryEmbedding(nn.Module):
108 query_pass = query[..., self.rotary_dim:]
109 key_pass = key[..., self.rotary_dim:]
110
110 - cos_sin = self.cos_sin_cache[positions]
111 + self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
112 + cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
113 + if offsets is not None else positions]
114 cos, sin = cos_sin.chunk(2, dim=-1)
115 if self.is_neox_style:
116 # NOTE(woosuk): Here we assume that the positions tensor has the
@@ -137,11 +140,19 @@ class RotaryEmbedding(nn.Module):
140 positions: torch.Tensor,
141 query: torch.Tensor,
142 key: torch.Tensor,
143 + offsets: Optional[torch.Tensor] = None,
144 ) -> Tuple[torch.Tensor, torch.Tensor]:
141 - # ops.rotary_embedding() is an in-place operation that
142 - # updates the query and key tensors.
143 - ops.rotary_embedding(positions, query, key, self.head_size,
144 - self.cos_sin_cache, self.is_neox_style)
145 + self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
146 + # ops.rotary_embedding()/batched_rotary_embedding() are in-place operations that
147 + # update the query and key tensors.
148 + if offsets is not None:
149 + ops.batched_rotary_embedding(positions, query, key, self.head_size,
150 + self.cos_sin_cache,
151 + self.is_neox_style, self.rotary_dim,
152 + offsets)
153 + else:
154 + ops.rotary_embedding(positions, query, key, self.head_size,
155 + self.cos_sin_cache, self.is_neox_style)
156 return query, key
157
158
@@ -158,27 +169,32 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
169 max_position_embeddings: int,
170 base: int,
171 is_neox_style: bool,
161 - scaling_factor: float,
172 + scaling_factors: Union[List[float], float],
173 ) -> None:
163 - self.scaling_factor = scaling_factor
174 + if isinstance(scaling_factors, float):
175 + scaling_factors = [scaling_factors]
176 + self.scaling_factors = scaling_factors
177 super().__init__(head_size, rotary_dim, max_position_embeddings, base,
178 is_neox_style)
179
180 def _compute_cos_sin_cache(self) -> torch.Tensor:
181 inv_freq = self._compute_inv_freq(self.base)
169 - # NOTE(woosuk): self.max_position_embeddings is the original
170 - # maximum length before applying the rope scaling.
171 - # Thus, the maximum length after applying the rope scaling is
172 - # self.max_position_embeddings * self.scaling_factor.
173 - max_len = self.max_position_embeddings * self.scaling_factor
174 - t = torch.arange(max_len, dtype=torch.float)
175 - t = t / self.scaling_factor
176 -
177 - freqs = torch.einsum("i,j -> ij", t, inv_freq)
178 - cos = freqs.cos()
179 - sin = freqs.sin()
180 - cache = torch.cat((cos, sin), dim=-1)
181 - return cache
182 + cache_list = []
183 + for scaling_factor in self.scaling_factors:
184 + # NOTE(woosuk): self.max_position_embeddings is the original
185 + # maximum length before applying the rope scaling.
186 + # Thus, the maximum length after applying the rope scaling is
187 + # self.max_position_embeddings * self.scaling_factor.
188 + max_len = self.max_position_embeddings * scaling_factor
189 + t = torch.arange(max_len, dtype=torch.float)
190 + t = t / scaling_factor
191 +
192 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
193 + cos = freqs.cos()
194 + sin = freqs.sin()
195 + cache = torch.cat((cos, sin), dim=-1)
196 + cache_list.append(cache)
197 + return torch.cat(cache_list, dim=0)
198
199
200 class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):