A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add batched RoPE kernel (#3095)
Terry committed
Mar 13, 2024 at 13:45 UTC
7e9bd08f60a4b18e3646ff986caeacde9ffffa53
6 files changed
+417
-37
benchmarks/kernels/benchmark_rope.py
new
+120
@@ -0,0 +1,120 @@
1
+from typing import Optional
2
+
3
+import argparse
4
+import torch
5
+import nvtx
6
+from itertools import accumulate
7
+from vllm.model_executor.layers.rotary_embedding import get_rope
8
+
9
+
10
+def benchmark_rope_kernels_multi_lora(
11
+ is_neox_style: bool,
12
+ batch_size: int,
13
+ seq_len: int,
14
+ num_heads: int,
15
+ head_size: int,
16
+ rotary_dim: Optional[int],
17
+ dtype: torch.dtype,
18
+ seed: int,
19
+ device: str,
20
+ max_position: int = 8192,
21
+ base: int = 10000,
22
+) -> None:
23
+ torch.random.manual_seed(seed)
24
+ if torch.cuda.is_available():
25
+ torch.cuda.manual_seed(seed)
26
+ torch.set_default_device(device)
27
+ if rotary_dim is None:
28
+ rotary_dim = head_size
29
+ # silulating serving 4 LoRAs
30
+ scaling_factors = [1, 2, 4, 8]
31
+ # batched RoPE can take multiple scaling factors
32
+ batched_rope = get_rope(head_size, rotary_dim, max_position, base,
33
+ is_neox_style, {
34
+ "type": "linear",
35
+ "factor": tuple(scaling_factors)
36
+ })
37
+ # non-batched RoPE takes only one scaling factor, we create multiple
38
+ # instances to simulate the same behavior
39
+ non_batched_ropes = []
40
+ for scaling_factor in scaling_factors:
41
+ non_batched_ropes.append(
42
+ get_rope(head_size, rotary_dim, max_position, base, is_neox_style,
43
+ {
44
+ "type": "linear",
45
+ "factor": (scaling_factor, )
46
+ }))
47
+
48
+ positions = torch.randint(0, max_position, (batch_size, seq_len))
49
+ query = torch.randn(batch_size,
50
+ seq_len,
51
+ num_heads * head_size,
52
+ dtype=dtype)
53
+ key = torch.randn_like(query)
54
+
55
+ # create query offsets for batched RoPE, we concat multiple kv cache
56
+ # together and each query needs to find the right kv cache of its type
57
+ offset_map = torch.tensor(
58
+ list(
59
+ accumulate([0] + [
60
+ max_position * scaling_factor * 2
61
+ for scaling_factor in scaling_factors[:-1]
62
+ ])))
63
+ query_types = torch.randint(0,
64
+ len(scaling_factors), (batch_size, seq_len),
65
+ device=device)
66
+ # map query types to offsets
67
+ query_offsets = offset_map[query_types]
68
+ # the kernel takes flattened offsets
69
+ flatten_offsets = query_offsets.flatten()
70
+
71
+ # batched queries of the same type together for non-batched RoPE
72
+ queries = [query[query_types == i] for i in range(len(scaling_factors))]
73
+ keys = [key[query_types == i] for i in range(len(scaling_factors))]
74
+ packed_qkr = zip(queries, keys, non_batched_ropes)
75
+ # synchronize before start timing
76
+ torch.cuda.synchronize()
77
+ with nvtx.annotate("non-batched", color="yellow"):
78
+ for q, k, r in packed_qkr:
79
+ r.forward(positions, q, k)
80
+ torch.cuda.synchronize()
81
+ with nvtx.annotate("batched", color="green"):
82
+ batched_rope.forward(positions, query, key, flatten_offsets)
83
+ torch.cuda.synchronize()
84
+
85
+
86
+if __name__ == '__main__':
87
+ parser = argparse.ArgumentParser(
88
+ description="Benchmark the rotary embedding kernels.")
89
+ parser.add_argument("--is-neox-style", type=bool, default=True)
90
+ parser.add_argument("--batch-size", type=int, default=16)
91
+ parser.add_argument("--seq-len", type=int, default=512)
92
+ parser.add_argument("--num-heads", type=int, default=8)
93
+ parser.add_argument("--head-size",
94
+ type=int,
95
+ choices=[64, 80, 96, 112, 128, 256],
96
+ default=128)
97
+ parser.add_argument("--rotary-dim", type=int, choices=[16, 32], default=32)
98
+ parser.add_argument("--dtype",
99
+ type=str,
100
+ choices=["bfloat16", "float"],
101
+ default="float")
102
+ parser.add_argument("--seed", type=int, default=0)
103
+ parser.add_argument("--device",
104
+ type=str,
105
+ choices=["cuda:0", "cuda:1"],
106
+ default="cuda:0")
107
+ args = parser.parse_args()
108
+ print(args)
109
+
110
+ benchmark_rope_kernels_multi_lora(
111
+ is_neox_style=args.is_neox_style,
112
+ batch_size=args.batch_size,
113
+ seq_len=args.seq_len,
114
+ num_heads=args.num_heads,
115
+ head_size=args.head_size,
116
+ rotary_dim=args.rotary_dim,
117
+ dtype=getattr(torch, args.dtype),
118
+ seed=args.seed,
119
+ device=args.device,
120
+ )
csrc/ops.h
+10
@@ -53,6 +53,16 @@ void rotary_embedding(
53
torch::Tensor& cos_sin_cache,
54
bool is_neox);
55
56
+void batched_rotary_embedding(
57
+ torch::Tensor& positions,
58
+ torch::Tensor& query,
59
+ torch::Tensor& key,
60
+ int head_size,
61
+ torch::Tensor& cos_sin_cache,
62
+ bool is_neox,
63
+ int rot_dim,
64
+ torch::Tensor& cos_sin_cache_offsets);
65
+
66
void silu_and_mul(
67
torch::Tensor& out,
68
torch::Tensor& input);
csrc/pos_encoding_kernels.cu
+111
-15
@@ -8,7 +8,7 @@
8
namespace vllm {
9
10
template<typename scalar_t, bool IS_NEOX>
11
-inline __device__ void apply_rotary_embedding(
11
+inline __device__ void apply_token_rotary_embedding(
12
scalar_t* __restrict__ arr,
13
const scalar_t* __restrict__ cos_ptr,
14
const scalar_t* __restrict__ sin_ptr,
@@ -38,22 +38,18 @@ inline __device__ void apply_rotary_embedding(
38
}
39
40
template<typename scalar_t, bool IS_NEOX>
41
-__global__ void rotary_embedding_kernel(
42
- const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
41
+inline __device__ void apply_rotary_embedding(
42
scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
43
scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
45
- const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
46
- const int rot_dim,
47
- const int64_t query_stride,
48
- const int64_t key_stride,
44
+ const scalar_t* cache_ptr,
45
+ const int head_size,
46
const int num_heads,
47
const int num_kv_heads,
51
- const int head_size) {
52
- // Each thread block is responsible for one token.
53
- const int token_idx = blockIdx.x;
54
- int64_t pos = positions[token_idx];
55
- const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
56
-
48
+ const int rot_dim,
49
+ const int token_idx,
50
+ const int64_t query_stride,
51
+ const int64_t key_stride)
52
+{
53
const int embed_dim = rot_dim / 2;
54
const scalar_t* cos_ptr = cache_ptr;
55
const scalar_t* sin_ptr = cache_ptr + embed_dim;
@@ -63,7 +59,7 @@ __global__ void rotary_embedding_kernel(
59
const int head_idx = i / embed_dim;
60
const int64_t token_head = token_idx * query_stride + head_idx * head_size;
61
const int rot_offset = i % embed_dim;
66
- apply_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
62
+ apply_token_rotary_embedding<scalar_t, IS_NEOX>(query + token_head, cos_ptr,
63
sin_ptr, rot_offset, embed_dim);
64
}
65
@@ -72,11 +68,53 @@ __global__ void rotary_embedding_kernel(
68
const int head_idx = i / embed_dim;
69
const int64_t token_head = token_idx * key_stride + head_idx * head_size;
70
const int rot_offset = i % embed_dim;
75
- apply_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
71
+ apply_token_rotary_embedding<scalar_t, IS_NEOX>(key + token_head, cos_ptr,
72
sin_ptr, rot_offset, embed_dim);
73
}
74
}
75
76
+template<typename scalar_t, bool IS_NEOX>
77
+__global__ void rotary_embedding_kernel(
78
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
79
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
80
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
81
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
82
+ const int rot_dim,
83
+ const int64_t query_stride,
84
+ const int64_t key_stride,
85
+ const int num_heads,
86
+ const int num_kv_heads,
87
+ const int head_size) {
88
+ // Each thread block is responsible for one token.
89
+ const int token_idx = blockIdx.x;
90
+ int64_t pos = positions[token_idx];
91
+ const scalar_t* cache_ptr = cos_sin_cache + pos * rot_dim;
92
+
93
+ apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
94
+}
95
+
96
+template<typename scalar_t, bool IS_NEOX>
97
+__global__ void batched_rotary_embedding_kernel(
98
+ const int64_t* __restrict__ positions, // [batch_size, seq_len] or [num_tokens]
99
+ scalar_t* __restrict__ query, // [batch_size, seq_len, num_heads, head_size] or [num_tokens, num_heads, head_size]
100
+ scalar_t* __restrict__ key, // [batch_size, seq_len, num_kv_heads, head_size] or [num_tokens, num_kv_heads, head_size]
101
+ const scalar_t* __restrict__ cos_sin_cache, // [max_position, 2, rot_dim // 2]
102
+ const int64_t* __restrict__ cos_sin_cache_offsets, // [batch_size, seq_len] or [num_tokens]
103
+ const int rot_dim,
104
+ const int64_t query_stride,
105
+ const int64_t key_stride,
106
+ const int num_heads,
107
+ const int num_kv_heads,
108
+ const int head_size) {
109
+ // Each thread block is responsible for one token.
110
+ const int token_idx = blockIdx.x;
111
+ int64_t pos = positions[token_idx];
112
+ int64_t cos_sin_cache_offset = cos_sin_cache_offsets[token_idx];
113
+ const scalar_t* cache_ptr = cos_sin_cache + (cos_sin_cache_offset + pos) * rot_dim;
114
+
115
+ apply_rotary_embedding<scalar_t, IS_NEOX>(query, key, cache_ptr, head_size, num_heads, num_kv_heads, rot_dim, token_idx, query_stride, key_stride);
116
+}
117
+
118
} // namespace vllm
119
120
void rotary_embedding(
@@ -128,3 +166,61 @@ void rotary_embedding(
166
}
167
});
168
}
169
+
170
+/*
171
+Batched version of rotary embedding, pack multiple LoRAs together
172
+and process in batched manner.
173
+*/
174
+void batched_rotary_embedding(
175
+ torch::Tensor& positions, // [batch_size, seq_len] or [num_tokens]
176
+ torch::Tensor& query, // [batch_size, seq_len, num_heads * head_size] or [num_tokens, num_heads * head_size]
177
+ torch::Tensor& key, // [batch_size, seq_len, num_kv_heads * head_size] or [num_tokens, num_kv_heads * head_size]
178
+ int head_size,
179
+ torch::Tensor& cos_sin_cache, // [max_position, rot_dim]
180
+ bool is_neox,
181
+ int rot_dim,
182
+ torch::Tensor& cos_sin_cache_offsets // [num_tokens]
183
+) {
184
+ int64_t num_tokens = cos_sin_cache_offsets.size(0);
185
+ int num_heads = query.size(-1) / head_size;
186
+ int num_kv_heads = key.size(-1) / head_size;
187
+ int64_t query_stride = query.stride(-2);
188
+ int64_t key_stride = key.stride(-2);
189
+
190
+ dim3 grid(num_tokens);
191
+ dim3 block(std::min(num_heads * rot_dim / 2, 512));
192
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(query));
193
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
194
+ VLLM_DISPATCH_FLOATING_TYPES(
195
+ query.scalar_type(),
196
+ "rotary_embedding",
197
+ [&] {
198
+ if (is_neox) {
199
+ vllm::batched_rotary_embedding_kernel<scalar_t, true><<<grid, block, 0, stream>>>(
200
+ positions.data_ptr<int64_t>(),
201
+ query.data_ptr<scalar_t>(),
202
+ key.data_ptr<scalar_t>(),
203
+ cos_sin_cache.data_ptr<scalar_t>(),
204
+ cos_sin_cache_offsets.data_ptr<int64_t>(),
205
+ rot_dim,
206
+ query_stride,
207
+ key_stride,
208
+ num_heads,
209
+ num_kv_heads,
210
+ head_size);
211
+ } else {
212
+ vllm::batched_rotary_embedding_kernel<scalar_t, false><<<grid, block, 0, stream>>>(
213
+ positions.data_ptr<int64_t>(),
214
+ query.data_ptr<scalar_t>(),
215
+ key.data_ptr<scalar_t>(),
216
+ cos_sin_cache.data_ptr<scalar_t>(),
217
+ cos_sin_cache_offsets.data_ptr<int64_t>(),
218
+ rot_dim,
219
+ query_stride,
220
+ key_stride,
221
+ num_heads,
222
+ num_kv_heads,
223
+ head_size);
224
+ }
225
+ });
226
+}
csrc/pybind.cpp
+5
@@ -56,6 +56,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
56
&rotary_embedding,
57
"Apply GPT-NeoX or GPT-J style rotary embedding to query and key");
58
59
+ ops.def(
60
+ "batched_rotary_embedding",
61
+ &batched_rotary_embedding,
62
+ "Apply GPT-NeoX or GPT-J style rotary embedding to query and key (supports multiple loras)");
63
+
64
// Quantization ops
65
#ifndef USE_ROCM
66
ops.def("awq_gemm", &awq_gemm, "Quantized GEMM for AWQ");
tests/kernels/test_pos_encoding.py
+134
-1
@@ -1,8 +1,9 @@
1
-from typing import Optional
1
+from typing import List, Optional
2
3
import pytest
4
import torch
5
from allclose_default import get_default_atol, get_default_rtol
6
+from itertools import accumulate
7
from vllm.model_executor.layers.rotary_embedding import get_rope
8
9
IS_NEOX_STYLE = [True, False]
@@ -72,3 +73,135 @@ def test_rotary_embedding(
73
ref_key,
74
atol=get_default_atol(out_key),
75
rtol=get_default_rtol(out_key))
76
+
77
+
78
+@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
79
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
80
+@pytest.mark.parametrize("seq_len", SEQ_LENS)
81
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
82
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
83
+@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
84
+@pytest.mark.parametrize("dtype", DTYPES)
85
+@pytest.mark.parametrize("seed", SEEDS)
86
+@pytest.mark.parametrize("device", CUDA_DEVICES)
87
+@torch.inference_mode()
88
+def test_batched_rotary_embedding(
89
+ is_neox_style: bool,
90
+ batch_size: int,
91
+ seq_len: int,
92
+ num_heads: int,
93
+ head_size: int,
94
+ rotary_dim: Optional[int],
95
+ dtype: torch.dtype,
96
+ seed: int,
97
+ device: str,
98
+ max_position: int = 8192,
99
+ base: int = 10000,
100
+) -> None:
101
+ torch.random.manual_seed(seed)
102
+ if torch.cuda.is_available():
103
+ torch.cuda.manual_seed(seed)
104
+ torch.set_default_device(device)
105
+ if rotary_dim is None:
106
+ rotary_dim = head_size
107
+ rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
108
+ "type": "linear",
109
+ "factor": (1, )
110
+ })
111
+ rope = rope.to(dtype=dtype)
112
+
113
+ positions = torch.randint(0, max_position, (batch_size, seq_len))
114
+ query = torch.randn(batch_size,
115
+ seq_len,
116
+ num_heads * head_size,
117
+ dtype=dtype)
118
+ key = torch.randn_like(query)
119
+
120
+ # NOTE(woosuk): The reference implementation should be executed first
121
+ # because the custom kernel is in-place.
122
+ ref_query, ref_key = rope._forward(positions, query, key)
123
+ out_query, out_key = rope.forward(positions,
124
+ query,
125
+ key,
126
+ offsets=torch.zeros(batch_size * seq_len,
127
+ dtype=int,
128
+ device=device))
129
+ # Compare the results.
130
+ assert torch.allclose(out_query,
131
+ ref_query,
132
+ atol=get_default_atol(out_query),
133
+ rtol=get_default_rtol(out_query))
134
+ assert torch.allclose(out_key,
135
+ ref_key,
136
+ atol=get_default_atol(out_key),
137
+ rtol=get_default_rtol(out_key))
138
+
139
+
140
+@pytest.mark.parametrize("is_neox_style", IS_NEOX_STYLE)
141
+@pytest.mark.parametrize("batch_size", BATCH_SIZES)
142
+@pytest.mark.parametrize("seq_len", SEQ_LENS)
143
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
144
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
145
+@pytest.mark.parametrize("rotary_dim", ROTARY_DIMS)
146
+@pytest.mark.parametrize("dtype", DTYPES)
147
+@pytest.mark.parametrize("seed", SEEDS)
148
+@pytest.mark.parametrize("device", CUDA_DEVICES)
149
+@torch.inference_mode()
150
+def test_batched_rotary_embedding_multi_lora(
151
+ is_neox_style: bool,
152
+ batch_size: int,
153
+ seq_len: int,
154
+ num_heads: int,
155
+ head_size: int,
156
+ rotary_dim: Optional[int],
157
+ dtype: torch.dtype,
158
+ seed: int,
159
+ device: str,
160
+ max_position: int = 8192,
161
+ base: int = 10000,
162
+) -> None:
163
+ torch.random.manual_seed(seed)
164
+ if torch.cuda.is_available():
165
+ torch.cuda.manual_seed(seed)
166
+ torch.set_default_device(device)
167
+ if rotary_dim is None:
168
+ rotary_dim = head_size
169
+ scaling_factors: List[int] = [1, 2, 4]
170
+ rope = get_rope(head_size, rotary_dim, max_position, base, is_neox_style, {
171
+ "type": "linear",
172
+ "factor": tuple(scaling_factors)
173
+ })
174
+ rope = rope.to(dtype=dtype)
175
+
176
+ positions = torch.randint(0, max_position, (batch_size, seq_len))
177
+ query = torch.randn(batch_size,
178
+ seq_len,
179
+ num_heads * head_size,
180
+ dtype=dtype)
181
+ key = torch.randn_like(query)
182
+
183
+ offset_map = torch.tensor(
184
+ list(
185
+ accumulate([0] + [
186
+ max_position * scaling_factor * 2
187
+ for scaling_factor in scaling_factors[:-1]
188
+ ])))
189
+ query_types = torch.randint(0,
190
+ len(scaling_factors), (batch_size, seq_len),
191
+ device=device)
192
+ query_offsets = offset_map[query_types]
193
+
194
+ # NOTE(woosuk): The reference implementation should be executed first
195
+ # because the custom kernel is in-place.
196
+ ref_query, ref_key = rope._forward(positions, query, key, query_offsets)
197
+ out_query, out_key = rope.forward(positions, query, key,
198
+ query_offsets.flatten())
199
+ # Compare the results.
200
+ assert torch.allclose(out_query,
201
+ ref_query,
202
+ atol=get_default_atol(out_query),
203
+ rtol=get_default_rtol(out_query))
204
+ assert torch.allclose(out_key,
205
+ ref_key,
206
+ atol=get_default_atol(out_key),
207
+ rtol=get_default_rtol(out_key))
vllm/model_executor/layers/rotary_embedding.py
+37
-21
@@ -22,7 +22,7 @@
22
# limitations under the License.
23
"""Rotary Positional Embeddings."""
24
import math
25
-from typing import Any, Dict, Optional, Tuple, Union
25
+from typing import Any, Dict, List, Optional, Tuple, Union
26
27
import torch
28
import torch.nn as nn
@@ -96,6 +96,7 @@ class RotaryEmbedding(nn.Module):
96
positions: torch.Tensor,
97
query: torch.Tensor,
98
key: torch.Tensor,
99
+ offsets: Optional[torch.Tensor] = None,
100
) -> Tuple[torch.Tensor, torch.Tensor]:
101
"""PyTorch-native implementation equivalent to forward()."""
102
query = query.view(*query.shape[:-1], -1, self.head_size)
@@ -107,7 +108,9 @@ class RotaryEmbedding(nn.Module):
108
query_pass = query[..., self.rotary_dim:]
109
key_pass = key[..., self.rotary_dim:]
110
110
- cos_sin = self.cos_sin_cache[positions]
111
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
112
+ cos_sin = self.cos_sin_cache[torch.add(positions, offsets)
113
+ if offsets is not None else positions]
114
cos, sin = cos_sin.chunk(2, dim=-1)
115
if self.is_neox_style:
116
# NOTE(woosuk): Here we assume that the positions tensor has the
@@ -137,11 +140,19 @@ class RotaryEmbedding(nn.Module):
140
positions: torch.Tensor,
141
query: torch.Tensor,
142
key: torch.Tensor,
143
+ offsets: Optional[torch.Tensor] = None,
144
) -> Tuple[torch.Tensor, torch.Tensor]:
141
- # ops.rotary_embedding() is an in-place operation that
142
- # updates the query and key tensors.
143
- ops.rotary_embedding(positions, query, key, self.head_size,
144
- self.cos_sin_cache, self.is_neox_style)
145
+ self.cos_sin_cache = self.cos_sin_cache.to(positions.get_device())
146
+ # ops.rotary_embedding()/batched_rotary_embedding() are in-place operations that
147
+ # update the query and key tensors.
148
+ if offsets is not None:
149
+ ops.batched_rotary_embedding(positions, query, key, self.head_size,
150
+ self.cos_sin_cache,
151
+ self.is_neox_style, self.rotary_dim,
152
+ offsets)
153
+ else:
154
+ ops.rotary_embedding(positions, query, key, self.head_size,
155
+ self.cos_sin_cache, self.is_neox_style)
156
return query, key
157
158
@@ -158,27 +169,32 @@ class LinearScalingRotaryEmbedding(RotaryEmbedding):
169
max_position_embeddings: int,
170
base: int,
171
is_neox_style: bool,
161
- scaling_factor: float,
172
+ scaling_factors: Union[List[float], float],
173
) -> None:
163
- self.scaling_factor = scaling_factor
174
+ if isinstance(scaling_factors, float):
175
+ scaling_factors = [scaling_factors]
176
+ self.scaling_factors = scaling_factors
177
super().__init__(head_size, rotary_dim, max_position_embeddings, base,
178
is_neox_style)
179
180
def _compute_cos_sin_cache(self) -> torch.Tensor:
181
inv_freq = self._compute_inv_freq(self.base)
169
- # NOTE(woosuk): self.max_position_embeddings is the original
170
- # maximum length before applying the rope scaling.
171
- # Thus, the maximum length after applying the rope scaling is
172
- # self.max_position_embeddings * self.scaling_factor.
173
- max_len = self.max_position_embeddings * self.scaling_factor
174
- t = torch.arange(max_len, dtype=torch.float)
175
- t = t / self.scaling_factor
176
-
177
- freqs = torch.einsum("i,j -> ij", t, inv_freq)
178
- cos = freqs.cos()
179
- sin = freqs.sin()
180
- cache = torch.cat((cos, sin), dim=-1)
181
- return cache
182
+ cache_list = []
183
+ for scaling_factor in self.scaling_factors:
184
+ # NOTE(woosuk): self.max_position_embeddings is the original
185
+ # maximum length before applying the rope scaling.
186
+ # Thus, the maximum length after applying the rope scaling is
187
+ # self.max_position_embeddings * self.scaling_factor.
188
+ max_len = self.max_position_embeddings * scaling_factor
189
+ t = torch.arange(max_len, dtype=torch.float)
190
+ t = t / scaling_factor
191
+
192
+ freqs = torch.einsum("i,j -> ij", t, inv_freq)
193
+ cos = freqs.cos()
194
+ sin = freqs.sin()
195
+ cache = torch.cat((cos, sin), dim=-1)
196
+ cache_list.append(cache)
197
+ return torch.cat(cache_list, dim=0)
198
199
200
class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):