A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
Revert "[Kernel] Use flash-attn for decoding (#3648)" (#4820)
Lora 3 & 4 test seems to have illegal memory access failure after this commit; [2024-05-14 23:51:18,182 E 22 22] logging.cc:101: Unhandled exception: N3c105ErrorE. what(): CUDA error: an illegal memory access was encountered <br class="Apple-interchange-newline"> Exmaple: https://buildkite.com/vllm/ci/builds/7382#018f793d-1527-4e1c-ab59-c3a34ec55241 This reverts commit 1356df5. FILL IN THE PR DESCRIPTION HERE FIX #xxxx (link existing issues this PR will resolve)
SangBin Cho committed
May 15, 2024 at 11:52 UTC
8a7cc254a064b8d42bf4de7a9c3f29552240dfd9
6 files changed
+65
-313
tests/kernels/test_flash_attn.py
deleted
-209
@@ -1,209 +0,0 @@
1
-from typing import List, Optional, Tuple
2
-
3
-import pytest
4
-import torch
5
-from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
6
-
7
-NUM_HEADS = [(16, 16), (32, 8), (64, 8)]
8
-HEAD_SIZES = [128, 256]
9
-BLOCK_SIZES = [16, 32]
10
-DTYPES = [torch.float16, torch.bfloat16]
11
-
12
-
13
-def ref_paged_attn(
14
- query: torch.Tensor,
15
- key_cache: torch.Tensor,
16
- value_cache: torch.Tensor,
17
- query_lens: List[int],
18
- kv_lens: List[int],
19
- block_tables: torch.Tensor,
20
- scale: float,
21
- sliding_window: Optional[int] = None,
22
-) -> torch.Tensor:
23
- num_seqs = len(query_lens)
24
- block_tables = block_tables.cpu().numpy()
25
- _, block_size, num_kv_heads, head_size = key_cache.shape
26
-
27
- outputs = []
28
- start_idx = 0
29
- for i in range(num_seqs):
30
- query_len = query_lens[i]
31
- kv_len = kv_lens[i]
32
- q = query[start_idx:start_idx + query_len]
33
- q *= scale
34
-
35
- num_kv_blocks = (kv_len + block_size - 1) // block_size
36
- block_indices = block_tables[i, :num_kv_blocks]
37
-
38
- k = key_cache[block_indices].view(-1, num_kv_heads, head_size)
39
- k = k[:kv_len]
40
- v = value_cache[block_indices].view(-1, num_kv_heads, head_size)
41
- v = v[:kv_len]
42
-
43
- if q.shape[1] != k.shape[1]:
44
- k = torch.repeat_interleave(k, q.shape[1] // k.shape[1], dim=1)
45
- v = torch.repeat_interleave(v, q.shape[1] // v.shape[1], dim=1)
46
- attn = torch.einsum("qhd,khd->hqk", q, k).float()
47
- empty_mask = torch.ones(query_len, kv_len)
48
- mask = torch.triu(empty_mask, diagonal=kv_len - query_len + 1).bool()
49
- if sliding_window is not None:
50
- sliding_window_mask = torch.triu(empty_mask,
51
- diagonal=kv_len -
52
- (query_len + sliding_window) +
53
- 1).bool().logical_not()
54
- mask |= sliding_window_mask
55
- attn.masked_fill_(mask, float("-inf"))
56
- attn = torch.softmax(attn, dim=-1).to(v.dtype)
57
- out = torch.einsum("hqk,khd->qhd", attn, v)
58
-
59
- outputs.append(out)
60
- start_idx += query_len
61
-
62
- return torch.cat(outputs, dim=0)
63
-
64
-
65
-@pytest.mark.parametrize("kv_lens", [[1328, 18, 463], [1, 54, 293, 70]])
66
-@pytest.mark.parametrize("num_heads", NUM_HEADS)
67
-@pytest.mark.parametrize("head_size", HEAD_SIZES)
68
-@pytest.mark.parametrize("block_size", BLOCK_SIZES)
69
-@pytest.mark.parametrize("dtype", DTYPES)
70
-@torch.inference_mode
71
-def test_flash_attn_with_paged_kv(
72
- kv_lens: List[Tuple[int, int]],
73
- num_heads: Tuple[int, int],
74
- head_size: int,
75
- dtype: torch.dtype,
76
- block_size: int,
77
-) -> None:
78
- torch.set_default_device("cuda")
79
- torch.cuda.manual_seed_all(0)
80
- num_blocks = 128
81
- num_seqs = len(kv_lens)
82
- num_query_heads = num_heads[0]
83
- num_kv_heads = num_heads[1]
84
- assert num_query_heads % num_kv_heads == 0
85
- max_kv_len = max(kv_lens)
86
- scale = head_size**-0.5
87
-
88
- query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
89
- key_cache = torch.randn(num_blocks,
90
- block_size,
91
- num_kv_heads,
92
- head_size,
93
- dtype=dtype)
94
- value_cache = torch.randn_like(key_cache)
95
- kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
96
-
97
- max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
98
- block_tables = torch.randint(0,
99
- num_blocks,
100
- (num_seqs, max_num_blocks_per_seq),
101
- dtype=torch.int32)
102
-
103
- output = flash_attn_with_kvcache(
104
- q=query.unsqueeze(1),
105
- k_cache=key_cache,
106
- v_cache=value_cache,
107
- softmax_scale=scale,
108
- causal=True,
109
- block_table=block_tables,
110
- cache_seqlens=kv_lens_tensor,
111
- ).squeeze(1)
112
-
113
- ref_output = ref_paged_attn(
114
- query=query,
115
- key_cache=key_cache,
116
- value_cache=value_cache,
117
- query_lens=[1] * num_seqs,
118
- kv_lens=kv_lens,
119
- block_tables=block_tables,
120
- scale=scale,
121
- )
122
- assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
123
- f"{torch.max(torch.abs(output - ref_output))}"
124
-
125
-
126
-@pytest.mark.parametrize("seq_lens", [[(1, 1328), (5, 18), (129, 463)]])
127
-@pytest.mark.parametrize("num_heads", NUM_HEADS)
128
-@pytest.mark.parametrize("head_size", HEAD_SIZES)
129
-@pytest.mark.parametrize("block_size", BLOCK_SIZES)
130
-@pytest.mark.parametrize("sliding_window", [None])
131
-@pytest.mark.parametrize("dtype", DTYPES)
132
-@torch.inference_mode
133
-def test_varlen_with_paged_kv(
134
- seq_lens: List[Tuple[int, int]],
135
- num_heads: Tuple[int, int],
136
- head_size: int,
137
- sliding_window: Optional[int],
138
- dtype: torch.dtype,
139
- block_size: int,
140
-) -> None:
141
- torch.set_default_device("cuda")
142
- torch.cuda.manual_seed_all(0)
143
- num_blocks = 128
144
- num_seqs = len(seq_lens)
145
- query_lens = [x[0] for x in seq_lens]
146
- kv_lens = [x[1] for x in seq_lens]
147
- num_query_heads = num_heads[0]
148
- num_kv_heads = num_heads[1]
149
- assert num_query_heads % num_kv_heads == 0
150
- max_query_len = max(query_lens)
151
- max_kv_len = max(kv_lens)
152
- window_size = ((sliding_window,
153
- sliding_window) if sliding_window is not None else
154
- (-1, -1))
155
- scale = head_size**-0.5
156
-
157
- query = torch.randn(sum(query_lens),
158
- num_query_heads,
159
- head_size,
160
- dtype=dtype)
161
- key_cache = torch.randn(num_blocks,
162
- block_size,
163
- num_kv_heads,
164
- head_size,
165
- dtype=dtype)
166
- value_cache = torch.randn_like(key_cache)
167
- # Normalize the scale of the key and value caches to mitigate
168
- # numerical instability.
169
- key_cache /= head_size**0.5
170
- value_cache /= head_size**0.5
171
- cu_query_lens = torch.tensor([0] + query_lens,
172
- dtype=torch.int32).cumsum(dim=0,
173
- dtype=torch.int32)
174
- cu_kv_lens = torch.tensor([0] + kv_lens,
175
- dtype=torch.int32).cumsum(dim=0,
176
- dtype=torch.int32)
177
-
178
- max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
179
- block_tables = torch.randint(0,
180
- num_blocks,
181
- (num_seqs, max_num_blocks_per_seq),
182
- dtype=torch.int32)
183
-
184
- output = flash_attn_varlen_func(
185
- q=query,
186
- k=key_cache,
187
- v=value_cache,
188
- cu_seqlens_q=cu_query_lens,
189
- cu_seqlens_k=cu_kv_lens,
190
- max_seqlen_q=max_query_len,
191
- max_seqlen_k=max_kv_len,
192
- softmax_scale=scale,
193
- causal=True,
194
- window_size=window_size,
195
- block_table=block_tables,
196
- )
197
-
198
- ref_output = ref_paged_attn(
199
- query=query,
200
- key_cache=key_cache,
201
- value_cache=value_cache,
202
- query_lens=query_lens,
203
- kv_lens=kv_lens,
204
- block_tables=block_tables,
205
- scale=scale,
206
- sliding_window=sliding_window,
207
- )
208
- assert torch.allclose(output, ref_output, atol=1e-2, rtol=1e-2), \
209
- f"{torch.max(torch.abs(output - ref_output))}"
tests/models/test_big_models.py
+1
-1
@@ -12,7 +12,7 @@ MODELS = [
12
# "Deci/DeciLM-7b", # Broken
13
# "tiiuae/falcon-7b", # Broken
14
"EleutherAI/gpt-j-6b",
15
- # "mosaicml/mpt-7b", # Broken
15
+ "mosaicml/mpt-7b",
16
# "Qwen/Qwen1.5-0.5B" # Broken,
17
]
18
tests/models/test_fp8.py
+5
-5
@@ -25,18 +25,18 @@ EXPECTED_STRS_MAP = {
25
'LLaMA is a high-throughput and memory-efficient inference and serving engine for Large Language Models (',
26
'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
27
'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
28
- 'A neural network is a complex system modeled after the human brain, consisting of interconnected nodes or "ne',
29
- 'Zeta-5, a highly advanced robot designed for menial labor, whirred to a',
30
- 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
28
+ 'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
29
+ 'Zeta-5, a highly advanced robot designed for menial labor, whirred and beep',
30
+ 'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. Here',
31
'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
32
- 'Here are the translations:\n\n**Japanese:** (Haya aki no tori, guri o',
32
+ 'Here are the translations:\n\n**Japanese:** (Haya tori, nemuri nemuri)\n\n**'
33
],
34
"meta-llama/Meta-Llama-3-8B-Instruct": [
35
'LLM (Large Language Model) is a type of artificial intelligence (AI) model that is trained',
36
'Here are the major milestones in the development of artificial intelligence (AI) from 1950 to ',
37
'Artificial intelligence (AI) and human intelligence (HI) differ significantly in how they process information.',
38
'A neural network is a complex system modeled after the human brain, composed of interconnected nodes or "ne',
39
- 'In the vast, sterile laboratory, Robot 3456-Alpha, or "Alpha" for short',
39
+ 'In the year 2154, the robotics lab at NeuroSpark Industries was on the cusp of',
40
'The COVID-19 pandemic has had a profound impact on global economic structures and future business models. The',
41
'The Mona Lisa, painted by Leonardo da Vinci in the early 16th century, is one of',
42
'Here are the translations:\n\n**Japanese:** (Haya aki wa mushi o tsukamu'
vllm/attention/backends/flash_attn.py
+55
-73
@@ -1,16 +1,20 @@
1
-"""Attention layer with FlashAttention."""
1
+"""Attention layer with Flash and PagedAttention.
2
+
3
+NOTE(woosuk): At the moment, this file includes a lot of duplicated code from
4
+XFormers backend. The duplicated code will be removed once we use flash-attn or
5
+flashinfer for all the attention operations.
6
+"""
7
from dataclasses import dataclass
8
from typing import List, Optional, Tuple, Type
9
10
import torch
6
-from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
11
+from vllm_flash_attn import flash_attn_varlen_func
12
8
-from vllm._C import cache_ops
13
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
AttentionMetadata,
15
AttentionMetadataPerStage)
12
-
13
-_SUPPORTED_HEAD_SIZES = [32, 64, 96, 128, 160, 192, 224, 256]
16
+from vllm.attention.ops.paged_attn import (PagedAttention,
17
+ PagedAttentionMetadata)
18
19
20
class FlashAttentionBackend(AttentionBackend):
@@ -34,9 +38,8 @@ class FlashAttentionBackend(AttentionBackend):
38
num_kv_heads: int,
39
head_size: int,
40
) -> Tuple[int, ...]:
37
- if block_size % 16 != 0:
38
- raise ValueError("Block size must be a multiple of 16.")
39
- return (2, num_blocks, block_size, num_kv_heads, head_size)
41
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
42
+ num_kv_heads, head_size)
43
44
@staticmethod
45
def swap_blocks(
@@ -44,26 +47,19 @@ class FlashAttentionBackend(AttentionBackend):
47
dst_kv_cache: torch.Tensor,
48
src_to_dst: torch.Tensor,
49
) -> None:
47
- src_key_cache = src_kv_cache[0]
48
- dst_key_cache = dst_kv_cache[0]
49
- cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
50
-
51
- src_value_cache = src_kv_cache[1]
52
- dst_value_cache = dst_kv_cache[1]
53
- cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
50
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
51
52
@staticmethod
53
def copy_blocks(
54
kv_caches: List[torch.Tensor],
55
src_to_dists: torch.Tensor,
56
) -> None:
60
- key_caches = [kv_cache[0] for kv_cache in kv_caches]
61
- value_caches = [kv_cache[1] for kv_cache in kv_caches]
62
- cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
57
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
58
59
60
@dataclass
66
-class FlashAttentionMetadata(AttentionMetadataPerStage):
61
+class FlashAttentionMetadata(AttentionMetadataPerStage,
62
+ PagedAttentionMetadata):
63
"""Metadata for FlashAttentionBackend.
64
65
NOTE: Any python object stored here is not updated when it is
@@ -109,14 +105,6 @@ class FlashAttentionMetadata(AttentionMetadataPerStage):
105
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
106
use_cuda_graph: bool
107
112
- # (batch_size, max_blocks_per_seq).
113
- # Block addresses per sequence. (Seq id -> list of physical block)
114
- # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
115
- # in the kv cache. Each block can contain up to block_size tokens.
116
- # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
117
- # captured.
118
- block_tables: Optional[torch.Tensor]
119
-
108
109
class FlashAttentionImpl(AttentionImpl):
110
"""
@@ -168,15 +156,11 @@ class FlashAttentionImpl(AttentionImpl):
156
assert self.num_heads % self.num_kv_heads == 0
157
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
158
171
- if sliding_window is not None:
172
- # NOTE(woosuk): flash-attn's sliding window does not work with
173
- # paged KV cache.
174
- raise ValueError(
175
- "Sliding window is not supported in FlashAttention.")
176
- if head_size not in _SUPPORTED_HEAD_SIZES:
159
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
160
+ if head_size not in suppored_head_sizes:
161
raise ValueError(
178
- f"Head size {head_size} is not supported by FlashAttention. "
179
- f"Supported head sizes are: {_SUPPORTED_HEAD_SIZES}.")
162
+ f"Head size {head_size} is not supported by PagedAttention. "
163
+ f"Supported head sizes are: {suppored_head_sizes}.")
164
165
def forward(
166
self,
@@ -187,20 +171,17 @@ class FlashAttentionImpl(AttentionImpl):
171
attn_metadata: AttentionMetadata[FlashAttentionMetadata],
172
kv_scale: float = 1.0,
173
) -> torch.Tensor:
190
- """Forward pass with FlashAttention.
174
+ """Forward pass with FlashAttention and PagedAttention.
175
176
Args:
177
query: shape = [num_tokens, num_heads * head_size]
178
key: shape = [num_tokens, num_kv_heads * head_size]
179
value: shape = [num_tokens, num_kv_heads * head_size]
196
- kv_cache = [2, num_blocks, block_size, num_kv_heads, head_size]
180
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
181
attn_metadata: Metadata for attention.
182
Returns:
183
shape = [num_tokens, num_heads * head_size]
184
"""
201
- # NOTE(woosuk): FlashAttention does not support FP8 KV cache.
202
- assert kv_scale == 1.0, "kv_scale is not supported in FlashAttention."
203
-
185
num_tokens, hidden_size = query.shape
186
# Reshape the query, key, and value tensors.
187
query = query.view(-1, self.num_heads, self.head_size)
@@ -208,20 +189,16 @@ class FlashAttentionImpl(AttentionImpl):
189
value = value.view(-1, self.num_kv_heads, self.head_size)
190
191
if kv_cache is not None:
211
- key_cache = kv_cache[0]
212
- value_cache = kv_cache[1]
192
+ key_cache, value_cache = PagedAttention.split_kv_cache(
193
+ kv_cache, self.num_kv_heads, self.head_size)
194
195
# Reshape the input keys and values and store them in the cache.
196
# If kv_cache is not provided, the new key and value tensors are
197
# not cached. This happens during the initial memory profiling run.
217
- cache_ops.reshape_and_cache_flash(
218
- key,
219
- value,
220
- key_cache,
221
- value_cache,
222
- attn_metadata.slot_mapping.flatten(),
223
- self.kv_cache_dtype,
224
- )
198
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
199
+ value_cache,
200
+ attn_metadata.slot_mapping,
201
+ self.kv_cache_dtype, kv_scale)
202
203
num_prefill_tokens = attn_metadata.num_prefill_tokens
204
num_decode_tokens = attn_metadata.num_decode_tokens
@@ -241,8 +218,7 @@ class FlashAttentionImpl(AttentionImpl):
218
219
if prefill_meta := attn_metadata.prefill_metadata:
220
# Prompt run.
244
- if (kv_cache is None or prefill_meta.block_tables is None
245
- or prefill_meta.block_tables.numel() == 0):
221
+ if kv_cache is None or prefill_meta.block_tables.numel() == 0:
222
# normal attention
223
# When block_tables are not filled, it means q and k are the
224
# prompt, and they have the same length.
@@ -263,32 +239,38 @@ class FlashAttentionImpl(AttentionImpl):
239
output[:num_prefill_tokens] = out
240
else:
241
# prefix-enabled attention
266
- output[:num_prefill_tokens] = flash_attn_varlen_func(
267
- q=query,
268
- k=key_cache,
269
- v=value_cache,
270
- cu_seqlens_q=prefill_meta.subquery_start_loc,
271
- max_seqlen_q=prefill_meta.max_query_len,
272
- cu_seqlens_k=prefill_meta.seq_start_loc,
273
- max_seqlen_k=prefill_meta.max_seq_len,
274
- softmax_scale=self.scale,
275
- causal=True,
276
- alibi_slopes=self.alibi_slopes,
277
- block_table=prefill_meta.block_tables,
242
+ # TODO(Hai) this triton kernel has regression issue (broke) to
243
+ # deal with different data types between KV and FP8 KV cache,
244
+ # to be addressed separately.
245
+ output[:num_prefill_tokens] = PagedAttention.forward_prefix(
246
+ query,
247
+ key,
248
+ value,
249
+ key_cache,
250
+ value_cache,
251
+ prefill_meta.block_tables,
252
+ prefill_meta.subquery_start_loc,
253
+ prefill_meta.seq_lens_tensor,
254
+ prefill_meta.context_lens_tensor,
255
+ prefill_meta.max_query_len,
256
+ self.alibi_slopes,
257
+ self.sliding_window[0],
258
)
279
-
259
if decode_meta := attn_metadata.decode_metadata:
260
# Decoding run.
282
- output[num_prefill_tokens:] = flash_attn_with_kvcache(
283
- decode_query.unsqueeze(1),
261
+ output[num_prefill_tokens:] = PagedAttention.forward_decode(
262
+ decode_query,
263
key_cache,
264
value_cache,
286
- block_table=decode_meta.block_tables,
287
- cache_seqlens=decode_meta.seq_lens_tensor,
288
- softmax_scale=self.scale,
289
- causal=True,
290
- alibi_slopes=self.alibi_slopes,
291
- ).squeeze(1)
265
+ decode_meta.block_tables,
266
+ decode_meta.seq_lens_tensor,
267
+ decode_meta.max_seq_len,
268
+ self.kv_cache_dtype,
269
+ self.num_kv_heads,
270
+ self.scale,
271
+ self.alibi_slopes,
272
+ kv_scale,
273
+ )
274
275
# Reshape the output tensor.
276
return output.view(num_tokens, hidden_size)
vllm/attention/selector.py
-14
@@ -93,20 +93,6 @@ def _which_attn_to_use(
93
"torch.float16 or torch.bfloat16.")
94
return _Backend.XFORMERS
95
96
- if kv_cache_dtype is not None and kv_cache_dtype.startswith("fp8"):
97
- logger.info("Cannot use FlashAttention-2 backend for FP8 KV cache.")
98
- return _Backend.XFORMERS
99
-
100
- if block_size % 16 != 0:
101
- logger.info("Cannot use FlashAttention-2 backend for block size not "
102
- "divisible by 16.")
103
- return _Backend.XFORMERS
104
-
105
- if sliding_window is not None:
106
- logger.info(
107
- "Cannot use FlashAttention-2 backend due to sliding window.")
108
- return _Backend.XFORMERS
109
-
96
try:
97
import vllm_flash_attn # noqa: F401
98
except ImportError:
vllm/worker/model_runner.py
+4
-11
@@ -266,27 +266,20 @@ class ModelRunner:
266
# Prefix is not supported with sliding_window
267
context_len = len(computed_block_nums) * self.block_size
268
prompt_tokens = prompt_tokens[context_len:]
269
- if self.attn_backend.get_name() == "flash-attn":
270
- # NOTE(woosuk): For flash-attn, the block table should
271
- # include the entries for the incoming prefill tokens.
272
- # TODO(woosuk): This is a temporary fix. We should
273
- # provide a unified interface for different backends.
274
- block_table = seq_group_metadata.block_tables[seq_id]
275
- else:
276
- block_table = computed_block_nums
269
+ prefix_block_tables.append(computed_block_nums)
270
elif self.scheduler_config.chunked_prefill_enabled:
271
if seq_group_metadata.block_tables is not None:
272
# Prefill has chunked before.
273
block_table = seq_group_metadata.block_tables[seq_id]
274
+ prefix_block_tables.append(block_table)
275
else:
276
# The first prefill.
283
- block_table = []
277
+ prefix_block_tables.append([])
278
else:
285
- block_table = []
279
+ prefix_block_tables.append([])
280
# Right now, prefill start is always 0. However, this
281
# assumption can be changed once chunked prefill is introduced.
282
assert context_len == 0
289
- prefix_block_tables.append(block_table)
283
284
# actual prompt lens
285
context_lens.append(context_len)