| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | import pytest |
| 5 | import torch |
| 6 | |
| 7 | from tests.v1.attention.test_attention_backends import BATCH_SPECS |
| 8 | from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata |
| 9 | from vllm.v1.attention.backends.utils import ( |
| 10 | split_decodes_and_prefills, |
| 11 | ) |
| 12 | from vllm.v1.worker.ubatch_utils import ( |
| 13 | UBatchSlice, |
| 14 | _make_metadata_with_slice, |
| 15 | maybe_create_ubatch_slices, |
| 16 | slice_query_start_locs, |
| 17 | split_attn_metadata, |
| 18 | ) |
| 19 | |
| 20 | |
| 21 | @pytest.fixture |
| 22 | def sample_query_start_loc(): |
| 23 | """Sample query_start_loc tensor for testing.""" |
| 24 | return torch.tensor([0, 5, 12, 20, 35, 50]) |
| 25 | |
| 26 | |
| 27 | def test_basic_slice_middle(sample_query_start_loc): |
| 28 | """Test slicing from middle of tensor.""" |
| 29 | req_slice = slice(1, 3) # slice from index 1 to 3 |
| 30 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 31 | |
| 32 | expected = torch.tensor([0, 7, 15]) |
| 33 | assert torch.equal(result, expected) |
| 34 | |
| 35 | |
| 36 | def test_slice_from_beginning(sample_query_start_loc): |
| 37 | """Test slicing from the beginning of tensor.""" |
| 38 | req_slice = slice(0, 2) # slice from index 0 to 2 |
| 39 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 40 | |
| 41 | expected = torch.tensor([0, 5, 12]) |
| 42 | assert torch.equal(result, expected) |
| 43 | |
| 44 | |
| 45 | def test_slice_to_end(sample_query_start_loc): |
| 46 | """Test slicing to the end of tensor.""" |
| 47 | req_slice = slice(3, 5) # slice from index 3 to 5 (last index) |
| 48 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 49 | |
| 50 | expected = torch.tensor([0, 15, 30]) |
| 51 | assert torch.equal(result, expected) |
| 52 | |
| 53 | |
| 54 | def test_single_element_slice(sample_query_start_loc): |
| 55 | """Test slice that results in single element.""" |
| 56 | req_slice = slice(2, 3) # slice from index 2 to 3 |
| 57 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 58 | |
| 59 | expected = torch.tensor([0, 8]) |
| 60 | assert torch.equal(result, expected) |
| 61 | |
| 62 | |
| 63 | def test_full_tensor_slice(sample_query_start_loc): |
| 64 | """Test slicing the entire tensor.""" |
| 65 | req_slice = slice(0, 5) # slice entire tensor |
| 66 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 67 | |
| 68 | expected = torch.tensor([0, 5, 12, 20, 35, 50]) |
| 69 | assert torch.equal(result, expected) |
| 70 | |
| 71 | |
| 72 | def test_slice_bounds_edge_cases(sample_query_start_loc): |
| 73 | # Test slice that goes exactly to the last element |
| 74 | req_slice = slice(4, 5) # Last index |
| 75 | result = slice_query_start_locs(sample_query_start_loc, req_slice) |
| 76 | |
| 77 | expected = torch.tensor([0, 15]) |
| 78 | assert torch.equal(result, expected) |
| 79 | |
| 80 | |
| 81 | @pytest.fixture |
| 82 | def small_decode_metadata(): |
| 83 | """Create metadata for small decode batch.""" |
| 84 | batch_spec = BATCH_SPECS["small_decode"] |
| 85 | device = torch.device("cpu") |
| 86 | return create_common_attn_metadata(batch_spec, block_size=16, device=device) |
| 87 | |
| 88 | |
| 89 | @pytest.fixture |
| 90 | def large_decode_metadata(): |
| 91 | """Create metadata for small decode batch.""" |
| 92 | batch_spec = BATCH_SPECS["large_decode"] |
| 93 | device = torch.device("cpu") |
| 94 | return create_common_attn_metadata(batch_spec, block_size=16, device=device) |
| 95 | |
| 96 | |
| 97 | @pytest.fixture |
| 98 | def mixed_small_metadata(): |
| 99 | """Create metadata for mixed small batch.""" |
| 100 | batch_spec = BATCH_SPECS["mixed_small"] |
| 101 | device = torch.device("cpu") |
| 102 | return create_common_attn_metadata(batch_spec, block_size=16, device=device) |
| 103 | |
| 104 | |
| 105 | # Tests for _make_metadata_with_slice |
| 106 | def test_make_metadata_with_slice_decode_batch(small_decode_metadata): |
| 107 | """Test slicing decode batch metadata.""" |
| 108 | # Split first request only |
| 109 | ubatch_slice = UBatchSlice(slice(0, 1), slice(0, 1)) |
| 110 | |
| 111 | result = _make_metadata_with_slice(ubatch_slice, small_decode_metadata) |
| 112 | |
| 113 | # Check sliced results |
| 114 | assert result.num_reqs == 1 # slice(0, 1) gives 1 requests |
| 115 | assert result.num_actual_tokens == 1 # slice(0, 1) gives 1 token |
| 116 | assert result.max_query_len == 1 |
| 117 | assert torch.equal(result.query_start_loc, torch.tensor([0, 1])) |
| 118 | assert torch.equal(result.seq_lens, torch.tensor([32])) |
| 119 | |
| 120 | |
| 121 | def test_make_metadata_with_slice_mixed_batch(mixed_small_metadata): |
| 122 | """Test slicing mixed batch metadata.""" |
| 123 | ubatch_slice = UBatchSlice(slice(1, 3), slice(1, 7)) # Requests 1-3, tokens 1-7 |
| 124 | |
| 125 | result = _make_metadata_with_slice(ubatch_slice, mixed_small_metadata) |
| 126 | |
| 127 | assert result.num_reqs == 2 # slice(1, 3) gives 2 requests |
| 128 | assert result.num_actual_tokens == 6 # slice(1, 7) gives 6 tokens |
| 129 | assert result.max_query_len == 5 |
| 130 | assert torch.equal(result.query_start_loc, torch.tensor([0, 1, 6])) |
| 131 | assert torch.equal(result.seq_lens, torch.tensor([40, 48])) |
| 132 | |
| 133 | |
| 134 | def test_split_attn_metadata_decode_batch(large_decode_metadata): |
| 135 | """Test splitting decode batch into two equal parts.""" |
| 136 | num_tokens = large_decode_metadata.num_reqs |
| 137 | mid_point = num_tokens // 2 |
| 138 | ubatch_slices = [ |
| 139 | UBatchSlice(slice(0, mid_point), slice(0, mid_point)), |
| 140 | UBatchSlice(slice(mid_point, num_tokens), slice(mid_point, num_tokens)), |
| 141 | ] |
| 142 | |
| 143 | results = split_attn_metadata(ubatch_slices, large_decode_metadata) |
| 144 | |
| 145 | assert len(results) == 2 |
| 146 | |
| 147 | # Check first split |
| 148 | assert results[0].num_reqs == mid_point |
| 149 | assert results[0].num_actual_tokens == mid_point |
| 150 | assert torch.equal(results[0].seq_lens, torch.tensor([2048] * mid_point)) |
| 151 | |
| 152 | # Check second split |
| 153 | assert results[1].num_reqs == mid_point |
| 154 | assert results[1].num_actual_tokens == mid_point |
| 155 | assert torch.equal(results[1].seq_lens, torch.tensor([2048] * mid_point)) |
| 156 | |
| 157 | |
| 158 | def apply_split_decodes_and_prefills( |
| 159 | query_lens: list[int], |
| 160 | decode_threshold: int, |
| 161 | require_uniform: bool, |
| 162 | padded_num_tokens: int | None = None, |
| 163 | is_prefilling: list[bool] | None = None, |
| 164 | treat_short_extends_as_decodes: bool = True, |
| 165 | ): |
| 166 | """Helper function to apply split_decodes_and_prefills and return |
| 167 | the results.""" |
| 168 | device = torch.device("cpu") |
| 169 | seq_lens = [10 * (i + 1) for i in range(len(query_lens))] |
| 170 | common_metadata = create_common_attn_metadata( |
| 171 | BatchSpec(seq_lens=seq_lens, query_lens=query_lens), |
| 172 | block_size=16, |
| 173 | device=device, |
| 174 | ) |
| 175 | |
| 176 | if padded_num_tokens is not None: |
| 177 | common_metadata.num_actual_tokens = padded_num_tokens |
| 178 | if is_prefilling is not None: |
| 179 | common_metadata.is_prefilling = torch.tensor(is_prefilling) |
| 180 | |
| 181 | return split_decodes_and_prefills( |
| 182 | common_metadata, |
| 183 | decode_threshold=decode_threshold, |
| 184 | require_uniform=require_uniform, |
| 185 | treat_short_extends_as_decodes=treat_short_extends_as_decodes, |
| 186 | ) |
| 187 | |
| 188 | |
| 189 | def test_split_decodes_and_prefills_nonuniform_all_ones(): |
| 190 | query_lens = [1, 1, 1] |
| 191 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 192 | apply_split_decodes_and_prefills(query_lens, 1, False) |
| 193 | ) |
| 194 | assert num_decodes == 3 |
| 195 | assert num_prefills == 0 |
| 196 | assert num_decode_tokens == 3 |
| 197 | assert num_prefill_tokens == 0 |
| 198 | |
| 199 | |
| 200 | def test_split_decodes_and_prefills_nonuniform_all_short_decodes(): |
| 201 | query_lens = [1, 2, 1, 3, 2, 1, 2] |
| 202 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 203 | apply_split_decodes_and_prefills(query_lens, 3, False) |
| 204 | ) |
| 205 | assert num_decodes == 7 |
| 206 | assert num_prefills == 0 |
| 207 | assert num_decode_tokens == sum(query_lens) |
| 208 | assert num_prefill_tokens == 0 |
| 209 | |
| 210 | |
| 211 | def test_split_decodes_and_prefills_nonuniform_all_prefills(): |
| 212 | query_lens = [4, 5, 6, 7] |
| 213 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 214 | apply_split_decodes_and_prefills(query_lens, 3, False) |
| 215 | ) |
| 216 | assert num_decodes == 0 |
| 217 | assert num_prefills == 4 |
| 218 | assert num_decode_tokens == 0 |
| 219 | assert num_prefill_tokens == sum(query_lens) |
| 220 | |
| 221 | |
| 222 | def test_split_decodes_and_prefills_nonuniform_mixed_batch(): |
| 223 | query_lens = [2, 1, 3, 4, 5, 6, 7, 8] |
| 224 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 225 | apply_split_decodes_and_prefills(query_lens, 4, False) |
| 226 | ) |
| 227 | assert num_decodes == 4 # 2, 1, 3, 4 are all <= 4 |
| 228 | assert num_prefills == 4 # 5, 6, 7, 8 are all > 4 |
| 229 | assert num_decode_tokens == 10 # 2 + 1 + 3 + 4 |
| 230 | assert num_prefill_tokens == 26 # 5 + 6 + 7 + 8 |
| 231 | |
| 232 | |
| 233 | def test_split_decodes_and_prefills_uniform_all_ones(): |
| 234 | query_lens = [1, 1, 1] |
| 235 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 236 | apply_split_decodes_and_prefills(query_lens, 1, True) |
| 237 | ) |
| 238 | assert num_decodes == 3 |
| 239 | assert num_prefills == 0 |
| 240 | assert num_decode_tokens == 3 |
| 241 | assert num_prefill_tokens == 0 |
| 242 | |
| 243 | |
| 244 | def test_split_decodes_and_prefills_uniform_short_extend(): |
| 245 | result = apply_split_decodes_and_prefills( |
| 246 | [1, 1], |
| 247 | decode_threshold=1, |
| 248 | require_uniform=True, |
| 249 | is_prefilling=[False, True], |
| 250 | treat_short_extends_as_decodes=False, |
| 251 | ) |
| 252 | assert result == (1, 1, 1, 1) |
| 253 | |
| 254 | |
| 255 | def test_split_decodes_and_prefills_uniform_all_short_decodes(): |
| 256 | query_lens = [2, 2, 1, 3, 2, 1, 2] |
| 257 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 258 | apply_split_decodes_and_prefills(query_lens, 3, True) |
| 259 | ) |
| 260 | assert num_decodes == 2 |
| 261 | assert num_prefills == 5 |
| 262 | assert num_decode_tokens == 4 |
| 263 | assert num_prefill_tokens == (1 + 3 + 2 + 1 + 2) |
| 264 | |
| 265 | |
| 266 | def test_split_decodes_and_prefills_uniform_all_prefills(): |
| 267 | query_lens = [4, 5, 6, 7] |
| 268 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 269 | apply_split_decodes_and_prefills(query_lens, 3, True) |
| 270 | ) |
| 271 | assert num_decodes == 0 |
| 272 | assert num_prefills == 4 |
| 273 | assert num_decode_tokens == 0 |
| 274 | assert num_prefill_tokens == sum(query_lens) |
| 275 | |
| 276 | |
| 277 | def test_split_decodes_and_prefills_uniform_mixed_batch_all_uniform_decodes(): |
| 278 | query_lens = [2, 2, 2, 4, 5, 6, 7, 8] |
| 279 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 280 | apply_split_decodes_and_prefills(query_lens, 4, True) |
| 281 | ) |
| 282 | assert num_decodes == 3 # 2, 2, 2 are all <= 4 and uniform |
| 283 | assert num_prefills == 5 # 4, 5, 6, 7, 8 are all > 4 |
| 284 | assert num_decode_tokens == 6 # 2 + 2 + 2 |
| 285 | assert num_prefill_tokens == 30 # 4 + 5 + 6 + 7 + 8 |
| 286 | |
| 287 | |
| 288 | def test_split_decodes_and_prefills_uniform_mixed_batch_non_uniform_decodes(): |
| 289 | query_lens = [2, 1, 2, 4, 5, 6, 7, 8] |
| 290 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 291 | apply_split_decodes_and_prefills(query_lens, 4, True) |
| 292 | ) |
| 293 | assert num_decodes == 1 # only the first 2 is taken as decode |
| 294 | assert num_prefills == 7 # 1, 2, 4, 5, 6, 7, 8 are all > 4 or non-uniform |
| 295 | assert num_decode_tokens == 2 # only the first 2 |
| 296 | assert num_prefill_tokens == (sum(query_lens) - 2) # rest of the tokens |
| 297 | |
| 298 | |
| 299 | def test_split_decodes_and_prefills_uniform_padded_batch_all_same(): |
| 300 | """Uniform batch where all query lengths are identical with 0 length padded reqs.""" |
| 301 | # All query lengths are 2, with decode_threshold=3 (so 2 <= 3) |
| 302 | # This triggers the padded uniform path at line 891 |
| 303 | query_lens = [2, 2, 2, 0] |
| 304 | padded_num_tokens = 8 |
| 305 | num_decodes, num_prefills, num_decode_tokens, num_prefill_tokens = ( |
| 306 | apply_split_decodes_and_prefills(query_lens, 3, True, padded_num_tokens) |
| 307 | ) |
| 308 | # With uniform batch, all requests are treated as decodes |
| 309 | assert num_decodes == 4 |
| 310 | assert num_prefills == 0 |
| 311 | assert num_decode_tokens == padded_num_tokens |
| 312 | assert num_prefill_tokens == 0 |
| 313 | |
| 314 | |
| 315 | @pytest.mark.parametrize( |
| 316 | "seq_lens,query_lens,split_point,expected_first_reqs,expected_second_reqs", |
| 317 | [ |
| 318 | # Split in the middle of request 1 |
| 319 | ([32, 40], [8, 8], 12, 2, 1), |
| 320 | # Split inside the first request |
| 321 | ([32, 40], [8, 8], 4, 1, 2), |
| 322 | ], |
| 323 | ) |
| 324 | def test_prefill_split_across_ubatches( |
| 325 | seq_lens, query_lens, split_point, expected_first_reqs, expected_second_reqs |
| 326 | ): |
| 327 | """Test splitting a prefill across ubatches.""" |
| 328 | import numpy as np |
| 329 | |
| 330 | device = torch.device("cpu") |
| 331 | batch_spec = BatchSpec(seq_lens=seq_lens, query_lens=query_lens) |
| 332 | common = create_common_attn_metadata(batch_spec, block_size=16, device=device) |
| 333 | |
| 334 | num_scheduled_tokens = np.array(query_lens, dtype=np.int32) |
| 335 | qsl_np = common.query_start_loc_cpu.numpy() |
| 336 | num_tokens = common.num_actual_tokens |
| 337 | |
| 338 | ubatch_slices, _ = maybe_create_ubatch_slices( |
| 339 | True, |
| 340 | num_scheduled_tokens, |
| 341 | num_tokens, |
| 342 | batch_spec.batch_size, |
| 343 | split_point=split_point, |
| 344 | num_ubatches=2, |
| 345 | ) |
| 346 | assert ubatch_slices is not None and len(ubatch_slices) == 2 |
| 347 | |
| 348 | first_meta = _make_metadata_with_slice(ubatch_slices[0], common) |
| 349 | second_meta = _make_metadata_with_slice(ubatch_slices[1], common) |
| 350 | |
| 351 | # Token counts match the split |
| 352 | assert first_meta.num_actual_tokens == split_point |
| 353 | assert second_meta.num_actual_tokens == num_tokens - split_point |
| 354 | |
| 355 | # Number of requests per ubatch |
| 356 | assert first_meta.num_reqs == expected_first_reqs |
| 357 | assert second_meta.num_reqs == expected_second_reqs |
| 358 | |
| 359 | # Identify which request is split and how many tokens are in the first chunk |
| 360 | split_req_idx = int(np.searchsorted(qsl_np, split_point, side="right") - 1) |
| 361 | tokens_in_first_chunk = split_point - int(qsl_np[split_req_idx]) |
| 362 | orig_q_lens = common.query_start_loc_cpu[1:] - common.query_start_loc_cpu[:-1] |
| 363 | |
| 364 | # Check query length continuity: first-chunk + second-chunk == original qlen |
| 365 | # First ubatch last request query length |
| 366 | qlen_first_last = int( |
| 367 | first_meta.query_start_loc_cpu[-1] - first_meta.query_start_loc_cpu[-2] |
| 368 | ) |
| 369 | # Second ubatch first request query length |
| 370 | qlen_second_first = int( |
| 371 | second_meta.query_start_loc_cpu[1] - second_meta.query_start_loc_cpu[0] |
| 372 | ) |
| 373 | assert qlen_first_last == tokens_in_first_chunk |
| 374 | assert qlen_first_last + qlen_second_first == int(orig_q_lens[split_req_idx]) |
| 375 | |
| 376 | # Check seq_lens adjustments |
| 377 | # Context lengths per original request |
| 378 | context_lens = [s - q for s, q in zip(seq_lens, query_lens)] |
| 379 | |
| 380 | # First ubatch: last request's seq_len should be |
| 381 | # context + tokens_in_first_chunk |
| 382 | expected_seqlen = context_lens[split_req_idx] + tokens_in_first_chunk |
| 383 | assert int(first_meta.seq_lens[-1]) == expected_seqlen |
| 384 | |
| 385 | # For full preceding requests in first ubatch, seq_lens should match |
| 386 | # originals |
| 387 | for i in range(first_meta.num_reqs - 1): |
| 388 | assert int(first_meta.seq_lens[i]) == seq_lens[i] |
| 389 | |
| 390 | # Second ubatch: first request (continuation) seq_len should be full |
| 391 | # original |
| 392 | assert int(second_meta.seq_lens[0]) == seq_lens[split_req_idx] |
| 393 | # Any following full requests in second ubatch should match originals |
| 394 | for j in range(1, second_meta.num_reqs): |
| 395 | # Map to original request index |
| 396 | orig_idx = split_req_idx + j |
| 397 | assert int(second_meta.seq_lens[j]) == seq_lens[orig_idx] |
| 398 | |
| 399 | |
| 400 | def test_build_attention_metadata_zeros_stale_is_prefilling(): |
| 401 | """_build_attention_metadata zeroes is_prefilling for padded rows.""" |
| 402 | from unittest.mock import MagicMock, patch |
| 403 | |
| 404 | from vllm.v1.attention.backend import CommonAttentionMetadata |
| 405 | from vllm.v1.worker.gpu_model_runner import GPUModelRunner |
| 406 | |
| 407 | num_reqs = 3 |
| 408 | num_reqs_padded = 5 |
| 409 | |
| 410 | # Real rows [0-2] have known computed/prompt values; padded rows [3-4] |
| 411 | # carry stale data from a prior prefill (num_computed < num_prompt → True). |
| 412 | num_computed = torch.tensor([50, 100, 200, 10, 20], dtype=torch.int32) |
| 413 | num_prompt = torch.tensor([50, 200, 200, 100, 200], dtype=torch.int32) |
| 414 | |
| 415 | runner = MagicMock() |
| 416 | runner.kv_cache_config.kv_cache_groups = [ |
| 417 | MagicMock() |
| 418 | ] # non-empty: skip early return |
| 419 | runner.attn_groups = [[]] # empty inner list: inner loop never runs |
| 420 | runner.input_batch.num_computed_tokens_cpu_tensor = num_computed |
| 421 | runner.input_batch.num_prompt_tokens_cpu_tensor = num_prompt |
| 422 | runner.optimistic_seq_lens_cpu = torch.tensor([100, 200, 300, 0, 0]) |
| 423 | runner.query_start_loc.gpu = torch.zeros(num_reqs_padded + 1, dtype=torch.int32) |
| 424 | runner.query_start_loc.cpu = torch.zeros(num_reqs_padded + 1, dtype=torch.int32) |
| 425 | runner.seq_lens = torch.zeros(num_reqs_padded, dtype=torch.int32) |
| 426 | runner.positions = torch.zeros(num_reqs_padded, dtype=torch.int64) |
| 427 | runner.routed_experts_initialized = False |
| 428 | runner.use_async_spec_decode = False |
| 429 | runner.dcp_world_size = 1 |
| 430 | runner.speculative_config = None |
| 431 | runner.is_mm_prefix_lm = False |
| 432 | runner._get_encoder_seq_lens.return_value = (None, None) |
| 433 | |
| 434 | # Intercept CommonAttentionMetadata construction to capture is_prefilling. |
| 435 | # With speculative_config=None the constructor is called exactly once (for |
| 436 | # cm_base), so captured reflects what the fix produced before storage. |
| 437 | captured_is_prefilling = None |
| 438 | original_init = CommonAttentionMetadata.__init__ |
| 439 | |
| 440 | def capturing_init(self, *args, **kwargs): |
| 441 | nonlocal captured_is_prefilling |
| 442 | if "is_prefilling" in kwargs: |
| 443 | captured_is_prefilling = kwargs["is_prefilling"] |
| 444 | original_init(self, *args, **kwargs) |
| 445 | |
| 446 | with patch.object(CommonAttentionMetadata, "__init__", capturing_init): |
| 447 | GPUModelRunner._build_attention_metadata( |
| 448 | runner, |
| 449 | num_tokens=num_reqs, |
| 450 | num_reqs=num_reqs, |
| 451 | max_query_len=1, |
| 452 | num_tokens_padded=num_reqs_padded, |
| 453 | num_reqs_padded=num_reqs_padded, |
| 454 | slot_mappings={0: torch.zeros(num_reqs_padded, dtype=torch.int64)}, |
| 455 | ) |
| 456 | |
| 457 | assert captured_is_prefilling is not None |
| 458 | assert not captured_is_prefilling[0] # decode (50 >= 50) |
| 459 | assert captured_is_prefilling[1] # prefill (100 < 200) |
| 460 | assert not captured_is_prefilling[2] # decode (200 >= 200) |
| 461 | assert not captured_is_prefilling[3] # stale data (10 < 100) zeroed |
| 462 | assert not captured_is_prefilling[4] # stale data (20 < 200) zeroed |