A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 11:00 UTC
[v1][KVCacheManager] pass num_new_computed_tokens to kv cache manager (#18001)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
Chen Zhang committed
May 14, 2025 at 10:09 UTC
f2ae883b672493ff83d1f372b6960a21d93b5ea9
3 files changed
+119
-53
tests/v1/core/test_prefix_caching.py
+93
-33
@@ -81,7 +81,9 @@ def test_prefill(hash_algo):
81
assert len(manager.req_to_block_hashes[req0.request_id]) == 3
82
assert not computed_blocks.blocks
83
assert num_computed_tokens == 0
84
- blocks = manager.allocate_slots(req0, 55, computed_blocks)
84
+ blocks = manager.allocate_slots(req0, 55,
85
+ len(computed_blocks.blocks) * 16,
86
+ computed_blocks)
87
assert blocks.get_block_ids() == [1, 2, 3, 4]
88
89
# Check full block metadata
@@ -108,7 +110,9 @@ def test_prefill(hash_algo):
110
assert computed_blocks.get_block_ids() == [1, 2, 3]
111
assert num_computed_tokens == 3 * 16
112
num_new_tokens = 53 - 3 * 16
111
- blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
113
+ blocks = manager.allocate_slots(req1, num_new_tokens,
114
+ len(computed_blocks.blocks) * 16,
115
+ computed_blocks)
116
assert blocks.get_block_ids() == [5]
117
for block in computed_blocks.blocks:
118
assert block.ref_cnt == 2
@@ -140,7 +144,9 @@ def test_prefill(hash_algo):
144
assert computed_blocks.get_block_ids() == [1, 2, 3]
145
assert num_computed_tokens == 3 * 16
146
num_new_tokens = 53 - 3 * 16
143
- blocks = manager.allocate_slots(req2, num_new_tokens, computed_blocks)
147
+ blocks = manager.allocate_slots(req2, num_new_tokens,
148
+ len(computed_blocks.blocks) * 16,
149
+ computed_blocks)
150
assert blocks.get_block_ids() == [6]
151
152
# Although we only have 6 free blocks, we have 8 blocks in
@@ -161,7 +167,9 @@ def test_prefill(hash_algo):
167
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
168
assert not computed_blocks.blocks
169
assert num_computed_tokens == 0
164
- blocks = manager.allocate_slots(req3, 16 * 10, computed_blocks)
170
+ blocks = manager.allocate_slots(req3, 16 * 10,
171
+ len(computed_blocks.blocks) * 16,
172
+ computed_blocks)
173
# This block ID order also checks the eviction order.
174
assert blocks.get_block_ids() == [7, 8, 9, 10, 4, 5, 6, 3, 2, 1]
175
assert manager.block_pool.free_block_queue.num_free_blocks == 0
@@ -197,7 +205,9 @@ def test_prefill_plp():
205
assert len(manager.req_to_block_hashes[req0.request_id]) == 0
206
assert not computed_blocks.blocks
207
assert num_computed_tokens == 0
200
- blocks = manager.allocate_slots(req0, 55, computed_blocks)
208
+ blocks = manager.allocate_slots(req0, 55,
209
+ len(computed_blocks.blocks) * 16,
210
+ computed_blocks)
211
assert blocks.get_block_ids() == [1, 2, 3, 4]
212
req0_block_hashes = [b.block_hash for b in blocks.blocks]
213
@@ -226,7 +236,9 @@ def test_prefill_plp():
236
assert computed_blocks.get_block_ids() == [1, 2, 3]
237
assert num_computed_tokens == 3 * 16
238
num_new_tokens = 53 - 3 * 16
229
- blocks = manager.allocate_slots(req1, num_new_tokens, computed_blocks)
239
+ blocks = manager.allocate_slots(req1, num_new_tokens,
240
+ len(computed_blocks.blocks) * 16,
241
+ computed_blocks)
242
assert blocks.get_block_ids() == [5]
243
for block in computed_blocks.blocks:
244
assert block.ref_cnt == 2
@@ -259,7 +271,9 @@ def test_prefill_plp():
271
assert len(manager.req_to_block_hashes[req2.request_id]) == 0
272
assert not computed_blocks.blocks
273
assert num_computed_tokens == 0
262
- blocks = manager.allocate_slots(req2, 55, computed_blocks)
274
+ blocks = manager.allocate_slots(req2, 55,
275
+ len(computed_blocks.blocks) * 16,
276
+ computed_blocks)
277
block_ids = blocks.get_block_ids()
278
# Duplicate cached blocks have different ids but same hashes vs request #0
279
assert [b.block_hash for b in blocks.blocks] == req0_block_hashes
@@ -290,14 +304,18 @@ def test_decode():
304
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
305
assert not computed_blocks.blocks
306
assert num_computed_tokens == 0
293
- blocks = manager.allocate_slots(req0, 55, computed_blocks)
307
+ blocks = manager.allocate_slots(req0, 55,
308
+ len(computed_blocks.blocks) * 16,
309
+ computed_blocks)
310
assert blocks.get_block_ids() == [1, 2, 3, 4]
311
312
# Append slots without allocating a new block.
313
req0.num_computed_tokens = 55
314
for _ in range(4):
315
req0.append_output_token_ids(8)
300
- new_blocks = manager.allocate_slots(req0, 4)
316
+ new_blocks = manager.allocate_slots(req0, 4,
317
+ len(computed_blocks.blocks) * 16,
318
+ computed_blocks)
319
assert new_blocks is not None and len(new_blocks.blocks) == 0
320
assert manager.single_type_manager.req_to_blocks[
321
req0.request_id][-1].block_hash is None
@@ -308,7 +326,9 @@ def test_decode():
326
# the preallocated block.
327
for _ in range(9 + 10):
328
req0.append_output_token_ids(7)
311
- new_blocks = manager.allocate_slots(req0, 19)
329
+ new_blocks = manager.allocate_slots(req0, 19,
330
+ len(computed_blocks.blocks) * 16,
331
+ computed_blocks)
332
assert new_blocks is not None and len(new_blocks.blocks) == 1
333
assert manager.single_type_manager.req_to_blocks[
334
req0.request_id][-2].block_hash is not None
@@ -328,7 +348,9 @@ def test_evict():
348
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
349
assert not computed_blocks.blocks
350
assert num_computed_tokens == 0
331
- blocks = manager.allocate_slots(req0, 5 * 16 + 7, computed_blocks)
351
+ blocks = manager.allocate_slots(req0, 5 * 16 + 7,
352
+ len(computed_blocks.blocks) * 16,
353
+ computed_blocks)
354
assert len(blocks.blocks) == 6 # 5 full + 1 partial
355
356
# 3 blocks.
@@ -337,7 +359,9 @@ def test_evict():
359
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
360
assert not computed_blocks.blocks
361
assert num_computed_tokens == 0
340
- blocks = manager.allocate_slots(req1, 3 * 16, computed_blocks)
362
+ blocks = manager.allocate_slots(req1, 3 * 16,
363
+ len(computed_blocks.blocks) * 16,
364
+ computed_blocks)
365
assert len(blocks.blocks) == 3 # 3 full blocks
366
last_token_id += 3 * 16
367
@@ -357,7 +381,9 @@ def test_evict():
381
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
382
assert computed_blocks.get_block_ids() == [1, 2]
383
assert num_computed_tokens == 2 * 16
360
- blocks = manager.allocate_slots(req2, 3, computed_blocks)
384
+ blocks = manager.allocate_slots(req2, 3,
385
+ len(computed_blocks.blocks) * 16,
386
+ computed_blocks)
387
assert blocks.get_block_ids() == [10]
388
assert manager.block_pool.free_block_queue.num_free_blocks == 7
389
@@ -380,7 +406,9 @@ def test_hash_block_correct_reuse():
406
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
407
assert not computed_blocks.blocks
408
assert num_computed_tokens == 0
383
- blocks = manager.allocate_slots(req, num_tokens, computed_blocks)
409
+ blocks = manager.allocate_slots(req, num_tokens,
410
+ len(computed_blocks.blocks) * 16,
411
+ computed_blocks)
412
assert len(blocks.blocks) == 1
413
414
# Deallocate the block.
@@ -392,7 +420,9 @@ def test_hash_block_correct_reuse():
420
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
421
assert not computed_blocks.blocks
422
assert num_computed_tokens == 0
395
- blocks = manager.allocate_slots(req, num_tokens - 1, computed_blocks)
423
+ blocks = manager.allocate_slots(req, num_tokens - 1,
424
+ len(computed_blocks.blocks) * 16,
425
+ computed_blocks)
426
assert len(blocks.blocks) == 1
427
428
assert manager.block_pool.blocks[
@@ -417,7 +447,9 @@ def test_computed_blocks_not_evicted():
447
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
448
assert not computed_blocks.blocks
449
assert num_computed_tokens == 0
420
- blocks = manager.allocate_slots(req0, num_tokens, computed_blocks)
450
+ blocks = manager.allocate_slots(req0, num_tokens,
451
+ len(computed_blocks.blocks) * 16,
452
+ computed_blocks)
453
assert len(blocks.blocks) == 1
454
assert blocks.blocks[0].block_id == 1
455
@@ -426,7 +458,9 @@ def test_computed_blocks_not_evicted():
458
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
459
assert not computed_blocks.blocks
460
assert num_computed_tokens == 0
429
- blocks = manager.allocate_slots(req1, num_tokens, computed_blocks)
461
+ blocks = manager.allocate_slots(req1, num_tokens,
462
+ len(computed_blocks.blocks) * 16,
463
+ computed_blocks)
464
assert len(blocks.blocks) == 1
465
assert blocks.blocks[0].block_id == 2
466
@@ -443,6 +477,7 @@ def test_computed_blocks_not_evicted():
477
assert num_computed_tokens == block_size
478
479
blocks = manager.allocate_slots(req2, num_tokens * 2 - num_tokens,
480
+ len(computed_blocks.blocks) * 16,
481
computed_blocks)
482
assert len(blocks.blocks) == 1
483
assert blocks.blocks[0].block_id == 2
@@ -464,7 +499,9 @@ def test_basic_prefix_caching_disabled():
499
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
500
assert not computed_blocks.blocks
501
assert num_computed_tokens == 0
467
- blocks = manager.allocate_slots(req1, 10, computed_blocks)
502
+ blocks = manager.allocate_slots(req1, 10,
503
+ len(computed_blocks.blocks) * 16,
504
+ computed_blocks)
505
assert len(blocks.blocks) == 3
506
507
# Free the blocks.
@@ -475,7 +512,9 @@ def test_basic_prefix_caching_disabled():
512
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
513
assert not computed_blocks.blocks
514
assert num_computed_tokens == 0
478
- blocks = manager.allocate_slots(req2, 16, computed_blocks)
515
+ blocks = manager.allocate_slots(req2, 16,
516
+ len(computed_blocks.blocks) * 16,
517
+ computed_blocks)
518
assert len(blocks.blocks) == 4
519
520
# New requests should not have any blocks.
@@ -483,7 +522,9 @@ def test_basic_prefix_caching_disabled():
522
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req3)
523
assert not computed_blocks.blocks
524
assert num_computed_tokens == 0
486
- blocks = manager.allocate_slots(req3, 4, computed_blocks)
525
+ blocks = manager.allocate_slots(req3, 4,
526
+ len(computed_blocks.blocks) * 16,
527
+ computed_blocks)
528
assert not blocks
529
530
@@ -581,14 +622,18 @@ def test_mm_prefix_caching():
622
assert block_hashes[1].extra_keys == ("aaa", "bbb")
623
assert block_hashes[2].extra_keys == ("bbb", )
624
584
- blocks = manager.allocate_slots(req0, 59, computed_blocks)
625
+ blocks = manager.allocate_slots(req0, 59,
626
+ len(computed_blocks.blocks) * 16,
627
+ computed_blocks)
628
assert blocks.get_block_ids() == [1, 2, 3, 4]
629
req0.num_computed_tokens = 59
630
631
# Append slots without allocating a new block.
632
for _ in range(5):
633
req0.append_output_token_ids(8)
591
- new_blocks = manager.allocate_slots(req0, 5)
634
+ new_blocks = manager.allocate_slots(req0, 5,
635
+ len(computed_blocks.blocks) * 16,
636
+ computed_blocks)
637
assert new_blocks is not None and len(new_blocks.blocks) == 0
638
639
# The just completed block should have hashes with extra keys.
@@ -638,14 +683,18 @@ def test_cache_key_salting():
683
assert block_hashes[1].extra_keys is None
684
assert block_hashes[2].extra_keys is None
685
641
- blocks = manager.allocate_slots(req0, 59, computed_blocks)
686
+ blocks = manager.allocate_slots(req0, 59,
687
+ len(computed_blocks.blocks) * 16,
688
+ computed_blocks)
689
assert blocks.get_block_ids() == [1, 2, 3, 4]
690
req0.num_computed_tokens = 59
691
692
# Append slots without allocating a new block.
693
for _ in range(5):
694
req0.append_output_token_ids(8)
648
- new_blocks = manager.allocate_slots(req0, 5)
695
+ new_blocks = manager.allocate_slots(req0, 5,
696
+ len(computed_blocks.blocks) * 16,
697
+ computed_blocks)
698
assert new_blocks is not None and len(new_blocks.blocks) == 0
699
700
# Now one more block that should not have extra keys.
@@ -691,7 +740,8 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
740
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req0)
741
assert not computed_blocks.blocks
742
assert num_computed_tokens == 0
694
- manager.allocate_slots(req0, 48, computed_blocks)
743
+ manager.allocate_slots(req0, 48,
744
+ len(computed_blocks.blocks) * 16, computed_blocks)
745
block_part0 = manager.single_type_manager.req_to_blocks[req0.request_id]
746
747
# | Common-0 | Common-1 | Common-2 | Req1-3 | Req1-4 | Req1-5 | ... |
@@ -699,7 +749,8 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
749
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req1)
750
assert computed_blocks.blocks == block_part0
751
assert num_computed_tokens == 3 * 16
702
- manager.allocate_slots(req1, 48, computed_blocks)
752
+ manager.allocate_slots(req1, 48,
753
+ len(computed_blocks.blocks) * 16, computed_blocks)
754
block_part1 = manager.single_type_manager.req_to_blocks[req1.request_id]
755
# | Common-0 | Common-1 | Common-2 | Req1-3 (F) | Req1-4 (F) |
756
# | Req1-5(F)| ... |
@@ -713,7 +764,8 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
764
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req2)
765
assert not computed_blocks.blocks
766
assert num_computed_tokens == 0
716
- manager.allocate_slots(req2, block_size * 2, computed_blocks)
767
+ manager.allocate_slots(req2, block_size * 2,
768
+ len(computed_blocks.blocks) * 16, computed_blocks)
769
770
# Req3 is Req2 + 3 new blocks, so the first 6 blocks are computed,
771
# but it cannot be allocated due to insufficient free blocks (2).
@@ -724,7 +776,9 @@ def test_prefill_not_enough_free_blocks_with_computed_blocks():
776
assert computed_blocks.blocks == block_part1
777
assert num_computed_tokens == 6 * 16
778
# Req3 cannot be allocated.
727
- assert manager.allocate_slots(req3, 48, computed_blocks) is None
779
+ assert manager.allocate_slots(req3, 48,
780
+ len(computed_blocks.blocks) * 16,
781
+ computed_blocks) is None
782
# Block 0-2 are used by Req 1.
783
assert {block.ref_cnt for block in block_part1[:3]} == {1}
784
# Block 3-5 are free.
@@ -751,7 +805,9 @@ def test_reset_prefix_cache():
805
computed_blocks, _ = manager.get_computed_blocks(req1)
806
assert len(manager.req_to_block_hashes[req1.request_id]) == 3
807
assert len(computed_blocks.blocks) == 3
754
- blocks = manager.allocate_slots(req1, 7, computed_blocks)
808
+ blocks = manager.allocate_slots(req1, 7,
809
+ len(computed_blocks.blocks) * 16,
810
+ computed_blocks)
811
assert blocks.get_block_ids() == [5]
812
813
# Failed to reset prefix cache because some blocks are not freed yet.
@@ -782,7 +838,8 @@ def test_prefix_cache_stats_disabled():
838
computed_blocks, num_computed_tokens = manager.get_computed_blocks(req)
839
assert not computed_blocks.blocks
840
assert num_computed_tokens == 0
785
- manager.allocate_slots(req, 16, computed_blocks)
841
+ manager.allocate_slots(req, 16,
842
+ len(computed_blocks.blocks) * 16, computed_blocks)
843
manager.reset_prefix_cache()
844
845
# Ensure prefix_cache_stats remains None
@@ -860,7 +917,8 @@ def test_eagle_enabled_removes_last_block():
917
918
# Prime the cache
919
computed_blocks, _ = manager.get_computed_blocks(req)
863
- manager.allocate_slots(req, len(token_ids), computed_blocks)
920
+ manager.allocate_slots(req, len(token_ids),
921
+ len(computed_blocks.blocks) * 16, computed_blocks)
922
manager.free(req)
923
924
# New request with same tokens + Eagle enabled
@@ -889,7 +947,8 @@ def test_eagle_with_partial_blocks():
947
948
# Prime the cache
949
computed_blocks, _ = manager.get_computed_blocks(req)
892
- manager.allocate_slots(req, len(token_ids), computed_blocks)
950
+ manager.allocate_slots(req, len(token_ids),
951
+ len(computed_blocks.blocks) * 16, computed_blocks)
952
manager.free(req)
953
954
# New request with Eagle enabled
@@ -928,7 +987,8 @@ def test_eagle_with_sliding_window():
987
988
# Prime the cache
989
computed_blocks, _ = manager.get_computed_blocks(req)
931
- manager.allocate_slots(req, len(token_ids), computed_blocks)
990
+ manager.allocate_slots(req, len(token_ids),
991
+ len(computed_blocks.blocks) * 16, computed_blocks)
992
# record the block hash of the first block in the request for later use
993
block_hash_first_block = manager.req_to_block_hashes[req.request_id][0]
994
assert block_hash_first_block is not None
vllm/v1/core/kv_cache_manager.py
+6
-10
@@ -121,13 +121,6 @@ class KVCacheManager:
121
- A list of blocks that are computed for the request.
122
- The number of computed tokens.
123
"""
124
-
125
- # Request already has blocks from async load via KVConnector.
126
- num_existing_blocks = len(
127
- self.single_type_manager.req_to_blocks[request.request_id])
128
- if num_existing_blocks > 0:
129
- return KVCacheBlocks.create_empty(), request.num_computed_tokens
130
-
124
# Prefix caching is disabled or
125
# When the request requires prompt logprobs, we skip prefix caching.
126
if (not self.enable_caching
@@ -172,6 +165,7 @@ class KVCacheManager:
165
self,
166
request: Request,
167
num_new_tokens: int,
168
+ num_new_computed_tokens: int = 0,
169
new_computed_blocks: Optional[KVCacheBlocks] = None,
170
num_lookahead_tokens: int = 0,
171
delay_cache_blocks: bool = False,
@@ -183,8 +177,10 @@ class KVCacheManager:
177
num_new_tokens: The number of tokens to allocate, including external
178
tokens. Note that this does not include tokens that have
179
already been computed locally (i.e. new_computed_blocks).
186
- new_computed_blocks: The new computed blocks just hitting the
187
- prefix caching.
180
+ num_new_computed_tokens: The number of new computed tokens just
181
+ hitting the prefix caching, excluding external tokens.
182
+ new_computed_blocks: The cached blocks for the above new computed
183
+ tokens.
184
num_lookahead_tokens: The number of speculative tokens to allocate.
185
This is used by spec decode proposers with kv-cache such
186
as eagle.
@@ -229,7 +225,7 @@ class KVCacheManager:
225
# The number of computed tokens is the number of computed tokens plus
226
# the new prefix caching hits
227
num_computed_tokens = (request.num_computed_tokens +
232
- len(new_computed_block_list) * self.block_size)
228
+ num_new_computed_tokens)
229
num_tokens_need_slot = min(
230
num_computed_tokens + num_new_tokens + num_lookahead_tokens,
231
self.max_model_len)
vllm/v1/core/sched/scheduler.py
+20
-10
@@ -18,7 +18,7 @@ from vllm.logger import init_logger
18
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
19
from vllm.v1.core.encoder_cache_manager import (EncoderCacheManager,
20
compute_encoder_budget)
21
-from vllm.v1.core.kv_cache_manager import KVCacheManager
21
+from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
22
from vllm.v1.core.sched.interface import SchedulerInterface
23
from vllm.v1.core.sched.output import (CachedRequestData, NewRequestData,
24
SchedulerOutput)
@@ -311,12 +311,14 @@ class Scheduler(SchedulerInterface):
311
break
312
313
request = self.waiting[0]
314
-
314
+ num_prealloc_computed_tokens = 0
315
# P/D: skip request if still waiting for remote kvs.
316
if request.status == RequestStatus.WAITING_FOR_REMOTE_KVS:
317
is_ready = self._update_waiting_for_remote_kv(request)
318
if is_ready:
319
request.status = RequestStatus.WAITING
320
+ num_prealloc_computed_tokens = (
321
+ request.num_computed_tokens)
322
else:
323
self.waiting.popleft()
324
skipped_waiting_requests.appendleft(request)
@@ -345,18 +347,25 @@ class Scheduler(SchedulerInterface):
347
continue
348
349
# Get already-cached tokens.
348
- new_computed_blocks, num_computed_tokens = \
349
- self.kv_cache_manager.get_computed_blocks(
350
- request)
350
+ if num_prealloc_computed_tokens == 0:
351
+ new_computed_blocks, num_native_computed_tokens = \
352
+ self.kv_cache_manager.get_computed_blocks(
353
+ request)
354
+ else:
355
+ # P/D: skip checking prefix cache if loaded from remote kvs.
356
+ new_computed_blocks = KVCacheBlocks.create_empty()
357
+ num_native_computed_tokens = 0
358
359
# Get externally-cached tokens if using a KVConnector.
353
- num_external_tokens, load_kv_async = (
360
+ num_external_computed_tokens, load_kv_async = (
361
(0, False) if self.connector is None else
362
self.connector.get_num_new_matched_tokens(
356
- request, num_computed_tokens))
363
+ request, num_native_computed_tokens))
364
365
# Total computed tokens (local + external).
359
- num_computed_tokens += num_external_tokens
366
+ num_computed_tokens = (num_native_computed_tokens +
367
+ num_external_computed_tokens +
368
+ num_prealloc_computed_tokens)
369
370
encoder_inputs_to_schedule = None
371
new_encoder_budget = encoder_budget
@@ -390,7 +399,8 @@ class Scheduler(SchedulerInterface):
399
400
new_blocks = self.kv_cache_manager.allocate_slots(
401
request,
393
- num_new_tokens + num_external_tokens,
402
+ num_new_tokens + num_external_computed_tokens,
403
+ num_native_computed_tokens,
404
new_computed_blocks,
405
num_lookahead_tokens=self.num_lookahead_tokens,
406
delay_cache_blocks=load_kv_async,
@@ -406,7 +416,7 @@ class Scheduler(SchedulerInterface):
416
self.connector.update_state_after_alloc(
417
request,
418
new_computed_blocks + new_blocks,
409
- num_external_tokens,
419
+ num_external_computed_tokens,
420
)
421
422
self.waiting.popleft()