A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 06:45 UTC
[Perf] Avoid forward scan for async output placeholders (#42938)
Izik Golan committed
May 20, 2026 at 06:16 UTC
2ae910ed88121d7c3acdcb9bab14cd968257b6e6
1 file changed
+6
-1
vllm/v1/worker/gpu_input_batch.py
+6
-1
@@ -1050,7 +1050,12 @@ class InputBatch:
1050
# output placeholders (tokens can be discarded after kv-load
1051
# failure) or a larger number (async spec decode adds optimistic
1052
# placeholders that may exceed the actual acceptance count).
1053
- first_placeholder = req_output_token_ids.index(-1)
1053
+ first_placeholder = len(req_output_token_ids)
1054
+ while (
1055
+ first_placeholder > 0
1056
+ and req_output_token_ids[first_placeholder - 1] == -1
1057
+ ):
1058
+ first_placeholder -= 1
1059
num_placeholders = len(req_output_token_ids) - first_placeholder
1060
num_to_replace = min(num_sampled_ids, num_placeholders)
1061
del new_ids[num_to_replace:]