A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 07:15 UTC
Fix the order of _free_encoder_inputs (#38907)
Signed-off-by: Tianyu Guo <guoty9@mail2.sysu.edu.cn> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Tianyu Guo committed
Apr 11, 2026 at 13:47 UTC
8d0aabdde9558a337a256375724d8c8ab1ebfd95
1 file changed
+4
-8
vllm/v1/core/sched/scheduler.py
+4
-8
@@ -996,14 +996,6 @@ class Scheduler(SchedulerInterface):
996
request.use_structured_output and not request.is_prefill_chunk
997
)
998
999
- # NOTE: _free_encoder_inputs relies on num_computed_tokens, which
1000
- # may be updated again in _update_from_output for speculative
1001
- # decoding. However, it is safe to call the method here because
1002
- # encoder inputs are always part of the prompt, not the output,
1003
- # and thus are unaffected by speculative decoding.
1004
- if request.has_encoder_inputs:
1005
- self._free_encoder_inputs(request)
1006
-
999
# Clear the finished request IDs.
1000
# NOTE: We shouldn't do self.finished_req_ids.clear() here because
1001
# it will also affect the scheduler output.
@@ -1389,6 +1381,10 @@ class Scheduler(SchedulerInterface):
1381
request_id=req_id,
1382
)
1383
1384
+ # Free encoder inputs only after the step has actually executed.
1385
+ if request.has_encoder_inputs:
1386
+ self._free_encoder_inputs(request)
1387
+
1388
stopped = False
1389
new_logprobs = None
1390
new_token_ids = generated_token_ids