A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Optimize] Don't send token ids when kv connector is not used (#20586)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jul 7, 2025 at 19:04 UTC
31c5d0a1b79b0b4621d71a069990d8af43976b08
1 file changed
+5
-1
vllm/v1/core/sched/scheduler.py
+5
-1
@@ -621,6 +621,7 @@ class Scheduler(SchedulerInterface):
621
new_block_ids: list[tuple[list[int], ...]] = []
622
num_computed_tokens: list[int] = []
623
624
+ use_connector = self.connector is not None
625
for req in itertools.chain(running_reqs, resumed_reqs):
626
req_id = req.request_id
627
req_ids.append(req_id)
@@ -635,7 +636,10 @@ class Scheduler(SchedulerInterface):
636
token_ids = req.all_token_ids[req.num_computed_tokens:req.
637
num_computed_tokens + num_tokens]
638
new_token_ids.append(token_ids)
638
- else:
639
+ elif use_connector:
640
+ # When using a KVConnector, we add a placeholder to avoid index
641
+ # out of bounds errors. TODO: Remove this once the KVConnector
642
+ # is updated to handle token IDs properly.
643
new_token_ids.append([])
644
new_block_ids.append(req_to_new_block_ids[req_id])
645
num_computed_tokens.append(req.num_computed_tokens)