A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 07:15 UTC
[Core] Fix abrupt request abort (#18485)
Signed-off-by: nicklucche <nlucches@redhat.com> Signed-off-by: Nick Hill <nhill@redhat.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Nicolò Lucchesi committed
Jun 7, 2025 at 01:27 UTC
b6a3a9f76db737e8b3679d824b8cd744e883e7fd
2 files changed
+9
-7
vllm/v1/core/kv_cache_coordinator.py
+1
-1
@@ -164,7 +164,7 @@ class KVCacheCoordinator(ABC):
164
Get the blocks for the request.
165
"""
166
return [
167
- manager.req_to_blocks[request_id]
167
+ manager.req_to_blocks.get(request_id) or []
168
for manager in self.single_type_managers
169
]
170
vllm/v1/core/sched/scheduler.py
+8
-6
@@ -76,6 +76,9 @@ class Scheduler(SchedulerInterface):
76
# KV Connector pushes/pull of remote KVs for P/D and offloading.
77
self.connector = None
78
if self.vllm_config.kv_transfer_config is not None:
79
+ assert len(self.kv_cache_config.kv_cache_groups) == 1, (
80
+ "Multiple KV cache groups are not currently supported "
81
+ "with KV connectors")
82
self.connector = KVConnectorFactory.create_connector_v1(
83
config=self.vllm_config, role=KVConnectorRole.SCHEDULER)
84
@@ -985,9 +988,8 @@ class Scheduler(SchedulerInterface):
988
"""
989
if self.connector is None:
990
return False, None
988
- assert len(self.kv_cache_config.kv_cache_groups
989
- ) == 1, "KV connector only supports one KV cache group now"
990
- block_ids = self.kv_cache_manager.get_block_ids(request.request_id)[0]
991
+
992
+ (block_ids, ) = self.kv_cache_manager.get_block_ids(request.request_id)
993
return self.connector.request_finished(request, block_ids)
994
995
def _update_waiting_for_remote_kv(self, request: Request) -> bool:
@@ -1002,12 +1004,12 @@ class Scheduler(SchedulerInterface):
1004
and the request state will be moved back to WAITING from
1005
WAITING_FOR_REMOTE_KV.
1006
"""
1007
+ assert self.connector is not None
1008
if request.request_id not in self.finished_recving_kv_req_ids:
1009
return False
1007
- assert len(self.kv_cache_config.kv_cache_groups
1008
- ) == 1, "KV connector only supports one KV cache group now"
1010
+
1011
# Now that the blocks are ready, actually cache them.
1010
- block_ids = self.kv_cache_manager.get_block_ids(request.request_id)[0]
1012
+ (block_ids, ) = self.kv_cache_manager.get_block_ids(request.request_id)
1013
num_computed_tokens = len(block_ids) * self.block_size
1014
# Handle the case where num request tokens less then one block.
1015
num_computed_tokens = min(num_computed_tokens, request.num_tokens)