A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[BugFix] Honor `enable_caching` in connector-delayed kvcache load case (#19435)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Jun 13, 2025 at 02:46 UTC
7e8d97dd3f0aaf05265f947997310ca3827d3c06
2 files changed
+6
-4
vllm/v1/core/kv_cache_manager.py
+5
-4
@@ -381,10 +381,11 @@ class KVCacheManager:
381
self.coordinator.get_blocks(request_id)).get_block_ids()
382
383
def cache_blocks(self, request: Request, num_computed_tokens: int) -> None:
384
- """Cache the blocks for the request."""
385
- block_hashes = self.req_to_block_hashes[request.request_id]
386
- self.coordinator.cache_blocks(request, block_hashes,
387
- num_computed_tokens)
384
+ """Cache the blocks for the request, if enabled."""
385
+ if self.enable_caching:
386
+ block_hashes = self.req_to_block_hashes[request.request_id]
387
+ self.coordinator.cache_blocks(request, block_hashes,
388
+ num_computed_tokens)
389
390
def create_empty_block_list(self) -> KVCacheBlocks:
391
"""Creates a new KVCacheBlocks instance with no blocks."""
vllm/v1/core/sched/scheduler.py
+1
@@ -1015,6 +1015,7 @@ class Scheduler(SchedulerInterface):
1015
num_computed_tokens = min(num_computed_tokens, request.num_tokens)
1016
if num_computed_tokens == request.num_tokens:
1017
num_computed_tokens -= 1
1018
+ # This will cache the blocks iff caching is enabled.
1019
self.kv_cache_manager.cache_blocks(request, num_computed_tokens)
1020
1021
# Update the request state for scheduling.