A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[V1][BugFix] Free encoder cache for aborted requests (#12545)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jan 29, 2025 at 13:47 UTC
e0cc5f259a8bec0d66ed0bc3e25ca245377679a1
2 files changed
+16
-7
vllm/v1/core/encoder_cache_manager.py
+8
-1
@@ -38,7 +38,8 @@ class EncoderCacheManager:
38
def get_cached_input_ids(self, request: Request) -> Set[int]:
39
return self.cached.get(request.request_id, set())
40
41
- def free(self, request: Request, input_id: int) -> None:
41
+ def free_encoder_input(self, request: Request, input_id: int) -> None:
42
+ """Free a single encoder input id for the request."""
43
req_id = request.request_id
44
if req_id not in self.cached:
45
return
@@ -49,6 +50,12 @@ class EncoderCacheManager:
50
self.num_free_slots += request.get_num_encoder_tokens(input_id)
51
self.freed.append((req_id, input_id))
52
53
+ def free(self, request: Request) -> None:
54
+ """Free all cached input ids for the request."""
55
+ input_ids = self.get_cached_input_ids(request)
56
+ for input_id in input_ids:
57
+ self.free_encoder_input(request, input_id)
58
+
59
def get_freed_ids(self) -> List[Tuple[str, int]]:
60
freed = self.freed
61
self.freed = []
vllm/v1/core/scheduler.py
+8
-6
@@ -202,7 +202,7 @@ class Scheduler:
202
# which have output tokens.
203
num_new_tokens = request.num_tokens - num_computed_tokens
204
if num_new_tokens == 0:
205
- # The happens when prompt length is divisible by the block
205
+ # This happens when prompt length is divisible by the block
206
# size and all blocks are cached. Now we force to recompute
207
# the last block. Note that we have to re-compute an entire
208
# block because allocate_slots() assumes num_computed_tokens
@@ -269,6 +269,7 @@ class Scheduler:
269
270
# Get the longest common prefix among all requests in the running queue.
271
# This can be potentially used for cascade attention.
272
+ num_common_prefix_blocks = 0
273
if self.running:
274
any_request = self.running[0]
275
num_common_prefix_blocks = (
@@ -433,7 +434,8 @@ class Scheduler:
434
if start_pos + num_tokens <= request.num_computed_tokens:
435
# The encoder output is already processed and stored
436
# in the decoder's KV cache.
436
- self.encoder_cache_manager.free(request, input_id)
437
+ self.encoder_cache_manager.free_encoder_input(
438
+ request, input_id)
439
440
if request.num_computed_tokens == request.num_tokens:
441
req_index = model_runner_output.req_id_to_index[req_id]
@@ -445,8 +447,10 @@ class Scheduler:
447
# TODO: Update the KV cache manager for prefix caching.
448
449
# Check for stop and update request state.
448
- # This must be called before me make the EngineCoreOutput.
450
+ # This must be called before we make the EngineCoreOutput.
451
stopped = self._check_stop(request)
452
+ if stopped:
453
+ self._free_request(request)
454
455
# Add EngineCoreOutput for this Request.
456
output = EngineCoreOutput(
@@ -472,7 +476,6 @@ class Scheduler:
476
if (request.num_tokens >= self.max_model_len
477
or request.num_output_tokens >= request.max_tokens):
478
request.status = RequestStatus.FINISHED_LENGTH_CAPPED
475
- self._free_request(request)
479
return True
480
481
sampling_params = request.sampling_params
@@ -480,13 +483,11 @@ class Scheduler:
483
if (not sampling_params.ignore_eos
484
and last_token_id == request.eos_token_id):
485
request.status = RequestStatus.FINISHED_STOPPED
483
- self._free_request(request)
486
return True
487
488
if last_token_id in (sampling_params.stop_token_ids or ()):
489
request.status = RequestStatus.FINISHED_STOPPED
490
request.stop_reason = last_token_id
489
- self._free_request(request)
491
return True
492
return False
493
@@ -525,6 +526,7 @@ class Scheduler:
526
def _free_request(self, request: Request) -> None:
527
assert request.is_finished()
528
self.kv_cache_manager.free(request)
529
+ self.encoder_cache_manager.free(request)
530
self.running_reqs_data.pop(request.request_id, None)
531
del self.requests[request.request_id]
532
self.finished_req_ids.add(request.request_id)