@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e0cc5f259a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[V1][BugFix] Free encoder cache for aborted requests (#12545)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Jan 29, 2025 at 13:47 UTC e0cc5f259a8bec0d66ed0bc3e25ca245377679a1
2 files changed +16 -7
vllm/v1/core/encoder_cache_manager.py
+8 -1
@@ -38,7 +38,8 @@ class EncoderCacheManager:
38 def get_cached_input_ids(self, request: Request) -> Set[int]:
39 return self.cached.get(request.request_id, set())
40
41 - def free(self, request: Request, input_id: int) -> None:
41 + def free_encoder_input(self, request: Request, input_id: int) -> None:
42 + """Free a single encoder input id for the request."""
43 req_id = request.request_id
44 if req_id not in self.cached:
45 return
@@ -49,6 +50,12 @@ class EncoderCacheManager:
50 self.num_free_slots += request.get_num_encoder_tokens(input_id)
51 self.freed.append((req_id, input_id))
52
53 + def free(self, request: Request) -> None:
54 + """Free all cached input ids for the request."""
55 + input_ids = self.get_cached_input_ids(request)
56 + for input_id in input_ids:
57 + self.free_encoder_input(request, input_id)
58 +
59 def get_freed_ids(self) -> List[Tuple[str, int]]:
60 freed = self.freed
61 self.freed = []
vllm/v1/core/scheduler.py
+8 -6
@@ -202,7 +202,7 @@ class Scheduler:
202 # which have output tokens.
203 num_new_tokens = request.num_tokens - num_computed_tokens
204 if num_new_tokens == 0:
205 - # The happens when prompt length is divisible by the block
205 + # This happens when prompt length is divisible by the block
206 # size and all blocks are cached. Now we force to recompute
207 # the last block. Note that we have to re-compute an entire
208 # block because allocate_slots() assumes num_computed_tokens
@@ -269,6 +269,7 @@ class Scheduler:
269
270 # Get the longest common prefix among all requests in the running queue.
271 # This can be potentially used for cascade attention.
272 + num_common_prefix_blocks = 0
273 if self.running:
274 any_request = self.running[0]
275 num_common_prefix_blocks = (
@@ -433,7 +434,8 @@ class Scheduler:
434 if start_pos + num_tokens <= request.num_computed_tokens:
435 # The encoder output is already processed and stored
436 # in the decoder's KV cache.
436 - self.encoder_cache_manager.free(request, input_id)
437 + self.encoder_cache_manager.free_encoder_input(
438 + request, input_id)
439
440 if request.num_computed_tokens == request.num_tokens:
441 req_index = model_runner_output.req_id_to_index[req_id]
@@ -445,8 +447,10 @@ class Scheduler:
447 # TODO: Update the KV cache manager for prefix caching.
448
449 # Check for stop and update request state.
448 - # This must be called before me make the EngineCoreOutput.
450 + # This must be called before we make the EngineCoreOutput.
451 stopped = self._check_stop(request)
452 + if stopped:
453 + self._free_request(request)
454
455 # Add EngineCoreOutput for this Request.
456 output = EngineCoreOutput(
@@ -472,7 +476,6 @@ class Scheduler:
476 if (request.num_tokens >= self.max_model_len
477 or request.num_output_tokens >= request.max_tokens):
478 request.status = RequestStatus.FINISHED_LENGTH_CAPPED
475 - self._free_request(request)
479 return True
480
481 sampling_params = request.sampling_params
@@ -480,13 +483,11 @@ class Scheduler:
483 if (not sampling_params.ignore_eos
484 and last_token_id == request.eos_token_id):
485 request.status = RequestStatus.FINISHED_STOPPED
483 - self._free_request(request)
486 return True
487
488 if last_token_id in (sampling_params.stop_token_ids or ()):
489 request.status = RequestStatus.FINISHED_STOPPED
490 request.stop_reason = last_token_id
489 - self._free_request(request)
491 return True
492 return False
493
@@ -525,6 +526,7 @@ class Scheduler:
526 def _free_request(self, request: Request) -> None:
527 assert request.is_finished()
528 self.kv_cache_manager.free(request)
529 + self.encoder_cache_manager.free(request)
530 self.running_reqs_data.pop(request.request_id, None)
531 del self.requests[request.request_id]
532 self.finished_req_ids.add(request.request_id)