@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3770071eb4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[V1][Bugfix] Clean up requests when aborted (#9629)

Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Woosuk Kwon committed Oct 23, 2024 at 23:33 UTC 3770071eb4dc97eb728ad68adde027769ee31afe
1 file changed +12 -3
vllm/v1/engine/llm_engine.py
+12 -3
@@ -300,6 +300,7 @@ class LLMEngine:
300 def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
301 self.scheduler.finish_requests(request_id,
302 RequestStatus.FINISHED_ABORTED)
303 + self._free_request(request_id)
304
305 def get_num_unfinished_requests(self) -> int:
306 """Gets the number of unfinished requests."""
@@ -361,6 +362,11 @@ class LLMEngine:
362 num_reqs = len(detokenizer_output.req_ids)
363 for i in range(num_reqs):
364 req_id = detokenizer_output.req_ids[i]
365 + if req_id not in self.requests:
366 + # The request has been aborted while the detokenizer was
367 + # processing the outputs.
368 + continue
369 +
370 req = self.requests[req_id]
371 req.output_text += detokenizer_output.detokenized_texts[i]
372
@@ -373,9 +379,7 @@ class LLMEngine:
379 req_outputs.append(req_output)
380
381 if finished:
376 - del self.requests[req_id]
377 - del self.num_lagged_steps[req_id]
378 - del self.request_outputs[req_id]
382 + self._free_request(req_id)
383 return req_outputs
384
385 def terminate_detokenizer(self) -> None:
@@ -440,6 +444,11 @@ class LLMEngine:
444 req_output.finished = finished
445 return req_output
446
447 + def _free_request(self, request_id: str) -> None:
448 + self.requests.pop(request_id, None)
449 + self.num_lagged_steps.pop(request_id, None)
450 + self.request_outputs.pop(request_id, None)
451 +
452 def check_health(self) -> None:
453 if self.tokenizer:
454 self.tokenizer.check_health()