A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[V1][Bugfix] Clean up requests when aborted (#9629)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Oct 23, 2024 at 23:33 UTC
3770071eb4dc97eb728ad68adde027769ee31afe
1 file changed
+12
-3
vllm/v1/engine/llm_engine.py
+12
-3
@@ -300,6 +300,7 @@ class LLMEngine:
300
def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
301
self.scheduler.finish_requests(request_id,
302
RequestStatus.FINISHED_ABORTED)
303
+ self._free_request(request_id)
304
305
def get_num_unfinished_requests(self) -> int:
306
"""Gets the number of unfinished requests."""
@@ -361,6 +362,11 @@ class LLMEngine:
362
num_reqs = len(detokenizer_output.req_ids)
363
for i in range(num_reqs):
364
req_id = detokenizer_output.req_ids[i]
365
+ if req_id not in self.requests:
366
+ # The request has been aborted while the detokenizer was
367
+ # processing the outputs.
368
+ continue
369
+
370
req = self.requests[req_id]
371
req.output_text += detokenizer_output.detokenized_texts[i]
372
@@ -373,9 +379,7 @@ class LLMEngine:
379
req_outputs.append(req_output)
380
381
if finished:
376
- del self.requests[req_id]
377
- del self.num_lagged_steps[req_id]
378
- del self.request_outputs[req_id]
382
+ self._free_request(req_id)
383
return req_outputs
384
385
def terminate_detokenizer(self) -> None:
@@ -440,6 +444,11 @@ class LLMEngine:
444
req_output.finished = finished
445
return req_output
446
447
+ def _free_request(self, request_id: str) -> None:
448
+ self.requests.pop(request_id, None)
449
+ self.num_lagged_steps.pop(request_id, None)
450
+ self.request_outputs.pop(request_id, None)
451
+
452
def check_health(self) -> None:
453
if self.tokenizer:
454
self.tokenizer.check_health()