A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Abort when coroutine is cancelled (#1020)
Jerry Yang committed
Sep 15, 2023 at 08:40 UTC
b9fe4616f98b77b4b9458bce203aa6544cb31ef2
1 file changed
+3
-2
vllm/engine/async_llm_engine.py
+3
-2
@@ -395,8 +395,9 @@ class AsyncLLMEngine:
395
396
async for request_output in stream:
397
yield request_output
398
- except Exception as e:
399
- # If there is an exception, abort the request.
398
+ except (Exception, asyncio.CancelledError) as e:
399
+ # If there is an exception or coroutine is cancelled, abort the
400
+ # request.
401
self._abort(request_id)
402
raise e
403