A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
[V1] Avoid socket errors during shutdown when requests are in in-flight (#16807)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Apr 22, 2025 at 21:36 UTC
b2f195c4294ce72a97d4fdc6ca672ed85b916bd8
2 files changed
+7
-5
vllm/v1/engine/core.py
+1
-1
@@ -384,7 +384,7 @@ class EngineCoreProc(EngineCore):
384
385
except SystemExit:
386
logger.debug("EngineCore exiting.")
387
-
387
+ raise
388
except Exception as e:
389
if engine_core is None:
390
logger.exception("EngineCore failed to start.")
vllm/v1/engine/core_client.py
+6
-4
@@ -312,6 +312,7 @@ class BackgroundResources:
312
def __call__(self):
313
"""Clean up background resources."""
314
315
+ self.engine_dead = True
316
for core_engine in self.core_engines:
317
core_engine.close()
318
@@ -564,7 +565,7 @@ class SyncMPClient(MPClient):
565
self._send_input(EngineCoreRequestType.ADD, request)
566
567
def abort_requests(self, request_ids: list[str]) -> None:
567
- if len(request_ids) > 0:
568
+ if request_ids and not self.resources.engine_dead:
569
self._send_input(EngineCoreRequestType.ABORT, request_ids)
570
571
def profile(self, is_start: bool = True) -> None:
@@ -735,7 +736,7 @@ class AsyncMPClient(MPClient):
736
self._ensure_output_queue_task()
737
738
async def abort_requests_async(self, request_ids: list[str]) -> None:
738
- if len(request_ids) > 0:
739
+ if request_ids and not self.resources.engine_dead:
740
await self._send_input(EngineCoreRequestType.ABORT, request_ids)
741
742
async def profile_async(self, is_start: bool = True) -> None:
@@ -902,5 +903,6 @@ class DPAsyncMPClient(AsyncMPClient):
903
904
async def _abort_requests(self, request_ids: list[str],
905
engine: CoreEngine) -> None:
905
- await self._send_input(EngineCoreRequestType.ABORT, request_ids,
906
- engine)
906
+ if not self.resources.engine_dead:
907
+ await self._send_input(EngineCoreRequestType.ABORT, request_ids,
908
+ engine)