A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Clear engine reference in AsyncEngineRPCServer (#7618)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
Rui Qiao committed
Aug 16, 2024 at 20:44 UTC
bae888cb8e01b89ad58e3fdaf14f21c109c5fad7
1 file changed
+2
vllm/entrypoints/openai/rpc/server.py
+2
@@ -38,6 +38,8 @@ class AsyncEngineRPCServer:
38
self.socket.close()
39
self.context.destroy()
40
self.engine.shutdown_background_loop()
41
+ # Clear the engine reference so that it can be GC'ed.
42
+ self.engine = None
43
44
async def get_model_config(self, identity):
45
"""Send the ModelConfig"""