A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Add `shutdown()` method to `ExecutorBase` (#4349)
Nick Hill committed
Apr 25, 2024 at 16:32 UTC
15e7c675b0dc36109c7b591f856f102e96493a94
2 files changed
+13
vllm/engine/llm_engine.py
+6
@@ -287,6 +287,12 @@ class LLMEngine:
287
# the closure used to initialize Ray worker actors
288
raise RuntimeError("LLMEngine should not be pickled!")
289
290
+ def __del__(self):
291
+ # Shutdown model executor when engine is garbage collected
292
+ # Use getattr since __init__ can fail before the field is set
293
+ if model_executor := getattr(self, "model_executor", None):
294
+ model_executor.shutdown()
295
+
296
def get_tokenizer(self) -> "PreTrainedTokenizer":
297
return self.tokenizer.get_lora_tokenizer(None)
298
vllm/executor/executor_base.py
+7
@@ -95,6 +95,13 @@ class ExecutorBase(ABC):
95
exception."""
96
raise NotImplementedError
97
98
+ def shutdown(self) -> None:
99
+ """Shutdown the executor."""
100
+ return
101
+
102
+ def __del__(self):
103
+ self.shutdown()
104
+
105
106
class ExecutorAsyncBase(ExecutorBase):
107