@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15e7c675b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Add `shutdown()` method to `ExecutorBase` (#4349)

Nick Hill committed Apr 25, 2024 at 16:32 UTC 15e7c675b0dc36109c7b591f856f102e96493a94
2 files changed +13
vllm/engine/llm_engine.py
+6
@@ -287,6 +287,12 @@ class LLMEngine:
287 # the closure used to initialize Ray worker actors
288 raise RuntimeError("LLMEngine should not be pickled!")
289
290 + def __del__(self):
291 + # Shutdown model executor when engine is garbage collected
292 + # Use getattr since __init__ can fail before the field is set
293 + if model_executor := getattr(self, "model_executor", None):
294 + model_executor.shutdown()
295 +
296 def get_tokenizer(self) -> "PreTrainedTokenizer":
297 return self.tokenizer.get_lora_tokenizer(None)
298
vllm/executor/executor_base.py
+7
@@ -95,6 +95,13 @@ class ExecutorBase(ABC):
95 exception."""
96 raise NotImplementedError
97
98 + def shutdown(self) -> None:
99 + """Shutdown the executor."""
100 + return
101 +
102 + def __del__(self):
103 + self.shutdown()
104 +
105
106 class ExecutorAsyncBase(ExecutorBase):
107