@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3f50c148fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] add wake_up doc and some sanity check (#12361)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 24, 2025 at 02:00 UTC 3f50c148fd6476fc9099c04add39e873d9972663
2 files changed +12
vllm/entrypoints/llm.py
+3
@@ -1157,6 +1157,9 @@ class LLM:
1157 self.llm_engine.sleep(level=level)
1158
1159 def wake_up(self):
1160 + """
1161 + Wake up the engine from sleep mode. See the :meth:`sleep` method
1162 + for more details."""
1163 self.llm_engine.wake_up()
1164
1165 # LEGACY
vllm/executor/executor_base.py
+9
@@ -47,6 +47,7 @@ class ExecutorBase(ABC):
47 self.prompt_adapter_config = vllm_config.prompt_adapter_config
48 self.observability_config = vllm_config.observability_config
49 self._init_executor()
50 + self.is_sleeping = False
51
52 @abstractmethod
53 def _init_executor(self) -> None:
@@ -194,10 +195,18 @@ class ExecutorBase(ABC):
195 self.collective_rpc("stop_profile")
196
197 def sleep(self, level: int = 1):
198 + if self.is_sleeping:
199 + logger.warning("Executor is already sleeping.")
200 + return
201 self.collective_rpc("sleep", kwargs=dict(level=level))
202 + self.is_sleeping = True
203
204 def wake_up(self):
205 + if not self.is_sleeping:
206 + logger.warning("Executor is not sleeping.")
207 + return
208 self.collective_rpc("wake_up")
209 + self.is_sleeping = False
210
211 def save_sharded_state(
212 self,