A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[core] add wake_up doc and some sanity check (#12361)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Jan 24, 2025 at 02:00 UTC
3f50c148fd6476fc9099c04add39e873d9972663
2 files changed
+12
vllm/entrypoints/llm.py
+3
@@ -1157,6 +1157,9 @@ class LLM:
1157
self.llm_engine.sleep(level=level)
1158
1159
def wake_up(self):
1160
+ """
1161
+ Wake up the engine from sleep mode. See the :meth:`sleep` method
1162
+ for more details."""
1163
self.llm_engine.wake_up()
1164
1165
# LEGACY
vllm/executor/executor_base.py
+9
@@ -47,6 +47,7 @@ class ExecutorBase(ABC):
47
self.prompt_adapter_config = vllm_config.prompt_adapter_config
48
self.observability_config = vllm_config.observability_config
49
self._init_executor()
50
+ self.is_sleeping = False
51
52
@abstractmethod
53
def _init_executor(self) -> None:
@@ -194,10 +195,18 @@ class ExecutorBase(ABC):
195
self.collective_rpc("stop_profile")
196
197
def sleep(self, level: int = 1):
198
+ if self.is_sleeping:
199
+ logger.warning("Executor is already sleeping.")
200
+ return
201
self.collective_rpc("sleep", kwargs=dict(level=level))
202
+ self.is_sleeping = True
203
204
def wake_up(self):
205
+ if not self.is_sleeping:
206
+ logger.warning("Executor is not sleeping.")
207
+ return
208
self.collective_rpc("wake_up")
209
+ self.is_sleeping = False
210
211
def save_sharded_state(
212
self,