@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 556ef7f714
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Misc] Log time consumption of sleep and wake-up (#13115)

Signed-off-by: Jun Duan <jun.duan.phd@outlook.com>

Jun Duan committed Feb 14, 2025 at 07:10 UTC 556ef7f714e09e8090cc535d045b3e1184653a12
1 file changed +9
vllm/executor/executor_base.py
+9
@@ -1,6 +1,7 @@
1 # SPDX-License-Identifier: Apache-2.0
2
3 import asyncio
4 +import time
5 from abc import ABC, abstractmethod
6 from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
7 Union)
@@ -200,15 +201,23 @@ class ExecutorBase(ABC):
201 if self.is_sleeping:
202 logger.warning("Executor is already sleeping.")
203 return
204 + time_before_sleep = time.perf_counter()
205 self.collective_rpc("sleep", kwargs=dict(level=level))
206 + time_after_sleep = time.perf_counter()
207 self.is_sleeping = True
208 + logger.info("It took %.6f seconds to fall asleep.",
209 + time_after_sleep - time_before_sleep)
210
211 def wake_up(self):
212 if not self.is_sleeping:
213 logger.warning("Executor is not sleeping.")
214 return
215 + time_before_wakeup = time.perf_counter()
216 self.collective_rpc("wake_up")
217 + time_after_wakeup = time.perf_counter()
218 self.is_sleeping = False
219 + logger.info("It took %.6f seconds to wake up.",
220 + time_after_wakeup - time_before_wakeup)
221
222 def save_sharded_state(
223 self,