A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Frontend] Expose do_log_stats interval to env (#22905)
Signed-off-by: Csrayz <jover@cmbchina.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Csrayz committed
Aug 15, 2025 at 21:00 UTC
a0632a3e03f9287a0e3966e8c76064984cfdc855
3 files changed
+9
-1
docs/usage/troubleshooting.md
+1
@@ -35,6 +35,7 @@ You can check if this is happening by trying the old defaults with `--generation
35
If other strategies don't solve the problem, it's likely that the vLLM instance is stuck somewhere. You can use the following environment variables to help debug the issue:
36
37
- `export VLLM_LOGGING_LEVEL=DEBUG` to turn on more logging.
38
+- `export VLLM_LOG_STATS_INTERVAL=1.` to get log statistics more frequently for tracking running queue, waiting queue and cache hit states.
39
- `export CUDA_LAUNCH_BLOCKING=1` to identify which CUDA kernel is causing the problem.
40
- `export NCCL_DEBUG=TRACE` to turn on more logging for NCCL.
41
- `export VLLM_TRACE_FUNCTION=1` to record all function calls for inspection in the log files to tell which function crashes or hangs. Do not use this flag unless absolutely needed for debugging, it will cause significant delays in startup time.
vllm/entrypoints/openai/api_server.py
+1
-1
@@ -126,7 +126,7 @@ async def lifespan(app: FastAPI):
126
127
async def _force_log():
128
while True:
129
- await asyncio.sleep(10.)
129
+ await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
130
await engine_client.do_log_stats()
131
132
task = asyncio.create_task(_force_log())
vllm/envs.py
+7
@@ -38,6 +38,7 @@ if TYPE_CHECKING:
38
VLLM_LOGGING_PREFIX: str = ""
39
VLLM_LOGGING_CONFIG_PATH: Optional[str] = None
40
VLLM_LOGITS_PROCESSOR_THREADS: Optional[int] = None
41
+ VLLM_LOG_STATS_INTERVAL: float = 10.
42
VLLM_TRACE_FUNCTION: int = 0
43
VLLM_ATTENTION_BACKEND: Optional[str] = None
44
VLLM_USE_FLASHINFER_SAMPLER: Optional[bool] = None
@@ -436,6 +437,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
437
lambda: int(os.getenv("VLLM_LOGITS_PROCESSOR_THREADS", "0"))
438
if "VLLM_LOGITS_PROCESSOR_THREADS" in os.environ else None,
439
440
+ # If set, vllm will log stats at this interval in seconds
441
+ # If not set, vllm will log stats every 10 seconds.
442
+ "VLLM_LOG_STATS_INTERVAL":
443
+ lambda: val if (val := float(os.getenv("VLLM_LOG_STATS_INTERVAL", "10.")))
444
+ > 0. else 10.,
445
+
446
# Trace function calls
447
# If set to 1, vllm will trace function calls
448
# Useful for debugging