A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Frontend] Make TIMEOUT_KEEP_ALIVE configurable through env var (#18472)
Signed-off-by: liusiqian <liusiqian@tal.com>
liusiqian-tal committed
Jun 10, 2025 at 05:41 UTC
31f58be96ae3e5dfc542604553ecead06cb5c487
4 files changed
+14
-11
tests/async_engine/api_server_async_engine.py
+6
-6
@@ -8,6 +8,7 @@ import uvicorn
8
from fastapi.responses import JSONResponse, Response
9
10
import vllm.entrypoints.api_server
11
+import vllm.envs as envs
12
from vllm.engine.arg_utils import AsyncEngineArgs
13
from vllm.engine.async_llm_engine import AsyncLLMEngine
14
from vllm.utils import FlexibleArgumentParser
@@ -46,9 +47,8 @@ if __name__ == "__main__":
47
engine_args = AsyncEngineArgs.from_cli_args(args)
48
engine = AsyncLLMEngineWithStats.from_engine_args(engine_args)
49
vllm.entrypoints.api_server.engine = engine
49
- uvicorn.run(
50
- app,
51
- host=args.host,
52
- port=args.port,
53
- log_level="debug",
54
- timeout_keep_alive=vllm.entrypoints.api_server.TIMEOUT_KEEP_ALIVE)
50
+ uvicorn.run(app,
51
+ host=args.host,
52
+ port=args.port,
53
+ log_level="debug",
54
+ timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE)
vllm/entrypoints/api_server.py
+2
-2
@@ -17,6 +17,7 @@ from typing import Any, Optional
17
from fastapi import FastAPI, Request
18
from fastapi.responses import JSONResponse, Response, StreamingResponse
19
20
+import vllm.envs as envs
21
from vllm.engine.arg_utils import AsyncEngineArgs
22
from vllm.engine.async_llm_engine import AsyncLLMEngine
23
from vllm.entrypoints.launcher import serve_http
@@ -29,7 +30,6 @@ from vllm.version import __version__ as VLLM_VERSION
30
31
logger = init_logger("vllm.entrypoints.api_server")
32
32
-TIMEOUT_KEEP_ALIVE = 5 # seconds.
33
app = FastAPI()
34
engine = None
35
@@ -134,7 +134,7 @@ async def run_server(args: Namespace,
134
host=args.host,
135
port=args.port,
136
log_level=args.log_level,
137
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
137
+ timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
138
ssl_keyfile=args.ssl_keyfile,
139
ssl_certfile=args.ssl_certfile,
140
ssl_ca_certs=args.ssl_ca_certs,
vllm/entrypoints/openai/api_server.py
+1
-3
@@ -103,8 +103,6 @@ from vllm.utils import (Device, FlexibleArgumentParser, get_open_zmq_ipc_path,
103
from vllm.v1.metrics.prometheus import get_prometheus_registry
104
from vllm.version import __version__ as VLLM_VERSION
105
106
-TIMEOUT_KEEP_ALIVE = 5 # seconds
107
-
106
prometheus_multiproc_dir: tempfile.TemporaryDirectory
107
108
# Cannot use __name__ (https://github.com/vllm-project/vllm/pull/4765)
@@ -1360,7 +1358,7 @@ async def run_server_worker(listen_address,
1358
# NOTE: When the 'disable_uvicorn_access_log' value is True,
1359
# no access log will be output.
1360
access_log=not args.disable_uvicorn_access_log,
1363
- timeout_keep_alive=TIMEOUT_KEEP_ALIVE,
1361
+ timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
1362
ssl_keyfile=args.ssl_keyfile,
1363
ssl_certfile=args.ssl_certfile,
1364
ssl_ca_certs=args.ssl_ca_certs,
vllm/envs.py
+5
@@ -71,6 +71,7 @@ if TYPE_CHECKING:
71
VERBOSE: bool = False
72
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
73
VLLM_RPC_TIMEOUT: int = 10000 # ms
74
+ VLLM_HTTP_TIMEOUT_KEEP_ALIVE: int = 5 # seconds
75
VLLM_PLUGINS: Optional[list[str]] = None
76
VLLM_LORA_RESOLVER_CACHE_DIR: Optional[str] = None
77
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
@@ -557,6 +558,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
558
"VLLM_RPC_TIMEOUT":
559
lambda: int(os.getenv("VLLM_RPC_TIMEOUT", "10000")),
560
561
+ # Timeout in seconds for keeping HTTP connections alive in API server
562
+ "VLLM_HTTP_TIMEOUT_KEEP_ALIVE":
563
+ lambda: int(os.environ.get("VLLM_HTTP_TIMEOUT_KEEP_ALIVE", "5")),
564
+
565
# a list of plugin names to load, separated by commas.
566
# if this is not set, it means all plugins will be loaded
567
# if this is set to an empty string, no plugins will be loaded