A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[NIXL] Increase default KV block eviction timeout on P (#25897)
Signed-off-by: NickLucche <nlucches@redhat.com>
Nicolò Lucchesi committed
Sep 29, 2025 at 23:35 UTC
2e4fe48c370e833350eae092eddd1490b65ff529
2 files changed
+3
-3
docs/features/nixl_connector_usage.md
+1
-1
@@ -84,7 +84,7 @@ python tests/v1/kv_connector/nixl_integration/toy_proxy_server.py \
84
- Connection info is passed via KVTransferParams from prefiller to decoder for handshake
85
86
- `VLLM_NIXL_ABORT_REQUEST_TIMEOUT`: Timeout (in seconds) for automatically releasing the prefiller’s KV cache for a particular request. (Optional)
87
- - Default: 120
87
+ - Default: 480
88
- If a request is aborted and the decoder has not yet read the KV-cache blocks through the nixl channel, the prefill instance will release its KV-cache blocks after this timeout to avoid holding them indefinitely.
89
90
## Multi-Instance Setup
vllm/envs.py
+2
-2
@@ -174,7 +174,7 @@ if TYPE_CHECKING:
174
"NONE"] = "NONE"
175
VLLM_ROCM_QUICK_REDUCE_CAST_BF16_TO_FP16: bool = True
176
VLLM_ROCM_QUICK_REDUCE_MAX_SIZE_BYTES_MB: Optional[int] = None
177
- VLLM_NIXL_ABORT_REQUEST_TIMEOUT: int = 120
177
+ VLLM_NIXL_ABORT_REQUEST_TIMEOUT: int = 480
178
VLLM_USE_CUDNN_PREFILL: bool = False
179
VLLM_ENABLE_CUDAGRAPH_GC: bool = False
180
VLLM_LOOPBACK_IP: str = ""
@@ -1330,7 +1330,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
1330
# consumer. This is only applicable when using NixlConnector in a
1331
# disaggregated decode-prefill setup.
1332
"VLLM_NIXL_ABORT_REQUEST_TIMEOUT":
1333
- lambda: int(os.getenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", "120")),
1333
+ lambda: int(os.getenv("VLLM_NIXL_ABORT_REQUEST_TIMEOUT", "480")),
1334
1335
# Controls whether or not to use cudnn prefill
1336
"VLLM_USE_CUDNN_PREFILL":