A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 7, 22:45 UTC
[PD] Change kv_load_failure_policy Default from "recompute" to "fail" (#34896)
Signed-off-by: NickLucche <nlucches@redhat.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Nicolò Lucchesi committed
Feb 21, 2026 at 10:34 UTC
ab6f3487a6146b325cb836711e34f40f341278e4
5 files changed
+10
-7
docs/features/nixl_connector_usage.md
+2
-2
@@ -197,8 +197,8 @@ For multi-host DP deployment, only need to provide the host/port of the head ins
197
198
The `kv_load_failure_policy` setting controls how the system handles failures when the decoder instance loads KV cache blocks from the prefiller instance:
199
200
-- **fail** (recommended): Immediately fail the request with an error when KV load fails. This prevents performance degradation by avoiding recomputation of prefill work on the decode instance.
201
-- **recompute** (default): Recompute failed blocks locally on the decode instance. This may cause performance _jitter_ on decode instances as the scheduled prefill will delay and interfere with other decodes. Furthermore, decode instances are typically configured with low-latency optimizations.
200
+- **fail** (default): Immediately fail the request with an error when KV load fails. This prevents performance degradation by avoiding recomputation of prefill work on the decode instance.
201
+- **recompute**: Recompute failed blocks locally on the decode instance. This may cause performance _jitter_ on decode instances as the scheduled prefill will delay and interfere with other decodes. Furthermore, decode instances are typically configured with low-latency optimizations.
202
203
!!! warning
204
Using `kv_load_failure_policy="recompute"` can lead to performance degradation in production deployments. When KV loads fail, the decode instance will execute prefill work with decode-optimized configurations, which is inefficient and defeats the purpose of disaggregated prefilling. This also increases tail latency for other ongoing decode requests.
examples/offline_inference/kv_load_failure_recovery/decode_example.py
+1
@@ -42,6 +42,7 @@ def main():
42
"async_load": args.async_load,
43
},
44
kv_connector_module_path="load_recovery_example_connector",
45
+ kv_load_failure_policy="recompute",
46
)
47
out_file = (
48
"async_decode_recovered_output.txt"
tests/v1/kv_connector/unit/test_kv_load_failure_recovery.py
+1
-1
@@ -30,7 +30,7 @@ def _make_get_num_new_matched_tokens(
30
31
@pytest.fixture
32
def scheduler():
33
- vllm_config = create_vllm_config()
33
+ vllm_config = create_vllm_config(kv_load_failure_policy="recompute")
34
return create_scheduler(vllm_config)
35
36
tests/v1/kv_connector/unit/utils.py
+3
-1
@@ -5,7 +5,7 @@ from collections import defaultdict
5
from collections.abc import Callable
6
from dataclasses import dataclass
7
from itertools import chain, count
8
-from typing import Any
8
+from typing import Any, Literal
9
10
import torch
11
@@ -96,6 +96,7 @@ def create_vllm_config(
96
cache_dtype: str = "auto",
97
hf_overrides: dict[str, Any] | None = None,
98
attention_backend: str | None = None,
99
+ kv_load_failure_policy: Literal["recompute", "fail"] = "fail",
100
) -> VllmConfig:
101
"""Initialize VllmConfig For Testing."""
102
model_config = ModelConfig(
@@ -125,6 +126,7 @@ def create_vllm_config(
126
kv_role="kv_both",
127
enable_permute_local_kv=enable_permute_local_kv,
128
kv_connector_extra_config=kv_connector_extra_config or {},
129
+ kv_load_failure_policy=kv_load_failure_policy,
130
)
131
attention_config = AttentionConfig(backend=attention_backend)
132
return VllmConfig(
vllm/config/kv_transfer.py
+3
-3
@@ -61,10 +61,10 @@ class KVTransferConfig:
61
enable_permute_local_kv: bool = False
62
"""Experiment feature flag to enable HND to NHD KV Transfer"""
63
64
- kv_load_failure_policy: Literal["recompute", "fail"] = "recompute"
64
+ kv_load_failure_policy: Literal["recompute", "fail"] = "fail"
65
"""Policy for handling KV cache load failures.
66
- 'recompute': reschedule the request to recompute failed blocks (default)
67
- 'fail': immediately fail the request with an error finish reason"""
66
+ 'recompute': reschedule the request to recompute failed blocks
67
+ 'fail': immediately fail the request with an error finish reason (default)"""
68
69
def compute_hash(self) -> str:
70
"""