A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 02:45 UTC
[UX] Use kv_offloading_backend=native by default (#32421)
Signed-off-by: mgoin <mgoin64@gmail.com>
Michael Goin committed
Jan 15, 2026 at 13:55 UTC
1be5a73571ab2e2570b79a7474c98c496cc80de2
4 files changed
+28
-15
tests/v1/kv_connector/unit/test_config.py
+18
-1
@@ -19,7 +19,8 @@ pytestmark = pytest.mark.cpu_test
19
("lmcache", 4.0, 1, 1, "LMCacheConnectorV1", 4.0),
20
# size per rank: 8.0 GiB / (2 * 2) = 2.0 GiB
21
("lmcache", 8.0, 2, 2, "LMCacheConnectorV1", 2.0),
22
- (None, None, 1, 1, None, None),
22
+ # When kv_offloading_size is None, offloading is disabled (backend is ignored)
23
+ ("native", None, 1, 1, None, None),
24
],
25
)
26
def test_kv_connector(
@@ -62,3 +63,19 @@ def test_kv_connector(
63
assert kv_connector_extra_config["lmcache.max_local_cpu_size"] == expected_bytes
64
# Existing config should be replaced
65
assert "existing_key" not in kv_connector_extra_config
66
+
67
+
68
+def test_kv_offloading_size_only_uses_native_default():
69
+ """Test that setting only kv_offloading_size enables native offloading."""
70
+ vllm_config = VllmConfig(
71
+ cache_config=CacheConfig(
72
+ kv_offloading_size=4.0,
73
+ # kv_offloading_backend not set, should default to "native"
74
+ ),
75
+ )
76
+
77
+ kv_transfer_config = vllm_config.kv_transfer_config
78
+ kv_connector_extra_config = kv_transfer_config.kv_connector_extra_config
79
+ assert kv_transfer_config.kv_connector == "OffloadingConnector"
80
+ assert kv_transfer_config.kv_role == "kv_both"
81
+ assert kv_connector_extra_config["cpu_bytes_to_use"] == 4.0 * (1 << 30)
vllm/config/cache.py
+5
-5
@@ -152,13 +152,13 @@ class CacheConfig:
152
kv_offloading_size: float | None = None
153
"""Size of the KV cache offloading buffer in GiB. When TP > 1, this is
154
the total buffer size summed across all TP ranks. By default, this is set
155
- to None, which means no KV offloading is enabled. When set with
156
- kv_offloading_backend, vLLM will enable KV cache offloading to CPU"""
155
+ to None, which means no KV offloading is enabled. When set, vLLM will
156
+ enable KV cache offloading to CPU using the kv_offloading_backend."""
157
158
- kv_offloading_backend: KVOffloadingBackend | None = None
158
+ kv_offloading_backend: KVOffloadingBackend = "native"
159
"""The backend to use for KV cache offloading. Supported backends include
160
- 'native' (vLLM native CPU offloading), 'lmcache' This option must be used
161
- together with kv_offloading_size."""
160
+ 'native' (vLLM native CPU offloading), 'lmcache'.
161
+ KV offloading is only activated when kv_offloading_size is set."""
162
163
def compute_hash(self) -> str:
164
"""
vllm/config/vllm.py
+4
-6
@@ -498,17 +498,15 @@ class VllmConfig:
498
Right now, this function reads the offloading settings from
499
CacheConfig and configures the KVTransferConfig accordingly.
500
"""
501
- if (kv_offloading_backend := self.cache_config.kv_offloading_backend) is None:
501
+ # KV offloading is only activated when kv_offloading_size is set.
502
+ if (kv_offloading_size := self.cache_config.kv_offloading_size) is None:
503
return
504
505
+ kv_offloading_backend = self.cache_config.kv_offloading_backend
506
+
507
# If no KVTransferConfig is provided, create a default one.
508
if self.kv_transfer_config is None:
509
self.kv_transfer_config = KVTransferConfig()
507
-
508
- if (kv_offloading_size := self.cache_config.kv_offloading_size) is None:
509
- raise ValueError(
510
- "You must set kv_offloading_size when kv_offloading_backend is set."
511
- )
510
num_kv_ranks = (
511
self.parallel_config.tensor_parallel_size
512
* self.parallel_config.pipeline_parallel_size
vllm/engine/arg_utils.py
+1
-3
@@ -574,9 +574,7 @@ class EngineArgs:
574
optimization_level: OptimizationLevel = VllmConfig.optimization_level
575
576
kv_offloading_size: float | None = CacheConfig.kv_offloading_size
577
- kv_offloading_backend: KVOffloadingBackend | None = (
578
- CacheConfig.kv_offloading_backend
579
- )
577
+ kv_offloading_backend: KVOffloadingBackend = CacheConfig.kv_offloading_backend
578
tokens_only: bool = False
579
580
def __post_init__(self):