A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 05:00 UTC
[KV Offload] Return None from lookup() for in-flight blocks (#41795)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Ronen Schaffer committed
May 6, 2026 at 17:31 UTC
f39bcf1e30693f2e620e3a2ec732c5f7192408fb
2 files changed
+11
-7
tests/v1/kv_offload/cpu/test_manager.py
+6
-6
@@ -163,9 +163,9 @@ def test_cpu_manager():
163
),
164
)
165
166
- # lookup [1, 2] -> not ready
167
- assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is False
168
- assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is False
166
+ # lookup [1, 2] -> write in-flight, not yet ready
167
+ assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is None
168
+ assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is None
169
170
# no events so far
171
assert list(cpu_manager.take_events()) == []
@@ -296,9 +296,9 @@ class TestARCPolicy:
296
),
297
)
298
299
- # lookup [1, 2] -> not ready
300
- assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is False
301
- assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is False
299
+ # lookup [1, 2] -> write in-flight, not yet ready
300
+ assert cpu_manager.lookup(to_key(1), _EMPTY_REQ_CTX) is None
301
+ assert cpu_manager.lookup(to_key(2), _EMPTY_REQ_CTX) is None
302
303
# no events so far
304
assert list(cpu_manager.take_events()) == []
vllm/v1/kv_offload/cpu/manager.py
+5
-1
@@ -86,7 +86,11 @@ class CPUOffloadingManager(OffloadingManager):
86
87
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
88
block = self._policy.get(key)
89
- return block is not None and block.is_ready
89
+ if block is None:
90
+ return False
91
+ if not block.is_ready:
92
+ return None # write in-flight; caller should retry
93
+ return True
94
95
def prepare_load(
96
self,