A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 13:30 UTC
[KV Offload] Rename `SecondaryTierManager.get_finished()` to `get_finished_jobs()` (#43870)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Ronen Schaffer committed
May 28, 2026 at 19:00 UTC
4bfa0f2b1458be320fa39c6fa54be5f83cef2444
6 files changed
+15
-15
tests/v1/kv_offload/test_fs_tier.py
+2
-2
@@ -72,14 +72,14 @@ def make_job(
72
73
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
74
"""
75
- Call get_finished() repeatedly until no new results arrive for 5
75
+ Call get_finished_jobs() repeatedly until no new results arrive for 5
76
consecutive rounds or max_rounds is reached.
77
"""
78
results = []
79
idle = 0
80
for _ in range(max_rounds):
81
time.sleep(0.01)
82
- new = list(tier.get_finished())
82
+ new = list(tier.get_finished_jobs())
83
results.extend(new)
84
if new:
85
idle = 0
tests/v1/kv_offload/test_tiering_offloading.py
+1
-1
@@ -184,7 +184,7 @@ class TestTieringOffloadingManager:
184
185
# End of step 1: _maybe_process_finished_jobs() was already called by
186
# prepare_store() above (setting the per-step flag), so take_events()
187
- # does NOT poll get_finished() again — cascade completions remain
187
+ # does NOT poll get_finished_jobs() again — cascade completions remain
188
# unprocessed until the next step.
189
list(self.manager.take_events())
190
vllm/v1/kv_offload/tiering/base.py
+4
-4
@@ -50,7 +50,7 @@ class SecondaryTierManager(ABC):
50
51
IMPORTANT: All methods run in the Scheduler process and must be
52
lightweight and non-blocking. submit_load() and submit_store() submit
53
- async jobs; get_finished() polls for completion.
53
+ async jobs; get_finished_jobs() polls for completion.
54
"""
55
56
def __init__(
@@ -106,7 +106,7 @@ class SecondaryTierManager(ABC):
106
3. Allocating space in this tier
107
4. Submitting the async transfer (read from primary via block_ids)
108
109
- Report completion via ``get_finished()``.
109
+ Report completion via ``get_finished_jobs()``.
110
111
Args:
112
job_metadata: Job metadata including job_id, keys, and block_ids
@@ -131,7 +131,7 @@ class SecondaryTierManager(ABC):
131
The implementation must copy data from this tier into the
132
primary-tier slots identified by ``block_ids``.
133
134
- Report completion via ``get_finished()``.
134
+ Report completion via ``get_finished_jobs()``.
135
136
Args:
137
job_metadata: Job metadata including job_id, keys, and block_ids
@@ -140,7 +140,7 @@ class SecondaryTierManager(ABC):
140
pass
141
142
@abstractmethod
143
- def get_finished(self) -> Iterable[JobResult]:
143
+ def get_finished_jobs(self) -> Iterable[JobResult]:
144
"""
145
Return all jobs (loads and stores) that completed since the last call.
146
vllm/v1/kv_offload/tiering/example/manager.py
+2
-2
@@ -61,7 +61,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
61
# key -> True (only care about presence)
62
self.blocks: dict[OffloadKey, bool] = {}
63
64
- # Completed jobs waiting to be retrieved by get_finished()
64
+ # Completed jobs waiting to be retrieved by get_finished_jobs()
65
self.completed_jobs: list[JobResult] = []
66
67
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
@@ -120,7 +120,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
120
121
self.completed_jobs.append(JobResult(job_id=job_metadata.job_id, success=True))
122
123
- def get_finished(self) -> Iterable[JobResult]:
123
+ def get_finished_jobs(self) -> Iterable[JobResult]:
124
"""
125
Poll for finished jobs.
126
vllm/v1/kv_offload/tiering/fs/manager.py
+2
-2
@@ -47,7 +47,7 @@ class FileSystemTierManager(SecondaryTierManager):
47
queue, so neither starves.
48
49
submit_store / submit_load are non-blocking: they enqueue tasks and return.
50
- get_finished() polls job completion and returns completed JobResults.
50
+ get_finished_jobs() polls job completion and returns completed JobResults.
51
52
"""
53
@@ -131,7 +131,7 @@ class FileSystemTierManager(SecondaryTierManager):
131
)
132
self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks)
133
134
- def get_finished(self) -> Iterable[JobResult]:
134
+ def get_finished_jobs(self) -> Iterable[JobResult]:
135
"""
136
Collect completed jobs from the finished-jobs queue.
137
"""
vllm/v1/kv_offload/tiering/manager.py
+4
-4
@@ -113,7 +113,7 @@ class TieringOffloadingManager(OffloadingManager):
113
114
Key internal state:
115
- Minimal state tracking; relies on secondary tiers to report completion
116
- via get_finished()
116
+ via get_finished_jobs()
117
- Secondary tiers return JobResult objects containing all necessary
118
information
119
- job_id_counter: monotonically increasing counter for job IDs
@@ -182,14 +182,14 @@ class TieringOffloadingManager(OffloadingManager):
182
Unconditionally poll all secondary tiers for completed jobs.
183
184
This method:
185
- 1. Calls get_finished() on each secondary tier
185
+ 1. Calls get_finished_jobs() on each secondary tier
186
2. For completed stores (primary→secondary): calls primary.complete_read()
187
to decrement ref_cnt
188
3. For completed loads (secondary→primary): calls primary.complete_write()
189
to make blocks available
190
"""
191
for i, tier in enumerate(self.secondary_tiers):
192
- for completed_job in tier.get_finished():
192
+ for completed_job in tier.get_finished_jobs():
193
job_id = completed_job.job_id
194
job_metadata = self._transfer_jobs.pop(job_id, None)
195
assert job_metadata is not None, (
@@ -464,7 +464,7 @@ class TieringOffloadingManager(OffloadingManager):
464
tier.submit_store(job_metadata)
465
466
# Note: The async transfers are now in flight. Their completion is
467
- # tracked via get_finished() / _maybe_process_finished_jobs().
467
+ # tracked via get_finished_jobs() / _maybe_process_finished_jobs().
468
469
def take_events(self) -> Iterable[OffloadingEvent]:
470
"""