A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 22:00 UTC
[KV offload] Offloading connector async scheduling support (#27648)
Signed-off-by: KevinCheung2259 <2651309292@qq.com> Co-authored-by: Nick Hill <nhill@redhat.com>
Yue Zhang committed
Nov 2, 2025 at 05:08 UTC
685c99ee77b4818dcdd15b30fe0e0eff0d5d22ec
1 file changed
+2
-2
vllm/distributed/kv_transfer/kv_connector/v1/offloading_connector.py
+2
-2
@@ -274,8 +274,8 @@ class OffloadingConnectorScheduler:
274
if num_new_blocks <= 0:
275
continue
276
277
- num_gpu_blocks = num_blocks * self.block_size_factor
278
- assert len(req.block_hashes) >= num_gpu_blocks
277
+ # NOTE: In async scheduling, placeholders may temporarily make
278
+ # len(req.block_hashes) < num_blocks * self.block_size_factor.
279
280
new_block_hashes = self._get_block_hashes(
281
req, start_idx=start_block_idx, end_idx=num_blocks