@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2729087efe
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix a ref count bug in swapping

Woosuk Kwon committed Feb 14, 2023 at 09:20 UTC 2729087efe1f3e75383eceee09273168da1b8809
1 file changed +2
cacheflow/master/block_manager.py
+2
@@ -166,6 +166,7 @@ class BlockSpaceManager:
166 for cpu_block in block_table:
167 if cpu_block in mapping:
168 gpu_block = mapping[cpu_block]
169 + gpu_block.ref_count += 1
170 else:
171 gpu_block = self.gpu_allocator.allocate()
172 mapping[cpu_block] = gpu_block
@@ -196,6 +197,7 @@ class BlockSpaceManager:
197 for gpu_block in block_table:
198 if gpu_block in mapping:
199 cpu_block = mapping[gpu_block]
200 + cpu_block.ref_count += 1
201 else:
202 cpu_block = self.cpu_allocator.allocate()
203 mapping[gpu_block] = cpu_block