A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix a ref count bug in swapping
Woosuk Kwon committed
Feb 14, 2023 at 09:20 UTC
2729087efe1f3e75383eceee09273168da1b8809
1 file changed
+2
cacheflow/master/block_manager.py
+2
@@ -166,6 +166,7 @@ class BlockSpaceManager:
166
for cpu_block in block_table:
167
if cpu_block in mapping:
168
gpu_block = mapping[cpu_block]
169
+ gpu_block.ref_count += 1
170
else:
171
gpu_block = self.gpu_allocator.allocate()
172
mapping[cpu_block] = gpu_block
@@ -196,6 +197,7 @@ class BlockSpaceManager:
197
for gpu_block in block_table:
198
if gpu_block in mapping:
199
cpu_block = mapping[gpu_block]
200
+ cpu_block.ref_count += 1
201
else:
202
cpu_block = self.cpu_allocator.allocate()
203
mapping[gpu_block] = cpu_block