@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be1e2163c9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

Fix memory leak in swapping

Woosuk Kwon committed Feb 14, 2023 at 08:45 UTC be1e2163c9f9519310abe2519caa36b0a6966a1b
1 file changed +9 -9
cacheflow/master/block_manager.py
+9 -9
@@ -165,10 +165,11 @@ class BlockSpaceManager:
165
166 for cpu_block in block_table:
167 if cpu_block in mapping:
168 - new_block_table.append(mapping[cpu_block])
169 - continue
170 - gpu_block = self.gpu_allocator.allocate()
171 - mapping[cpu_block] = gpu_block
168 + gpu_block = mapping[cpu_block]
169 + else:
170 + gpu_block = self.gpu_allocator.allocate()
171 + mapping[cpu_block] = gpu_block
172 + new_block_table.append(gpu_block)
173 # Free the CPU block swapped in to GPU.
174 self.cpu_allocator.free(cpu_block)
175 self.block_tables[seq.seq_id] = new_block_table
@@ -194,12 +195,11 @@ class BlockSpaceManager:
195
196 for gpu_block in block_table:
197 if gpu_block in mapping:
197 - new_block_table.append(mapping[gpu_block])
198 - continue
199 - cpu_block = self.cpu_allocator.allocate()
198 + cpu_block = mapping[gpu_block]
199 + else:
200 + cpu_block = self.cpu_allocator.allocate()
201 + mapping[gpu_block] = cpu_block
202 new_block_table.append(cpu_block)
201 -
202 - mapping[gpu_block] = cpu_block
203 # Free the GPU block swapped out to CPU.
204 self.gpu_allocator.free(gpu_block)
205 self.block_tables[seq.seq_id] = new_block_table