A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
Fix memory leak in swapping
Woosuk Kwon committed
Feb 14, 2023 at 08:45 UTC
be1e2163c9f9519310abe2519caa36b0a6966a1b
1 file changed
+9
-9
cacheflow/master/block_manager.py
+9
-9
@@ -165,10 +165,11 @@ class BlockSpaceManager:
165
166
for cpu_block in block_table:
167
if cpu_block in mapping:
168
- new_block_table.append(mapping[cpu_block])
169
- continue
170
- gpu_block = self.gpu_allocator.allocate()
171
- mapping[cpu_block] = gpu_block
168
+ gpu_block = mapping[cpu_block]
169
+ else:
170
+ gpu_block = self.gpu_allocator.allocate()
171
+ mapping[cpu_block] = gpu_block
172
+ new_block_table.append(gpu_block)
173
# Free the CPU block swapped in to GPU.
174
self.cpu_allocator.free(cpu_block)
175
self.block_tables[seq.seq_id] = new_block_table
@@ -194,12 +195,11 @@ class BlockSpaceManager:
195
196
for gpu_block in block_table:
197
if gpu_block in mapping:
197
- new_block_table.append(mapping[gpu_block])
198
- continue
199
- cpu_block = self.cpu_allocator.allocate()
198
+ cpu_block = mapping[gpu_block]
199
+ else:
200
+ cpu_block = self.cpu_allocator.allocate()
201
+ mapping[gpu_block] = cpu_block
202
new_block_table.append(cpu_block)
201
-
202
- mapping[gpu_block] = cpu_block
203
# Free the GPU block swapped out to CPU.
204
self.gpu_allocator.free(gpu_block)
205
self.block_tables[seq.seq_id] = new_block_table