A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fixes #1556 double free (#3347)
Breno Faria committed
Mar 13, 2024 at 01:30 UTC
49a3c8662ba745503890ab8b3c502aad7e1a0a19
2 files changed
+102
-2
tests/core/test_block_manager.py
+87
@@ -274,3 +274,90 @@ def test_reset():
274
# Resetting block manager frees all allocated blocks.
275
block_manager.reset()
276
assert block_manager.get_num_free_gpu_blocks() == original_blocks
277
+
278
+
279
+def test_sliding_window_multi_seq():
280
+ """
281
+ Tests that memory allocation and deallocation is handled
282
+ correctly with multiple sequences that exceed the sliding
283
+ window's capacity.
284
+ """
285
+ block_size = 1
286
+ num_cpu_blocks = 8
287
+ num_gpu_blocks = 8
288
+ sliding_window = 2
289
+ block_manager = BlockSpaceManager(block_size,
290
+ num_cpu_blocks,
291
+ num_gpu_blocks,
292
+ sliding_window=sliding_window,
293
+ watermark=0)
294
+
295
+ assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
296
+
297
+ parent = Sequence(1, "one two three", [0, 1, 2], block_size)
298
+ seq_group = SequenceGroup("1", [parent], SamplingParams(), time.time(),
299
+ None)
300
+ block_manager.allocate(seq_group)
301
+
302
+ # assert the number of blocks allocated is correct
303
+ # the parent seq has len 3, but since sliding_window is 2,
304
+ # we will use at most 2 blocks
305
+ assert block_manager.get_num_free_gpu_blocks(
306
+ ) == num_gpu_blocks - sliding_window
307
+
308
+ # Fork prompt and copy block tables.
309
+ child = parent.fork(2)
310
+ block_manager.fork(parent, child)
311
+
312
+ # assert the number of blocks allocated is correct
313
+ # forking does not increase memory consumption
314
+ assert block_manager.get_num_free_gpu_blocks(
315
+ ) == num_gpu_blocks - sliding_window
316
+
317
+ # assert both parent and child share all blocks
318
+ assert block_manager.get_block_table(
319
+ parent) == block_manager.get_block_table(child)
320
+
321
+ token_id = 4
322
+ # Append token to child. Block is shared so copy on write occurs.
323
+ child.append_token_id(token_id, {token_id: Logprob(0.0)})
324
+ block_manager.append_slot(child)
325
+
326
+ # assert the number of blocks allocated is correct
327
+ # we will use now one block more. Each seq will use 2 blocks,
328
+ # but only one can be shared
329
+ assert block_manager.get_num_free_gpu_blocks(
330
+ ) == num_gpu_blocks - sliding_window - 1
331
+
332
+ token_id = 5
333
+ parent.append_token_id(token_id, {token_id: Logprob(0.0)})
334
+ block_manager.append_slot(parent)
335
+
336
+ # assert the number of blocks allocated is correct
337
+ # no change, because both sequences are still just sharing one block
338
+ assert block_manager.get_num_free_gpu_blocks(
339
+ ) == num_gpu_blocks - sliding_window - 1
340
+
341
+ block_table_parent = block_manager.get_block_table(parent)
342
+ block_table_child = block_manager.get_block_table(child)
343
+
344
+ assert block_table_parent != block_table_child
345
+
346
+ # assert both blocks are sharing the second-last block
347
+ assert block_table_parent[-2] == block_table_child[-2]
348
+
349
+ # now let's clean up...
350
+ block_manager.free(parent)
351
+
352
+ # assert the number of blocks allocated is correct
353
+ # We have freed one seq, reducing the ref count of two blocks by one.
354
+ # One of the two was only used by the parent seq, so this is now free.
355
+ # The child seq still consumes sliding_window blocks
356
+ assert block_manager.get_num_free_gpu_blocks(
357
+ ) == num_gpu_blocks - sliding_window
358
+
359
+ # free all blocks
360
+ block_manager.free(child)
361
+
362
+ # assert all blocks are free now
363
+ assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
vllm/core/block_manager.py
+15
-2
@@ -312,7 +312,12 @@ class BlockSpaceManager:
312
# Thus, it is always safe from OOM.
313
src_block_table = self.block_tables[parent_seq.seq_id]
314
self.block_tables[child_seq.seq_id] = src_block_table.copy()
315
- for block in src_block_table:
315
+ # When using a sliding window, blocks will be eventually reused.
316
+ # In this case the block tables will contain repeated blocks.
317
+ # When forking, we must make sure that each block's `ref_count`
318
+ # is only incremented by one, so we deduplicate them by wrapping
319
+ # them in a set.
320
+ for block in set(src_block_table):
321
block.ref_count += 1
322
323
def _get_physical_blocks(
@@ -393,7 +398,15 @@ class BlockSpaceManager:
398
return block_number_mapping
399
400
def _free_block_table(self, block_table: BlockTable) -> None:
396
- for block in set(block_table):
401
+ # when using a sliding window, each seq will only use up
402
+ # to `self.block_sliding_window` blocks. When freeing
403
+ # the block table, we must make sure to not free blocks more
404
+ # than once. If no sliding window is used, there is no block
405
+ # reuse in the block table, so we must free all blocks.
406
+ blocks_to_free = (block_table[-self.block_sliding_window:]
407
+ if self.block_sliding_window is not None else
408
+ block_table)
409
+ for block in set(blocks_to_free):
410
if block.device == Device.GPU:
411
self.gpu_allocator.free(block)
412
else: