@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 49a3c8662b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fixes #1556 double free (#3347)

Breno Faria committed Mar 13, 2024 at 01:30 UTC 49a3c8662ba745503890ab8b3c502aad7e1a0a19
2 files changed +102 -2
tests/core/test_block_manager.py
+87
@@ -274,3 +274,90 @@ def test_reset():
274 # Resetting block manager frees all allocated blocks.
275 block_manager.reset()
276 assert block_manager.get_num_free_gpu_blocks() == original_blocks
277 +
278 +
279 +def test_sliding_window_multi_seq():
280 + """
281 + Tests that memory allocation and deallocation is handled
282 + correctly with multiple sequences that exceed the sliding
283 + window's capacity.
284 + """
285 + block_size = 1
286 + num_cpu_blocks = 8
287 + num_gpu_blocks = 8
288 + sliding_window = 2
289 + block_manager = BlockSpaceManager(block_size,
290 + num_cpu_blocks,
291 + num_gpu_blocks,
292 + sliding_window=sliding_window,
293 + watermark=0)
294 +
295 + assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
296 +
297 + parent = Sequence(1, "one two three", [0, 1, 2], block_size)
298 + seq_group = SequenceGroup("1", [parent], SamplingParams(), time.time(),
299 + None)
300 + block_manager.allocate(seq_group)
301 +
302 + # assert the number of blocks allocated is correct
303 + # the parent seq has len 3, but since sliding_window is 2,
304 + # we will use at most 2 blocks
305 + assert block_manager.get_num_free_gpu_blocks(
306 + ) == num_gpu_blocks - sliding_window
307 +
308 + # Fork prompt and copy block tables.
309 + child = parent.fork(2)
310 + block_manager.fork(parent, child)
311 +
312 + # assert the number of blocks allocated is correct
313 + # forking does not increase memory consumption
314 + assert block_manager.get_num_free_gpu_blocks(
315 + ) == num_gpu_blocks - sliding_window
316 +
317 + # assert both parent and child share all blocks
318 + assert block_manager.get_block_table(
319 + parent) == block_manager.get_block_table(child)
320 +
321 + token_id = 4
322 + # Append token to child. Block is shared so copy on write occurs.
323 + child.append_token_id(token_id, {token_id: Logprob(0.0)})
324 + block_manager.append_slot(child)
325 +
326 + # assert the number of blocks allocated is correct
327 + # we will use now one block more. Each seq will use 2 blocks,
328 + # but only one can be shared
329 + assert block_manager.get_num_free_gpu_blocks(
330 + ) == num_gpu_blocks - sliding_window - 1
331 +
332 + token_id = 5
333 + parent.append_token_id(token_id, {token_id: Logprob(0.0)})
334 + block_manager.append_slot(parent)
335 +
336 + # assert the number of blocks allocated is correct
337 + # no change, because both sequences are still just sharing one block
338 + assert block_manager.get_num_free_gpu_blocks(
339 + ) == num_gpu_blocks - sliding_window - 1
340 +
341 + block_table_parent = block_manager.get_block_table(parent)
342 + block_table_child = block_manager.get_block_table(child)
343 +
344 + assert block_table_parent != block_table_child
345 +
346 + # assert both blocks are sharing the second-last block
347 + assert block_table_parent[-2] == block_table_child[-2]
348 +
349 + # now let's clean up...
350 + block_manager.free(parent)
351 +
352 + # assert the number of blocks allocated is correct
353 + # We have freed one seq, reducing the ref count of two blocks by one.
354 + # One of the two was only used by the parent seq, so this is now free.
355 + # The child seq still consumes sliding_window blocks
356 + assert block_manager.get_num_free_gpu_blocks(
357 + ) == num_gpu_blocks - sliding_window
358 +
359 + # free all blocks
360 + block_manager.free(child)
361 +
362 + # assert all blocks are free now
363 + assert block_manager.get_num_free_gpu_blocks() == num_gpu_blocks
vllm/core/block_manager.py
+15 -2
@@ -312,7 +312,12 @@ class BlockSpaceManager:
312 # Thus, it is always safe from OOM.
313 src_block_table = self.block_tables[parent_seq.seq_id]
314 self.block_tables[child_seq.seq_id] = src_block_table.copy()
315 - for block in src_block_table:
315 + # When using a sliding window, blocks will be eventually reused.
316 + # In this case the block tables will contain repeated blocks.
317 + # When forking, we must make sure that each block's `ref_count`
318 + # is only incremented by one, so we deduplicate them by wrapping
319 + # them in a set.
320 + for block in set(src_block_table):
321 block.ref_count += 1
322
323 def _get_physical_blocks(
@@ -393,7 +398,15 @@ class BlockSpaceManager:
398 return block_number_mapping
399
400 def _free_block_table(self, block_table: BlockTable) -> None:
396 - for block in set(block_table):
401 + # when using a sliding window, each seq will only use up
402 + # to `self.block_sliding_window` blocks. When freeing
403 + # the block table, we must make sure to not free blocks more
404 + # than once. If no sliding window is used, there is no block
405 + # reuse in the block table, so we must free all blocks.
406 + blocks_to_free = (block_table[-self.block_sliding_window:]
407 + if self.block_sliding_window is not None else
408 + block_table)
409 + for block in set(blocks_to_free):
410 if block.device == Device.GPU:
411 self.gpu_allocator.free(block)
412 else: