@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c8f26bb636
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix][Core] Fix BlockManagerV2 when Encoder Input is None (#9103)

sroy745 committed Oct 6, 2024 at 20:52 UTC c8f26bb63694adb4202ab275efb0759c13edcaa8
3 files changed +3 -8
vllm/core/block/block_table.py
-2
@@ -220,7 +220,6 @@ class BlockTable:
220 occupied by each block. After freeing all the blocks, the `_blocks` list
221 is set to `None`.
222 """
223 - assert self._is_allocated
223 for block in self.blocks:
224 self._allocator.free(block)
225 self._blocks.reset()
@@ -239,7 +238,6 @@ class BlockTable:
238 List[int]: A list of physical block indices for the blocks in the
239 BlockTable.
240 """
242 - assert self._is_allocated
241 return self._blocks.ids()
242
243 def get_unseen_token_ids(self, sequence_token_ids: List[int]) -> List[int]:
vllm/core/block_manager_v2.py
+3 -1
@@ -151,7 +151,9 @@ class BlockSpaceManagerV2(BlockSpaceManager):
151 block_allocator=self.block_allocator,
152 max_block_sliding_window=self.max_block_sliding_window,
153 )
154 - block_table.allocate(seq.get_token_ids())
154 + if seq.get_token_ids():
155 + # Add blocks to the block table only if the sequence is non empty.
156 + block_table.allocate(seq.get_token_ids())
157
158 return block_table
159
vllm/engine/arg_utils.py
-5
@@ -903,11 +903,6 @@ class EngineArgs:
903 "--enable-prefix-caching is currently not "
904 "supported for multimodal models and has been disabled.")
905 self.enable_prefix_caching = False
906 - if model_config.is_encoder_decoder_model:
907 - logger.warning(
908 - "Block Manager v2 does not support encoder-decoder models"
909 - " currently. Using Block Manager v1 as fallback.")
910 - self.use_v2_block_manager = False
906
907 cache_config = CacheConfig(
908 block_size=self.block_size if self.device != "neuron" else