@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dac914b0d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Bugfix] use blockmanagerv1 for encoder-decoder (#9084)

Co-authored-by: Roger Wang <ywang@roblox.com>

Chen Zhang committed Oct 4, 2024 at 21:45 UTC dac914b0d6bc36de4eb4bf70a9d20954560893ea
1 file changed +5
vllm/engine/arg_utils.py
+5
@@ -903,6 +903,11 @@ class EngineArgs:
903 "--enable-prefix-caching is currently not "
904 "supported for multimodal models and has been disabled.")
905 self.enable_prefix_caching = False
906 + if model_config.is_encoder_decoder_model:
907 + logger.warning(
908 + "Block Manager v2 does not support encoder-decoder models"
909 + " currently. Using Block Manager v1 as fallback.")
910 + self.use_v2_block_manager = False
911
912 cache_config = CacheConfig(
913 block_size=self.block_size if self.device != "neuron" else