@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c16369455f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Hotfix][Core][VLM] Disable chunked prefill by default and prefix caching for multimodal models (#8425)

Roger Wang committed Sep 12, 2024 at 14:06 UTC c16369455f9568b709d286be0857375a860842ab
2 files changed +13 -3
vllm/engine/arg_utils.py
+11 -1
@@ -843,6 +843,13 @@ class EngineArgs:
843 device_config = DeviceConfig(device=self.device)
844 model_config = self.create_model_config()
845
846 + if model_config.is_multimodal_model:
847 + if self.enable_prefix_caching:
848 + logger.warning(
849 + "--enable-prefix-caching is currently not "
850 + "supported for multimodal models and has been disabled.")
851 + self.enable_prefix_caching = False
852 +
853 cache_config = CacheConfig(
854 block_size=self.block_size if self.device != "neuron" else
855 self.max_model_len, # neuron needs block_size = max_model_len
@@ -874,7 +881,10 @@ class EngineArgs:
881 # If not explicitly set, enable chunked prefill by default for
882 # long context (> 32K) models. This is to avoid OOM errors in the
883 # initial memory profiling phase.
877 - if use_long_context:
884 +
885 + # Chunked prefill is currently disabled for multimodal models by
886 + # default.
887 + if use_long_context and not model_config.is_multimodal_model:
888 is_gpu = device_config.device_type == "cuda"
889 use_sliding_window = (model_config.get_sliding_window()
890 is not None)
vllm/model_executor/models/__init__.py
+2 -2
@@ -90,12 +90,12 @@ _MULTIMODAL_MODELS = {
90 "PaliGemmaForConditionalGeneration": ("paligemma",
91 "PaliGemmaForConditionalGeneration"),
92 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93 - "UltravoxModel": ("ultravox", "UltravoxModel"),
94 - "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
93 "PixtralForConditionalGeneration": ("pixtral",
94 "PixtralForConditionalGeneration"),
95 + "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
96 "Qwen2VLForConditionalGeneration": ("qwen2_vl",
97 "Qwen2VLForConditionalGeneration"),
98 + "UltravoxModel": ("ultravox", "UltravoxModel"),
99 }
100 _CONDITIONAL_GENERATION_MODELS = {
101 "BartModel": ("bart", "BartForConditionalGeneration"),