A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Hotfix][Core][VLM] Disable chunked prefill by default and prefix caching for multimodal models (#8425)
Roger Wang committed
Sep 12, 2024 at 14:06 UTC
c16369455f9568b709d286be0857375a860842ab
2 files changed
+13
-3
vllm/engine/arg_utils.py
+11
-1
@@ -843,6 +843,13 @@ class EngineArgs:
843
device_config = DeviceConfig(device=self.device)
844
model_config = self.create_model_config()
845
846
+ if model_config.is_multimodal_model:
847
+ if self.enable_prefix_caching:
848
+ logger.warning(
849
+ "--enable-prefix-caching is currently not "
850
+ "supported for multimodal models and has been disabled.")
851
+ self.enable_prefix_caching = False
852
+
853
cache_config = CacheConfig(
854
block_size=self.block_size if self.device != "neuron" else
855
self.max_model_len, # neuron needs block_size = max_model_len
@@ -874,7 +881,10 @@ class EngineArgs:
881
# If not explicitly set, enable chunked prefill by default for
882
# long context (> 32K) models. This is to avoid OOM errors in the
883
# initial memory profiling phase.
877
- if use_long_context:
884
+
885
+ # Chunked prefill is currently disabled for multimodal models by
886
+ # default.
887
+ if use_long_context and not model_config.is_multimodal_model:
888
is_gpu = device_config.device_type == "cuda"
889
use_sliding_window = (model_config.get_sliding_window()
890
is not None)
vllm/model_executor/models/__init__.py
+2
-2
@@ -90,12 +90,12 @@ _MULTIMODAL_MODELS = {
90
"PaliGemmaForConditionalGeneration": ("paligemma",
91
"PaliGemmaForConditionalGeneration"),
92
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93
- "UltravoxModel": ("ultravox", "UltravoxModel"),
94
- "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
93
"PixtralForConditionalGeneration": ("pixtral",
94
"PixtralForConditionalGeneration"),
95
+ "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
96
"Qwen2VLForConditionalGeneration": ("qwen2_vl",
97
"Qwen2VLForConditionalGeneration"),
98
+ "UltravoxModel": ("ultravox", "UltravoxModel"),
99
}
100
_CONDITIONAL_GENERATION_MODELS = {
101
"BartModel": ("bart", "BartForConditionalGeneration"),