A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[V1]Enable APC by default only for text models (#10148)
Signed-off-by: Roger Wang <ywang@roblox.com>
Roger Wang committed
Nov 8, 2024 at 06:39 UTC
208ce622c712fef75623f785597dbbd698700fa6
1 file changed
+4
-1
vllm/v1/engine/llm_engine.py
+4
-1
@@ -65,7 +65,10 @@ class LLMEngine:
65
elif usage_context == UsageContext.OPENAI_API_SERVER:
66
scheduler_config.max_num_seqs = 1024
67
scheduler_config.max_num_batched_tokens = 2048
68
- cache_config.enable_prefix_caching = True
68
+
69
+ # TODO (ywang96): Enable APC by default when VLM supports it.
70
+ if not model_config.is_multimodal_model:
71
+ cache_config.enable_prefix_caching = True
72
73
logger.info(
74
"Initializing an LLM engine (v%s) with config: "