@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 208ce622c7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[V1]Enable APC by default only for text models (#10148)

Signed-off-by: Roger Wang <ywang@roblox.com>

Roger Wang committed Nov 8, 2024 at 06:39 UTC 208ce622c712fef75623f785597dbbd698700fa6
1 file changed +4 -1
vllm/v1/engine/llm_engine.py
+4 -1
@@ -65,7 +65,10 @@ class LLMEngine:
65 elif usage_context == UsageContext.OPENAI_API_SERVER:
66 scheduler_config.max_num_seqs = 1024
67 scheduler_config.max_num_batched_tokens = 2048
68 - cache_config.enable_prefix_caching = True
68 +
69 + # TODO (ywang96): Enable APC by default when VLM supports it.
70 + if not model_config.is_multimodal_model:
71 + cache_config.enable_prefix_caching = True
72
73 logger.info(
74 "Initializing an LLM engine (v%s) with config: "