A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
Disable chunked prefill and/or prefix caching when MLA is enabled (#12642)
From @mgoin in https://github.com/vllm-project/vllm/pull/12638 I cannot push to that branch, therefore a new PR to unblock release. --------- Signed-off-by: mgoin <michael@neuralmagic.com> Signed-off-by: simon-mo <simon.mo@hey.com> Co-authored-by: mgoin <michael@neuralmagic.com>
Simon Mo committed
Jan 31, 2025 at 23:46 UTC
4f4d427ac2cee0f8ff7f79103001f6617fa8989c
1 file changed
+10
vllm/config.py
+10
@@ -3252,6 +3252,16 @@ class VllmConfig:
3252
3253
current_platform.check_and_update_config(self)
3254
3255
+ # If MLA is enabled, force disable chunked prefill and prefix caching
3256
+ if self.model_config and self.model_config.use_mla:
3257
+ logger.info("MLA is enabled; forcing chunked prefill and prefix "
3258
+ "caching to be disabled.")
3259
+ self.scheduler_config.enable_chunked_prefill = False
3260
+ self.scheduler_config.chunked_prefill_enabled = False
3261
+
3262
+ if self.cache_config is not None:
3263
+ self.cache_config.enable_prefix_caching = False
3264
+
3265
if not self.instance_id:
3266
self.instance_id = random_uuid()[:5]
3267