A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[KV Sharing] Raise error if using eagle with fast prefill (#24350)
Signed-off-by: Yong Hoon Shin <yhshin@meta.com>
Yong Hoon Shin committed
Sep 5, 2025 at 20:22 UTC
3c529fc9945964819dc17b9910ad6ccdbf231413
2 files changed
+18
-7
vllm/config/__init__.py
+18
@@ -3665,6 +3665,24 @@ class VllmConfig:
3665
" Disabling `torch.compile`.")
3666
self.compilation_config.level = CompilationLevel.NO_COMPILATION
3667
3668
+ if self.cache_config.kv_sharing_fast_prefill:
3669
+ if not envs.VLLM_USE_V1:
3670
+ raise NotImplementedError(
3671
+ "Fast prefill optimization for KV sharing is not supported "
3672
+ "in V0 currently.")
3673
+
3674
+ if self.speculative_config is not None and \
3675
+ self.speculative_config.use_eagle():
3676
+ raise NotImplementedError(
3677
+ "Fast prefill optimization for KV sharing is not "
3678
+ "compatible with EAGLE as EAGLE requires correct logits "
3679
+ "for all tokens while fast prefill gives incorrect logits "
3680
+ "for prompt tokens.")
3681
+
3682
+ logger.warning_once(
3683
+ "--kv-sharing-fast-prefill requires changes on model side for "
3684
+ "correctness and to realize prefill savings. ")
3685
+
3686
if ((not envs.VLLM_USE_V1) and self.lora_config is not None
3687
and self.compilation_config.level
3688
!= CompilationLevel.NO_COMPILATION):
vllm/config/cache.py
-7
@@ -145,19 +145,12 @@ class CacheConfig:
145
146
self._verify_cache_dtype()
147
self._verify_prefix_caching()
148
- self._verify_kv_sharing_fast_prefill()
148
149
def metrics_info(self):
150
# convert cache_config to dict(key: str, value: str) for prometheus
151
# metrics info
152
return {key: str(value) for key, value in self.__dict__.items()}
153
155
- def _verify_kv_sharing_fast_prefill(self) -> None:
156
- if self.kv_sharing_fast_prefill and not envs.VLLM_USE_V1:
157
- raise NotImplementedError(
158
- "Fast prefill optimization for KV sharing is not supported "
159
- "in V0 currently.")
160
-
154
@model_validator(mode='after')
155
def _verify_args(self) -> Self:
156
if self.cpu_offload_gb < 0: