A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Remove block size constraint (#11723)
Cody Yu committed
Jan 5, 2025 at 20:49 UTC
408e5600158bfa34306cfbd034a3779e488752fa
1 file changed
-5
vllm/config.py
-5
@@ -1015,11 +1015,6 @@ class CacheConfig:
1015
raise ValueError(
1016
"GPU memory utilization must be less than 1.0. Got "
1017
f"{self.gpu_memory_utilization}.")
1018
- from vllm.platforms import current_platform
1019
- if (current_platform.is_cuda() and self.block_size is not None
1020
- and self.block_size > 32):
1021
- raise ValueError("CUDA Paged Attention kernel only supports "
1022
- f"block sizes up to 32. Got {self.block_size}.")
1018
1019
def _verify_cache_dtype(self) -> None:
1020
if self.cache_dtype == "auto":