@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 408e560015
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Remove block size constraint (#11723)

Cody Yu committed Jan 5, 2025 at 20:49 UTC 408e5600158bfa34306cfbd034a3779e488752fa
1 file changed -5
vllm/config.py
-5
@@ -1015,11 +1015,6 @@ class CacheConfig:
1015 raise ValueError(
1016 "GPU memory utilization must be less than 1.0. Got "
1017 f"{self.gpu_memory_utilization}.")
1018 - from vllm.platforms import current_platform
1019 - if (current_platform.is_cuda() and self.block_size is not None
1020 - and self.block_size > 32):
1021 - raise ValueError("CUDA Paged Attention kernel only supports "
1022 - f"block sizes up to 32. Got {self.block_size}.")
1018
1019 def _verify_cache_dtype(self) -> None:
1020 if self.cache_dtype == "auto":