@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 866fa4550d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Restore support for larger block sizes (#11259)

Signed-off-by: Konrad Zawora <kzawora@habana.ai>

Konrad Zawora committed Dec 18, 2024 at 01:39 UTC 866fa4550d572f4ff3521ccf503e0df2e76591a1
2 files changed +8 -2
vllm/config.py
+4
@@ -917,6 +917,10 @@ class CacheConfig:
917 raise ValueError(
918 "GPU memory utilization must be less than 1.0. Got "
919 f"{self.gpu_memory_utilization}.")
920 + if (current_platform.is_cuda() and self.block_size is not None
921 + and self.block_size > 32):
922 + raise ValueError("CUDA Paged Attention kernel only supports "
923 + f"block sizes up to 32. Got {self.block_size}.")
924
925 def _verify_cache_dtype(self) -> None:
926 if self.cache_dtype == "auto":
vllm/engine/arg_utils.py
+4 -2
@@ -424,10 +424,12 @@ class EngineArgs:
424 parser.add_argument('--block-size',
425 type=int,
426 default=EngineArgs.block_size,
427 - choices=[8, 16, 32],
427 + choices=[8, 16, 32, 64, 128],
428 help='Token block size for contiguous chunks of '
429 'tokens. This is ignored on neuron devices and '
430 - 'set to max-model-len')
430 + 'set to max-model-len. On CUDA devices, '
431 + 'only block sizes up to 32 are supported. '
432 + 'On HPU devices, block size defaults to 128.')
433
434 parser.add_argument(
435 "--enable-prefix-caching",