@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 69ba344de8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix block size validation (#10938)

chenqianfzh committed Dec 15, 2024 at 16:38 UTC 69ba344de8683ec4d3d42d11ae4e147a2a302da8
1 file changed +1 -1
vllm/engine/arg_utils.py
+1 -1
@@ -425,7 +425,7 @@ class EngineArgs:
425 parser.add_argument('--block-size',
426 type=int,
427 default=EngineArgs.block_size,
428 - choices=[8, 16, 32, 64, 128],
428 + choices=[8, 16, 32],
429 help='Token block size for contiguous chunks of '
430 'tokens. This is ignored on neuron devices and '
431 'set to max-model-len')