A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Bugfix] Fix block size validation (#10938)
chenqianfzh committed
Dec 15, 2024 at 16:38 UTC
69ba344de8683ec4d3d42d11ae4e147a2a302da8
1 file changed
+1
-1
vllm/engine/arg_utils.py
+1
-1
@@ -425,7 +425,7 @@ class EngineArgs:
425
parser.add_argument('--block-size',
426
type=int,
427
default=EngineArgs.block_size,
428
- choices=[8, 16, 32, 64, 128],
428
+ choices=[8, 16, 32],
429
help='Token block size for contiguous chunks of '
430
'tokens. This is ignored on neuron devices and '
431
'set to max-model-len')