A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 13:30 UTC
`CacheConfig.block_size` should always be `int` when used (#17052)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Apr 23, 2025 at 16:50 UTC
f3a21e9c6835f04ecb5da1b953cb650986bf2209
1 file changed
+5
-2
vllm/config.py
+5
-2
@@ -1261,11 +1261,14 @@ PrefixCachingHashAlgo = Literal["builtin", "sha256"]
1261
class CacheConfig:
1262
"""Configuration for the KV cache."""
1263
1264
- block_size: Optional[BlockSize] = None
1264
+ block_size: BlockSize = None # type: ignore
1265
"""Size of a contiguous cache block in number of tokens. This is ignored on
1266
neuron devices and set to `--max-model-len`. On CUDA devices, only block
1267
sizes up to 32 are supported. On HPU devices, block size defaults to 128.
1268
- """
1268
+
1269
+ This config has no static default. If left unspecified by the user, it will
1270
+ be set in `Platform.check_and_update_configs()` based on the current
1271
+ platform."""
1272
gpu_memory_utilization: float = 0.9
1273
"""The fraction of GPU memory to be used for the model executor, which can
1274
range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory