A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 04:45 UTC
[MISC] add cpu_kvcache_space_bytes to CacheConfig (#19812)
Signed-off-by: Andy Xie <andy.xning@gmail.com>
Ning Xie committed
Jun 22, 2025 at 13:39 UTC
2bb246b8f7b8dd220008ff7bd735249b362c799a
1 file changed
+2
vllm/config.py
+2
@@ -1506,6 +1506,8 @@ class CacheConfig:
1506
"""This enables dynamic calculation of `k_scale` and `v_scale` when
1507
kv_cache_dtype is fp8. If `False`, the scales will be loaded from the model
1508
checkpoint if available. Otherwise, the scales will default to 1.0."""
1509
+ cpu_kvcache_space_bytes: Optional[int] = None
1510
+ """(CPU backend only) CPU key-value cache space."""
1511
1512
# Will be set after profiling.
1513
num_gpu_blocks: Optional[int] = field(default=None, init=False)