| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from collections.abc import Callable |
| 5 | from dataclasses import field |
| 6 | from functools import cache |
| 7 | from typing import Any, ClassVar, Literal |
| 8 | |
| 9 | from pydantic import Field, field_validator, model_validator |
| 10 | |
| 11 | from vllm.config.utils import config |
| 12 | from vllm.logger import init_logger |
| 13 | from vllm.utils.torch_utils import ( |
| 14 | is_quantized_kv_cache, |
| 15 | kv_cache_uses_per_token_head_scales, |
| 16 | ) |
| 17 | from vllm.v1.kv_cache_layout import KVCacheLayout |
| 18 | |
| 19 | logger = init_logger(__name__) |
| 20 | |
| 21 | _LAYOUT_COMPAT_ALIASES = { |
| 22 | "NHD": "LBNHC", |
| 23 | "HND": "LBHNC", |
| 24 | } |
| 25 | |
| 26 | |
| 27 | @cache |
| 28 | def _layout_from_name(layout_name: str) -> KVCacheLayout: |
| 29 | layout_name = _LAYOUT_COMPAT_ALIASES.get(layout_name, layout_name) |
| 30 | try: |
| 31 | return KVCacheLayout[layout_name] |
| 32 | except KeyError: |
| 33 | raise ValueError( |
| 34 | f"Unknown KV cache layout {layout_name!r}. " |
| 35 | f"Valid layouts: {[m.name for m in KVCacheLayout]}" |
| 36 | ) from None |
| 37 | |
| 38 | |
| 39 | CacheDType = Literal[ |
| 40 | "auto", |
| 41 | "float16", |
| 42 | "bfloat16", |
| 43 | "fp8", |
| 44 | "fp8_e4m3", |
| 45 | "fp8_e5m2", |
| 46 | "fp8_inc", |
| 47 | "fp8_ds_mla", |
| 48 | "nvfp4_ds_mla", |
| 49 | "turboquant_k8v4", |
| 50 | "turboquant_4bit_nc", |
| 51 | "turboquant_k3v4_nc", |
| 52 | "turboquant_3bit_nc", |
| 53 | "int4_per_token_head", |
| 54 | "int8_per_token_head", |
| 55 | "fp8_per_token_head", |
| 56 | "nvfp4", |
| 57 | "nvfp4_4over6", |
| 58 | ] |
| 59 | |
| 60 | |
| 61 | MambaDType = Literal["auto", "float32", "float16", "bfloat16"] |
| 62 | MambaCacheMode = Literal["align", "none"] |
| 63 | PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] |
| 64 | KVOffloadingBackend = Literal["native", "lmcache"] |
| 65 | |
| 66 | |
| 67 | @config |
| 68 | class CacheConfig: |
| 69 | """Configuration for the KV cache.""" |
| 70 | |
| 71 | DEFAULT_BLOCK_SIZE: ClassVar[int] = 16 |
| 72 | |
| 73 | block_size: int = Field(default=None, gt=0) # type: ignore[assignment] |
| 74 | """Size of a contiguous cache block in number of tokens. |
| 75 | Accepts None (meaning "use default"). After construction, always int.""" |
| 76 | user_specified_block_size: bool = field(default=False, init=False) |
| 77 | """Whether block_size was explicitly provided. Derived automatically.""" |
| 78 | kv_cache_layout: str | None = field(default=None, init=False) |
| 79 | """Resolved physical KV cache layout name (a ``KVCacheLayout`` member). |
| 80 | |
| 81 | ``None`` means the layout has not been resolved yet. The engine core |
| 82 | resolves it once (``resolve_kv_cache_layout``) before memory profiling, and |
| 83 | every worker process adopts the resolved name — via the |
| 84 | ``set_kv_cache_layout`` RPC, or ``KVCacheConfig.kv_cache_layout`` for |
| 85 | workers spawned after resolution — before the KV cache is allocated. Once |
| 86 | set the value is final and read with ``get_resolved_kv_cache_layout``, |
| 87 | which raises on ``None``. Tests and standalone tools may pre-set a value, |
| 88 | which resolution then honors as-is.""" |
| 89 | prefix_match_unit: int | None = Field(default=None, gt=0) |
| 90 | """The finest token boundary (in tokens) a prefix-cache hit can land on. |
| 91 | |
| 92 | Prefix-cache keys are computed every `prefix_match_unit` tokens. It can |
| 93 | be set finer than the physical KV cache block sizes (e.g. 32 vs a |
| 94 | 1024-token hybrid-model block) as long as every KV cache group's |
| 95 | `block_size` is divisible by it, enabling cache hits at boundaries |
| 96 | inside a physical block. It controls matching granularity only, not how |
| 97 | often states are stored. |
| 98 | |
| 99 | This equals to the `hash_block_size` used throughout the KV cache code. |
| 100 | """ |
| 101 | gpu_memory_utilization: float = Field(default=0.92, gt=0, le=1) |
| 102 | """The fraction of GPU memory to be used for the model executor, which can |
| 103 | range from 0 to 1. For example, a value of 0.5 would imply 50% GPU memory |
| 104 | utilization. If unspecified, will use the default value of 0.92. This is a |
| 105 | per-instance limit, and only applies to the current vLLM instance. It does |
| 106 | not matter if you have another vLLM instance running on the same GPU. For |
| 107 | example, if you have two vLLM instances running on the same GPU, you can |
| 108 | set the GPU memory utilization to 0.5 for each instance. On non-GPU |
| 109 | installs, this value controls the corresponding device memory utilization. |
| 110 | """ |
| 111 | |
| 112 | @property |
| 113 | def device_memory_utilization(self) -> float: |
| 114 | """Device-neutral alias for ``gpu_memory_utilization``.""" |
| 115 | return self.gpu_memory_utilization |
| 116 | |
| 117 | @device_memory_utilization.setter |
| 118 | def device_memory_utilization(self, value: float) -> None: |
| 119 | self.gpu_memory_utilization = value |
| 120 | |
| 121 | cache_dtype: CacheDType = "auto" |
| 122 | """Data type for kv cache storage. If "auto", will use model data type. |
| 123 | CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports |
| 124 | fp8 (=fp8_e4m3). Intel Gaudi (HPU) supports fp8 (using fp8_inc). |
| 125 | Some models (namely DeepSeekV3.2) default to fp8, set to bfloat16 to use |
| 126 | bfloat16 instead, this is an invalid option for models that do not default |
| 127 | to fp8. |
| 128 | "nvfp4_4over6" uses the NVFP4 layout and selects between max/6 and max/4 |
| 129 | scales per 16 values by minimizing squared reconstruction error. |
| 130 | """ |
| 131 | is_attention_free: bool = False |
| 132 | """Whether the model is attention-free. This is primarily set in |
| 133 | `ModelConfig` and that value should be manually duplicated here.""" |
| 134 | num_gpu_blocks_override: int | None = None |
| 135 | """Number of GPU blocks to use. This overrides the profiled `num_gpu_blocks` |
| 136 | if specified. Does nothing if `None`. Used for testing preemption.""" |
| 137 | sliding_window: int | None = None |
| 138 | """Sliding window size for the KV cache. This is primarily set in |
| 139 | `ModelConfig` and that value should be manually duplicated here.""" |
| 140 | enable_prefix_caching: bool = True |
| 141 | """Whether to enable prefix caching.""" |
| 142 | prefix_caching_hash_algo: PrefixCachingHashAlgo = "sha256" |
| 143 | """Set the hash algorithm for prefix caching: |
| 144 | |
| 145 | - "sha256" uses Pickle for object serialization before hashing. This is the current |
| 146 | default, as SHA256 is the most secure choice to avoid potential hash collisions. |
| 147 | - "sha256_cbor" provides a reproducible, cross-language compatible hash. It |
| 148 | serializes objects using canonical CBOR and hashes them with SHA-256. |
| 149 | - "xxhash" uses Pickle serialization with xxHash (128-bit) for faster, |
| 150 | non-cryptographic hashing. Requires the optional ``xxhash`` package. |
| 151 | IMPORTANT: Use of a hashing algorithm that is not considered cryptographically |
| 152 | secure theoretically increases the risk of hash collisions, which can cause |
| 153 | undefined behavior or even leak private information in multi-tenant environments. |
| 154 | Even if collisions are still very unlikely, it is important to consider your |
| 155 | security risk tolerance against the performance benefits before turning this on. |
| 156 | - "xxhash_cbor" combines canonical CBOR serialization with xxHash for |
| 157 | reproducible hashing. Requires the optional ``xxhash`` package.""" |
| 158 | prefix_cache_retention_interval: int | None = Field(default=0, ge=0) |
| 159 | """Token interval between retained sliding-window and Mamba prefix-cache |
| 160 | checkpoints. ``0`` retains only semantic checkpoints, including the latest |
| 161 | replay boundary and shared-prefix junctions. Positive values additionally |
| 162 | retain periodic checkpoints at the specified interval, which must be a |
| 163 | multiple of the scheduler block size. ``None`` retains checkpoints densely. |
| 164 | Applies only to sliding-window and Mamba cache groups.""" |
| 165 | kv_cache_dtype_skip_layers: list[str] = field(default_factory=list) |
| 166 | """Layer patterns to skip KV cache quantization. Accepts layer indices |
| 167 | (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window').""" |
| 168 | mamba_page_size_padded: int | None = None |
| 169 | """ Optional override for mamba page size; used by hybrid mamba/attention |
| 170 | models to ensure exact alignment with attention page size.""" |
| 171 | skip_page_size_padded: int | None = None |
| 172 | """Optional override for the page size of layers skipped from KV cache |
| 173 | quantization (``--kv-cache-dtype-skip-layers``); set during block-size |
| 174 | alignment so unquantized skip layers pad up to the quantized primary's |
| 175 | page.""" |
| 176 | mamba_block_size: int | None = Field(default=None, gt=0) |
| 177 | """Size of a contiguous cache block in number of tokens for mamba cache. |
| 178 | Can be set only when prefix caching is enabled. |
| 179 | Value must be a multiple of 8 to align with causal_conv1d kernel.""" |
| 180 | mamba_cache_dtype: MambaDType = "auto" |
| 181 | """The data type to use for the Mamba cache (both the conv as well as the |
| 182 | ssm state). If set to 'auto', the data type will be inferred from the model |
| 183 | config.""" |
| 184 | mamba_ssm_cache_dtype: MambaDType = "auto" |
| 185 | """The data type to use for the Mamba cache (ssm state only, conv state will |
| 186 | still be controlled by mamba_cache_dtype). If set to 'auto', the data type |
| 187 | for the ssm state will be determined by mamba_cache_dtype.""" |
| 188 | mamba_cache_mode: MambaCacheMode = "none" |
| 189 | """The cache strategy for Mamba layers: |
| 190 | |
| 191 | - "none": set when prefix caching is disabled. |
| 192 | - "align": only cache the mamba state of the last token of each scheduler step and |
| 193 | when the token is at position i * block_size. This is the default when prefix |
| 194 | caching is enabled. |
| 195 | """ |
| 196 | enable_mamba_shared_prefix_checkpoint: bool = False |
| 197 | """Also register a Mamba "align" checkpoint at the shared-prefix junction -- |
| 198 | where an EAGLE/MTP sibling was observed to resume -- instead of only at the |
| 199 | prompt tail. Off by default; only takes effect with `mamba_cache_mode` |
| 200 | "align", EAGLE on the Mamba group, and a prefix match unit smaller than the |
| 201 | Mamba block size.""" |
| 202 | replayssm_buffer_len: int = Field(default=16, gt=0) |
| 203 | """ReplaySSM logical history length B for Mamba2. Triton uses B physical |
| 204 | rows and FlashInfer uses B+T, where T is the target verification length. |
| 205 | Kimi-K3 speculative decode does not use B. Default 16.""" |
| 206 | use_replayssm: bool = False |
| 207 | """Use the ReplaySSM Mamba2 decode kernel: cache recent SSM inputs and skip |
| 208 | the per-step full-state store, writing the checkpoint back only on flush. |
| 209 | Requires mamba_cache_mode 'none' or 'align' (prefix caching) and the Triton |
| 210 | or FlashInfer mamba backend. Mamba2 speculative decode requires FlashInfer |
| 211 | and mamba_cache_mode 'none'. In align mode flushes are most efficient when |
| 212 | mamba_block_size is a multiple of replayssm_buffer_len, but this is not |
| 213 | required.""" |
| 214 | use_kda_recoverssm: bool = field(default=False, init=False) |
| 215 | """Whether Kimi-K3 KDA uses RecoverSSM speculative decode.""" |
| 216 | |
| 217 | # Will be set after profiling. |
| 218 | num_gpu_blocks: int | None = field(default=None, init=False) |
| 219 | """The number of blocks to allocate for GPU memory.""" |
| 220 | num_cpu_blocks: int | None = field(default=None, init=False) |
| 221 | """The number of blocks to allocate for CPU memory.""" |
| 222 | |
| 223 | # Set after KV cache initialization. |
| 224 | effective_attention_block_size: int | None = field(default=None, init=False) |
| 225 | """Full-attention block size in tokens, including DCP, or None if unavailable.""" |
| 226 | kv_cache_size_tokens: int | None = field(default=None, init=False) |
| 227 | """Per-DP-engine KV cache capacity in tokens (group-aware). Uses |
| 228 | group-aware capacity since num_gpu_blocks * block_size can be wrong |
| 229 | for hybrid models where requests occupy multiple KV cache groups.""" |
| 230 | kv_cache_max_concurrency: float | None = field(default=None, init=False) |
| 231 | """Per-DP-engine maximum concurrency at max_model_len tokens.""" |
| 232 | |
| 233 | kv_sharing_fast_prefill: bool = False |
| 234 | """In some KV sharing setups, e.g. YOCO (https://arxiv.org/abs/2405.05254), |
| 235 | some layers can skip tokens corresponding to prefill. This flag enables |
| 236 | attention metadata for eligible layers to be overridden with metadata |
| 237 | necessary for implementing this optimization in some models (e.g. Gemma3n) |
| 238 | """ |
| 239 | |
| 240 | swa_bounded_replay: bool = True |
| 241 | """Keep the sliding-window KV of models that support it (DeepSeek-V4.1) |
| 242 | out of prefix caching and rebuild it after a prefix hit by recomputing the |
| 243 | hit's last window. The layers past the last KV-source layer then also run |
| 244 | eager prefill steps on each request's trailing window only. Requires model |
| 245 | runner V2.""" |
| 246 | |
| 247 | kv_cache_memory_bytes: int | None = None |
| 248 | """Size of KV Cache per GPU in bytes. By default, this is set to None |
| 249 | and vllm can automatically infer the kv cache size based on |
| 250 | gpu_memory_utilization. However, users may want to manually specify |
| 251 | the kv cache memory size. kv_cache_memory_bytes allows more fine-grain |
| 252 | control of how much memory gets used when compared with using |
| 253 | gpu_memory_utilization. Note that kv_cache_memory_bytes |
| 254 | (when not-None) ignores gpu_memory_utilization""" |
| 255 | |
| 256 | kv_offloading_size: float | None = None |
| 257 | """Size of the KV cache offloading buffer in GiB. When TP > 1, this is |
| 258 | the total buffer size summed across all TP ranks. By default, this is set |
| 259 | to None, which means no KV offloading is enabled. When set, vLLM will |
| 260 | enable KV cache offloading to CPU using the kv_offloading_backend.""" |
| 261 | |
| 262 | kv_offloading_backend: KVOffloadingBackend = "native" |
| 263 | """The backend to use for KV cache offloading. Supported backends include |
| 264 | 'native' (vLLM native CPU offloading), 'lmcache'. |
| 265 | KV offloading is only activated when kv_offloading_size is set.""" |
| 266 | |
| 267 | def compute_hash(self) -> str: |
| 268 | """WARNING: Whenever a new field is added to this config, |
| 269 | ensure that it is included in the factors list if |
| 270 | it affects the computation graph. |
| 271 | |
| 272 | Provide a hash that uniquely identifies all the configs |
| 273 | that affect the structure of the computation |
| 274 | graph from input ids/embeddings to the final hidden states, |
| 275 | excluding anything before input ids/embeddings and after |
| 276 | the final hidden states. |
| 277 | """ |
| 278 | ignored_factors = { |
| 279 | # Runtime/derived knobs that don't affect compiled graph shape |
| 280 | "gpu_memory_utilization", |
| 281 | "kv_cache_memory_bytes", |
| 282 | "is_attention_free", |
| 283 | "num_gpu_blocks_override", |
| 284 | "enable_prefix_caching", |
| 285 | "prefix_caching_hash_algo", |
| 286 | "prefix_cache_retention_interval", |
| 287 | # Prefix-caching implementation detail (doesn't affect compiled graph). |
| 288 | "prefix_match_unit", |
| 289 | "enable_mamba_shared_prefix_checkpoint", |
| 290 | "mamba_page_size_padded", |
| 291 | "skip_page_size_padded", |
| 292 | "user_specified_block_size", |
| 293 | "_block_size_resolved", |
| 294 | # Post-init/derived counters |
| 295 | "num_gpu_blocks", |
| 296 | "num_cpu_blocks", |
| 297 | "effective_attention_block_size", |
| 298 | "kv_cache_size_tokens", |
| 299 | "kv_cache_max_concurrency", |
| 300 | # Feature toggles not impacting compiled graph shape |
| 301 | "kv_sharing_fast_prefill", |
| 302 | "swa_bounded_replay", |
| 303 | } |
| 304 | |
| 305 | from vllm.config.utils import get_hash_factors, hash_factors |
| 306 | |
| 307 | factors = get_hash_factors(self, ignored_factors) |
| 308 | return hash_factors(factors) |
| 309 | |
| 310 | def metrics_info(self): |
| 311 | # convert cache_config to dict(key: str, value: str) for prometheus |
| 312 | # metrics info |
| 313 | return {key: str(value) for key, value in self.__dict__.items()} |
| 314 | |
| 315 | _block_size_resolved: bool = field(default=False, init=False) |
| 316 | """Guard against pydantic re-running _apply_block_size_default.""" |
| 317 | |
| 318 | @field_validator("block_size", mode="wrap") |
| 319 | @classmethod |
| 320 | def _skip_none_validation(cls, value: Any, handler: Callable) -> Any: |
| 321 | if value is None: |
| 322 | return value |
| 323 | return handler(value) |
| 324 | |
| 325 | @model_validator(mode="after") |
| 326 | def _apply_block_size_default(self) -> "CacheConfig": |
| 327 | # Pydantic re-runs validators when CacheConfig is nested inside |
| 328 | # another pydantic model (e.g. VllmConfig). Guard against that. |
| 329 | if self._block_size_resolved: |
| 330 | return self |
| 331 | self._block_size_resolved = True |
| 332 | if self.block_size is None: |
| 333 | self.block_size = self.DEFAULT_BLOCK_SIZE |
| 334 | else: |
| 335 | self.user_specified_block_size = True |
| 336 | return self |
| 337 | |
| 338 | @field_validator("cache_dtype", mode="after") |
| 339 | @classmethod |
| 340 | def _validate_cache_dtype(cls, cache_dtype: CacheDType) -> CacheDType: |
| 341 | if kv_cache_uses_per_token_head_scales(cache_dtype): |
| 342 | logger.info( |
| 343 | "Using %s data type to store kv cache. It reduces the GPU " |
| 344 | "memory footprint and boosts the performance. " |
| 345 | "Dynamic per-token-head scales will be computed at runtime.", |
| 346 | str(cache_dtype), |
| 347 | ) |
| 348 | elif is_quantized_kv_cache(cache_dtype): |
| 349 | logger.info( |
| 350 | "Using %s data type to store kv cache. It reduces the GPU " |
| 351 | "memory footprint and boosts the performance. " |
| 352 | "Meanwhile, it may cause accuracy drop without a proper " |
| 353 | "scaling factor", |
| 354 | str(cache_dtype), |
| 355 | ) |
| 356 | return cache_dtype |
| 357 | |
| 358 | def get_resolved_kv_cache_layout(self) -> KVCacheLayout: |
| 359 | if self.kv_cache_layout is None: |
| 360 | raise ValueError( |
| 361 | "KV cache layout has not been resolved yet; it is resolved once " |
| 362 | "by the engine core (resolve_kv_cache_layout) unless explicitly " |
| 363 | "set by the user." |
| 364 | ) |
| 365 | return _layout_from_name(self.kv_cache_layout) |