A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 11, 00:00 UTC
Correct position of docstring of class attributes (#31209)
Signed-off-by: Weida Hong <wdhongtw@google.com>
Weida Hong committed
Dec 23, 2025 at 18:08 UTC
73cfb7a722e36c8429e7fdb5306be2d981c3f8dd
2 files changed
+13
-10
vllm/forward_context.py
+1
-1
@@ -186,6 +186,7 @@ class DPMetadata:
186
class ForwardContext:
187
# copy from vllm_config.compilation_config.static_forward_context
188
no_compile_layers: dict[str, Any]
189
+ attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]]
190
"""
191
Type Dict[str, AttentionMetadata] for v1, map from layer_name of each
192
attention layer to its attention metadata
@@ -193,7 +194,6 @@ class ForwardContext:
194
for each microbatch.
195
Set dynamically for each forward pass
196
"""
196
- attn_metadata: dict[str, AttentionMetadata] | list[dict[str, AttentionMetadata]]
197
# TODO: remove after making all virtual_engines share the same kv cache
198
virtual_engine: int # set dynamically for each forward pass
199
# set dynamically for each forward pass
vllm/v1/kv_cache_interface.py
+12
-9
@@ -80,17 +80,20 @@ class AttentionSpec(KVCacheSpec):
80
81
@dataclass(frozen=True)
82
class FullAttentionSpec(AttentionSpec):
83
- sliding_window: int | None = None
84
- attention_chunk_size: int | None = None
83
"""
86
- When hybrid allocator is disabled and the model contains both full
87
- attention layers and sliding window attention layers, sliding
88
- window attention are regarded as full attention in KV cache manager
89
- (blocks are allocated for all tokens), while computed as sliding window
84
+ When hybrid allocator is disabled and the model contains both full
85
+ attention layers and sliding window attention layers, sliding
86
+ window attention are regarded as full attention in KV cache manager
87
+ (blocks are allocated for all tokens), while computed as sliding window
88
attention in model runner.
89
In this case, we use FullAttentionSpec and record the sliding window size.
90
+ """
91
+
92
+ sliding_window: int | None = None
93
+ """
94
Default to None for not using sliding window attention.
95
"""
96
+ attention_chunk_size: int | None = None
97
98
def max_memory_usage_bytes(self, vllm_config: VllmConfig) -> int:
99
max_model_len = vllm_config.model_config.max_model_len
@@ -390,10 +393,11 @@ class KVCacheConfig:
393
The KV cache configuration of a model.
394
"""
395
393
- """The number of KV cache blocks"""
396
num_blocks: int
395
- """How should model runner initialize the KV cache tensors for each layer"""
397
+ """The number of KV cache blocks"""
398
kv_cache_tensors: list[KVCacheTensor]
399
+ """How should model runner initialize the KV cache tensors for each layer"""
400
+ kv_cache_groups: list[KVCacheGroupSpec]
401
"""
402
The kv cache groups of the model.
403
For models with only one type of attention, there is only one group that
@@ -401,4 +405,3 @@ class KVCacheConfig:
405
For models with multiple types of attention, there will be multiple groups,
406
see `_get_kv_cache_config_uniform_page_size` for more details.
407
"""
404
- kv_cache_groups: list[KVCacheGroupSpec]