A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 22:30 UTC
[Misc][UX] Suppress confusing `num_gpu_blocks` log lines (#40402)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Matthew Bonanni committed
Apr 20, 2026 at 18:32 UTC
c075702eaee56ca2f4a315e742f3c89f5e4c6e71
2 files changed
+29
-11
vllm/v1/core/kv_cache_utils.py
+28
-10
@@ -820,24 +820,31 @@ def get_max_concurrency_for_kv_cache_config(
820
return max_concurrency
821
822
823
-def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
823
+def may_override_num_blocks(
824
+ vllm_config: VllmConfig, num_blocks: int, suppress_log: bool = False
825
+) -> int:
826
"""
827
Override the number of kv cache blocks if `num_gpu_blocks_override` is set.
828
"""
829
if vllm_config.cache_config.num_gpu_blocks_override is not None:
830
num_gpu_blocks_override = vllm_config.cache_config.num_gpu_blocks_override
829
- logger.info(
830
- "Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
831
- num_blocks,
832
- num_gpu_blocks_override,
833
- )
831
+ if not suppress_log:
832
+ logger.info(
833
+ "Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
834
+ num_blocks,
835
+ num_gpu_blocks_override,
836
+ )
837
num_blocks = num_gpu_blocks_override
838
839
return num_blocks
840
841
842
def get_num_blocks(
840
- vllm_config: VllmConfig, num_layers: int, available_memory: int, page_size: int
843
+ vllm_config: VllmConfig,
844
+ num_layers: int,
845
+ available_memory: int,
846
+ page_size: int,
847
+ suppress_log: bool = False,
848
) -> int:
849
"""
850
Get the number of kv cache blocks.
@@ -847,10 +854,14 @@ def get_num_blocks(
854
num_layers: The number of layers
855
available_memory: Memory available for KV cache in bytes.
856
page_size: The page size of the KV cache.
857
+ suppress_log: Whether to suppress override log messages. Used when creating a
858
+ temporary/dummy KV cache config, e.g. during CG memory profiling
859
"""
860
num_blocks = int(available_memory // page_size // num_layers)
861
num_blocks = max(num_blocks, 0)
853
- num_blocks = may_override_num_blocks(vllm_config, num_blocks)
862
+ num_blocks = may_override_num_blocks(
863
+ vllm_config, num_blocks, suppress_log=suppress_log
864
+ )
865
return num_blocks
866
867
@@ -1082,6 +1093,7 @@ def get_kv_cache_config_from_groups(
1093
vllm_config: VllmConfig,
1094
kv_cache_groups: list[KVCacheGroupSpec],
1095
available_memory: int,
1096
+ suppress_log: bool = False,
1097
) -> KVCacheConfig:
1098
"""
1099
Generate the KV cache configuration from the KV cache groups and spec
@@ -1113,7 +1125,9 @@ def get_kv_cache_config_from_groups(
1125
num_blocks = (
1126
available_memory // kv_cache_groups[0].kv_cache_spec.page_size_bytes
1127
)
1116
- num_blocks = may_override_num_blocks(vllm_config, num_blocks)
1128
+ num_blocks = may_override_num_blocks(
1129
+ vllm_config, num_blocks, suppress_log=suppress_log
1130
+ )
1131
per_layer_specs = kv_cache_groups[0].kv_cache_spec.kv_cache_specs
1132
kv_cache_tensors = [
1133
KVCacheTensor(
@@ -1138,7 +1152,11 @@ def get_kv_cache_config_from_groups(
1152
)
1153
assert group_size > 0, "group_size must be greater than 0"
1154
num_blocks = get_num_blocks(
1141
- vllm_config, group_size, available_memory, page_size
1155
+ vllm_config,
1156
+ group_size,
1157
+ available_memory,
1158
+ page_size,
1159
+ suppress_log=suppress_log,
1160
)
1161
kv_cache_tensors = []
1162
for i in range(group_size):
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -5853,7 +5853,7 @@ class GPUModelRunner(
5853
saved_override = self.cache_config.num_gpu_blocks_override
5854
self.cache_config.num_gpu_blocks_override = min_blocks
5855
minimal_config = get_kv_cache_config_from_groups(
5856
- self.vllm_config, kv_cache_groups, available_memory=0
5856
+ self.vllm_config, kv_cache_groups, available_memory=0, suppress_log=True
5857
)
5858
self.cache_config.num_gpu_blocks_override = saved_override
5859