[Bugfix] Make KV cache and MFU log lines backend-neutral (#55650)
Signed-off-by: Ianniu <985683179yzy@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Zhiyao(Ian) Yu committed
Sep 14, 2026 at 14:00 UTC
e85c8826ce2a810367e8a70eedb987e31b140800
3 files changed
+10
-4
vllm/v1/core/kv_cache_utils.py
+3
-1
@@ -2391,9 +2391,11 @@ def update_kv_cache_capacity(
2391
vllm_config.cache_config.kv_cache_size_tokens = num_tokens
2392
vllm_config.cache_config.kv_cache_max_concurrency = max_concurrency
2393
max_model_len = vllm_config.model_config.max_model_len
2394
+ device_type = vllm_config.device_config.device_type
2395
logger.info_once(
2395
- "GPU KV cache size: %s tokens, "
2396
+ "%s KV cache size: %s tokens, "
2397
"Maximum concurrency for %s tokens per request: %.2fx",
2398
+ "GPU" if device_type == "cuda" else device_type.upper(),
2399
f"{num_tokens:,}",
2400
f"{max_model_len:,}",
2401
max_concurrency,
vllm/v1/metrics/loggers.py
+6
-2
@@ -101,6 +101,8 @@ class LoggingStatLogger(StatLoggerBase):
101
def __init__(self, vllm_config: VllmConfig, engine_index: int = 0):
102
self.engine_index = engine_index
103
self.vllm_config = vllm_config
104
+ device_type = vllm_config.device_config.device_type
105
+ self.kv_cache_device = "GPU" if device_type == "cuda" else device_type.upper()
106
self._reset(time.monotonic())
107
108
self.last_scheduler_stats = SchedulerStats()
@@ -184,7 +186,7 @@ class LoggingStatLogger(StatLoggerBase):
186
"%sIteration(%d): %d context requests, %d context tokens, "
187
"%d generation requests, %d generation tokens, "
188
"iteration elapsed time: %.2f ms%s, "
187
- "GPU KV cache usage: %.1f%%%s",
189
+ "%s KV cache usage: %.1f%%%s",
190
self._log_prefix_for_engine(engine_idx),
191
details.iteration_index,
192
details.num_ctx_requests,
@@ -193,6 +195,7 @@ class LoggingStatLogger(StatLoggerBase):
195
details.num_generation_tokens,
196
details.elapsed_ms,
197
" (dummy)" if details.is_dummy else "",
198
+ self.kv_cache_device,
199
scheduler_stats.kv_cache_usage * 100,
200
encoder_msg,
201
)
@@ -287,12 +290,13 @@ class LoggingStatLogger(StatLoggerBase):
290
291
log_parts.extend(
292
[
290
- "GPU KV cache usage: %.1f%%",
293
+ "%s KV cache usage: %.1f%%",
294
"Prefix cache hit rate: %.1f%%",
295
]
296
)
297
log_args.extend(
298
[
299
+ self.kv_cache_device,
300
self.last_scheduler_stats.kv_cache_usage * 100,
301
self.prefix_caching_metrics.hit_rate * 100,
302
]
vllm/v1/metrics/perf.py
+1
-1
@@ -1749,7 +1749,7 @@ class PerfMetricsLogging:
1749
)
1750
1751
log_fn(
1752
- "%sMFU: %.1f TF/s/GPU %.1f GB/s/GPU",
1752
+ "%sMFU: %.1f TF/s/device %.1f GB/s/device",
1753
log_prefix,
1754
avg_tflops_per_gpu,
1755
avg_gbps_per_gpu,