A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Metrics] Add test for multi-modal cache stats logging (#26588)
Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Mark McLoughlin committed
Oct 10, 2025 at 17:00 UTC
e5192819208c4d68194844b7dfafbc00020d0dea
3 files changed
+30
-4
tests/entrypoints/llm/test_mm_cache_stats.py
+24
@@ -1,10 +1,14 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
+import logging
5
+
6
import pytest
7
+import regex as re
8
9
from vllm import LLM
10
from vllm.entrypoints.chat_utils import ChatCompletionMessageParam
11
+from vllm.v1.metrics import loggers as stat_loggers
12
from vllm.v1.metrics.reader import Counter, Metric
13
14
from ..openai.test_vision import TEST_IMAGE_ASSETS
@@ -37,12 +41,27 @@ def _get_mm_cache_stats(metrics: list[Metric]):
41
return mm_cache_queries, mm_cache_hits
42
43
44
+def _get_mm_cache_log(llm: LLM, caplog_vllm: pytest.LogCaptureFixture) -> float:
45
+ caplog_vllm.clear()
46
+ with caplog_vllm.at_level(logging.INFO, logger=stat_loggers.__name__):
47
+ llm.llm_engine.do_log_stats()
48
+
49
+ assert len(caplog_vllm.records) == 1
50
+ msg = caplog_vllm.records[0].getMessage()
51
+
52
+ assert "MM cache hit rate" in msg
53
+ match = re.search(r"MM cache hit rate: ([0-9.]+)%", msg)
54
+ assert match is not None
55
+ return float(match.group(1))
56
+
57
+
58
@pytest.mark.parametrize("image_urls", [TEST_IMAGE_ASSETS[:2]], indirect=True)
59
@pytest.mark.parametrize("mm_processor_cache_type", ["lru", "shm"])
60
def test_mm_cache_stats(
61
num_gpus_available,
62
image_urls,
63
mm_processor_cache_type,
64
+ caplog_vllm,
65
):
66
llm = LLM(
67
model="llava-hf/llava-1.5-7b-hf",
@@ -56,12 +75,15 @@ def test_mm_cache_stats(
75
76
llm.chat(_make_messages(image_urls[0]))
77
assert _get_mm_cache_stats(llm.get_metrics()) == (1, 0)
78
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
79
80
llm.chat(_make_messages(image_urls[1]))
81
assert _get_mm_cache_stats(llm.get_metrics()) == (2, 0)
82
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
83
84
llm.chat(_make_messages(image_urls[0]))
85
assert _get_mm_cache_stats(llm.get_metrics()) == (3, 1)
86
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(33.3)
87
88
# NOTE: This only resets hit rate stats in CachingMetrics
89
# The raw queries and hits counts remain unaffected
@@ -69,6 +91,8 @@ def test_mm_cache_stats(
91
92
llm.chat(_make_messages(image_urls[0]))
93
assert _get_mm_cache_stats(llm.get_metrics()) == (4, 1)
94
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
95
96
llm.chat(_make_messages(image_urls[1]))
97
assert _get_mm_cache_stats(llm.get_metrics()) == (5, 1)
98
+ assert _get_mm_cache_log(llm, caplog_vllm) == pytest.approx(0.0)
vllm/v1/metrics/loggers.py
+1
-4
@@ -60,7 +60,6 @@ class LoggingStatLogger(StatLoggerBase):
60
self._reset(time.monotonic())
61
62
self.last_scheduler_stats = SchedulerStats()
63
- self.last_mm_cache_stats: Optional[MultiModalCacheStats] = None
63
64
# Caching metrics. This cannot be reset.
65
# TODO: Make the interval configurable.
@@ -115,8 +114,6 @@ class LoggingStatLogger(StatLoggerBase):
114
if mm_cache_stats:
115
self.mm_caching_metrics.observe(mm_cache_stats)
116
118
- self.last_mm_cache_stats = mm_cache_stats
119
-
117
def log(self):
118
now = time.monotonic()
119
prompt_throughput = self._get_throughput(self.num_prompt_tokens, now)
@@ -157,7 +154,7 @@ class LoggingStatLogger(StatLoggerBase):
154
scheduler_stats.kv_cache_usage * 100,
155
self.prefix_caching_metrics.hit_rate * 100,
156
]
160
- if self.last_mm_cache_stats:
157
+ if not self.mm_caching_metrics.empty:
158
log_parts.append("MM cache hit rate: %.1f%%")
159
log_args.append(self.mm_caching_metrics.hit_rate * 100)
160
vllm/v1/metrics/stats.py
+5
@@ -96,6 +96,11 @@ class CachingMetrics:
96
self.aggregated_query_hit = 0
97
self.query_queue.clear()
98
99
+ @property
100
+ def empty(self) -> bool:
101
+ """Return true if no requests have been observed."""
102
+ return self.aggregated_requests == 0
103
+
104
@property
105
def hit_rate(self) -> float:
106
"""Calculate the hit rate for the past N requests."""