[Test][Core] Compute the expected hybrid prefix-cache hit instead of hard-coding it (#57467)
Signed-off-by: Lin, Fanli <fanli.lin@intel.com>
Fanli Lin committed
Sep 20, 2026 at 10:16 UTC
708c2495c0c6d0fc576224ecfc54efacc6414981
2 files changed
+53
-11
.buildkite/intel_jobs/test-intel.yaml
+1
-1
@@ -98,7 +98,7 @@ steps:
98
- >-
99
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
100
'cd tests && pip install dpctl &&
101
- pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py --ignore=v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py &&
101
+ pytest -v -s v1/core --ignore=v1/core/test_reset_prefix_cache_e2e.py --ignore=v1/core/test_scheduler_e2e.py &&
102
pytest -v -s v1/engine --ignore=v1/engine/test_output_processor.py &&
103
pytest -v -s v1/sample --ignore=v1/sample/test_logprobs.py --ignore=v1/sample/test_logprobs_e2e.py -k "not test_topk_only and not test_topp_only and not test_topk_and_topp" &&
104
pytest -v -s v1/worker --ignore=v1/worker/test_gpu_model_runner.py --ignore=v1/worker/test_worker_memory_snapshot.py &&
tests/v1/core/prefix_cache/test_hybrid_prefix_cache_hit_rate.py
+52
-10
@@ -4,6 +4,7 @@
4
5
import asyncio
6
import json
7
+from math import lcm
8
from uuid import uuid4
9
10
import httpx
@@ -12,6 +13,7 @@ from openai.types.chat import ChatCompletionMessageParam
13
from prometheus_client.parser import text_string_to_metric_families
14
15
from tests.utils import RemoteOpenAIServer
16
+from vllm.utils.math_utils import round_down
17
18
MODEL = "Qwen/Qwen3.5-0.8B"
19
NUM_CONVERSATIONS = 8
@@ -34,6 +36,29 @@ def _user_turns(conversation: int) -> list[str]:
36
]
37
38
39
+async def _scheduler_block_size(client: httpx.AsyncClient) -> int:
40
+ """Return the coarsest boundary an expected cache hit can land on.
41
+
42
+ Mirrors ``scheduler_block_size`` in ``resolve_kv_cache_block_sizes``, which
43
+ the engine keeps internal: the LCM of the resolved group block sizes. A
44
+ finer ``prefix_match_unit`` only adds hits, so this stays conservative.
45
+ """
46
+ response = await client.get("/metrics")
47
+ response.raise_for_status()
48
+ for family in text_string_to_metric_families(response.text):
49
+ for sample in family.samples:
50
+ if sample.name != "vllm:cache_config_info":
51
+ continue
52
+ sizes = [
53
+ int(value)
54
+ for name in ("block_size", "mamba_block_size")
55
+ if (value := sample.labels.get(name, "None")) != "None"
56
+ ]
57
+ assert sizes, f"no block size in {sample.labels}"
58
+ return lcm(*sizes)
59
+ raise AssertionError("missing vllm:cache_config_info metric")
60
+
61
+
62
async def _prefix_cache_counters(
63
client: httpx.AsyncClient,
64
) -> tuple[float, float]:
@@ -83,13 +108,13 @@ def server(request):
108
109
@pytest.mark.asyncio
110
@pytest.mark.parametrize(
86
- "server, min_hit_rate",
87
- # MTP re-prefills an extra block; both floors reject one more missed block.
88
- [pytest.param(False, 0.90, id="base"), pytest.param(True, 0.75, id="mtp")],
111
+ "server, dropped_blocks",
112
+ # MTP's prefill lookahead pollutes the tail block, so EAGLE drops one.
113
+ [pytest.param(False, 0, id="base"), pytest.param(True, 1, id="mtp")],
114
indirect=["server"],
115
)
116
async def test_prefix_cache_hit_rate(
92
- server: RemoteOpenAIServer, min_hit_rate: float
117
+ server: RemoteOpenAIServer, dropped_blocks: int
118
) -> None:
119
# Isolate sessions so another conversation cannot hide a cache miss.
120
salts = [uuid4().hex for _ in range(NUM_CONVERSATIONS)]
@@ -102,6 +127,8 @@ async def test_prefix_cache_hit_rate(
127
server.get_async_client() as client,
128
httpx.AsyncClient(base_url=server.url_root, timeout=METRICS_TIMEOUT) as metrics,
129
):
130
+ block_size = await _scheduler_block_size(metrics)
131
+ print(f"{block_size=}")
132
133
async def run_conversation(session: int, *, replay: bool) -> tuple[int, int]:
134
history = histories[session]
@@ -126,15 +153,30 @@ async def test_prefix_cache_hit_rate(
153
assert cached is not None, context
154
assert 0 <= cached < prompt, context
155
assert prompt >= MIN_PROMPT_TOKENS, context
129
- if not replay and turn == 0:
130
- assert cached == 0, context
131
- else:
132
- assert cached / prompt >= min_hit_rate, (
133
- f"{context}, {min_hit_rate=}"
134
- )
156
if replay:
157
assert prompt == prompt_lengths[session][turn], context
158
+ # The identical request computed this whole prompt, but at
159
+ # least one token is always recomputed.
160
+ computed = prompt - 1
161
+ elif turn:
162
+ computed = prompt_lengths[session][turn - 1]
163
else:
164
+ assert cached == 0, context
165
+ computed = 0
166
+ # Every token below the last reachable boundary must be reused.
167
+ floor = max(
168
+ 0, round_down(computed, block_size) - dropped_blocks * block_size
169
+ )
170
+ print(
171
+ f"{session=} {turn=} {replay=} {prompt=} {cached=} {floor=} "
172
+ f"hit_rate={cached / prompt:.4f}"
173
+ )
174
+ assert cached >= floor, (
175
+ f"{context}, {computed=}, {block_size=}, "
176
+ f"hit_rate={cached / prompt:.4f}, "
177
+ f"expected_hit_rate={floor / prompt:.4f}"
178
+ )
179
+ if not replay:
180
prompt_lengths[session].append(prompt)
181
history.append(
182
{