[CI] Share BF16 baselines across quantization comparison tests (#58469)
Signed-off-by: Aarushi Jain <Aarushi.Jain2@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Aarushi Jain committed
Sep 23, 2026 at 19:26 UTC
e0f03eb4f48ed6c0e193573e5cf1811a06a23e58
3 files changed
+53
-12
tests/models/quantization/conftest.py
new
+41
@@ -0,0 +1,41 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+"""Shared BF16 baselines for the quantization comparison tests.
4
+
5
+Several tests here quantize the same model and compare against an
6
+unquantized run of it. Computing that reference once per session avoids
7
+booting an identical engine for every quantization scheme under test.
8
+
9
+Only the outputs are cached; each engine is still torn down as usual, so
10
+tests keep the isolation ``VllmRunner.__exit__`` provides.
11
+"""
12
+
13
+import pytest
14
+
15
+
16
+@pytest.fixture(scope="session")
17
+def quant_baseline_logprobs(vllm_runner):
18
+ """Greedy logprobs for an unquantized model, computed once per config."""
19
+ cache: dict = {}
20
+
21
+ def baseline(
22
+ model: str,
23
+ prompts: list[str],
24
+ *,
25
+ max_model_len: int,
26
+ max_tokens: int,
27
+ num_logprobs: int,
28
+ ):
29
+ key = (model, max_model_len, max_tokens, num_logprobs, tuple(prompts))
30
+ if key not in cache:
31
+ with vllm_runner(
32
+ model,
33
+ max_model_len=max_model_len,
34
+ enforce_eager=True,
35
+ ) as vllm_model:
36
+ cache[key] = vllm_model.generate_greedy_logprobs(
37
+ prompts, max_tokens, num_logprobs
38
+ )
39
+ return cache[key]
40
+
41
+ return baseline
tests/models/quantization/test_fp8_per_channel.py
+6
-6
@@ -37,6 +37,7 @@ NUM_LOG_PROBS = 8
37
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
38
def test_fp8_per_channel_logprobs(
39
vllm_runner,
40
+ quant_baseline_logprobs,
41
example_prompts,
42
model: str,
43
monkeypatch: pytest.MonkeyPatch,
@@ -52,14 +53,13 @@ def test_fp8_per_channel_logprobs(
53
with monkeypatch.context() as m:
54
m.setenv("TOKENIZERS_PARALLELISM", "true")
55
55
- with vllm_runner(
56
+ baseline_outputs = quant_baseline_logprobs(
57
model,
58
+ example_prompts,
59
max_model_len=MAX_MODEL_LEN,
58
- enforce_eager=True,
59
- ) as vllm_model:
60
- baseline_outputs = vllm_model.generate_greedy_logprobs(
61
- example_prompts, MAX_TOKENS, NUM_LOG_PROBS
62
- )
60
+ max_tokens=MAX_TOKENS,
61
+ num_logprobs=NUM_LOG_PROBS,
62
+ )
63
64
with vllm_runner(
65
model,
tests/models/quantization/test_mxfp8.py
+6
-6
@@ -42,6 +42,7 @@ NUM_LOG_PROBS = 8
42
@pytest.mark.parametrize("model", [DENSE_MODEL, MOE_MODEL], ids=["dense", "moe"])
43
def test_mxfp8_logprobs(
44
vllm_runner,
45
+ quant_baseline_logprobs,
46
example_prompts,
47
model: str,
48
monkeypatch: pytest.MonkeyPatch,
@@ -56,14 +57,13 @@ def test_mxfp8_logprobs(
57
with monkeypatch.context() as m:
58
m.setenv("TOKENIZERS_PARALLELISM", "true")
59
59
- with vllm_runner(
60
+ baseline_outputs = quant_baseline_logprobs(
61
model,
62
+ example_prompts,
63
max_model_len=MAX_MODEL_LEN,
62
- enforce_eager=True,
63
- ) as vllm_model:
64
- baseline_outputs = vllm_model.generate_greedy_logprobs(
65
- example_prompts, MAX_TOKENS, NUM_LOG_PROBS
66
- )
64
+ max_tokens=MAX_TOKENS,
65
+ num_logprobs=NUM_LOG_PROBS,
66
+ )
67
68
with vllm_runner(
69
model,