A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Bugfix] Fix speculative decode seeded test (#6743)
Nick Hill committed
Jul 24, 2024 at 08:58 UTC
2cf0df3381b74f93dce3215bf5a043b5c47c55f4
2 files changed
+19
-6
tests/spec_decode/e2e/conftest.py
+2
-1
@@ -191,7 +191,8 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
191
and llm.llm_engine.log_stats):
192
for sate_logger in llm.llm_engine.stat_loggers.values():
193
sate_logger.local_interval = 0
194
- set_random_seed(seed)
194
+ if seed is not None:
195
+ set_random_seed(seed)
196
197
yield llm
198
del llm
tests/spec_decode/e2e/test_seed.py
+17
-5
@@ -21,7 +21,8 @@ from .conftest import run_equality_correctness_test
21
"num_speculative_tokens": 3,
22
}])
23
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24
-@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
24
+@pytest.mark.parametrize("baseline_llm_kwargs", [{"seed": 1}])
25
+@pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
26
@pytest.mark.parametrize("batch_size", [1, 8, 32])
27
@pytest.mark.parametrize("temperature", [0.1, 1.0])
28
@pytest.mark.parametrize(
@@ -30,15 +31,26 @@ from .conftest import run_equality_correctness_test
31
# Use smaller output len for fast test.
32
10,
33
])
33
-@pytest.mark.parametrize("seed", [1])
34
-def test_seeded_consistency(baseline_llm_generator, batch_size: int,
35
- temperature: float, output_len: int):
34
+@pytest.mark.parametrize("seed", [None])
35
+def test_seeded_consistency(baseline_llm_generator, test_llm_generator,
36
+ batch_size: int, temperature: float,
37
+ output_len: int):
38
"""Verify outputs are consistent across multiple runs with same seed
39
"""
40
run_equality_correctness_test(baseline_llm_generator,
39
- baseline_llm_generator,
41
+ test_llm_generator,
42
batch_size,
43
max_output_len=output_len,
44
temperature=temperature,
45
seeded=True,
46
force_output_len=True)
47
+
48
+ # Ensure this same test does fail if we _don't_ include per-request seeds
49
+ with pytest.raises(AssertionError):
50
+ run_equality_correctness_test(baseline_llm_generator,
51
+ test_llm_generator,
52
+ batch_size,
53
+ max_output_len=output_len,
54
+ temperature=temperature,
55
+ seeded=False,
56
+ force_output_len=True)