@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2cf0df3381
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Bugfix] Fix speculative decode seeded test (#6743)

Nick Hill committed Jul 24, 2024 at 08:58 UTC 2cf0df3381b74f93dce3215bf5a043b5c47c55f4
2 files changed +19 -6
tests/spec_decode/e2e/conftest.py
+2 -1
@@ -191,7 +191,8 @@ def create_llm_generator(baseline_or_test, request, common_llm_kwargs,
191 and llm.llm_engine.log_stats):
192 for sate_logger in llm.llm_engine.stat_loggers.values():
193 sate_logger.local_interval = 0
194 - set_random_seed(seed)
194 + if seed is not None:
195 + set_random_seed(seed)
196
197 yield llm
198 del llm
tests/spec_decode/e2e/test_seed.py
+17 -5
@@ -21,7 +21,8 @@ from .conftest import run_equality_correctness_test
21 "num_speculative_tokens": 3,
22 }])
23 @pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
24 -@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
24 +@pytest.mark.parametrize("baseline_llm_kwargs", [{"seed": 1}])
25 +@pytest.mark.parametrize("test_llm_kwargs", [{"seed": 5}])
26 @pytest.mark.parametrize("batch_size", [1, 8, 32])
27 @pytest.mark.parametrize("temperature", [0.1, 1.0])
28 @pytest.mark.parametrize(
@@ -30,15 +31,26 @@ from .conftest import run_equality_correctness_test
31 # Use smaller output len for fast test.
32 10,
33 ])
33 -@pytest.mark.parametrize("seed", [1])
34 -def test_seeded_consistency(baseline_llm_generator, batch_size: int,
35 - temperature: float, output_len: int):
34 +@pytest.mark.parametrize("seed", [None])
35 +def test_seeded_consistency(baseline_llm_generator, test_llm_generator,
36 + batch_size: int, temperature: float,
37 + output_len: int):
38 """Verify outputs are consistent across multiple runs with same seed
39 """
40 run_equality_correctness_test(baseline_llm_generator,
39 - baseline_llm_generator,
41 + test_llm_generator,
42 batch_size,
43 max_output_len=output_len,
44 temperature=temperature,
45 seeded=True,
46 force_output_len=True)
47 +
48 + # Ensure this same test does fail if we _don't_ include per-request seeds
49 + with pytest.raises(AssertionError):
50 + run_equality_correctness_test(baseline_llm_generator,
51 + test_llm_generator,
52 + batch_size,
53 + max_output_len=output_len,
54 + temperature=temperature,
55 + seeded=False,
56 + force_output_len=True)