A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[Bugfix] Fix the speculative decoding test by setting the target dtype (#19633)
Lu Fang committed
Jun 14, 2025 at 11:57 UTC
06be85882814f5772f54f5e15c975c43f43361a9
1 file changed
+24
tests/spec_decode/e2e/test_multistep_correctness.py
+24
@@ -57,6 +57,9 @@ from .conftest import (get_output_from_llm_generator,
57
58
# Skip cuda graph recording for fast test.
59
"enforce_eager": True,
60
+
61
+ # The original model is float32, keep it for numerical stability.
62
+ "dtype": "float32",
63
}])
64
@pytest.mark.parametrize(
65
"per_test_common_llm_kwargs",
@@ -139,6 +142,9 @@ def test_spec_decode_e2e_with_detokenization(test_llm_generator,
142
143
# Print spec metrics.
144
"disable_log_stats": False,
145
+
146
+ # The original model is float32, keep it for numerical stability.
147
+ "dtype": "float32",
148
}])
149
@pytest.mark.parametrize(
150
"per_test_common_llm_kwargs",
@@ -216,6 +222,9 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_bs1(
222
223
# Print spec metrics.
224
"disable_log_stats": False,
225
+
226
+ # The original model is float32, keep it for numerical stability.
227
+ "dtype": "float32",
228
}])
229
@pytest.mark.parametrize(
230
"per_test_common_llm_kwargs",
@@ -279,6 +288,9 @@ def test_spec_decode_e2e_greedy_correctness_tiny_model_large_bs(
288
[{
289
# Skip cuda graph recording for fast test.
290
"enforce_eager": True,
291
+
292
+ # The original model is float32, keep it for numerical stability.
293
+ "dtype": "float32",
294
}])
295
@pytest.mark.parametrize(
296
"per_test_common_llm_kwargs",
@@ -464,6 +476,8 @@ def test_spec_decode_e2e_greedy_correctness_real_model_large_bs(
476
477
# Skip cuda graph recording for fast test.
478
"enforce_eager": True,
479
+ # The original model is float32, keep it for numerical stability.
480
+ "dtype": "float32",
481
}])
482
@pytest.mark.parametrize("per_test_common_llm_kwargs", [
483
{
@@ -523,6 +537,8 @@ def test_spec_decode_e2e_greedy_correctness_with_preemption(
537
538
# Skip cuda graph recording for fast test.
539
"enforce_eager": True,
540
+ # The original model is float32, keep it for numerical stability.
541
+ "dtype": "float32",
542
}])
543
@pytest.mark.parametrize(
544
"per_test_common_llm_kwargs",
@@ -589,6 +605,8 @@ def test_spec_decode_different_block_size(vllm_runner, common_llm_kwargs,
605
606
# Skip cuda graph recording for fast test.
607
"enforce_eager": True,
608
+ # The original model is float32, keep it for numerical stability.
609
+ "dtype": "float32",
610
}])
611
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
612
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -655,6 +673,8 @@ def test_skip_speculation(vllm_runner, common_llm_kwargs,
673
674
# Skip cuda graph recording for fast test.
675
"enforce_eager": True,
676
+ # The original model is float32, keep it for numerical stability.
677
+ "dtype": "float32",
678
}])
679
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
680
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -706,6 +726,8 @@ def test_disable_speculation(vllm_runner, common_llm_kwargs,
726
727
# Skip cuda graph recording for fast test.
728
"enforce_eager": True,
729
+ # The original model is float32, keep it for numerical stability.
730
+ "dtype": "float32",
731
}])
732
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
733
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
@@ -763,6 +785,8 @@ def test_many_k(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
785
786
# Skip cuda graph recording for fast test.
787
"enforce_eager": True,
788
+ # The original model is float32, keep it for numerical stability.
789
+ "dtype": "float32",
790
}])
791
@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
792
@pytest.mark.parametrize("baseline_llm_kwargs", [{}])