A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Speculative decoding] CUDA graph support (#4295)
Co-authored-by: Cade Daniel <edacih@gmail.com>
heeju-kim2 committed
May 11, 2024 at 02:36 UTC
2e7796f2cf4537dd3b08d5de3aa8349f5db1a168
1 file changed
+37
tests/spec_decode/e2e/test_multistep_correctness.py
+37
@@ -611,3 +611,40 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
611
batch_size,
612
max_output_len=output_len,
613
force_output_len=True)
614
+
615
+
616
+@pytest.mark.parametrize(
617
+ "common_llm_kwargs",
618
+ [{
619
+ # Required for spec decode.
620
+ "use_v2_block_manager": True,
621
+
622
+ # Verify equality when cuda graphs allowed.
623
+ "enforce_eager": False,
624
+ "model": "JackFram/llama-68m",
625
+ }])
626
+@pytest.mark.parametrize(
627
+ "per_test_common_llm_kwargs",
628
+ [
629
+ {
630
+ # Identical models.
631
+ "speculative_model": "JackFram/llama-68m",
632
+ "num_speculative_tokens": 5,
633
+ },
634
+ ])
635
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
636
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
637
+@pytest.mark.parametrize("batch_size", [8])
638
+@pytest.mark.parametrize("output_len", [32])
639
+@pytest.mark.parametrize("seed", [1])
640
+def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
641
+ batch_size, output_len):
642
+ """Verify spec decode equality when cuda graphs are enabled.
643
+ """
644
+ run_greedy_equality_correctness_test(
645
+ baseline_llm_generator,
646
+ test_llm_generator,
647
+ batch_size,
648
+ max_output_len=output_len,
649
+ force_output_len=True,
650
+ )