@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2e7796f2cf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Speculative decoding] CUDA graph support (#4295)

Co-authored-by: Cade Daniel <edacih@gmail.com>

heeju-kim2 committed May 11, 2024 at 02:36 UTC 2e7796f2cf4537dd3b08d5de3aa8349f5db1a168
1 file changed +37
tests/spec_decode/e2e/test_multistep_correctness.py
+37
@@ -611,3 +611,40 @@ def test_many_k(baseline_llm_generator, test_llm_generator, batch_size: int,
611 batch_size,
612 max_output_len=output_len,
613 force_output_len=True)
614 +
615 +
616 +@pytest.mark.parametrize(
617 + "common_llm_kwargs",
618 + [{
619 + # Required for spec decode.
620 + "use_v2_block_manager": True,
621 +
622 + # Verify equality when cuda graphs allowed.
623 + "enforce_eager": False,
624 + "model": "JackFram/llama-68m",
625 + }])
626 +@pytest.mark.parametrize(
627 + "per_test_common_llm_kwargs",
628 + [
629 + {
630 + # Identical models.
631 + "speculative_model": "JackFram/llama-68m",
632 + "num_speculative_tokens": 5,
633 + },
634 + ])
635 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
636 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
637 +@pytest.mark.parametrize("batch_size", [8])
638 +@pytest.mark.parametrize("output_len", [32])
639 +@pytest.mark.parametrize("seed", [1])
640 +def test_spec_decode_cuda_graph(baseline_llm_generator, test_llm_generator,
641 + batch_size, output_len):
642 + """Verify spec decode equality when cuda graphs are enabled.
643 + """
644 + run_greedy_equality_correctness_test(
645 + baseline_llm_generator,
646 + test_llm_generator,
647 + batch_size,
648 + max_output_len=output_len,
649 + force_output_len=True,
650 + )