@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6c1208d083
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Core] Add Sliding Window Support with Flashinfer (#10462)

Signed-off-by: Pavani Majety <pmajety@nvidia.com>

Pavani Majety committed Nov 20, 2024 at 19:56 UTC 6c1208d083fbaaf89c6d812f4d3424e15182f652
2 files changed +18 -7
tests/core/block/e2e/test_correctness_sliding_window.py
+10 -2
@@ -3,6 +3,7 @@ from typing import List
3
4 import pytest
5
6 +from tests.kernels.utils import override_backend_env_variable
7 from vllm import LLM, SamplingParams
8
9 from .conftest import get_text_from_llm_generator
@@ -28,8 +29,9 @@ BLOCK_SIZE = 16
29 @pytest.mark.parametrize("test_llm_kwargs", [{}])
30 @pytest.mark.parametrize("batch_size", [5])
31 @pytest.mark.parametrize("seed", [1])
32 +@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
33 def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
32 - batch_size, seed):
34 + batch_size, seed, backend, monkeypatch):
35 """
36 The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
37 asks for value of one of them (which is outside the sliding window).
@@ -38,6 +40,8 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
40
41 Additionally, we compare the results of the v1 and v2 managers.
42 """
43 + override_backend_env_variable(monkeypatch, backend)
44 +
45 sampling_params = SamplingParams(
46 max_tokens=1024,
47 ignore_eos=True,
@@ -84,7 +88,9 @@ def test_sliding_window_retrival(baseline_llm_generator, test_llm_generator,
88 @pytest.mark.parametrize("test_llm_kwargs", [{"enable_chunked_prefill": True}])
89 @pytest.mark.parametrize("batch_size", [5])
90 @pytest.mark.parametrize("seed", [1])
87 -def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
91 +@pytest.mark.parametrize("backend", ["FLASH_ATTN", "FLASHINFER", "XFORMERS"])
92 +def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed,
93 + backend, monkeypatch):
94 """
95 This is similar to test_sliding_window_retrival, however, it doesn't
96 compare against the v1 block manager since v1 doesn't support
@@ -93,6 +99,8 @@ def test_sliding_window_chunked_prefill(test_llm_generator, batch_size, seed):
99 The results with and without chunked prefill are not the same due to
100 numerical instabilities.
101 """
102 + override_backend_env_variable(monkeypatch, backend)
103 +
104 sampling_params = SamplingParams(
105 max_tokens=10,
106 ignore_eos=True,
vllm/attention/backends/flashinfer.py
+8 -5
@@ -757,9 +757,8 @@ class FlashInferImpl(AttentionImpl):
757 if alibi_slopes is not None:
758 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
759 self.alibi_slopes = alibi_slopes
760 - if sliding_window is not None:
761 - raise ValueError("Sliding window is not supported in FlashInfer.")
762 - self.sliding_window = (-1, -1)
760 + self.sliding_window = ((sliding_window - 1,
761 + 0) if sliding_window is not None else (-1, -1))
762 self.kv_cache_dtype = kv_cache_dtype
763 self.logits_soft_cap = logits_soft_cap
764
@@ -865,6 +864,8 @@ def unified_flash_infer(
864 assert query.shape[0] == num_prefill_tokens
865 assert decode_query.shape[0] == num_decode_tokens
866
867 + window_left = window_size[0] if window_size is not None else -1
868 +
869 prefill_output: Optional[torch.Tensor] = None
870 decode_output: Optional[torch.Tensor] = None
871 if prefill_meta := attn_metadata.prefill_metadata:
@@ -895,7 +896,8 @@ def unified_flash_infer(
896 logits_soft_cap=logits_soft_cap,
897 causal=True,
898 k_scale=k_scale,
898 - v_scale=v_scale)
899 + v_scale=v_scale,
900 + window_left=window_left)
901 if decode_meta := attn_metadata.decode_metadata:
902 assert attn_metadata.decode_metadata is not None
903 assert attn_metadata.decode_metadata.decode_wrapper is not None
@@ -905,7 +907,8 @@ def unified_flash_infer(
907 sm_scale=softmax_scale,
908 logits_soft_cap=logits_soft_cap,
909 k_scale=k_scale,
908 - v_scale=v_scale)
910 + v_scale=v_scale,
911 + window_left=window_left)
912
913 if prefill_output is None and decode_output is not None:
914 # Decode only batch.