@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54bd9a03c4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

register custom op for flash attn and use from torch.ops (#7536)

youkaichao committed Aug 15, 2024 at 22:38 UTC 54bd9a03c4b2da0fd0b0e17b0552bbb0d517a081
5 files changed +220 -41
.buildkite/test-pipeline.yaml
+7
@@ -163,6 +163,13 @@ steps:
163 - pytest -v -s models/test_oot_registration.py # it needs a clean process
164 - pytest -v -s models -m \"not vlm\" --ignore=models/test_oot_registration.py
165
166 +- label: torch compile integration test
167 + source_file_dependencies:
168 + - vllm/
169 + commands:
170 + - pytest -v -s ./compile/test_full_graph.py
171 +
172 +
173 - label: Vision Language Models Test # 42min
174 mirror_hardwares: [amd]
175 source_file_dependencies:
tests/compile/test_full_graph.py new
+20
@@ -0,0 +1,20 @@
1 +import os
2 +
3 +import pytest
4 +
5 +
6 +@pytest.mark.parametrize("model", ["meta-llama/Meta-Llama-3-8B"])
7 +def test_full_graph(model):
8 + # make sure these models can be captured in full graph mode
9 + os.environ["VLLM_TEST_DYNAMO_GRAPH_CAPTURE"] = "1"
10 +
11 + from vllm import LLM, SamplingParams
12 + prompts = [
13 + "Hello, my name is",
14 + "The president of the United States is",
15 + "The capital of France is",
16 + "The future of AI is",
17 + ]
18 + sampling_params = SamplingParams(temperature=0)
19 + llm = LLM(model="meta-llama/Meta-Llama-3-8B")
20 + llm.generate(prompts, sampling_params)
tests/kernels/test_flash_attn.py
+61 -12
@@ -2,13 +2,16 @@ from typing import List, Optional, Tuple
2
3 import pytest
4 import torch
5 -from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
5
7 -NUM_HEADS = [(16, 16), (32, 8), (64, 8)]
6 +import vllm.attention.backends.flash_attn # noqa: F401
7 +
8 +NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
9 HEAD_SIZES = [128, 256]
10 BLOCK_SIZES = [16, 32]
11 DTYPES = [torch.float16, torch.bfloat16]
11 -NUM_BLOCKS = 32768 # Large enough to test overflow in index calculation.
12 +# one value large enough to test overflow in index calculation.
13 +# one value small enough to test the schema op check
14 +NUM_BLOCKS = [32768, 2048]
15
16
17 def ref_paged_attn(
@@ -72,6 +75,7 @@ def ref_paged_attn(
75 @pytest.mark.parametrize("block_size", BLOCK_SIZES)
76 @pytest.mark.parametrize("dtype", DTYPES)
77 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
78 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
79 @torch.inference_mode()
80 def test_flash_attn_with_paged_kv(
81 kv_lens: List[int],
@@ -80,6 +84,7 @@ def test_flash_attn_with_paged_kv(
84 dtype: torch.dtype,
85 block_size: int,
86 soft_cap: Optional[float],
87 + num_blocks: int,
88 ) -> None:
89 torch.set_default_device("cuda")
90 torch.cuda.manual_seed_all(0)
@@ -91,7 +96,7 @@ def test_flash_attn_with_paged_kv(
96 scale = head_size**-0.5
97
98 query = torch.randn(num_seqs, num_query_heads, head_size, dtype=dtype)
94 - key_cache = torch.randn(NUM_BLOCKS,
99 + key_cache = torch.randn(num_blocks,
100 block_size,
101 num_kv_heads,
102 head_size,
@@ -101,14 +106,14 @@ def test_flash_attn_with_paged_kv(
106
107 max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
108 block_tables = torch.randint(0,
104 - NUM_BLOCKS,
109 + num_blocks,
110 (num_seqs, max_num_blocks_per_seq),
111 dtype=torch.int32)
112
108 - output = flash_attn_with_kvcache(
109 - q=query.unsqueeze(1),
110 - k_cache=key_cache,
111 - v_cache=value_cache,
113 + output = torch.ops.vllm.flash_attn_with_kvcache(
114 + decode_query=query.unsqueeze(1),
115 + key_cache=key_cache,
116 + value_cache=value_cache,
117 softmax_scale=scale,
118 causal=True,
119 block_table=block_tables,
@@ -116,6 +121,25 @@ def test_flash_attn_with_paged_kv(
121 softcap=soft_cap if soft_cap is not None else 0,
122 ).squeeze(1)
123
124 + if num_blocks <= 2048:
125 + test_utils = ["test_faketensor", "test_schema"]
126 + else:
127 + test_utils = ["test_faketensor"]
128 +
129 + torch.library.opcheck(torch.ops.vllm.flash_attn_with_kvcache,
130 + args=tuple(),
131 + kwargs=dict(
132 + decode_query=query.unsqueeze(1),
133 + key_cache=key_cache,
134 + value_cache=value_cache,
135 + softmax_scale=scale,
136 + causal=True,
137 + block_table=block_tables,
138 + cache_seqlens=kv_lens_tensor,
139 + softcap=soft_cap if soft_cap is not None else 0,
140 + ),
141 + test_utils=test_utils)
142 +
143 ref_output = ref_paged_attn(
144 query=query,
145 key_cache=key_cache,
@@ -137,6 +161,7 @@ def test_flash_attn_with_paged_kv(
161 @pytest.mark.parametrize("sliding_window", [None])
162 @pytest.mark.parametrize("dtype", DTYPES)
163 @pytest.mark.parametrize("soft_cap", [None, 10.0, 50.0])
164 +@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
165 @torch.inference_mode()
166 def test_varlen_with_paged_kv(
167 seq_lens: List[Tuple[int, int]],
@@ -146,6 +171,7 @@ def test_varlen_with_paged_kv(
171 dtype: torch.dtype,
172 block_size: int,
173 soft_cap: Optional[float],
174 + num_blocks: int,
175 ) -> None:
176 torch.set_default_device("cuda")
177 torch.cuda.manual_seed_all(0)
@@ -166,7 +192,7 @@ def test_varlen_with_paged_kv(
192 num_query_heads,
193 head_size,
194 dtype=dtype)
169 - key_cache = torch.randn(NUM_BLOCKS,
195 + key_cache = torch.randn(num_blocks,
196 block_size,
197 num_kv_heads,
198 head_size,
@@ -181,11 +207,11 @@ def test_varlen_with_paged_kv(
207
208 max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
209 block_tables = torch.randint(0,
184 - NUM_BLOCKS,
210 + num_blocks,
211 (num_seqs, max_num_blocks_per_seq),
212 dtype=torch.int32)
213
188 - output = flash_attn_varlen_func(
214 + output = torch.ops.vllm.flash_attn_varlen_func(
215 q=query,
216 k=key_cache,
217 v=value_cache,
@@ -200,6 +226,29 @@ def test_varlen_with_paged_kv(
226 softcap=soft_cap if soft_cap is not None else 0,
227 )
228
229 + if num_blocks <= 2048:
230 + test_utils = ["test_faketensor", "test_schema"]
231 + else:
232 + test_utils = ["test_faketensor"]
233 +
234 + torch.library.opcheck(torch.ops.vllm.flash_attn_varlen_func,
235 + args=tuple(),
236 + kwargs=dict(
237 + q=query,
238 + k=key_cache,
239 + v=value_cache,
240 + cu_seqlens_q=cu_query_lens,
241 + cu_seqlens_k=cu_kv_lens,
242 + max_seqlen_q=max_query_len,
243 + max_seqlen_k=max_kv_len,
244 + softmax_scale=scale,
245 + causal=True,
246 + window_size=window_size,
247 + block_table=block_tables,
248 + softcap=soft_cap if soft_cap is not None else 0,
249 + ),
250 + test_utils=test_utils)
251 +
252 ref_output = ref_paged_attn(
253 query=query,
254 key_cache=key_cache,
vllm/attention/backends/flash_attn.py
+129 -26
@@ -3,7 +3,6 @@ from dataclasses import dataclass
3 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type
4
5 import torch
6 -from vllm_flash_attn import flash_attn_varlen_func, flash_attn_with_kvcache
6
7 from vllm import _custom_ops as ops
8 from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
@@ -18,6 +17,108 @@ from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17 if TYPE_CHECKING:
18 from vllm.worker.model_runner import ModelInputForGPUBuilder
19
20 +from vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
21 +from vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache
22 +
23 +
24 +@torch.library.custom_op("vllm::flash_attn_varlen_func", mutates_args=[])
25 +def flash_attn_varlen_func(
26 + q: torch.Tensor,
27 + k: torch.Tensor,
28 + v: torch.Tensor,
29 + cu_seqlens_q: torch.Tensor,
30 + cu_seqlens_k: torch.Tensor,
31 + max_seqlen_q: int,
32 + max_seqlen_k: int,
33 + softmax_scale: Optional[float] = None,
34 + causal: bool = False,
35 + window_size: Optional[List[int]] = None,
36 + softcap: float = 0.0,
37 + alibi_slopes: Optional[torch.Tensor] = None,
38 + block_table: Optional[torch.Tensor] = None,
39 +) -> torch.Tensor:
40 + # custom op does not support tuple input
41 + real_window_size: Tuple[int, int]
42 + if window_size is None:
43 + real_window_size = (-1, -1)
44 + else:
45 + assert len(window_size) == 2
46 + real_window_size = (window_size[0], window_size[1])
47 + return _flash_attn_varlen_func(
48 + q=q,
49 + k=k,
50 + v=v,
51 + cu_seqlens_q=cu_seqlens_q,
52 + cu_seqlens_k=cu_seqlens_k,
53 + max_seqlen_q=max_seqlen_q,
54 + max_seqlen_k=max_seqlen_k,
55 + softmax_scale=softmax_scale,
56 + causal=causal,
57 + window_size=real_window_size,
58 + softcap=softcap,
59 + alibi_slopes=alibi_slopes,
60 + block_table=block_table,
61 + )
62 +
63 +
64 +@flash_attn_varlen_func.register_fake # type: ignore
65 +def _(
66 + q: torch.Tensor,
67 + k: torch.Tensor,
68 + v: torch.Tensor,
69 + cu_seqlens_q: torch.Tensor,
70 + cu_seqlens_k: torch.Tensor,
71 + max_seqlen_q: int,
72 + max_seqlen_k: int,
73 + softmax_scale: Optional[float] = None,
74 + causal: bool = False,
75 + window_size: Optional[List[int]] = None,
76 + softcap: float = 0.0,
77 + alibi_slopes: Optional[torch.Tensor] = None,
78 + block_table: Optional[torch.Tensor] = None,
79 +) -> torch.Tensor:
80 + return torch.empty_like(q)
81 +
82 +
83 +@torch.library.custom_op("vllm::flash_attn_with_kvcache", mutates_args=[])
84 +def flash_attn_with_kvcache(
85 + decode_query: torch.Tensor,
86 + key_cache: torch.Tensor,
87 + value_cache: torch.Tensor,
88 + cache_seqlens: Optional[torch.Tensor] = None,
89 + block_table: Optional[torch.Tensor] = None,
90 + softmax_scale: Optional[float] = None,
91 + causal: bool = False,
92 + alibi_slopes: Optional[torch.Tensor] = None,
93 + softcap: float = 0.0,
94 +) -> torch.Tensor:
95 + return _flash_attn_with_kvcache(
96 + decode_query,
97 + key_cache,
98 + value_cache,
99 + cache_seqlens=cache_seqlens,
100 + block_table=block_table,
101 + softmax_scale=softmax_scale,
102 + causal=causal,
103 + alibi_slopes=alibi_slopes,
104 + softcap=softcap,
105 + )
106 +
107 +
108 +@flash_attn_with_kvcache.register_fake # type: ignore
109 +def _(
110 + decode_query: torch.Tensor,
111 + key_cache: torch.Tensor,
112 + value_cache: torch.Tensor,
113 + cache_seqlens: Optional[torch.Tensor] = None,
114 + block_table: Optional[torch.Tensor] = None,
115 + softmax_scale: Optional[float] = None,
116 + causal: bool = False,
117 + alibi_slopes: Optional[torch.Tensor] = None,
118 + softcap: float = 0.0,
119 +) -> torch.Tensor:
120 + return torch.empty_like(decode_query)
121 +
122
123 class FlashAttentionBackend(AttentionBackend):
124
@@ -517,7 +618,7 @@ class FlashAttentionImpl(AttentionImpl):
618 # normal attention
619 # When block_tables are not filled, it means q and k are the
620 # prompt, and they have the same length.
520 - out = flash_attn_varlen_func(
621 + out = torch.ops.vllm.flash_attn_varlen_func(
622 q=query,
623 k=key,
624 v=value,
@@ -537,34 +638,36 @@ class FlashAttentionImpl(AttentionImpl):
638 # prefix-enabled attention
639 assert prefill_meta.seq_lens is not None
640 max_seq_len = max(prefill_meta.seq_lens)
540 - output[:num_prefill_tokens] = flash_attn_varlen_func(
541 - q=query,
542 - k=key_cache,
543 - v=value_cache,
544 - cu_seqlens_q=prefill_meta.query_start_loc,
545 - max_seqlen_q=prefill_meta.max_query_len,
546 - cu_seqlens_k=prefill_meta.seq_start_loc,
547 - max_seqlen_k=max_seq_len,
641 + output[:
642 + num_prefill_tokens] = torch.ops.vllm.flash_attn_varlen_func( # noqa
643 + q=query,
644 + k=key_cache,
645 + v=value_cache,
646 + cu_seqlens_q=prefill_meta.query_start_loc,
647 + max_seqlen_q=prefill_meta.max_query_len,
648 + cu_seqlens_k=prefill_meta.seq_start_loc,
649 + max_seqlen_k=max_seq_len,
650 + softmax_scale=self.scale,
651 + causal=True,
652 + alibi_slopes=self.alibi_slopes,
653 + block_table=prefill_meta.block_tables,
654 + softcap=self.logits_soft_cap,
655 + )
656 +
657 + if decode_meta := attn_metadata.decode_metadata:
658 + # Decoding run.
659 + output[
660 + num_prefill_tokens:] = torch.ops.vllm.flash_attn_with_kvcache(
661 + decode_query.unsqueeze(1),
662 + key_cache,
663 + value_cache,
664 + block_table=decode_meta.block_tables,
665 + cache_seqlens=decode_meta.seq_lens_tensor,
666 softmax_scale=self.scale,
667 causal=True,
668 alibi_slopes=self.alibi_slopes,
551 - block_table=prefill_meta.block_tables,
669 softcap=self.logits_soft_cap,
553 - )
554 -
555 - if decode_meta := attn_metadata.decode_metadata:
556 - # Decoding run.
557 - output[num_prefill_tokens:] = flash_attn_with_kvcache(
558 - decode_query.unsqueeze(1),
559 - key_cache,
560 - value_cache,
561 - block_table=decode_meta.block_tables,
562 - cache_seqlens=decode_meta.seq_lens_tensor,
563 - softmax_scale=self.scale,
564 - causal=True,
565 - alibi_slopes=self.alibi_slopes,
566 - softcap=self.logits_soft_cap,
567 - ).squeeze(1)
670 + ).squeeze(1)
671
672 # Reshape the output tensor.
673 return output.view(num_tokens, hidden_size)
vllm/attention/backends/flashinfer.py
+3 -3
@@ -4,9 +4,9 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
4 try:
5 from flashinfer import BatchDecodeWithPagedKVCacheWrapper
6 from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
7 - from vllm_flash_attn import flash_attn_varlen_func
7 +
8 + import vllm.attention.backends.flash_attn # noqa
9 except ImportError:
9 - flash_attn_varlen_func = None
10 BatchDecodeWithPagedKVCacheWrapper = None
11 BatchPrefillWithPagedKVCacheWrapper = None
12
@@ -520,7 +520,7 @@ class FlashInferImpl(AttentionImpl):
520 # This happens when vllm runs the profiling to
521 # determine the number of blocks.
522 if kv_cache is None:
523 - output = flash_attn_varlen_func(
523 + output = torch.ops.vllm.flash_attn_varlen_func(
524 q=query,
525 k=key,
526 v=value,