A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Kernel] Use flashinfer for decoding (#4353)
Co-authored-by: LiuXiaoxuanPKU <llilyliupku@gmail.com>
Lily Liu committed
May 3, 2024 at 15:51 UTC
43c413ec570e94869ee7b7d275de720219a34357
15 files changed
+600
-53
csrc/cache.h
+8
@@ -24,6 +24,14 @@ void reshape_and_cache(
24
const std::string& kv_cache_dtype,
25
const float kv_scale);
26
27
+void reshape_and_cache_flash(
28
+ torch::Tensor& key,
29
+ torch::Tensor& value,
30
+ torch::Tensor& key_cache,
31
+ torch::Tensor& value_cache,
32
+ torch::Tensor& slot_mapping,
33
+ const std::string& kv_cache_dtype);
34
+
35
// Just for unittest
36
void convert_fp8(
37
torch::Tensor& src_cache,
csrc/cache_kernels.cu
+80
@@ -215,6 +215,41 @@ __global__ void reshape_and_cache_kernel(
215
}
216
}
217
218
+template<typename scalar_t>
219
+__global__ void reshape_and_cache_flash_kernel(
220
+ const scalar_t* __restrict__ key, // [num_tokens, num_heads, head_size]
221
+ const scalar_t* __restrict__ value, // [num_tokens, num_heads, head_size]
222
+ scalar_t* __restrict__ k_cache, // [num_blocks, block_size, num_heads, head_size]
223
+ scalar_t* __restrict__ v_cache, // [num_blocks, block_size, num_heads, head_size]
224
+ const int64_t* __restrict__ slot_mapping, // [num_tokens]
225
+ const int block_stride,
226
+ const int key_stride,
227
+ const int value_stride,
228
+ const int num_heads,
229
+ const int head_size,
230
+ const int block_size) {
231
+ const int64_t token_idx = blockIdx.x;
232
+ const int64_t slot_idx = slot_mapping[token_idx];
233
+ // NOTE: slot_idx can be -1 if the token is padded
234
+ if (slot_idx < 0) {
235
+ return;
236
+ }
237
+ const int64_t block_idx = slot_idx / block_size;
238
+ const int64_t block_offset = slot_idx % block_size;
239
+ const int n = num_heads * head_size;
240
+ for (int i = threadIdx.x; i < n; i += blockDim.x) {
241
+ const int64_t src_key_idx = token_idx * key_stride + i;
242
+ const int64_t src_value_idx = token_idx * value_stride + i;
243
+ const int head_idx = i / head_size;
244
+ const int head_offset = i % head_size;
245
+ const int64_t tgt_value_idx = block_idx * block_stride
246
+ + block_offset * num_heads * head_size
247
+ + head_idx * head_size
248
+ + head_offset;
249
+ k_cache[tgt_value_idx] = key[src_key_idx];
250
+ v_cache[tgt_value_idx] = value[src_value_idx];
251
+ }
252
+}
253
} // namespace vllm
254
255
#define CALL_RESHAPE_AND_CACHE(KV_T, CACHE_T, IS_FP8_KV_CACHE) \
@@ -275,6 +310,51 @@ void reshape_and_cache(
310
}
311
}
312
313
+void reshape_and_cache_flash(
314
+ torch::Tensor& key, // [num_tokens, num_heads, head_size]
315
+ torch::Tensor& value, // [num_tokens, num_heads, head_size]
316
+ torch::Tensor& k_cache, // [num_blocks, block_size, num_heads, head_size]
317
+ torch::Tensor& v_cache, // [num_blocks, block_size, num_heads, head_size]
318
+ torch::Tensor& slot_mapping, // [num_tokens]
319
+ const std::string& kv_cache_dtype)
320
+{
321
+ // FIXME: only support auto datatype, does not support fp8
322
+ if (kv_cache_dtype != "auto") {
323
+ TORCH_CHECK(false, "Unsupported data type of kv cache: ", kv_cache_dtype);
324
+ }
325
+ int num_tokens = key.size(0);
326
+ int num_heads = key.size(1);
327
+ int head_size = key.size(2);
328
+ int block_size = k_cache.size(1);
329
+
330
+ int key_stride = key.stride(0);
331
+ int value_stride = value.stride(0);
332
+ int block_stride = k_cache.stride(0);
333
+ TORCH_CHECK(k_cache.stride(0) == v_cache.stride(0));
334
+
335
+ dim3 grid(num_tokens);
336
+ dim3 block(std::min(num_heads * head_size, 512));
337
+ const at::cuda::OptionalCUDAGuard device_guard(device_of(key));
338
+ const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
339
+ VLLM_DISPATCH_FLOATING_TYPES(
340
+ key.scalar_type(),
341
+ "reshape_and_cache_flash",
342
+ [&] {
343
+ vllm::reshape_and_cache_flash_kernel<scalar_t><<<grid, block, 0, stream>>>(
344
+ key.data_ptr<scalar_t>(),
345
+ value.data_ptr<scalar_t>(),
346
+ k_cache.data_ptr<scalar_t>(),
347
+ v_cache.data_ptr<scalar_t>(),
348
+ slot_mapping.data_ptr<int64_t>(),
349
+ block_stride,
350
+ key_stride,
351
+ value_stride,
352
+ num_heads,
353
+ head_size,
354
+ block_size);
355
+ });
356
+}
357
+
358
namespace vllm {
359
360
template<typename Tout, typename Tin>
csrc/pybind.cpp
+4
@@ -96,6 +96,10 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
96
"reshape_and_cache",
97
&reshape_and_cache,
98
"Reshape the key and value tensors and cache them");
99
+ cache_ops.def(
100
+ "reshape_and_cache_flash",
101
+ &reshape_and_cache_flash,
102
+ "Reshape the key and value tensors and cache them");
103
cache_ops.def(
104
"convert_fp8",
105
&convert_fp8,
tests/basic_correctness/test_basic_correctness.py
+11
-1
@@ -2,12 +2,15 @@
2
3
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4
"""
5
+import os
6
+
7
import pytest
8
9
MODELS = [
10
"facebook/opt-125m",
11
"meta-llama/Llama-2-7b-hf",
12
]
13
+VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
14
15
16
@pytest.mark.parametrize("model", MODELS)
@@ -23,11 +26,18 @@ def test_models(
26
max_tokens: int,
27
enforce_eager: bool,
28
) -> None:
29
+ backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
30
+ if backend_by_env_var == "FLASHINFER" and enforce_eager is False:
31
+ pytest.skip("Skipping non-eager test for FlashInferBackend.")
32
+
33
hf_model = hf_runner(model, dtype=dtype)
34
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
35
del hf_model
36
30
- vllm_model = vllm_runner(model, dtype=dtype, enforce_eager=enforce_eager)
37
+ vllm_model = vllm_runner(model,
38
+ dtype=dtype,
39
+ enforce_eager=enforce_eager,
40
+ gpu_memory_utilization=0.7)
41
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
42
del vllm_model
43
tests/distributed/test_basic_distributed_correctness.py
+9
-5
@@ -18,6 +18,7 @@ import torch
18
MODELS = [
19
os.environ["TEST_DIST_MODEL"],
20
]
21
+VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
22
23
24
@pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -33,16 +34,19 @@ def test_models(
34
dtype: str,
35
max_tokens: int,
36
) -> None:
37
+ enforce_eager = False
38
+ backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
39
+ if backend_by_env_var == "FLASHINFER":
40
+ enforce_eager = True
41
42
hf_model = hf_runner(model, dtype=dtype)
43
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
44
del hf_model
45
41
- vllm_model = vllm_runner(
42
- model,
43
- dtype=dtype,
44
- tensor_parallel_size=2,
45
- )
46
+ vllm_model = vllm_runner(model,
47
+ dtype=dtype,
48
+ tensor_parallel_size=2,
49
+ enforce_eager=enforce_eager)
50
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
51
del vllm_model
52
tests/kernels/conftest.py
+7
-1
@@ -1,8 +1,14 @@
1
import pytest
2
3
-from vllm.utils import create_kv_caches_with_random
3
+from vllm.utils import (create_kv_caches_with_random,
4
+ create_kv_caches_with_random_flash)
5
6
7
@pytest.fixture()
8
def kv_cache_factory():
9
return create_kv_caches_with_random
10
+
11
+
12
+@pytest.fixture()
13
+def kv_cache_factory_flashinfer():
14
+ return create_kv_caches_with_random_flash
tests/kernels/test_cache.py
+77
@@ -5,6 +5,7 @@ import pytest
5
import torch
6
7
from vllm import _custom_ops as ops
8
+from vllm._C import cache_ops
9
from vllm.utils import is_hip
10
11
COPYING_DIRECTION = [('cuda', 'cpu'), ('cuda', 'cuda'), ('cpu', 'cuda')]
@@ -191,6 +192,82 @@ def test_reshape_and_cache(
192
assert torch.allclose(value_cache, cloned_value_cache)
193
194
195
+@pytest.mark.parametrize("num_tokens", NUM_TOKENS)
196
+@pytest.mark.parametrize("num_heads", NUM_HEADS)
197
+@pytest.mark.parametrize("head_size", HEAD_SIZES)
198
+@pytest.mark.parametrize("block_size", BLOCK_SIZES)
199
+@pytest.mark.parametrize("num_blocks", NUM_BLOCKS)
200
+@pytest.mark.parametrize("dtype", DTYPES)
201
+@pytest.mark.parametrize("seed", SEEDS)
202
+@pytest.mark.parametrize("device", CUDA_DEVICES)
203
+@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
204
+@torch.inference_mode()
205
+def test_reshape_and_cache_flash(
206
+ kv_cache_factory_flashinfer,
207
+ num_tokens: int,
208
+ num_heads: int,
209
+ head_size: int,
210
+ block_size: int,
211
+ num_blocks: int,
212
+ dtype: torch.dtype,
213
+ seed: int,
214
+ device: str,
215
+ kv_cache_dtype: str,
216
+) -> None:
217
+ if kv_cache_dtype == "fp8":
218
+ pytest.skip()
219
+ random.seed(seed)
220
+ torch.random.manual_seed(seed)
221
+ torch.cuda.manual_seed(seed)
222
+
223
+ # Create a random slot mapping.
224
+ num_slots = block_size * num_blocks
225
+ slot_mapping = random.sample(range(num_slots), num_tokens)
226
+ slot_mapping = torch.tensor(slot_mapping, dtype=torch.long, device='cuda')
227
+
228
+ qkv = torch.randn(num_tokens,
229
+ 3,
230
+ num_heads,
231
+ head_size,
232
+ dtype=dtype,
233
+ device=device)
234
+ _, key, value = qkv.unbind(dim=1)
235
+
236
+ # Create the KV caches.
237
+ key_caches, value_caches = kv_cache_factory_flashinfer(
238
+ num_blocks,
239
+ block_size,
240
+ 1,
241
+ num_heads,
242
+ head_size,
243
+ kv_cache_dtype,
244
+ dtype,
245
+ )
246
+ key_cache, value_cache = key_caches[0], value_caches[0]
247
+
248
+ # Clone the KV caches.
249
+ cloned_key_cache = key_cache.clone()
250
+ cloned_value_cache = value_cache.clone()
251
+
252
+ # Call the reshape_and_cache kernel.
253
+ cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
254
+ slot_mapping, kv_cache_dtype)
255
+
256
+ # Run the reference implementation.
257
+ block_indicies = torch.div(slot_mapping, block_size, rounding_mode='floor')
258
+ block_indicies = block_indicies.cpu().tolist()
259
+ block_offsets = slot_mapping % block_size
260
+ block_offsets = block_offsets.cpu().tolist()
261
+ for i in range(num_tokens):
262
+ block_idx = block_indicies[i]
263
+ block_offset = block_offsets[i]
264
+ cloned_key_cache[block_idx, block_offset, :, :] = key[i]
265
+ cloned_value_cache[block_idx, block_offset, :, :] = value[i]
266
+
267
+ assert torch.allclose(key_cache, cloned_key_cache)
268
+ assert torch.allclose(value_cache, cloned_value_cache)
269
+
270
+
271
@pytest.mark.parametrize("direction", COPYING_DIRECTION)
272
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
273
@pytest.mark.parametrize("num_heads", NUM_HEADS)
vllm/_custom_ops.py
+12
@@ -222,6 +222,18 @@ def reshape_and_cache(
222
slot_mapping, kv_cache_dtype, kv_scale)
223
224
225
+def reshape_and_cache_flash(
226
+ key: torch.Tensor,
227
+ value: torch.Tensor,
228
+ key_cache: torch.Tensor,
229
+ value_cache: torch.Tensor,
230
+ slot_mapping: torch.Tensor,
231
+ kv_cache_dtype: str,
232
+) -> None:
233
+ vllm_cache_ops.reshape_and_cache_flash(key, value, key_cache, value_cache,
234
+ slot_mapping, kv_cache_dtype)
235
+
236
+
237
def copy_blocks(key_caches: torch.Tensor, value_caches: torch.Tensor,
238
block_mapping: torch.Tensor) -> None:
239
vllm_cache_ops.copy_blocks(key_caches, value_caches, block_mapping)
vllm/attention/backends/abstract.py
+9
-4
@@ -1,6 +1,7 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass, fields
3
-from typing import Any, Dict, Generic, List, Optional, Tuple, Type, TypeVar
3
+from typing import (Any, Dict, Generic, List, Optional, Set, Tuple, Type,
4
+ TypeVar)
5
6
import torch
7
@@ -15,7 +16,7 @@ class AttentionBackend(ABC):
16
17
@staticmethod
18
@abstractmethod
18
- def make_metadata(*args, **kwargs) -> "AttentionMetadata":
19
+ def make_metadata(*args, **kwargs) -> "AttentionMetadataPerStage":
20
raise NotImplementedError
21
22
@staticmethod
@@ -50,13 +51,17 @@ class AttentionBackend(ABC):
51
class AttentionMetadataPerStage:
52
"""Attention metadata for a specific stage. I.e., prefill or decode."""
53
53
- def asdict_zerocopy(self) -> Dict[str, Any]:
54
+ def asdict_zerocopy(self,
55
+ skip_fields: Optional[Set[str]] = None
56
+ ) -> Dict[str, Any]:
57
"""Similar to dataclasses.asdict, but avoids deepcopying."""
58
+ if skip_fields is None:
59
+ skip_fields = set()
60
# Note that if we add dataclasses as fields, they will need
61
# similar handling.
62
return {
63
field.name: getattr(self, field.name)
59
- for field in fields(self)
64
+ for field in fields(self) if field.name not in skip_fields
65
}
66
67
vllm/attention/backends/flashinfer.py
new
+220
@@ -0,0 +1,220 @@
1
+from dataclasses import dataclass
2
+from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
+
4
+try:
5
+ import flashinfer
6
+ from flash_attn import flash_attn_varlen_func
7
+ from flashinfer import BatchDecodeWithPagedKVCacheWrapper
8
+except ImportError:
9
+ flashinfer = None
10
+ flash_attn_varlen_func = None
11
+ BatchDecodeWithPagedKVCacheWrapper = None
12
+
13
+import torch
14
+
15
+from vllm import _custom_ops as ops
16
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
17
+ AttentionMetadata,
18
+ AttentionMetadataPerStage)
19
+
20
+
21
+class FlashInferBackend(AttentionBackend):
22
+
23
+ @staticmethod
24
+ def get_impl_cls() -> Type["FlashInferImpl"]:
25
+ return FlashInferImpl
26
+
27
+ @staticmethod
28
+ def make_metadata(*args, **kwargs) -> "FlashInferMetadata":
29
+ return FlashInferMetadata(*args, **kwargs)
30
+
31
+ @staticmethod
32
+ def get_kv_cache_shape(
33
+ num_blocks: int,
34
+ block_size: int,
35
+ num_kv_heads: int,
36
+ head_size: int,
37
+ ) -> Tuple[int, ...]:
38
+ return (num_blocks, 2, block_size, num_kv_heads, head_size)
39
+
40
+ @staticmethod
41
+ def swap_blocks(
42
+ src_kv_cache: torch.Tensor,
43
+ dst_kv_cache: torch.Tensor,
44
+ src_to_dst: Dict[int, int],
45
+ ) -> None:
46
+ raise NotImplementedError
47
+
48
+ @staticmethod
49
+ def copy_blocks(
50
+ kv_caches: List[torch.Tensor],
51
+ src_to_dists: Dict[int, List[int]],
52
+ ) -> None:
53
+ raise NotImplementedError
54
+
55
+ @staticmethod
56
+ def get_supported_head_sizes() -> List[int]:
57
+ return [64, 128, 256]
58
+
59
+
60
+@dataclass
61
+class FlashInferMetadata(AttentionMetadataPerStage):
62
+
63
+ is_prompt: bool
64
+
65
+ use_cuda_graph: bool = False
66
+
67
+ decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None
68
+
69
+ # Metadata for the prefill stage since we still
70
+ # use flash attention for prefill.
71
+ seq_start_loc: Optional[torch.Tensor] = None
72
+ max_seq_len: Optional[int] = None
73
+ block_tables: Optional[torch.Tensor] = None
74
+
75
+ # Metadata for the decode stage
76
+ # Workspace buffer required by the kernel, the buffer should not
77
+ # be allocated/deacollated by the FalshInfermetadata object.
78
+ workspace_buffer: Optional[torch.Tensor] = None
79
+ # An example for paged_kv_indices, paged_kv_indptr:
80
+ # request 1, page indices [0, 5, 8]
81
+ # request 2, page indices [1, 6, 7]
82
+ # request 3, page indices [3, 4]
83
+ # paged_kv_indices is a concatenation of page indices of all requests:
84
+ # [0, 5, 8, 1, 6, 7, 3, 4]
85
+ # paged_kv_indptr is used to index into paged_kv_indices:
86
+ # [0, 3, 6, 8]
87
+ # The indptr of the paged kv cache, shape: [batch_size + 1]
88
+ paged_kv_indptr: Optional[torch.Tensor] = None
89
+ # The page indices of the paged kv cache
90
+ paged_kv_indices: Optional[torch.Tensor] = None
91
+ # The number of entries in the last page of each request in
92
+ # the paged kv cache, shape: [batch_size]
93
+ paged_kv_last_page_len: Optional[torch.Tensor] = None
94
+ # The number of query/output heads
95
+ num_qo_heads: Optional[int] = None
96
+ # The number of key/value heads
97
+ num_kv_heads: Optional[int] = None
98
+ # The dimension of the attention heads
99
+ head_dim: Optional[int] = None
100
+ # Block size of vllm
101
+ page_size: Optional[int] = None
102
+ # The data type of the paged kv cache
103
+ data_type: torch.dtype = None
104
+
105
+ def __post_init__(self):
106
+ # Refer to
107
+ # https://github.com/flashinfer-ai/flashinfer/blob/3d55c71a62052c590c130897d3a3db49b14fcc34/include/flashinfer/utils.cuh#L157
108
+ supported_head_sizes = FlashInferBackend.get_supported_head_sizes()
109
+ if self.head_dim is not None and self.head_dim \
110
+ not in supported_head_sizes:
111
+ raise ValueError(
112
+ f"Only {supported_head_sizes} are supported for head_dim,",
113
+ f"received {self.head_dim}.")
114
+
115
+ # When using flashinfer, we are also creating the FlashInferMetadata,
116
+ # which will also call post_init by default, here we want to skip the
117
+ # post_init if it's the prefill phase.
118
+ if not self.is_prompt:
119
+ self.decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
120
+ self.workspace_buffer, "NHD")
121
+ self.decode_wrapper.begin_forward(
122
+ self.paged_kv_indptr,
123
+ self.paged_kv_indices,
124
+ self.paged_kv_last_page_len,
125
+ self.num_qo_heads,
126
+ self.num_kv_heads,
127
+ self.head_dim,
128
+ self.page_size,
129
+ # Disable flashinfer's pos encoding and use vllm's rope.
130
+ pos_encoding_mode="NONE",
131
+ data_type=self.data_type)
132
+
133
+ def asdict_zerocopy(self,
134
+ skip_fields: Optional[Set[str]] = None
135
+ ) -> Dict[str, Any]:
136
+ if skip_fields is None:
137
+ skip_fields = set()
138
+ # We need to skip the decode_wrapper field since it cannot be
139
+ # broadcasted with nccl when TP is enabled.
140
+ skip_fields.add('decode_wrapper')
141
+ return super().asdict_zerocopy(skip_fields)
142
+
143
+
144
+class FlashInferImpl(AttentionImpl):
145
+
146
+ def __init__(
147
+ self,
148
+ num_heads: int,
149
+ head_size: int,
150
+ scale: float,
151
+ num_kv_heads: Optional[int] = None,
152
+ alibi_slopes: Optional[List[float]] = None,
153
+ sliding_window: Optional[int] = None,
154
+ ) -> None:
155
+ if sliding_window is not None:
156
+ raise ValueError("Sliding window is not supported in FlashInfer.")
157
+ self.sliding_window = (-1, -1)
158
+ self.alibi_slopes = alibi_slopes
159
+ self.scale = scale
160
+ self.num_heads = num_heads
161
+ self.head_size = head_size
162
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
163
+
164
+ def forward(self, query: torch.Tensor, key: torch.Tensor,
165
+ value: torch.Tensor, kv_cache: Optional[torch.Tensor],
166
+ attn_metadata: AttentionMetadata[FlashInferMetadata],
167
+ kv_scale: float):
168
+ num_tokens, hidden_size = query.shape
169
+ query = query.view(-1, self.num_heads, self.head_size)
170
+ key = key.view(-1, self.num_kv_heads, self.head_size)
171
+ value = value.view(-1, self.num_kv_heads, self.head_size)
172
+
173
+ if attn_metadata.num_prefill_tokens > 0:
174
+ assert attn_metadata.num_decode_tokens == 0, (
175
+ "Chunked prefill is not supported with flashinfer yet.")
176
+ if attn_metadata.num_decode_tokens > 0:
177
+ assert attn_metadata.num_prefill_tokens == 0, (
178
+ "Chunked prefill is not supported with flashinfer yet.")
179
+
180
+ if kv_cache is not None:
181
+ # Use the same reshape and cache kernel as flash attention.
182
+ ops.reshape_and_cache_flash(
183
+ key,
184
+ value,
185
+ kv_cache[:, 0],
186
+ kv_cache[:, 1],
187
+ attn_metadata.slot_mapping.flatten(),
188
+ attn_metadata.kv_cache_dtype,
189
+ )
190
+
191
+ if prefill_meta := attn_metadata.prefill_metadata:
192
+ assert prefill_meta.block_tables is not None
193
+ if kv_cache is None or prefill_meta.block_tables.numel() == 0:
194
+ output = flash_attn_varlen_func(
195
+ q=query,
196
+ k=key,
197
+ v=value,
198
+ cu_seqlens_q=prefill_meta.seq_start_loc,
199
+ cu_seqlens_k=prefill_meta.seq_start_loc,
200
+ max_seqlen_q=prefill_meta.max_seq_len,
201
+ max_seqlen_k=prefill_meta.max_seq_len,
202
+ softmax_scale=self.scale,
203
+ causal=True,
204
+ window_size=self.sliding_window,
205
+ alibi_slopes=self.alibi_slopes,
206
+ )
207
+ else:
208
+ raise NotImplementedError(
209
+ "Prefix caching is not supported with flashinfer yet.")
210
+ else:
211
+ assert attn_metadata.decode_metadata is not None
212
+ assert attn_metadata.decode_metadata.decode_wrapper is not None
213
+ query = query.contiguous(
214
+ ) # Flashinfer requires query to be contiguous
215
+ output = attn_metadata.decode_metadata.decode_wrapper.forward(
216
+ query,
217
+ kv_cache,
218
+ sm_scale=self.scale,
219
+ )
220
+ return output.view(num_tokens, hidden_size)
vllm/attention/selector.py
+6
@@ -17,6 +17,7 @@ class _Backend(enum.Enum):
17
XFORMERS = enum.auto()
18
ROCM_FLASH = enum.auto()
19
TORCH_SDPA = enum.auto()
20
+ FLASHINFER = enum.auto()
21
22
23
@lru_cache(maxsize=None)
@@ -41,6 +42,11 @@ def get_attn_backend(dtype: torch.dtype) -> Type[AttentionBackend]:
42
logger.info("Using Torch SDPA backend.")
43
from vllm.attention.backends.torch_sdpa import TorchSDPABackend
44
return TorchSDPABackend
45
+ elif backend == _Backend.FLASHINFER:
46
+ logger.info("Using Flashinfer backend.")
47
+ logger.warning("Eager mode is enforced for the Flashinfer backend. ")
48
+ from vllm.attention.backends.flashinfer import FlashInferBackend
49
+ return FlashInferBackend
50
else:
51
raise ValueError("Invalid attention backend.")
52
vllm/config.py
+5
@@ -298,6 +298,11 @@ class ModelConfig:
298
return max(1,
299
total_num_kv_heads // parallel_config.tensor_parallel_size)
300
301
+ def get_num_attention_heads(self,
302
+ parallel_config: "ParallelConfig") -> int:
303
+ return self.hf_text_config.num_attention_heads // \
304
+ parallel_config.tensor_parallel_size
305
+
306
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
307
total_num_hidden_layers = self.hf_text_config.num_hidden_layers
308
return total_num_hidden_layers // parallel_config.pipeline_parallel_size
vllm/sequence.py
+3
-1
@@ -579,8 +579,10 @@ class SequenceGroupMetadata:
579
query tokens for prefill, we don't need sampling.
580
token_chunk_size: The number of tokens to be processed (per sequence).
581
None if chunking is not required.
582
- state: Internal state tied to this sequence group.
582
lora_request: LoRA request.
583
+ computed_block_nums: The block numbers that are already computed,
584
+ used in prefix caching.
585
+ state: Internal state tied to this sequence group.
586
multi_modal_data: Multi modal data.
587
"""
588
vllm/utils.py
+52
-15
@@ -355,21 +355,9 @@ def _generate_random_fp8(
355
del tensor_tmp
356
357
358
-def create_kv_caches_with_random(
359
- num_blocks: int,
360
- block_size: int,
361
- num_layers: int,
362
- num_heads: int,
363
- head_size: int,
364
- cache_dtype: Optional[Union[str, torch.dtype]],
365
- model_dtype: Optional[Union[str, torch.dtype]] = None,
366
- seed: int = 0,
367
- device: Optional[str] = "cuda",
368
-) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
369
- torch.random.manual_seed(seed)
370
- if torch.cuda.is_available():
371
- torch.cuda.manual_seed(seed)
372
-
358
+def get_kv_cache_torch_dtype(
359
+ cache_dtype: Optional[Union[str, torch.dtype]],
360
+ model_dtype: Optional[Union[str, torch.dtype]] = None) -> torch.dtype:
361
if isinstance(cache_dtype, str):
362
if cache_dtype == "auto":
363
if isinstance(model_dtype, str):
@@ -388,6 +376,55 @@ def create_kv_caches_with_random(
376
torch_dtype = cache_dtype
377
else:
378
raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
379
+ return torch_dtype
380
+
381
+
382
+def create_kv_caches_with_random_flash(
383
+ num_blocks: int,
384
+ block_size: int,
385
+ num_layers: int,
386
+ num_heads: int,
387
+ head_size: int,
388
+ cache_dtype: Optional[Union[str, torch.dtype]],
389
+ model_dtype: Optional[Union[str, torch.dtype]] = None,
390
+ seed: int = 0,
391
+ device: Optional[str] = "cuda",
392
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
393
+ assert cache_dtype != "fp8"
394
+ torch.random.manual_seed(seed)
395
+ if torch.cuda.is_available():
396
+ torch.cuda.manual_seed(seed)
397
+
398
+ torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
399
+ key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
400
+ scale = head_size**-0.5
401
+ key_caches, value_caches = [], []
402
+ for _ in range(num_layers):
403
+ key_value_cache = torch.empty(size=key_value_cache_shape,
404
+ dtype=torch_dtype,
405
+ device=device)
406
+ key_value_cache.uniform_(-scale, scale)
407
+ key_caches.append(key_value_cache[:, 0])
408
+ value_caches.append(key_value_cache[:, 1])
409
+ return key_caches, value_caches
410
+
411
+
412
+def create_kv_caches_with_random(
413
+ num_blocks: int,
414
+ block_size: int,
415
+ num_layers: int,
416
+ num_heads: int,
417
+ head_size: int,
418
+ cache_dtype: Optional[Union[str, torch.dtype]],
419
+ model_dtype: Optional[Union[str, torch.dtype]] = None,
420
+ seed: int = 0,
421
+ device: Optional[str] = "cuda",
422
+) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
423
+ torch.random.manual_seed(seed)
424
+ if torch.cuda.is_available():
425
+ torch.cuda.manual_seed(seed)
426
+
427
+ torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
428
429
scale = head_size**-0.5
430
x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
vllm/worker/model_runner.py
+97
-26
@@ -9,6 +9,7 @@ import torch.nn as nn
9
10
from vllm.attention import (AttentionMetadata, AttentionMetadataPerStage,
11
get_attn_backend)
12
+from vllm.attention.backends.flashinfer import FlashInferBackend
13
from vllm.config import (DeviceConfig, LoadConfig, LoRAConfig, ModelConfig,
14
ParallelConfig, SchedulerConfig, VisionLanguageConfig)
15
from vllm.distributed import broadcast_tensor_dict, with_pynccl_for_all_reduce
@@ -23,8 +24,8 @@ from vllm.model_executor.model_loader import get_model
24
from vllm.sampling_params import SamplingParams
25
from vllm.sequence import (MultiModalData, SamplerOutput, SequenceData,
26
SequenceGroupMetadata)
26
-from vllm.utils import (CudaMemoryProfiler, is_hip, is_pin_memory_available,
27
- make_tensor_with_pad)
27
+from vllm.utils import (CudaMemoryProfiler, get_kv_cache_torch_dtype, is_hip,
28
+ is_pin_memory_available, make_tensor_with_pad)
29
30
logger = init_logger(__name__)
31
@@ -155,6 +156,9 @@ class ModelRunner:
156
# (max batch size to capture, max context len to capture / block size).
157
self.graph_block_tables: torch.Tensor # Set after initial profiling.
158
159
+ # Set if the backend is flashinfer.
160
+ self.flashinfer_workspace_buffer: torch.Tensor
161
+
162
def load_model(self) -> None:
163
with CudaMemoryProfiler() as m:
164
self.model = get_model(
@@ -315,6 +319,7 @@ class ModelRunner:
319
320
# Compute the slot mapping.
321
block_table = seq_group_metadata.block_tables[seq_id]
322
+
323
# Mask the [0, start_idx) tokens of the prompt with _PAD_SLOT_ID,
324
# where start_idx is max(0, seq_len - sliding_window).
325
# For example, if the prompt len is 10, sliding window is 8, and
@@ -390,18 +395,26 @@ class ModelRunner:
395
dtype=seq_start_loc.dtype,
396
out=seq_start_loc[1:])
397
393
- attn_metadata = self.attn_backend.make_metadata(
394
- is_prompt=True,
395
- seq_lens=seq_lens,
396
- seq_lens_tensor=seq_lens_tensor,
397
- max_query_len=max_query_len,
398
- max_seq_len=max_seq_len,
399
- subquery_start_loc=subquery_start_loc,
400
- seq_start_loc=seq_start_loc,
401
- context_lens_tensor=context_lens_tensor,
402
- block_tables=block_tables,
403
- use_cuda_graph=False,
404
- )
398
+ if self.attn_backend is FlashInferBackend:
399
+ attn_metadata = self.attn_backend.make_metadata(
400
+ is_prompt=True,
401
+ use_cuda_graph=False,
402
+ seq_start_loc=seq_start_loc,
403
+ max_seq_len=max_seq_len,
404
+ block_tables=block_tables)
405
+ else:
406
+ attn_metadata = self.attn_backend.make_metadata(
407
+ is_prompt=True,
408
+ seq_lens=seq_lens,
409
+ seq_lens_tensor=seq_lens_tensor,
410
+ max_query_len=max_query_len,
411
+ max_seq_len=max_seq_len,
412
+ subquery_start_loc=subquery_start_loc,
413
+ seq_start_loc=seq_start_loc,
414
+ context_lens_tensor=context_lens_tensor,
415
+ block_tables=block_tables,
416
+ use_cuda_graph=False,
417
+ )
418
419
return PreparePromptMetadata(
420
input_tokens=input_tokens,
@@ -429,6 +442,24 @@ class ModelRunner:
442
lora_prompt_mapping: List[int] = []
443
lora_requests: Set[LoRARequest] = set()
444
445
+ # The following fields are only for flashinfer
446
+ # Please follow https://docs.flashinfer.ai/tutorials/kv_layout.html#page-layout
447
+ # for the precise definition of the following fields.
448
+ # An example:
449
+ # request 1, page indices [0, 5, 8]
450
+ # request 2, page indices [1, 6, 7]
451
+ # request 3, page indices [3, 4]
452
+ # paged_kv_indices is a concatenation of page indices of all requests:
453
+ # [0, 5, 8, 1, 6, 7, 3, 4]
454
+ # paged_kv_indptr is used to index into paged_kv_indices:
455
+ # [0, 3, 6, 8]
456
+ paged_kv_indices: List[int] = []
457
+ # 0 at the beginning of paged_kv_indptr indicates the start of the
458
+ # first request’s page indices in the paged_kv_indices list.
459
+ paged_kv_indptr: List[int] = [0]
460
+ # paged_kv_last_page_len is the length of the last page of each request
461
+ paged_kv_last_page_len: List[int] = []
462
+
463
if len(seq_group_metadata_list) == 0:
464
return PrepareDecodeMetadata.empty()
465
@@ -469,6 +500,13 @@ class ModelRunner:
500
block_table = block_table[-sliding_window_blocks:]
501
block_tables.append(block_table)
502
503
+ paged_kv_indices.extend(block_table)
504
+ paged_kv_indptr.append(paged_kv_indptr[-1] + len(block_table))
505
+ last_page_len = seq_data.get_len() % self.block_size
506
+ if last_page_len == 0:
507
+ last_page_len = self.block_size
508
+ paged_kv_last_page_len.append(last_page_len)
509
+
510
# vLLM uses cuda graph only for decoding requests.
511
# See `capture_model` API for more details.
512
# For decoding requests, batch_size == input_tokens.
@@ -518,18 +556,51 @@ class ModelRunner:
556
device=self.device,
557
)
558
521
- attn_metadata = self.attn_backend.make_metadata(
522
- is_prompt=False,
523
- seq_lens=None,
524
- seq_lens_tensor=seq_lens_tensor,
525
- max_query_len=None,
526
- max_seq_len=max_seq_len,
527
- subquery_start_loc=None,
528
- seq_start_loc=None,
529
- context_lens_tensor=None,
530
- block_tables=block_tables,
531
- use_cuda_graph=use_captured_graph,
532
- )
559
+ if self.attn_backend is FlashInferBackend:
560
+ if not hasattr(self, "flashinfer_workspace_buffer"):
561
+ # Allocate 16MB workspace buffer
562
+ # Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html
563
+ self.flashinfer_workspace_buffer = torch.empty(
564
+ 16 * 1024 * 1024, dtype=torch.uint8, device=self.device)
565
+ paged_kv_indptr = torch.tensor(paged_kv_indptr,
566
+ dtype=torch.int,
567
+ device=self.device)
568
+ paged_kv_indices = torch.tensor(paged_kv_indices,
569
+ dtype=torch.int,
570
+ device=self.device)
571
+ paged_kv_last_page_len = torch.tensor(paged_kv_last_page_len,
572
+ dtype=torch.int,
573
+ device=self.device)
574
+ kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
575
+ self.model_config.dtype)
576
+
577
+ attn_metadata = self.attn_backend.make_metadata(
578
+ is_prompt=False,
579
+ use_cuda_graph=False,
580
+ workspace_buffer=self.flashinfer_workspace_buffer,
581
+ paged_kv_indptr=paged_kv_indptr,
582
+ paged_kv_indices=paged_kv_indices,
583
+ paged_kv_last_page_len=paged_kv_last_page_len,
584
+ num_qo_heads=self.model_config.get_num_attention_heads(
585
+ self.parallel_config),
586
+ num_kv_heads=self.model_config.get_num_kv_heads(
587
+ self.parallel_config),
588
+ head_dim=self.model_config.get_head_size(),
589
+ page_size=self.block_size,
590
+ data_type=kv_cache_dtype)
591
+ else:
592
+ attn_metadata = self.attn_backend.make_metadata(
593
+ is_prompt=False,
594
+ seq_lens=None,
595
+ seq_lens_tensor=seq_lens_tensor,
596
+ max_query_len=None,
597
+ max_seq_len=max_seq_len,
598
+ subquery_start_loc=None,
599
+ seq_start_loc=None,
600
+ context_lens_tensor=None,
601
+ block_tables=block_tables,
602
+ use_cuda_graph=use_captured_graph,
603
+ )
604
return PrepareDecodeMetadata(
605
input_tokens=input_tokens,
606
input_positions=input_positions,