A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Kernel] Flashinfer for prefill & decode, with Cudagraph support for decode (#4628)
Co-authored-by: LiuXiaoxuanPKU <llilyliupku@gmail.com>, bong-furiosa <bongwon.jang@furiosa.ai>
Lily Liu committed
Jun 28, 2024 at 15:28 UTC
7041de43849fda7c8e931f0726f3db2a0d8015a4
7 files changed
+313
-117
.buildkite/test-pipeline.yaml
+3
@@ -211,3 +211,6 @@ steps:
211
- pytest -v -s distributed/test_custom_all_reduce.py
212
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
213
- TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=mp pytest -v -s distributed/test_basic_distributed_correctness.py
214
+ - pip install https://github.com/flashinfer-ai/flashinfer/releases/download/v0.0.5/flashinfer-0.0.5+cu121torch2.3-cp310-cp310-linux_x86_64.whl
215
+ - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=facebook/opt-125m DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
216
+ - VLLM_ATTENTION_BACKEND=FLASHINFER TEST_DIST_MODEL=meta-llama/Meta-Llama-3-8B DISTRIBUTED_EXECUTOR_BACKEND=ray pytest -v -s distributed/test_basic_distributed_correctness.py
\ No newline at end of file
requirements-test.txt
+1
-1
@@ -19,4 +19,4 @@ sentence-transformers # required for embedding
19
aiohttp
20
21
# quantization
22
-bitsandbytes==0.42.0
22
+bitsandbytes==0.42.0
\ No newline at end of file
tests/basic_correctness/test_basic_correctness.py
-6
@@ -2,7 +2,6 @@
2
3
Run `pytest tests/basic_correctness/test_basic_correctness.py`.
4
"""
5
-import os
5
import weakref
6
7
import pytest
@@ -13,7 +12,6 @@ MODELS = [
12
"facebook/opt-125m",
13
"meta-llama/Llama-2-7b-hf",
14
]
16
-VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
15
16
17
def test_vllm_gc_ed():
@@ -39,10 +37,6 @@ def test_models(
37
max_tokens: int,
38
enforce_eager: bool,
39
) -> None:
42
- backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43
- if backend_by_env_var == "FLASHINFER" and enforce_eager is False:
44
- pytest.skip("Skipping non-eager test for FlashInferBackend.")
45
-
40
with hf_runner(model, dtype=dtype) as hf_model:
41
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
42
tests/distributed/test_basic_distributed_correctness.py
-5
@@ -21,7 +21,6 @@ MODELS = [
21
os.environ["TEST_DIST_MODEL"],
22
]
23
DISTRIBUTED_EXECUTOR_BACKEND = "DISTRIBUTED_EXECUTOR_BACKEND"
24
-VLLM_ATTENTION_BACKEND = "VLLM_ATTENTION_BACKEND"
24
25
26
@pytest.mark.skipif(torch.cuda.device_count() < 2,
@@ -39,16 +38,12 @@ def test_models(
38
) -> None:
39
distributed_executor_backend = os.getenv(DISTRIBUTED_EXECUTOR_BACKEND)
40
42
- backend_by_env_var = os.getenv(VLLM_ATTENTION_BACKEND)
43
- enforce_eager = backend_by_env_var == "FLASHINFER"
44
-
41
with hf_runner(model, dtype=dtype) as hf_model:
42
hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
43
44
with vllm_runner(model,
45
dtype=dtype,
46
tensor_parallel_size=2,
51
- enforce_eager=enforce_eager,
47
distributed_executor_backend=distributed_executor_backend
48
) as vllm_model:
49
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
vllm/attention/backends/flashinfer.py
+58
-25
@@ -1,10 +1,16 @@
1
from dataclasses import dataclass
2
from typing import Any, Dict, List, Optional, Set, Tuple, Type
3
4
-import flashinfer
4
+try:
5
+ from flashinfer import BatchDecodeWithPagedKVCacheWrapper
6
+ from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
7
+ from vllm_flash_attn import flash_attn_varlen_func
8
+except ImportError:
9
+ flash_attn_varlen_func = None
10
+ BatchDecodeWithPagedKVCacheWrapper = None
11
+ BatchPrefillWithPagedKVCacheWrapper = None
12
+
13
import torch
6
-from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7
-from vllm_flash_attn import flash_attn_varlen_func
14
15
from vllm import _custom_ops as ops
16
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
@@ -60,19 +66,16 @@ class FlashInferMetadata(AttentionMetadata):
66
# requests only.
67
max_prefill_seq_len: int
68
63
- use_cuda_graph: bool = False
69
+ use_cuda_graph: bool = True
70
71
+ prefill_wrapper: Optional[BatchPrefillWithPagedKVCacheWrapper] = None
72
decode_wrapper: Optional[BatchDecodeWithPagedKVCacheWrapper] = None
73
67
- # Metadata for the prefill stage since we still
68
- # use flash attention for prefill.
74
+ # Metadata for the prefill stage
75
seq_start_loc: Optional[torch.Tensor] = None
76
+ query_start_loc: Optional[torch.Tensor] = None
77
block_tables: Optional[torch.Tensor] = None
78
72
- # Metadata for the decode stage
73
- # Workspace buffer required by the kernel, the buffer should not
74
- # be allocated/deacollated by the FalshInfermetadata object.
75
- workspace_buffer: Optional[torch.Tensor] = None
79
# An example for paged_kv_indices, paged_kv_indptr:
80
# request 1, page indices [0, 5, 8]
81
# request 2, page indices [1, 6, 7]
@@ -98,6 +101,7 @@ class FlashInferMetadata(AttentionMetadata):
101
page_size: Optional[int] = None
102
# The data type of the paged kv cache
103
data_type: torch.dtype = None
104
+ device: torch.device = torch.device("cuda")
105
106
def __post_init__(self):
107
# Refer to
@@ -109,13 +113,35 @@ class FlashInferMetadata(AttentionMetadata):
113
f"Only {supported_head_sizes} are supported for head_dim,",
114
f"received {self.head_dim}.")
115
112
- # When using flashinfer, we are also creating the FlashInferMetadata,
113
- # which will also call post_init by default, here we want to skip the
114
- # post_init if it's the prefill phase.
115
- if self.num_prefills == 0:
116
- assert self.num_decode_tokens > 0
117
- self.decode_wrapper = flashinfer.BatchDecodeWithPagedKVCacheWrapper(
118
- self.workspace_buffer, "NHD")
116
+ def begin_forward(self):
117
+ if self.num_prefill_tokens > 0:
118
+ if self.paged_kv_indices is None:
119
+ return
120
+
121
+ assert self.prefill_wrapper is not None
122
+ assert self.paged_kv_indices is not None
123
+ assert self.paged_kv_indptr is not None
124
+ assert self.paged_kv_last_page_len is not None
125
+ self.paged_kv_indices = self.paged_kv_indices.to(self.device)
126
+ self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
127
+ self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
128
+ self.device)
129
+ self.prefill_wrapper.begin_forward(
130
+ self.query_start_loc, self.paged_kv_indptr,
131
+ self.paged_kv_indices, self.paged_kv_last_page_len,
132
+ self.num_qo_heads, self.num_kv_heads, self.head_dim,
133
+ self.page_size)
134
+ else:
135
+ if not self.use_cuda_graph:
136
+ assert self.paged_kv_indices is not None
137
+ assert self.paged_kv_indptr is not None
138
+ assert self.paged_kv_last_page_len is not None
139
+ self.paged_kv_indices = self.paged_kv_indices.to(self.device)
140
+ self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
141
+ self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
142
+ self.device)
143
+
144
+ assert self.decode_wrapper is not None
145
self.decode_wrapper.begin_forward(
146
self.paged_kv_indptr,
147
self.paged_kv_indices,
@@ -133,8 +159,9 @@ class FlashInferMetadata(AttentionMetadata):
159
) -> Dict[str, Any]:
160
if skip_fields is None:
161
skip_fields = set()
136
- # We need to skip the decode_wrapper field since it cannot be
162
+ # We need to skip the prefill/decode_wrapper field since it cannot be
163
# broadcasted with nccl when TP is enabled.
164
+ skip_fields.add('prefill_wrapper')
165
skip_fields.add('decode_wrapper')
166
return super().asdict_zerocopy(skip_fields)
167
@@ -168,6 +195,7 @@ class FlashInferImpl(AttentionImpl):
195
alibi_slopes: Optional[List[float]],
196
sliding_window: Optional[int],
197
kv_cache_dtype: str,
198
+ blocksparse_params: Optional[Dict[str, Any]] = None,
199
) -> None:
200
self.num_heads = num_heads
201
self.head_size = head_size
@@ -217,10 +245,14 @@ class FlashInferImpl(AttentionImpl):
245
self.kv_cache_dtype,
246
)
247
248
+ query = query.contiguous(
249
+ ) # Flashinfer requires query to be contiguous
250
if prefill_meta := attn_metadata.prefill_metadata:
221
- # Prompt run.
222
- assert prefill_meta.block_tables is not None
223
- if kv_cache is None or prefill_meta.block_tables.numel() == 0:
251
+ # We will use flash attention for prefill
252
+ # when kv_cache is not provided.
253
+ # This happens when vllm runs the profiling to
254
+ # determine the number of blocks.
255
+ if kv_cache is None:
256
output = flash_attn_varlen_func(
257
q=query,
258
k=key,
@@ -235,13 +267,14 @@ class FlashInferImpl(AttentionImpl):
267
alibi_slopes=self.alibi_slopes,
268
)
269
else:
238
- raise NotImplementedError(
239
- "Prefix caching is not supported with flashinfer yet.")
270
+ assert prefill_meta is not None
271
+ assert prefill_meta.prefill_wrapper is not None
272
+ output = prefill_meta.prefill_wrapper.forward(query,
273
+ kv_cache,
274
+ causal=True)
275
else:
276
assert attn_metadata.decode_metadata is not None
277
assert attn_metadata.decode_metadata.decode_wrapper is not None
243
- query = query.contiguous(
244
- ) # Flashinfer requires query to be contiguous
278
output = attn_metadata.decode_metadata.decode_wrapper.forward(
279
query,
280
kv_cache,
vllm/attention/selector.py
+3
-2
@@ -77,8 +77,9 @@ def get_attn_backend(
77
return IpexAttnBackend
78
elif backend == _Backend.FLASHINFER:
79
logger.info("Using Flashinfer backend.")
80
- logger.warning("Eager mode is required for the Flashinfer backend. "
81
- "Please make sure --enforce-eager is set.")
80
+ logger.warning(("Flashinfer will be stuck on llma-2-7b,"
81
+ " please avoid using Flashinfer as the"
82
+ "backend when running on llma-2-7b."))
83
from vllm.attention.backends.flashinfer import FlashInferBackend
84
return FlashInferBackend
85
elif backend == _Backend.PALLAS:
vllm/worker/model_runner.py
+248
-78
@@ -10,6 +10,17 @@ import numpy as np
10
import torch
11
import torch.nn as nn
12
13
+try:
14
+ from flashinfer import BatchDecodeWithPagedKVCacheWrapper
15
+ from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
16
+ from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
17
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
18
+except ImportError:
19
+ BatchDecodeWithPagedKVCacheWrapper = None
20
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
21
+ BatchPrefillWithPagedKVCacheWrapper = None
22
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
23
+
24
from vllm.attention import AttentionMetadata, get_attn_backend
25
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
26
ModelConfig, ParallelConfig, SchedulerConfig,
@@ -198,11 +209,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
209
210
# Lazy initialization
211
self.model: nn.Module # Set after load_model
201
- # Set if the backend is flashinfer.
202
- self.flashinfer_workspace_buffer: torch.Tensor
212
# Set after load_model.
213
self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
214
215
+ self.flashinfer_decode_workspace_buffer = None
216
+ self.flashinfer_decode_wrapper = None
217
+ self.flashinfer_prefill_workspace_buffer = None
218
+ self.flashinfer_prefill_wrapper = None
219
+
220
def load_model(self) -> None:
221
with CudaMemoryProfiler() as m:
222
self.model = get_model(
@@ -450,15 +464,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
464
if curr_sliding_window_blocks is not None:
465
block_table = block_table[
466
-curr_sliding_window_blocks:]
453
- if self.attn_backend.get_name() == "flashinfer":
454
- paged_kv_indices.extend(block_table)
455
- paged_kv_indptr.append(paged_kv_indptr[-1] +
456
- len(block_table))
457
- last_page_len = seq_data.get_len(
458
- ) % self.block_size
459
- if last_page_len == 0:
460
- last_page_len = self.block_size
461
- paged_kv_last_page_len.append(last_page_len)
467
else:
468
# Only happens when memory profiling runs.
469
block_table = []
@@ -505,7 +510,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
510
for k, v in mm_kwargs.items():
511
multi_modal_kwargs_list[k].append(v)
512
508
- if _is_block_tables_empty(seq_group_metadata.block_tables):
513
+ is_profile_run = _is_block_tables_empty(
514
+ seq_group_metadata.block_tables)
515
+ if is_profile_run:
516
# During memory profiling, the block tables are not
517
# initialized yet. In this case, we just use a dummy
518
# slot mapping.
@@ -544,6 +551,27 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
551
slot = block_number * self.block_size + block_offset
552
slot_mapping.append(slot)
553
554
+ # Prepare input tensors for flashinfer
555
+ if self.attn_backend.get_name() == "flashinfer":
556
+ seq_len = seq_data.get_len()
557
+ # Get the number of valid blocks based on sequence length.
558
+ # If seq_len = 16, block_size = 16,
559
+ # block_table_bound is 1 with 1 valid block.
560
+ # If seq_len = 15, block_size = 16,
561
+ # block_table_bound is 0 + 1 with 1 valid block.
562
+ block_table_bound = seq_len // self.block_size + 1 \
563
+ if seq_len % self.block_size != 0 \
564
+ else seq_len // self.block_size
565
+
566
+ paged_kv_indices.extend(block_table[:block_table_bound])
567
+ paged_kv_indptr.append(paged_kv_indptr[-1] +
568
+ block_table_bound)
569
+
570
+ last_page_len = seq_len % self.block_size
571
+ if last_page_len == 0:
572
+ last_page_len = self.block_size
573
+ paged_kv_last_page_len.append(last_page_len)
574
+
575
batch_size = len(input_tokens)
576
max_query_len = max(query_lens)
577
max_prefill_seq_len = max(prefill_seq_lens, default=0)
@@ -566,6 +594,12 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
594
seq_lens.append(1)
595
block_tables.append([])
596
lora_index_mapping.append(0)
597
+
598
+ if self.attn_backend.get_name() == "flashinfer":
599
+ last_paged_kv_indptr = paged_kv_indptr[-1]
600
+ paged_kv_indptr.append(last_paged_kv_indptr)
601
+ paged_kv_last_page_len.append(0)
602
+
603
batch_size = graph_batch_size
604
num_decode_tokens = batch_size
605
@@ -589,9 +623,19 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
623
)
624
assert max_query_len > 0, ("query_lens: {}".format(query_lens))
625
626
+ context_lens_tensor = torch.tensor(context_lens,
627
+ dtype=torch.int,
628
+ device=self.device)
629
+
630
seq_lens_tensor = torch.tensor(seq_lens,
631
dtype=torch.int,
632
device=self.device)
633
+ query_lens_tensor = torch.tensor(query_lens,
634
+ dtype=torch.long,
635
+ device=self.device)
636
+ query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
637
+ dtype=torch.int32,
638
+ device=self.device)
639
seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
640
dtype=torch.int32,
641
device=self.device)
@@ -600,6 +644,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
644
dim=0,
645
dtype=seq_start_loc.dtype,
646
out=seq_start_loc[1:])
647
+ torch.cumsum(query_lens_tensor,
648
+ dim=0,
649
+ dtype=query_start_loc.dtype,
650
+ out=query_start_loc[1:])
651
652
input_tokens_tensor = torch.tensor(input_tokens,
653
dtype=torch.long,
@@ -612,30 +660,30 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
660
device=self.device)
661
662
if self.attn_backend.get_name() == "flashinfer":
615
- if not hasattr(self, "flashinfer_workspace_buffer"):
616
- # Allocate 16MB workspace buffer
617
- # Follow the example of flashinfer: https://docs.flashinfer.ai/api/python/decode.html
618
- self.flashinfer_workspace_buffer = torch.empty(
619
- 16 * 1024 * 1024, dtype=torch.uint8, device=self.device)
620
- paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
621
- dtype=torch.int,
622
- device=self.device)
623
- paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
624
- dtype=torch.int,
625
- device=self.device)
626
- paged_kv_last_page_len_tensor = torch.tensor(
627
- paged_kv_last_page_len, dtype=torch.int, device=self.device)
663
+ if len(paged_kv_indptr) > 0:
664
+ paged_kv_indices_tensor = torch.tensor(paged_kv_indices,
665
+ device='cpu',
666
+ dtype=torch.int)
667
+ paged_kv_indptr_tensor = torch.tensor(paged_kv_indptr,
668
+ device='cpu',
669
+ dtype=torch.int)
670
+ paged_kv_last_page_len_tensor = torch.tensor(
671
+ paged_kv_last_page_len, device='cpu', dtype=torch.int)
672
+ else:
673
+ paged_kv_indices_tensor = None
674
+ paged_kv_indptr_tensor = None
675
+ paged_kv_last_page_len_tensor = None
676
+
677
kv_cache_dtype = get_kv_cache_torch_dtype(self.kv_cache_dtype,
678
self.model_config.dtype)
679
+
680
attn_metadata = self.attn_backend.make_metadata(
681
num_prefills=num_prefills,
682
slot_mapping=slot_mapping_tensor,
683
num_prefill_tokens=num_prefill_tokens,
684
num_decode_tokens=num_decode_tokens,
635
- use_cuda_graph=False,
685
max_prefill_seq_len=max_prefill_seq_len,
686
block_tables=block_tables,
638
- workspace_buffer=self.flashinfer_workspace_buffer,
687
paged_kv_indptr=paged_kv_indptr_tensor,
688
paged_kv_indices=paged_kv_indices_tensor,
689
paged_kv_last_page_len=paged_kv_last_page_len_tensor,
@@ -644,25 +692,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
692
num_kv_heads=self.model_config.get_num_kv_heads(
693
self.parallel_config),
694
head_dim=self.model_config.get_head_size(),
647
- page_size=16,
695
+ page_size=self.block_size,
696
seq_start_loc=seq_start_loc,
649
- data_type=kv_cache_dtype)
650
- else:
651
- context_lens_tensor = torch.tensor(context_lens,
652
- dtype=torch.int,
653
- device=self.device)
654
- query_lens_tensor = torch.tensor(query_lens,
655
- dtype=torch.long,
656
- device=self.device)
657
- query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
658
- dtype=torch.int32,
659
- device=self.device)
660
-
661
- torch.cumsum(query_lens_tensor,
662
- dim=0,
663
- dtype=query_start_loc.dtype,
664
- out=query_start_loc[1:])
697
+ query_start_loc=query_start_loc,
698
+ device=self.device,
699
+ data_type=kv_cache_dtype,
700
+ use_cuda_graph=use_captured_graph)
701
702
+ else:
703
attn_metadata = self.attn_backend.make_metadata(
704
num_prefills=num_prefills,
705
slot_mapping=slot_mapping_tensor,
@@ -854,27 +891,97 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
891
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
892
]
893
894
+ if self.attn_backend.get_name() == "flashinfer":
895
+ # For flashinfer, different batch sizes will share the
896
+ # same workspace buffer.
897
+ decode_workspace_buffer = \
898
+ torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE,
899
+ dtype=torch.uint8,
900
+ device=self.device)
901
+ indices_buffer = torch.empty(max_batch_size *
902
+ self.cache_config.num_gpu_blocks,
903
+ dtype=torch.int32,
904
+ device=self.device)
905
+ indptr_buffer = torch.empty(max_batch_size + 1,
906
+ dtype=torch.int32,
907
+ device=self.device)
908
+ last_page_len_buffer = torch.empty(max_batch_size,
909
+ dtype=torch.int32,
910
+ device=self.device)
911
+
912
with graph_capture() as graph_capture_context:
913
# NOTE: Capturing the largest batch size first may help reduce the
914
# memory usage of CUDA graph.
915
for batch_size in reversed(batch_size_capture_list):
861
- # Create dummy attn_metadata.
862
- attn_metadata = self.attn_backend.make_metadata(
863
- num_prefills=0,
864
- num_prefill_tokens=0,
865
- num_decode_tokens=batch_size,
866
- slot_mapping=slot_mapping[:batch_size],
867
- seq_lens=None,
868
- seq_lens_tensor=seq_lens[:batch_size],
869
- max_query_len=None,
870
- max_prefill_seq_len=0,
871
- max_decode_seq_len=self.max_seq_len_to_capture,
872
- query_start_loc=None,
873
- seq_start_loc=None,
874
- context_lens_tensor=None,
875
- block_tables=block_tables[:batch_size],
876
- use_cuda_graph=True,
877
- )
916
+ if self.attn_backend.get_name() == "flashinfer":
917
+ indptr_buffer = indptr_buffer[:batch_size + 1]
918
+ last_page_len_buffer = last_page_len_buffer[:batch_size]
919
+
920
+ num_qo_heads = self.model_config.get_num_attention_heads(
921
+ self.parallel_config)
922
+ num_kv_heads = self.model_config.get_num_kv_heads(
923
+ self.parallel_config)
924
+ if num_qo_heads // num_kv_heads >= 4:
925
+ use_tensor_cores = True
926
+ else:
927
+ use_tensor_cores = False
928
+ decode_wrapper = \
929
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
930
+ decode_workspace_buffer, indptr_buffer, indices_buffer,
931
+ last_page_len_buffer, "NHD", use_tensor_cores)
932
+ kv_cache_dtype = get_kv_cache_torch_dtype(
933
+ self.kv_cache_dtype, self.model_config.dtype)
934
+
935
+ paged_kv_indptr_tensor_host = torch.arange(
936
+ 0, batch_size + 1, dtype=torch.int32)
937
+ paged_kv_indices_tensor_host = torch.arange(
938
+ 0, batch_size, dtype=torch.int32)
939
+ paged_kv_last_page_len_tensor_host = torch.full(
940
+ (batch_size, ), self.block_size, dtype=torch.int32)
941
+ query_start_loc_host = torch.arange(0,
942
+ batch_size + 1,
943
+ dtype=torch.int32)
944
+
945
+ attn_metadata = self.attn_backend.make_metadata(
946
+ num_prefills=0,
947
+ slot_mapping=slot_mapping[:batch_size],
948
+ num_prefill_tokens=0,
949
+ num_decode_tokens=batch_size,
950
+ max_prefill_seq_len=0,
951
+ block_tables=block_tables,
952
+ paged_kv_indptr=paged_kv_indptr_tensor_host,
953
+ paged_kv_indices=paged_kv_indices_tensor_host,
954
+ paged_kv_last_page_len=
955
+ paged_kv_last_page_len_tensor_host,
956
+ num_qo_heads=num_qo_heads,
957
+ num_kv_heads=num_kv_heads,
958
+ head_dim=self.model_config.get_head_size(),
959
+ page_size=self.block_size,
960
+ seq_start_loc=None,
961
+ query_start_loc=query_start_loc_host,
962
+ device=self.device,
963
+ data_type=kv_cache_dtype,
964
+ use_cuda_graph=True,
965
+ decode_wrapper=decode_wrapper,
966
+ prefill_wrapper=None)
967
+ attn_metadata.begin_forward()
968
+ else:
969
+ attn_metadata = self.attn_backend.make_metadata(
970
+ num_prefills=0,
971
+ num_prefill_tokens=0,
972
+ num_decode_tokens=batch_size,
973
+ slot_mapping=slot_mapping[:batch_size],
974
+ seq_lens=None,
975
+ seq_lens_tensor=seq_lens[:batch_size],
976
+ max_query_len=None,
977
+ max_prefill_seq_len=0,
978
+ max_decode_seq_len=self.max_seq_len_to_capture,
979
+ query_start_loc=None,
980
+ seq_start_loc=None,
981
+ context_lens_tensor=None,
982
+ block_tables=block_tables[:batch_size],
983
+ use_cuda_graph=True,
984
+ )
985
986
if self.lora_config:
987
lora_mapping = LoRAMapping(
@@ -883,8 +990,20 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
990
)
991
self.set_active_loras(set(), lora_mapping)
992
886
- graph_runner = CUDAGraphRunner(self.model)
887
- hidden_states = graph_runner.capture(
993
+ graph_runner = CUDAGraphRunner(self.model,
994
+ self.attn_backend.get_name())
995
+
996
+ if self.attn_backend.get_name() == "flashinfer":
997
+ graph_runner.flashinfer_indptr_buffer = indptr_buffer
998
+ graph_runner.flashinfer_indices_buffer = indices_buffer
999
+ graph_runner.flashinfer_last_page_len_buffer = \
1000
+ last_page_len_buffer
1001
+ graph_runner.flashinfer_decode_workspace_buffer = \
1002
+ decode_workspace_buffer
1003
+ graph_runner.flashinfer_decode_wrapper = \
1004
+ decode_wrapper
1005
+
1006
+ graph_runner.capture(
1007
input_tokens[:batch_size],
1008
input_positions[:batch_size],
1009
hidden_states[:batch_size]
@@ -918,11 +1037,12 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1037
self,
1038
tensor_dict: Dict[str, Any],
1039
) -> ModelInputForGPUWithSamplingMetadata:
921
- return (
1040
+ model_input = \
1041
ModelInputForGPUWithSamplingMetadata.from_broadcasted_tensor_dict(
1042
tensor_dict,
1043
attn_backend=self.attn_backend,
925
- ))
1044
+ )
1045
+ return model_input
1046
1047
def prepare_model_input(
1048
self,
@@ -970,6 +1090,36 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1090
self.set_active_loras(model_input.lora_requests,
1091
model_input.lora_mapping)
1092
1093
+ if self.attn_backend.get_name() == "flashinfer":
1094
+ assert model_input.attn_metadata is not None
1095
+ assert model_input.input_tokens is not None
1096
+ if self.flashinfer_decode_workspace_buffer is None:
1097
+ self.flashinfer_decode_workspace_buffer = torch.empty(
1098
+ FLASHINFER_WORKSPACE_BUFFER_SIZE,
1099
+ dtype=torch.uint8,
1100
+ device=self.device)
1101
+ self.flashinfer_decode_wrapper = \
1102
+ BatchDecodeWithPagedKVCacheWrapper(
1103
+ self.flashinfer_decode_workspace_buffer, "NHD")
1104
+ self.flashinfer_prefill_workspace_buffer = torch.empty(
1105
+ FLASHINFER_WORKSPACE_BUFFER_SIZE,
1106
+ dtype=torch.uint8,
1107
+ device=self.device)
1108
+ self.flashinfer_prefill_wrapper = \
1109
+ BatchPrefillWithPagedKVCacheWrapper(
1110
+ self.flashinfer_prefill_workspace_buffer, "NHD")
1111
+
1112
+ model_input.attn_metadata.prefill_wrapper = \
1113
+ self.flashinfer_prefill_wrapper
1114
+ if model_input.attn_metadata.use_cuda_graph:
1115
+ batch_size = model_input.input_tokens.shape[0]
1116
+ model_input.attn_metadata.decode_wrapper = self.graph_runners[
1117
+ batch_size].flashinfer_decode_wrapper
1118
+ else:
1119
+ model_input.attn_metadata.decode_wrapper = \
1120
+ self.flashinfer_decode_wrapper
1121
+ model_input.attn_metadata.begin_forward()
1122
+
1123
# Currently cuda graph is only supported by the decode phase.
1124
assert model_input.attn_metadata is not None
1125
prefill_meta = model_input.attn_metadata.prefill_metadata
@@ -1020,13 +1170,22 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1170
1171
class CUDAGraphRunner:
1172
1023
- def __init__(self, model: nn.Module):
1173
+ def __init__(self, model: nn.Module, backend_name: str):
1174
self.model = model
1175
+ self.backend_name = backend_name
1176
+
1177
self.input_buffers: Dict[str, torch.Tensor] = {}
1178
self.output_buffers: Dict[str, torch.Tensor] = {}
1179
1180
self._graph: Optional[torch.cuda.CUDAGraph] = None
1181
1182
+ self.flashinfer_decode_workspace_buffer: Optional[torch.Tensor] = None
1183
+ self.flashinfer_indptr_buffer: Optional[torch.Tensor] = None
1184
+ self.flashinfer_indices_buffer: Optional[torch.Tensor] = None
1185
+ self.flashinfer_last_page_len_buffer: Optional[torch.Tensor] = None
1186
+ self.flashinfer_decode_wrapper: Optional[
1187
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = None
1188
+
1189
@property
1190
def graph(self):
1191
assert self._graph is not None
@@ -1079,14 +1238,23 @@ class CUDAGraphRunner:
1238
torch.cuda.synchronize()
1239
1240
# Save the input and output buffers.
1082
- self.input_buffers = {
1083
- "input_ids": input_ids,
1084
- "positions": positions,
1085
- "kv_caches": kv_caches,
1086
- "slot_mapping": attn_metadata.slot_mapping,
1087
- "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
1088
- "block_tables": attn_metadata.decode_metadata.block_tables,
1089
- }
1241
+ if self.backend_name == "flashinfer":
1242
+ self.input_buffers = {
1243
+ "input_ids": input_ids,
1244
+ "positions": positions,
1245
+ "kv_caches": kv_caches,
1246
+ "slot_mapping": attn_metadata.slot_mapping,
1247
+ }
1248
+ else:
1249
+ self.input_buffers = {
1250
+ "input_ids": input_ids,
1251
+ "positions": positions,
1252
+ "kv_caches": kv_caches,
1253
+ "slot_mapping": attn_metadata.slot_mapping,
1254
+ "seq_lens_tensor":
1255
+ attn_metadata.decode_metadata.seq_lens_tensor,
1256
+ "block_tables": attn_metadata.decode_metadata.block_tables,
1257
+ }
1258
self.output_buffers = {"hidden_states": hidden_states}
1259
return hidden_states
1260
@@ -1106,10 +1274,12 @@ class CUDAGraphRunner:
1274
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1275
self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1276
non_blocking=True)
1109
- self.input_buffers["seq_lens_tensor"].copy_(
1110
- attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
1111
- self.input_buffers["block_tables"].copy_(
1112
- attn_metadata.decode_metadata.block_tables, non_blocking=True)
1277
+ if self.backend_name != "flashinfer":
1278
+ self.input_buffers["seq_lens_tensor"].copy_(
1279
+ attn_metadata.decode_metadata.seq_lens_tensor,
1280
+ non_blocking=True)
1281
+ self.input_buffers["block_tables"].copy_(
1282
+ attn_metadata.decode_metadata.block_tables, non_blocking=True)
1283
# Run the graph.
1284
self.graph.replay()
1285