A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Core] Add `AttentionState` abstraction (#7663)
Antoni Baum committed
Aug 20, 2024 at 11:50 UTC
3b682179dd4844fa346ef5e77706880c4d0b09da
16 files changed
+372
-247
tests/worker/test_model_input.py
+6
-1
@@ -5,6 +5,7 @@ import torch
5
6
from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
7
from vllm.attention.backends.abstract import AttentionBackend
8
+from vllm.attention.backends.utils import CommonAttentionState
9
from vllm.model_executor import SamplingMetadata
10
from vllm.model_executor.pooling_metadata import PoolingMetadata
11
from vllm.worker.embedding_model_runner import (
@@ -29,7 +30,11 @@ class MockAttentionBackend(AttentionBackend):
30
31
@staticmethod
32
def get_builder_cls() -> Type["AttentionMetadataBuilder"]:
32
- raise AttentionMetadataBuilder
33
+ return AttentionMetadataBuilder
34
+
35
+ @staticmethod
36
+ def get_state_cls() -> Type["CommonAttentionState"]:
37
+ return CommonAttentionState
38
39
@staticmethod
40
def get_kv_cache_shape(
vllm/attention/__init__.py
+2
-1
@@ -1,7 +1,7 @@
1
from vllm.attention.backends.abstract import (AttentionBackend,
2
AttentionMetadata,
3
AttentionMetadataBuilder,
4
- AttentionType)
4
+ AttentionState, AttentionType)
5
from vllm.attention.layer import Attention
6
from vllm.attention.selector import get_attn_backend
7
@@ -12,5 +12,6 @@ __all__ = [
12
"AttentionType",
13
"AttentionMetadataBuilder",
14
"Attention",
15
+ "AttentionState",
16
"get_attn_backend",
17
]
vllm/attention/backends/abstract.py
+50
-1
@@ -1,4 +1,5 @@
1
from abc import ABC, abstractmethod
2
+from contextlib import contextmanager
3
from dataclasses import dataclass, fields
4
from enum import Enum, auto
5
from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
@@ -7,7 +8,9 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, List, Optional, Set,
8
import torch
9
10
if TYPE_CHECKING:
10
- from vllm.worker.model_runner_base import ModelRunnerInputBuilderBase
11
+ from vllm.worker.model_runner_base import (ModelRunnerBase,
12
+ ModelRunnerInputBase,
13
+ ModelRunnerInputBuilderBase)
14
15
16
class AttentionType(Enum):
@@ -34,6 +37,11 @@ class AttentionBackend(ABC):
37
def get_metadata_cls() -> Type["AttentionMetadata"]:
38
raise NotImplementedError
39
40
+ @staticmethod
41
+ @abstractmethod
42
+ def get_state_cls() -> Type["AttentionState"]:
43
+ raise NotImplementedError
44
+
45
@classmethod
46
def make_metadata(cls, *args, **kwargs) -> "AttentionMetadata":
47
return cls.get_metadata_cls()(*args, **kwargs)
@@ -126,6 +134,47 @@ class AttentionMetadata:
134
T = TypeVar("T", bound=AttentionMetadata)
135
136
137
+class AttentionState(ABC, Generic[T]):
138
+ """Holds attention backend-specific objects reused during the
139
+ lifetime of the model runner."""
140
+
141
+ @abstractmethod
142
+ def __init__(self, runner: "ModelRunnerBase"):
143
+ ...
144
+
145
+ @abstractmethod
146
+ @contextmanager
147
+ def graph_capture(self, max_batch_size: int):
148
+ """Context manager used when capturing CUDA graphs."""
149
+ yield
150
+
151
+ @abstractmethod
152
+ def graph_clone(self, batch_size: int) -> "AttentionState[T]":
153
+ """Clone attention state to save in CUDA graph metadata."""
154
+ ...
155
+
156
+ @abstractmethod
157
+ def graph_capture_get_metadata_for_batch(self, batch_size: int) -> T:
158
+ """Get attention metadata for CUDA graph capture of batch_size."""
159
+ ...
160
+
161
+ @abstractmethod
162
+ def get_graph_input_buffers(self, attn_metadata: T) -> Dict[str, Any]:
163
+ """Get attention-specific input buffers for CUDA graph capture."""
164
+ ...
165
+
166
+ @abstractmethod
167
+ def prepare_graph_input_buffers(self, input_buffers: Dict[str, Any],
168
+ attn_metadata: T) -> None:
169
+ """In-place modify input buffers dict for CUDA graph replay."""
170
+ ...
171
+
172
+ @abstractmethod
173
+ def begin_forward(self, model_input: "ModelRunnerInputBase") -> None:
174
+ """Prepare state for forward pass."""
175
+ ...
176
+
177
+
178
class AttentionMetadataBuilder(ABC, Generic[T]):
179
"""Abstract class for attention metadata builders."""
180
vllm/attention/backends/blocksparse_attn.py
+6
-1
@@ -5,7 +5,8 @@ import torch
5
6
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7
AttentionMetadata, AttentionType)
8
-from vllm.attention.backends.utils import CommonMetadataBuilder
8
+from vllm.attention.backends.utils import (CommonAttentionState,
9
+ CommonMetadataBuilder)
10
from vllm.attention.ops.blocksparse_attention.interface import (
11
LocalStridedBlockSparseAttn, get_head_sliding_step)
12
from vllm.attention.ops.paged_attn import PagedAttention
@@ -98,6 +99,10 @@ class BlocksparseFlashAttentionBackend(AttentionBackend):
99
def get_builder_cls() -> Type["BlocksparseFlashAttentionMetadataBuilder"]:
100
return BlocksparseFlashAttentionMetadataBuilder
101
102
+ @staticmethod
103
+ def get_state_cls() -> Type["CommonAttentionState"]:
104
+ return CommonAttentionState
105
+
106
@staticmethod
107
def get_kv_cache_shape(
108
num_blocks: int,
vllm/attention/backends/flash_attn.py
+6
-1
@@ -9,7 +9,8 @@ from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
AttentionMetadata,
10
AttentionMetadataBuilder,
11
AttentionType)
12
-from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
12
+from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
13
+ compute_slot_mapping,
14
compute_slot_mapping_start_idx,
15
is_block_tables_empty)
16
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
@@ -142,6 +143,10 @@ class FlashAttentionBackend(AttentionBackend):
143
def get_builder_cls() -> Type["FlashAttentionMetadataBuilder"]:
144
return FlashAttentionMetadataBuilder
145
146
+ @staticmethod
147
+ def get_state_cls() -> Type["CommonAttentionState"]:
148
+ return CommonAttentionState
149
+
150
@staticmethod
151
def get_kv_cache_shape(
152
num_blocks: int,
vllm/attention/backends/flashinfer.py
+164
-1
@@ -1,14 +1,19 @@
1
+from contextlib import contextmanager
2
from dataclasses import dataclass
3
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Type
4
5
try:
6
from flashinfer import BatchDecodeWithPagedKVCacheWrapper
7
+ from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
8
from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
9
10
import vllm.attention.backends.flash_attn # noqa
11
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
12
except ImportError:
13
BatchDecodeWithPagedKVCacheWrapper = None
14
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
15
BatchPrefillWithPagedKVCacheWrapper = None
16
+ FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
17
18
import torch
19
@@ -16,7 +21,7 @@ from vllm import _custom_ops as ops
21
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
22
AttentionMetadata,
23
AttentionMetadataBuilder,
19
- AttentionType)
24
+ AttentionState, AttentionType)
25
from vllm.attention.backends.utils import (PAD_SLOT_ID, compute_slot_mapping,
26
compute_slot_mapping_start_idx,
27
is_block_tables_empty)
@@ -46,6 +51,10 @@ class FlashInferBackend(AttentionBackend):
51
def get_builder_cls() -> Type["FlashInferMetadataBuilder"]:
52
return FlashInferMetadataBuilder
53
54
+ @staticmethod
55
+ def get_state_cls() -> Type["FlashInferState"]:
56
+ return FlashInferState
57
+
58
@staticmethod
59
def get_kv_cache_shape(
60
num_blocks: int,
@@ -75,6 +84,160 @@ class FlashInferBackend(AttentionBackend):
84
return [64, 128, 256]
85
86
87
+class FlashInferState(AttentionState):
88
+
89
+ def __init__(self, runner):
90
+ self.runner = runner
91
+ self._is_graph_capturing = False
92
+ self._workspace_buffer = None
93
+ self._decode_wrapper = None
94
+ self._prefill_wrapper = None
95
+
96
+ def _get_workspace_buffer(self):
97
+ if self._workspace_buffer is None:
98
+ self._workspace_buffer = torch.empty(
99
+ FLASHINFER_WORKSPACE_BUFFER_SIZE,
100
+ dtype=torch.uint8,
101
+ device=self.runner.device)
102
+ return self._workspace_buffer
103
+
104
+ def _get_prefill_wrapper(self):
105
+ if self._prefill_wrapper is None:
106
+ self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
107
+ self._get_workspace_buffer(), "NHD")
108
+ return self._prefill_wrapper
109
+
110
+ def _get_decode_wrapper(self):
111
+ if self._decode_wrapper is None:
112
+ num_qo_heads = (self.runner.model_config.get_num_attention_heads(
113
+ self.runner.parallel_config))
114
+ num_kv_heads = self.runner.model_config.get_num_kv_heads(
115
+ self.runner.parallel_config)
116
+ use_tensor_cores = num_qo_heads // num_kv_heads >= 4
117
+ self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
118
+ self._get_workspace_buffer(),
119
+ "NHD",
120
+ use_tensor_cores=use_tensor_cores)
121
+ return self._decode_wrapper
122
+
123
+ @contextmanager
124
+ def graph_capture(self, max_batch_size: int):
125
+ self._is_graph_capturing = True
126
+ self._graph_decode_wrapper = None
127
+ self._graph_slot_mapping = torch.full((max_batch_size, ),
128
+ PAD_SLOT_ID,
129
+ dtype=torch.long,
130
+ device=self.runner.device)
131
+ self._graph_seq_lens = torch.ones(max_batch_size,
132
+ dtype=torch.int32,
133
+ device=self.runner.device)
134
+ self._graph_block_tables = torch.from_numpy(
135
+ self.runner.graph_block_tables).to(device=self.runner.device)
136
+ self._graph_decode_workspace_buffer = self._get_workspace_buffer()
137
+ self._graph_indices_buffer = torch.empty(
138
+ max_batch_size * self.runner.cache_config.num_gpu_blocks,
139
+ dtype=torch.int32,
140
+ device=self.runner.device)
141
+ self._graph_indptr_buffer = torch.empty(max_batch_size + 1,
142
+ dtype=torch.int32,
143
+ device=self.runner.device)
144
+ self._graph_last_page_len_buffer = torch.empty(
145
+ max_batch_size, dtype=torch.int32, device=self.runner.device)
146
+ yield
147
+ self._is_graph_capturing = False
148
+ del self._graph_slot_mapping
149
+ del self._graph_seq_lens
150
+ del self._graph_block_tables
151
+ del self._graph_decode_workspace_buffer
152
+ del self._graph_indices_buffer
153
+ del self._graph_indptr_buffer
154
+ del self._graph_last_page_len_buffer
155
+ del self._graph_decode_wrapper
156
+
157
+ def graph_clone(self, batch_size: int):
158
+ assert self._is_graph_capturing
159
+ state = self.__class__(self.runner)
160
+ state._workspace_buffer = self._graph_decode_workspace_buffer
161
+ state._decode_wrapper = self._graph_decode_wrapper
162
+ state._prefill_wrapper = self._get_prefill_wrapper()
163
+ return state
164
+
165
+ def graph_capture_get_metadata_for_batch(self, batch_size: int):
166
+ assert self._is_graph_capturing
167
+ _indptr_buffer = self._graph_indptr_buffer[:batch_size + 1]
168
+ _last_page_len_buffer = self._graph_last_page_len_buffer[:batch_size]
169
+
170
+ num_qo_heads = (self.runner.model_config.get_num_attention_heads(
171
+ self.runner.parallel_config))
172
+ num_kv_heads = self.runner.model_config.get_num_kv_heads(
173
+ self.runner.parallel_config)
174
+ use_tensor_cores = num_qo_heads // num_kv_heads >= 4
175
+ self._graph_decode_wrapper = \
176
+ CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
177
+ self._graph_decode_workspace_buffer, _indptr_buffer,
178
+ self._graph_indices_buffer, _last_page_len_buffer, "NHD",
179
+ use_tensor_cores)
180
+ kv_cache_dtype = get_kv_cache_torch_dtype(
181
+ self.runner.kv_cache_dtype, self.runner.model_config.dtype)
182
+
183
+ paged_kv_indptr_tensor_host = torch.arange(0,
184
+ batch_size + 1,
185
+ dtype=torch.int32)
186
+ paged_kv_indices_tensor_host = torch.arange(0,
187
+ batch_size,
188
+ dtype=torch.int32)
189
+ paged_kv_last_page_len_tensor_host = torch.full((batch_size, ),
190
+ self.runner.block_size,
191
+ dtype=torch.int32)
192
+ query_start_loc_host = torch.arange(0,
193
+ batch_size + 1,
194
+ dtype=torch.int32)
195
+
196
+ attn_metadata = self.runner.attn_backend.make_metadata(
197
+ num_prefills=0,
198
+ slot_mapping=self._graph_slot_mapping[:batch_size],
199
+ num_prefill_tokens=0,
200
+ num_decode_tokens=batch_size,
201
+ max_prefill_seq_len=0,
202
+ block_tables=self._graph_block_tables,
203
+ paged_kv_indptr=paged_kv_indptr_tensor_host,
204
+ paged_kv_indices=paged_kv_indices_tensor_host,
205
+ paged_kv_last_page_len=paged_kv_last_page_len_tensor_host,
206
+ num_qo_heads=num_qo_heads,
207
+ num_kv_heads=num_kv_heads,
208
+ head_dim=self.runner.model_config.get_head_size(),
209
+ page_size=self.runner.block_size,
210
+ seq_start_loc=None,
211
+ query_start_loc=query_start_loc_host,
212
+ device=self.runner.device,
213
+ data_type=kv_cache_dtype,
214
+ use_cuda_graph=True,
215
+ decode_wrapper=self._graph_decode_wrapper,
216
+ prefill_wrapper=None)
217
+ attn_metadata.begin_forward()
218
+ return attn_metadata
219
+
220
+ def get_graph_input_buffers(self, attn_metadata):
221
+ return {
222
+ "slot_mapping": attn_metadata.slot_mapping,
223
+ }
224
+
225
+ def prepare_graph_input_buffers(self, input_buffers, attn_metadata):
226
+ return
227
+
228
+ def begin_forward(self, model_input):
229
+ assert not self._is_graph_capturing
230
+ state = self
231
+ if model_input.attn_metadata.use_cuda_graph:
232
+ batch_size = model_input.input_tokens.shape[0]
233
+ state = (self.runner.graph_runners[model_input.virtual_engine]
234
+ [batch_size].attn_state)
235
+ model_input.attn_metadata.prefill_wrapper = state._get_prefill_wrapper(
236
+ )
237
+ model_input.attn_metadata.decode_wrapper = state._get_decode_wrapper()
238
+ model_input.attn_metadata.begin_forward()
239
+
240
+
241
@dataclass
242
class FlashInferMetadata(AttentionMetadata):
243
# Maximum sequence length among prefill batch. 0 if there are decoding
vllm/attention/backends/ipex_attn.py
+5
@@ -8,6 +8,7 @@ import torch
8
from vllm._ipex_ops import ipex_ops
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata, AttentionType)
11
+from vllm.attention.backends.utils import CommonAttentionState
12
from vllm.attention.ops.paged_attn import (PagedAttention,
13
PagedAttentionMetadata)
14
@@ -28,6 +29,10 @@ class IpexAttnBackend(AttentionBackend):
29
def get_metadata_cls() -> Type["IpexAttnMetadata"]:
30
return IpexAttnMetadata
31
32
+ @staticmethod
33
+ def get_state_cls() -> Type["CommonAttentionState"]:
34
+ return CommonAttentionState
35
+
36
@staticmethod
37
def get_kv_cache_shape(
38
num_blocks: int,
vllm/attention/backends/openvino.py
+6
-1
@@ -1,11 +1,12 @@
1
from dataclasses import dataclass
2
-from typing import List, Tuple
2
+from typing import List, Tuple, Type
3
4
import openvino as ov
5
import torch
6
7
from vllm.attention.backends.abstract import (AttentionBackend,
8
AttentionMetadata)
9
+from vllm.attention.backends.utils import CommonAttentionState
10
11
12
class OpenVINOAttentionBackend(AttentionBackend):
@@ -24,6 +25,10 @@ class OpenVINOAttentionBackend(AttentionBackend):
25
def make_metadata(*args, **kwargs) -> "AttentionMetadata":
26
raise NotImplementedError
27
28
+ @staticmethod
29
+ def get_state_cls() -> Type["CommonAttentionState"]:
30
+ return CommonAttentionState
31
+
32
@staticmethod
33
def make_openvino_metadata(*args, **kwargs) -> "OpenVINOAttentionMetadata":
34
return OpenVINOAttentionMetadata(*args, **kwargs)
vllm/attention/backends/pallas.py
+5
@@ -6,6 +6,7 @@ import torch_xla.experimental.custom_kernel # Required to register custom ops.
6
7
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
8
AttentionMetadata, AttentionType)
9
+from vllm.attention.backends.utils import CommonAttentionState
10
11
12
class PallasAttentionBackend(AttentionBackend):
@@ -18,6 +19,10 @@ class PallasAttentionBackend(AttentionBackend):
19
def get_metadata_cls() -> Type["PallasMetadata"]:
20
return PallasMetadata
21
22
+ @staticmethod
23
+ def get_state_cls() -> Type["CommonAttentionState"]:
24
+ return CommonAttentionState
25
+
26
@staticmethod
27
def get_kv_cache_shape(
28
num_blocks: int,
vllm/attention/backends/rocm_flash_attn.py
+6
-1
@@ -7,7 +7,8 @@ import torch
7
import vllm.envs as envs
8
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
9
AttentionMetadata, AttentionType)
10
-from vllm.attention.backends.utils import CommonMetadataBuilder
10
+from vllm.attention.backends.utils import (CommonAttentionState,
11
+ CommonMetadataBuilder)
12
from vllm.attention.ops.paged_attn import (PagedAttention,
13
PagedAttentionMetadata)
14
from vllm.logger import init_logger
@@ -33,6 +34,10 @@ class ROCmFlashAttentionBackend(AttentionBackend):
34
def get_builder_cls() -> Type["ROCmFlashAttentionMetadataBuilder"]:
35
return ROCmFlashAttentionMetadataBuilder
36
37
+ @staticmethod
38
+ def get_state_cls() -> Type["CommonAttentionState"]:
39
+ return CommonAttentionState
40
+
41
@staticmethod
42
def get_kv_cache_shape(
43
num_blocks: int,
vllm/attention/backends/torch_sdpa.py
+5
@@ -8,6 +8,7 @@ from torch.nn.functional import scaled_dot_product_attention
8
9
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
10
AttentionMetadata, AttentionType)
11
+from vllm.attention.backends.utils import CommonAttentionState
12
from vllm.attention.ops.paged_attn import PagedAttentionMetadata
13
from vllm.utils import is_cpu
14
@@ -34,6 +35,10 @@ class TorchSDPABackend(AttentionBackend):
35
def get_metadata_cls() -> Type["AttentionMetadata"]:
36
return TorchSDPAMetadata
37
38
+ @staticmethod
39
+ def get_state_cls() -> Type["CommonAttentionState"]:
40
+ return CommonAttentionState
41
+
42
@staticmethod
43
def get_kv_cache_shape(
44
num_blocks: int,
vllm/attention/backends/utils.py
+73
-2
@@ -1,12 +1,17 @@
1
"""Attention backend utils"""
2
-from typing import TYPE_CHECKING, Dict, List, Type, TypeVar, Union
2
+from contextlib import contextmanager
3
+from typing import TYPE_CHECKING, Any, Dict, List, Type, TypeVar, Union
4
5
import numpy as np
6
import torch
7
7
-from vllm.attention import AttentionMetadata, AttentionMetadataBuilder
8
+from vllm.attention import (AttentionMetadata, AttentionMetadataBuilder,
9
+ AttentionState)
10
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
11
12
+if TYPE_CHECKING:
13
+ from vllm.worker.model_runner_base import ModelRunnerBase
14
+
15
# Error string(s) for encoder/decoder
16
# unsupported attention scenarios
17
STR_NOT_IMPL_ENC_DEC_ROCM_HIP = ("ROCm/HIP is not currently supported "
@@ -269,3 +274,69 @@ class CommonMetadataBuilder(AttentionMetadataBuilder[TAttentionMetadata]):
274
block_tables=block_tables,
275
use_cuda_graph=use_captured_graph,
276
)
277
+
278
+
279
+class CommonAttentionState(AttentionState):
280
+
281
+ def __init__(self, runner: "ModelRunnerBase"):
282
+ self.runner = runner
283
+ self._is_graph_capturing = False
284
+
285
+ @contextmanager
286
+ def graph_capture(self, max_batch_size: int):
287
+ self._is_graph_capturing = True
288
+ self._graph_slot_mapping = torch.full((max_batch_size, ),
289
+ PAD_SLOT_ID,
290
+ dtype=torch.long,
291
+ device=self.runner.device)
292
+ self._graph_seq_lens = torch.ones(max_batch_size,
293
+ dtype=torch.int32,
294
+ device=self.runner.device)
295
+ self._graph_block_tables = torch.from_numpy(
296
+ self.runner.graph_block_tables).to(device=self.runner.device)
297
+ yield
298
+ self._is_graph_capturing = False
299
+ del self._graph_slot_mapping
300
+ del self._graph_seq_lens
301
+ del self._graph_block_tables
302
+
303
+ def graph_clone(self, batch_size: int) -> "CommonAttentionState":
304
+ assert self._is_graph_capturing
305
+ return self.__class__(self.runner)
306
+
307
+ def graph_capture_get_metadata_for_batch(self, batch_size: int):
308
+ assert self._is_graph_capturing
309
+ attn_metadata = self.runner.attn_backend.make_metadata(
310
+ num_prefills=0,
311
+ num_prefill_tokens=0,
312
+ num_decode_tokens=batch_size,
313
+ slot_mapping=self._graph_slot_mapping[:batch_size],
314
+ seq_lens=None,
315
+ seq_lens_tensor=self._graph_seq_lens[:batch_size],
316
+ max_query_len=None,
317
+ max_prefill_seq_len=0,
318
+ max_decode_seq_len=self.runner.max_seq_len_to_capture,
319
+ query_start_loc=None,
320
+ seq_start_loc=None,
321
+ context_lens_tensor=None,
322
+ block_tables=self._graph_block_tables[:batch_size],
323
+ use_cuda_graph=True,
324
+ )
325
+ return attn_metadata
326
+
327
+ def get_graph_input_buffers(self, attn_metadata) -> Dict[str, Any]:
328
+ return {
329
+ "slot_mapping": attn_metadata.slot_mapping,
330
+ "seq_lens_tensor": attn_metadata.decode_metadata.seq_lens_tensor,
331
+ "block_tables": attn_metadata.decode_metadata.block_tables,
332
+ }
333
+
334
+ def prepare_graph_input_buffers(self, input_buffers,
335
+ attn_metadata) -> None:
336
+ input_buffers["seq_lens_tensor"].copy_(
337
+ attn_metadata.decode_metadata.seq_lens_tensor, non_blocking=True)
338
+ input_buffers["block_tables"].copy_(
339
+ attn_metadata.decode_metadata.block_tables, non_blocking=True)
340
+
341
+ def begin_forward(self, model_input) -> None:
342
+ return
vllm/attention/backends/xformers.py
+6
-1
@@ -11,7 +11,8 @@ from xformers.ops.fmha.attn_bias import (AttentionBias,
11
12
from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
AttentionMetadata, AttentionType)
14
-from vllm.attention.backends.utils import CommonMetadataBuilder
14
+from vllm.attention.backends.utils import (CommonAttentionState,
15
+ CommonMetadataBuilder)
16
from vllm.attention.ops.paged_attn import (PagedAttention,
17
PagedAttentionMetadata)
18
from vllm.logger import init_logger
@@ -37,6 +38,10 @@ class XFormersBackend(AttentionBackend):
38
def get_builder_cls() -> Type["XFormersMetadataBuilder"]:
39
return XFormersMetadataBuilder
40
41
+ @staticmethod
42
+ def get_state_cls() -> Type["CommonAttentionState"]:
43
+ return CommonAttentionState
44
+
45
@staticmethod
46
def get_kv_cache_shape(
47
num_blocks: int,
vllm/spec_decode/draft_model_runner.py
+1
-46
@@ -11,17 +11,6 @@ except ModuleNotFoundError:
11
from vllm.attention.backends.rocm_flash_attn import (
12
ROCmFlashAttentionMetadata as FlashAttentionMetadata)
13
14
-try:
15
- from flashinfer import BatchDecodeWithPagedKVCacheWrapper
16
- from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
17
- from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
18
- FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
19
-except ImportError:
20
- BatchDecodeWithPagedKVCacheWrapper = None
21
- CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
22
- BatchPrefillWithPagedKVCacheWrapper = None
23
- FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
24
-
14
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
15
ModelConfig, ObservabilityConfig, ParallelConfig,
16
PromptAdapterConfig, SchedulerConfig)
@@ -90,11 +79,6 @@ class TP1DraftModelRunner(ModelRunner):
79
observability_config=observability_config,
80
)
81
93
- self.flashinfer_decode_workspace_buffer = None
94
- self.flashinfer_decode_wrapper = None
95
- self.flashinfer_prefill_workspace_buffer = None
96
- self.flashinfer_prefill_wrapper = None
97
-
82
def _update_sampling_metadata(self, sampling_metadata, num_seqs,
83
num_queries):
84
@@ -270,36 +254,7 @@ class TP1DraftModelRunner(ModelRunner):
254
model_input.prompt_adapter_requests,
255
model_input.prompt_adapter_mapping)
256
273
- if self.attn_backend.get_name() == "flashinfer":
274
- assert model_input.attn_metadata is not None
275
- assert model_input.input_tokens is not None
276
- if self.flashinfer_decode_workspace_buffer is None:
277
- self.flashinfer_decode_workspace_buffer = torch.empty(
278
- FLASHINFER_WORKSPACE_BUFFER_SIZE,
279
- dtype=torch.uint8,
280
- device=self.device)
281
- self.flashinfer_decode_wrapper = \
282
- BatchDecodeWithPagedKVCacheWrapper(
283
- self.flashinfer_decode_workspace_buffer, "NHD")
284
- self.flashinfer_prefill_workspace_buffer = torch.empty(
285
- FLASHINFER_WORKSPACE_BUFFER_SIZE,
286
- dtype=torch.uint8,
287
- device=self.device)
288
- self.flashinfer_prefill_wrapper = \
289
- BatchPrefillWithPagedKVCacheWrapper(
290
- self.flashinfer_prefill_workspace_buffer, "NHD")
291
-
292
- model_input.attn_metadata.prefill_wrapper = \
293
- self.flashinfer_prefill_wrapper
294
- if model_input.attn_metadata.use_cuda_graph:
295
- batch_size = model_input.input_tokens.shape[0]
296
- model_input.attn_metadata.decode_wrapper = \
297
- self.graph_runners[model_input.
298
- virtual_engine][batch_size].flashinfer_decode_wrapper
299
- else:
300
- model_input.attn_metadata.decode_wrapper = \
301
- self.flashinfer_decode_wrapper
302
- model_input.attn_metadata.begin_forward()
257
+ self.attn_state.begin_forward(model_input)
258
259
# Detect exec mode
260
assert model_input.attn_metadata is not None
vllm/worker/enc_dec_model_runner.py
+3
-2
@@ -6,6 +6,7 @@ import torch.distributed
6
7
from vllm.attention.backends.abstract import (AttentionBackend,
8
AttentionMetadata)
9
+from vllm.attention.backends.utils import PAD_SLOT_ID
10
from vllm.attention.selector import (_Backend, get_env_variable_attn_backend,
11
get_global_forced_attn_backend,
12
global_force_attn_backend)
@@ -20,7 +21,7 @@ from vllm.sampling_params import SamplingParams
21
from vllm.sequence import (IntermediateTensors, PoolerOutput, SamplerOutput,
22
SequenceGroupMetadata)
23
from vllm.utils import STR_NOT_IMPL_ENC_DEC_BACKEND, make_tensor_with_pad
23
-from vllm.worker.model_runner import (_PAD_SLOT_ID, GPUModelRunnerBase,
24
+from vllm.worker.model_runner import (GPUModelRunnerBase,
25
ModelInputForGPUBuilder,
26
ModelInputForGPUWithSamplingMetadata)
27
from vllm.worker.model_runner_base import (
@@ -395,7 +396,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
396
# initialized yet. In this case, we just use a dummy
397
# slot mapping.
398
# In embeddings, the block tables are {seq_id: None}.
398
- cross_slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
399
+ cross_slot_mapping.extend([PAD_SLOT_ID] * seq_len)
400
else:
401
for i in range(0, seq_len):
402
block_number = seq_group_metadata.cross_block_table[
vllm/worker/model_runner.py
+28
-188
@@ -13,19 +13,10 @@ import torch
13
import torch.distributed
14
import torch.nn as nn
15
16
-try:
17
- from flashinfer import BatchDecodeWithPagedKVCacheWrapper
18
- from flashinfer.decode import CUDAGraphBatchDecodeWithPagedKVCacheWrapper
19
- from flashinfer.prefill import BatchPrefillWithPagedKVCacheWrapper
20
- FLASHINFER_WORKSPACE_BUFFER_SIZE = 256 * 1024 * 1024
21
-except ImportError:
22
- BatchDecodeWithPagedKVCacheWrapper = None
23
- CUDAGraphBatchDecodeWithPagedKVCacheWrapper = None
24
- BatchPrefillWithPagedKVCacheWrapper = None
25
- FLASHINFER_WORKSPACE_BUFFER_SIZE = 0
26
-
16
import vllm.envs as envs
17
from vllm.attention import AttentionMetadata, get_attn_backend
18
+from vllm.attention.backends.abstract import AttentionState
19
+from vllm.attention.backends.utils import CommonAttentionState
20
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
21
ModelConfig, ObservabilityConfig, ParallelConfig,
22
PromptAdapterConfig, SchedulerConfig)
@@ -52,8 +43,7 @@ from vllm.sampling_params import SamplingParams
43
from vllm.sequence import (IntermediateTensors, SamplerOutput,
44
SequenceGroupMetadata)
45
from vllm.utils import (CudaMemoryProfiler, PyObjectCache, async_tensor_h2d,
55
- flatten_2d_lists, get_kv_cache_torch_dtype, is_hip,
56
- is_pin_memory_available)
46
+ flatten_2d_lists, is_hip, is_pin_memory_available)
47
from vllm.worker.model_runner_base import (
48
ModelRunnerBase, ModelRunnerInputBase, ModelRunnerInputBuilderBase,
49
_add_attn_metadata_broadcastable_dict,
@@ -66,7 +56,6 @@ if TYPE_CHECKING:
56
57
logger = init_logger(__name__)
58
69
-_PAD_SLOT_ID = -1
59
LORA_WARMUP_RANK = 8
60
_BATCH_SIZE_ALIGNMENT = 8
61
# Capture graphs for token size 1, 2, 4, 8, 16, 24, 32, 40, ..., 256.
@@ -858,6 +847,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
847
self.kv_cache_dtype,
848
self.block_size,
849
) if num_attn_heads else None
850
+ if self.attn_backend:
851
+ self.attn_state = self.attn_backend.get_state_cls()(
852
+ weakref.proxy(self))
853
+ else:
854
+ self.attn_state = CommonAttentionState(weakref.proxy(self))
855
856
# Multi-modal data support
857
self.input_registry = input_registry
@@ -872,11 +866,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
866
self.lora_manager: Optional[LRUCacheWorkerLoRAManager] = None
867
self.prompt_adapter_manager: LRUCacheWorkerPromptAdapterManager = None
868
875
- self.flashinfer_decode_workspace_buffer = None
876
- self.flashinfer_decode_wrapper = None
877
- self.flashinfer_prefill_workspace_buffer = None
878
- self.flashinfer_prefill_wrapper = None
879
-
869
set_cpu_offload_max_bytes(
870
int(self.cache_config.cpu_offload_gb * 1024**3))
871
@@ -1203,10 +1192,6 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1192
max_batch_size = max(_BATCH_SIZES_TO_CAPTURE)
1193
input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1194
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1206
- slot_mapping = torch.empty(max_batch_size, dtype=torch.long).cuda()
1207
- slot_mapping.fill_(_PAD_SLOT_ID)
1208
- seq_lens = torch.ones(max_batch_size, dtype=torch.int32).cuda()
1209
- block_tables = torch.from_numpy(self.graph_block_tables).cuda()
1195
intermediate_inputs = None
1196
if not get_pp_group().is_first_rank:
1197
intermediate_inputs = self.model.make_empty_intermediate_tensors(
@@ -1226,102 +1211,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1211
bs for bs in _BATCH_SIZES_TO_CAPTURE if bs <= graph_batch_size
1212
]
1213
1229
- if self.attn_backend.get_name() == "flashinfer":
1230
- # For flashinfer, different batch sizes will share the
1231
- # same workspace buffer.
1232
- decode_workspace_buffer = \
1233
- torch.empty(FLASHINFER_WORKSPACE_BUFFER_SIZE,
1234
- dtype=torch.uint8,
1235
- device=self.device)
1236
- indices_buffer = torch.empty(max_batch_size *
1237
- self.cache_config.num_gpu_blocks,
1238
- dtype=torch.int32,
1239
- device=self.device)
1240
- indptr_buffer = torch.empty(max_batch_size + 1,
1241
- dtype=torch.int32,
1242
- device=self.device)
1243
- last_page_len_buffer = torch.empty(max_batch_size,
1244
- dtype=torch.int32,
1245
- device=self.device)
1246
-
1247
- with graph_capture() as graph_capture_context:
1214
+ with self.attn_state.graph_capture(
1215
+ max_batch_size), graph_capture() as graph_capture_context:
1216
# NOTE: Capturing the largest batch size first may help reduce the
1217
# memory usage of CUDA graph.
1218
for virtual_engine in range(
1219
self.parallel_config.pipeline_parallel_size):
1220
for batch_size in reversed(batch_size_capture_list):
1253
- if self.attn_backend.get_name() == "flashinfer":
1254
- _indptr_buffer = indptr_buffer[:batch_size + 1]
1255
- _last_page_len_buffer = last_page_len_buffer[:
1256
- batch_size]
1257
-
1258
- num_qo_heads = (
1259
- self.model_config.get_num_attention_heads(
1260
- self.parallel_config))
1261
- num_kv_heads = self.model_config.get_num_kv_heads(
1262
- self.parallel_config)
1263
- if num_qo_heads // num_kv_heads >= 4:
1264
- use_tensor_cores = True
1265
- else:
1266
- use_tensor_cores = False
1267
- decode_wrapper = \
1268
- CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
1269
- decode_workspace_buffer, _indptr_buffer,
1270
- indices_buffer, _last_page_len_buffer, "NHD",
1271
- use_tensor_cores)
1272
- kv_cache_dtype = get_kv_cache_torch_dtype(
1273
- self.kv_cache_dtype, self.model_config.dtype)
1274
-
1275
- paged_kv_indptr_tensor_host = torch.arange(
1276
- 0, batch_size + 1, dtype=torch.int32)
1277
- paged_kv_indices_tensor_host = torch.arange(
1278
- 0, batch_size, dtype=torch.int32)
1279
- paged_kv_last_page_len_tensor_host = torch.full(
1280
- (batch_size, ), self.block_size, dtype=torch.int32)
1281
- query_start_loc_host = torch.arange(0,
1282
- batch_size + 1,
1283
- dtype=torch.int32)
1284
-
1285
- attn_metadata = self.attn_backend.make_metadata(
1286
- num_prefills=0,
1287
- slot_mapping=slot_mapping[:batch_size],
1288
- num_prefill_tokens=0,
1289
- num_decode_tokens=batch_size,
1290
- max_prefill_seq_len=0,
1291
- block_tables=block_tables,
1292
- paged_kv_indptr=paged_kv_indptr_tensor_host,
1293
- paged_kv_indices=paged_kv_indices_tensor_host,
1294
- paged_kv_last_page_len=
1295
- paged_kv_last_page_len_tensor_host,
1296
- num_qo_heads=num_qo_heads,
1297
- num_kv_heads=num_kv_heads,
1298
- head_dim=self.model_config.get_head_size(),
1299
- page_size=self.block_size,
1300
- seq_start_loc=None,
1301
- query_start_loc=query_start_loc_host,
1302
- device=self.device,
1303
- data_type=kv_cache_dtype,
1304
- use_cuda_graph=True,
1305
- decode_wrapper=decode_wrapper,
1306
- prefill_wrapper=None)
1307
- attn_metadata.begin_forward()
1308
- else:
1309
- attn_metadata = self.attn_backend.make_metadata(
1310
- num_prefills=0,
1311
- num_prefill_tokens=0,
1312
- num_decode_tokens=batch_size,
1313
- slot_mapping=slot_mapping[:batch_size],
1314
- seq_lens=None,
1315
- seq_lens_tensor=seq_lens[:batch_size],
1316
- max_query_len=None,
1317
- max_prefill_seq_len=0,
1318
- max_decode_seq_len=self.max_seq_len_to_capture,
1319
- query_start_loc=None,
1320
- seq_start_loc=None,
1321
- context_lens_tensor=None,
1322
- block_tables=block_tables[:batch_size],
1323
- use_cuda_graph=True,
1324
- )
1221
+ attn_metadata = (
1222
+ self.attn_state.graph_capture_get_metadata_for_batch(
1223
+ batch_size))
1224
1225
if self.lora_config:
1226
lora_mapping = LoRAMapping(
@@ -1339,17 +1238,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1238
set(), prompt_adapter_mapping)
1239
1240
graph_runner = CUDAGraphRunner(
1342
- self.model, self.attn_backend.get_name())
1343
-
1344
- if self.attn_backend.get_name() == "flashinfer":
1345
- graph_runner.flashinfer_indptr_buffer = _indptr_buffer
1346
- graph_runner.flashinfer_indices_buffer = indices_buffer
1347
- graph_runner.flashinfer_last_page_len_buffer = \
1348
- _last_page_len_buffer
1349
- graph_runner.flashinfer_decode_workspace_buffer = \
1350
- decode_workspace_buffer
1351
- graph_runner.flashinfer_decode_wrapper = \
1352
- decode_wrapper
1241
+ self.model, self.attn_backend.get_name(),
1242
+ self.attn_state.graph_clone(batch_size))
1243
1244
capture_inputs = {
1245
"input_ids":
@@ -1476,36 +1366,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1366
model_input.prompt_adapter_requests,
1367
model_input.prompt_adapter_mapping)
1368
1479
- if self.attn_backend.get_name() == "flashinfer":
1480
- assert model_input.attn_metadata is not None
1481
- assert model_input.input_tokens is not None
1482
- if self.flashinfer_decode_workspace_buffer is None:
1483
- self.flashinfer_decode_workspace_buffer = torch.empty(
1484
- FLASHINFER_WORKSPACE_BUFFER_SIZE,
1485
- dtype=torch.uint8,
1486
- device=self.device)
1487
- self.flashinfer_decode_wrapper = \
1488
- BatchDecodeWithPagedKVCacheWrapper(
1489
- self.flashinfer_decode_workspace_buffer, "NHD")
1490
- self.flashinfer_prefill_workspace_buffer = torch.empty(
1491
- FLASHINFER_WORKSPACE_BUFFER_SIZE,
1492
- dtype=torch.uint8,
1493
- device=self.device)
1494
- self.flashinfer_prefill_wrapper = \
1495
- BatchPrefillWithPagedKVCacheWrapper(
1496
- self.flashinfer_prefill_workspace_buffer, "NHD")
1497
-
1498
- model_input.attn_metadata.prefill_wrapper = \
1499
- self.flashinfer_prefill_wrapper
1500
- if model_input.attn_metadata.use_cuda_graph:
1501
- batch_size = model_input.input_tokens.shape[0]
1502
- model_input.attn_metadata.decode_wrapper = self.graph_runners[
1503
- model_input.
1504
- virtual_engine][batch_size].flashinfer_decode_wrapper
1505
- else:
1506
- model_input.attn_metadata.decode_wrapper = \
1507
- self.flashinfer_decode_wrapper
1508
- model_input.attn_metadata.begin_forward()
1369
+ self.attn_state.begin_forward(model_input)
1370
1371
# Currently cuda graph is only supported by the decode phase.
1372
assert model_input.attn_metadata is not None
@@ -1613,22 +1474,17 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1474
1475
class CUDAGraphRunner:
1476
1616
- def __init__(self, model: nn.Module, backend_name: str):
1477
+ def __init__(self, model: nn.Module, backend_name: str,
1478
+ attn_state: AttentionState):
1479
self.model = model
1480
self.backend_name = backend_name
1481
+ self.attn_state = attn_state
1482
1483
self.input_buffers: Dict[str, torch.Tensor] = {}
1484
self.output_buffers: Dict[str, torch.Tensor] = {}
1485
1486
self._graph: Optional[torch.cuda.CUDAGraph] = None
1487
1625
- self.flashinfer_decode_workspace_buffer: Optional[torch.Tensor] = None
1626
- self.flashinfer_indptr_buffer: Optional[torch.Tensor] = None
1627
- self.flashinfer_indices_buffer: Optional[torch.Tensor] = None
1628
- self.flashinfer_last_page_len_buffer: Optional[torch.Tensor] = None
1629
- self.flashinfer_decode_wrapper: Optional[
1630
- CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = None
1631
-
1488
@property
1489
def graph(self):
1490
assert self._graph is not None
@@ -1693,25 +1549,13 @@ class CUDAGraphRunner:
1549
torch.cuda.synchronize()
1550
1551
# Save the input and output buffers.
1696
- if self.backend_name == "flashinfer":
1697
- self.input_buffers = {
1698
- "input_ids": input_ids,
1699
- "positions": positions,
1700
- "kv_caches": kv_caches,
1701
- "slot_mapping": attn_metadata.slot_mapping,
1702
- **kwargs,
1703
- }
1704
- else:
1705
- self.input_buffers = {
1706
- "input_ids": input_ids,
1707
- "positions": positions,
1708
- "kv_caches": kv_caches,
1709
- "slot_mapping": attn_metadata.slot_mapping,
1710
- "seq_lens_tensor":
1711
- attn_metadata.decode_metadata.seq_lens_tensor,
1712
- "block_tables": attn_metadata.decode_metadata.block_tables,
1713
- **kwargs,
1714
- }
1552
+ self.input_buffers = {
1553
+ "input_ids": input_ids,
1554
+ "positions": positions,
1555
+ "kv_caches": kv_caches,
1556
+ **self.attn_state.get_graph_input_buffers(attn_metadata),
1557
+ **kwargs,
1558
+ }
1559
if intermediate_inputs is not None:
1560
self.input_buffers.update(intermediate_inputs.tensors)
1561
if get_pp_group().is_last_rank:
@@ -1739,12 +1583,8 @@ class CUDAGraphRunner:
1583
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1584
self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1585
non_blocking=True)
1742
- if self.backend_name != "flashinfer":
1743
- self.input_buffers["seq_lens_tensor"].copy_(
1744
- attn_metadata.decode_metadata.seq_lens_tensor,
1745
- non_blocking=True)
1746
- self.input_buffers["block_tables"].copy_(
1747
- attn_metadata.decode_metadata.block_tables, non_blocking=True)
1586
+ self.attn_state.prepare_graph_input_buffers(self.input_buffers,
1587
+ attn_metadata)
1588
if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1589
self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1590
**kwargs)