A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Core] Refactor Attention Take 2 (#3462)
Woosuk Kwon committed
Mar 24, 2024 at 21:39 UTC
925f3332cac488e5ad2dbc8f5c6d5f42d2556816
47 files changed
+1269
-1118
tests/kernels/test_prefix_prefill.py
+1
-2
@@ -3,8 +3,7 @@ import pytest
3
import time
4
5
import torch
6
-from vllm.model_executor.layers.attention.ops.prefix_prefill import (
7
- context_attention_fwd)
6
+from vllm.attention.ops.prefix_prefill import context_attention_fwd
7
from xformers import ops as xops
8
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
9
tests/samplers/test_beam_search.py
+7
@@ -2,7 +2,10 @@
2
3
Run `pytest tests/samplers/test_beam_search.py --forked`.
4
"""
5
+import gc
6
+
7
import pytest
8
+import torch
9
10
# FIXME(zhuohan): The test can not pass if we:
11
# 1. Increase max_tokens to 256.
@@ -36,6 +39,10 @@ def test_beam_search_single_input(
39
vllm_outputs = vllm_model.generate_beam_search(example_prompts, beam_width,
40
max_tokens)
41
del vllm_model
42
+ # NOTE(woosuk): For some reason, the following GC is required to avoid
43
+ # GPU OOM errors in the following tests using `vllm_runner`.
44
+ gc.collect()
45
+ torch.cuda.empty_cache()
46
47
for i in range(len(example_prompts)):
48
hf_output_ids, _ = hf_outputs[i]
tests/worker/test_model_runner.py
+30
-30
@@ -34,19 +34,19 @@ def test_prepare_prompt(batch_size):
34
expected_selected_token_indices.append(selected_token_start_idx +
35
prompt_len - 1)
36
selected_token_start_idx += prompt_len
37
- (input_tokens, input_positions, input_metadata, return_prompt_lens, _, _,
38
- _, _) = (model_runner._prepare_prompt(seq_group_metadata_list))
37
+ (input_tokens, input_positions, attn_metadata, return_prompt_lens, _, _, _,
38
+ _) = (model_runner._prepare_prompt(seq_group_metadata_list))
39
assert return_prompt_lens == prompt_lens
40
41
# Verify input metadata is correct for prompts.
42
device = model_runner.device
43
- assert input_metadata.is_prompt is True
44
- assert torch.allclose(input_metadata.prompt_lens_tensor,
43
+ assert attn_metadata.is_prompt is True
44
+ assert torch.allclose(attn_metadata.prompt_lens_tensor,
45
torch.tensor(prompt_lens, device=device))
46
- assert input_metadata.prompt_lens == prompt_lens
47
- assert input_metadata.num_prompt_tokens == sum(prompt_lens)
48
- assert input_metadata.num_generation_tokens == 0
49
- assert input_metadata.max_seq_len == max(prompt_lens)
46
+ assert attn_metadata.prompt_lens == prompt_lens
47
+ assert attn_metadata.num_prompt_tokens == sum(prompt_lens)
48
+ assert attn_metadata.num_generation_tokens == 0
49
+ assert attn_metadata.max_prompt_len == max(prompt_lens)
50
51
# Test subquery start locs.
52
start_idx = 0
@@ -55,7 +55,7 @@ def test_prepare_prompt(batch_size):
55
start_idx += prompt_len
56
start_loc.append(start_idx)
57
assert torch.allclose(
58
- input_metadata.subquery_start_loc,
58
+ attn_metadata.subquery_start_loc,
59
torch.tensor(start_loc, dtype=torch.int32, device=device))
60
61
# Test seq start locs. Note that for normal prefill it is
@@ -67,22 +67,22 @@ def test_prepare_prompt(batch_size):
67
seq_start_loc.append(start_idx)
68
69
assert torch.allclose(
70
- input_metadata.seq_start_loc,
70
+ attn_metadata.seq_start_loc,
71
torch.tensor(start_loc, dtype=torch.int32, device=device))
72
- assert input_metadata.max_context_len is None
72
+ assert attn_metadata.max_context_len is None
73
assert torch.allclose(
74
- input_metadata.context_lens,
75
- torch.zeros(input_metadata.context_lens.shape[0],
74
+ attn_metadata.context_lens,
75
+ torch.zeros(attn_metadata.context_lens.shape[0],
76
dtype=torch.int,
77
device=device))
78
79
expected = torch.tensor([[] for _ in range(len(seq_group_metadata_list))],
80
dtype=torch.int32,
81
device=model_runner.device)
82
- assert torch.allclose(input_metadata.block_tables, expected)
82
+ assert torch.allclose(attn_metadata.block_tables, expected)
83
# Cuda graph should not be used for prerill.
84
- assert input_metadata.use_cuda_graph is False
85
- assert input_metadata.kv_cache_dtype == "auto"
84
+ assert attn_metadata.use_cuda_graph is False
85
+ assert attn_metadata.kv_cache_dtype == "auto"
86
87
assert input_tokens.shape == (sum(prompt_lens), )
88
assert input_positions.shape == (sum(prompt_lens), )
@@ -140,34 +140,34 @@ def test_prepare_decode_cuda_graph(batch_size):
140
block_tables={0: [1]},
141
))
142
143
- input_tokens, input_positions, input_metadata, _, _, _ = (
143
+ input_tokens, input_positions, attn_metadata, _, _, _ = (
144
model_runner._prepare_decode(seq_group_metadata_list))
145
146
expected_bs = _get_graph_batch_size(len(seq_group_metadata_list))
147
# Verify input metadata is correct for prompts.
148
device = model_runner.device
149
- assert input_metadata.is_prompt is False
150
- assert input_metadata.prompt_lens is None
151
- assert input_metadata.num_prompt_tokens == 0
152
- assert input_metadata.num_generation_tokens == expected_bs
153
- assert input_metadata.max_seq_len is None
154
- assert input_metadata.subquery_start_loc is None
155
- assert input_metadata.seq_start_loc is None
156
- assert input_metadata.max_context_len == max(prompt_lens)
149
+ assert attn_metadata.is_prompt is False
150
+ assert attn_metadata.prompt_lens is None
151
+ assert attn_metadata.num_prompt_tokens == 0
152
+ assert attn_metadata.num_generation_tokens == expected_bs
153
+ assert attn_metadata.max_prompt_len is None
154
+ assert attn_metadata.subquery_start_loc is None
155
+ assert attn_metadata.seq_start_loc is None
156
+ assert attn_metadata.max_context_len == max(prompt_lens)
157
assert torch.allclose(
158
- input_metadata.context_lens[:len(prompt_lens)],
158
+ attn_metadata.context_lens[:len(prompt_lens)],
159
torch.tensor(prompt_lens, dtype=torch.int, device=device))
160
161
# block table's first index corresponds to each batch, meaning in
162
# decoding it is each token.
163
- assert input_metadata.block_tables.shape[0] == len(input_tokens)
163
+ assert attn_metadata.block_tables.shape[0] == len(input_tokens)
164
# Block table's second dim correspondsd to each token's block number.
165
# It is padded up to
166
- assert input_metadata.block_tables.shape[1] == (
166
+ assert attn_metadata.block_tables.shape[1] == (
167
model_runner.get_max_block_per_batch())
168
# Cuda graph should not be used for prerill.
169
- assert input_metadata.use_cuda_graph is True
170
- assert input_metadata.kv_cache_dtype == "auto"
169
+ assert attn_metadata.use_cuda_graph is True
170
+ assert attn_metadata.kv_cache_dtype == "auto"
171
172
assert input_tokens.shape == (expected_bs, )
173
assert input_positions.shape == (expected_bs, )
vllm/attention/__init__.py
new
+10
@@ -0,0 +1,10 @@
1
+from vllm.attention.backends.abstract import AttentionBackend, AttentionMetadata
2
+from vllm.attention.layer import Attention
3
+from vllm.attention.selector import get_attn_backend
4
+
5
+__all__ = [
6
+ "AttentionBackend",
7
+ "AttentionMetadata",
8
+ "Attention",
9
+ "get_attn_backend",
10
+]
vllm/attention/backends/__init__.py
renamed
vllm/attention/backends/abstract.py
new
+85
@@ -0,0 +1,85 @@
1
+from abc import ABC, abstractmethod
2
+from dataclasses import dataclass, fields
3
+from typing import Any, Dict, List, Optional, Tuple, Type
4
+
5
+import torch
6
+
7
+
8
+class AttentionBackend(ABC):
9
+ """Abstract class for attention backends."""
10
+
11
+ @staticmethod
12
+ @abstractmethod
13
+ def get_impl_cls() -> Type["AttentionImpl"]:
14
+ raise NotImplementedError
15
+
16
+ @staticmethod
17
+ @abstractmethod
18
+ def make_metadata(*args, **kwargs) -> "AttentionMetadata":
19
+ raise NotImplementedError
20
+
21
+ @staticmethod
22
+ @abstractmethod
23
+ def get_kv_cache_shape(
24
+ num_blocks: int,
25
+ block_size: int,
26
+ num_kv_heads: int,
27
+ head_size: int,
28
+ ) -> Tuple[int, ...]:
29
+ raise NotImplementedError
30
+
31
+ @staticmethod
32
+ @abstractmethod
33
+ def swap_blocks(
34
+ src_kv_cache: torch.Tensor,
35
+ dst_kv_cache: torch.Tensor,
36
+ src_to_dst: Dict[int, int],
37
+ ) -> None:
38
+ raise NotImplementedError
39
+
40
+ @staticmethod
41
+ @abstractmethod
42
+ def copy_blocks(
43
+ kv_caches: List[torch.Tensor],
44
+ src_to_dists: Dict[int, List[int]],
45
+ ) -> None:
46
+ raise NotImplementedError
47
+
48
+
49
+@dataclass
50
+class AttentionMetadata:
51
+
52
+ def asdict_zerocopy(self) -> Dict[str, Any]:
53
+ """Similar to dataclasses.asdict, but avoids deepcopying."""
54
+ # Note that if we add dataclasses as fields, they will need
55
+ # similar handling.
56
+ return {
57
+ field.name: getattr(self, field.name)
58
+ for field in fields(self)
59
+ }
60
+
61
+
62
+class AttentionImpl(ABC):
63
+
64
+ @abstractmethod
65
+ def __init__(
66
+ self,
67
+ num_heads: int,
68
+ head_size: int,
69
+ scale: float,
70
+ num_kv_heads: Optional[int] = None,
71
+ alibi_slopes: Optional[List[float]] = None,
72
+ sliding_window: Optional[int] = None,
73
+ ) -> None:
74
+ raise NotImplementedError
75
+
76
+ @abstractmethod
77
+ def forward(
78
+ self,
79
+ query: torch.Tensor,
80
+ key: torch.Tensor,
81
+ value: torch.Tensor,
82
+ kv_cache: torch.Tensor,
83
+ attn_metadata: AttentionMetadata,
84
+ ) -> torch.Tensor:
85
+ raise NotImplementedError
vllm/attention/backends/flash_attn.py
new
+238
@@ -0,0 +1,238 @@
1
+"""Attention layer with Flash and PagedAttention.
2
+
3
+NOTE(woosuk): At the moment, this file includes a lot of duplicated code from
4
+XFormers backend. The duplicated code will be removed once we use flash-attn or
5
+flashinfer for all the attention operations.
6
+"""
7
+from dataclasses import dataclass
8
+from typing import Dict, List, Optional, Tuple, Type
9
+
10
+from flash_attn import flash_attn_varlen_func
11
+import torch
12
+
13
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
14
+ AttentionMetadata)
15
+from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
16
+
17
+
18
+class FlashAttentionBackend(AttentionBackend):
19
+
20
+ @staticmethod
21
+ def get_impl_cls() -> Type["FlashAttentionImpl"]:
22
+ return FlashAttentionImpl
23
+
24
+ @staticmethod
25
+ def make_metadata(*args, **kwargs) -> "FlashAttentionMetadata":
26
+ return FlashAttentionMetadata(*args, **kwargs)
27
+
28
+ @staticmethod
29
+ def get_kv_cache_shape(
30
+ num_blocks: int,
31
+ block_size: int,
32
+ num_kv_heads: int,
33
+ head_size: int,
34
+ ) -> Tuple[int, ...]:
35
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
36
+ num_kv_heads, head_size)
37
+
38
+ @staticmethod
39
+ def swap_blocks(
40
+ src_kv_cache: torch.Tensor,
41
+ dst_kv_cache: torch.Tensor,
42
+ src_to_dst: Dict[int, int],
43
+ ) -> None:
44
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
45
+
46
+ @staticmethod
47
+ def copy_blocks(
48
+ kv_caches: List[torch.Tensor],
49
+ src_to_dists: Dict[int, List[int]],
50
+ ) -> None:
51
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
52
+
53
+
54
+@dataclass
55
+class FlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
56
+ """Metadata for FlashAttentionBackend.
57
+
58
+ NOTE: Any python object stored here is not updated when it is
59
+ cuda-graph replayed. If you have values that need to be changed
60
+ dynamically, it should be stored in tensor. The tensor has to be
61
+ updated from `CUDAGraphRunner.forward` API.
62
+ """
63
+ # Currently, input sequences can only contain all prompts
64
+ # or all decoding. True if all sequences are prompts.
65
+ is_prompt: bool
66
+ # (batch_size,). The prompt length per sequence. None if it is a decoding.
67
+ prompt_lens: Optional[List[int]]
68
+ # prompt_lens stored as a tensor.
69
+ prompt_lens_tensor: Optional[torch.Tensor]
70
+ # The number of prompt tokens. Doesn't include padding.
71
+ num_prompt_tokens: int
72
+ # The number of generation tokens. Doesn't include padding.
73
+ num_generation_tokens: int
74
+
75
+ # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
76
+ # |---------- N-1 iteration --------|
77
+ # |---------------- N iteration ---------------------|
78
+ # |- tokenA -|......................|-- newTokens ---|
79
+ # |---------- context_len ----------|
80
+ # |-------------------- seqlen ----------------------|
81
+ # |- subquery_len -|
82
+
83
+ # WARNING(sang): context_len has different definition depending on if it is
84
+ # prefill vs decoding. When it is prefill, it doesn't include new tokens.
85
+ # When it is for decoding, it includes a new token.
86
+
87
+ # Maximum subquery length in the batch.
88
+ max_subquery_len: Optional[int]
89
+ # Maximum prompt length in the batch.
90
+ max_prompt_len: Optional[int]
91
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
92
+ # the batch, used to index into subquery. E.g., if the subquery length
93
+ # is [4, 6], it is [0, 4, 10].
94
+ subquery_start_loc: Optional[torch.Tensor]
95
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
96
+ # the batch, used to index into sequence. E.g., if the sequence length is
97
+ # [4, 6], it is [0, 4, 10].
98
+ seq_start_loc: Optional[torch.Tensor]
99
+
100
+ # Whether or not if cuda graph is enabled.
101
+ # Cuda-graph is currently enabled for decoding only.
102
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
103
+ use_cuda_graph: bool
104
+
105
+
106
+class FlashAttentionImpl(AttentionImpl):
107
+ """
108
+ If the input tensors contain prompt tokens, the layout is as follows:
109
+ |<--------------- num_prompt_tokens -------------->|
110
+ |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
111
+
112
+ Otherwise, the layout is as follows:
113
+ |<------------------ num_generation_tokens (M) ----------------->|
114
+ |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
115
+
116
+ Generation tokens can contain padding when cuda-graph is used.
117
+ Currently, prompt tokens don't contain any padding.
118
+
119
+ The prompts might have different lengths, while the generation tokens
120
+ always have length 1.
121
+ """
122
+
123
+ def __init__(
124
+ self,
125
+ num_heads: int,
126
+ head_size: int,
127
+ scale: float,
128
+ num_kv_heads: Optional[int] = None,
129
+ alibi_slopes: Optional[List[float]] = None,
130
+ sliding_window: Optional[int] = None,
131
+ ) -> None:
132
+ self.num_heads = num_heads
133
+ self.head_size = head_size
134
+ self.scale = float(scale)
135
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
136
+ self.sliding_window = ((sliding_window, sliding_window)
137
+ if sliding_window is not None else (-1, -1))
138
+ if alibi_slopes is not None:
139
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
140
+ self.alibi_slopes = alibi_slopes
141
+
142
+ assert self.num_heads % self.num_kv_heads == 0
143
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
144
+
145
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
146
+ if head_size not in suppored_head_sizes:
147
+ raise ValueError(
148
+ f"Head size {head_size} is not supported by PagedAttention. "
149
+ f"Supported head sizes are: {suppored_head_sizes}.")
150
+
151
+ def forward(
152
+ self,
153
+ query: torch.Tensor,
154
+ key: torch.Tensor,
155
+ value: torch.Tensor,
156
+ kv_cache: torch.Tensor,
157
+ attn_metadata: FlashAttentionMetadata,
158
+ ) -> torch.Tensor:
159
+ """Forward pass with FlashAttention and PagedAttention.
160
+
161
+ Args:
162
+ query: shape = [num_tokens, num_heads * head_size]
163
+ key: shape = [num_tokens, num_kv_heads * head_size]
164
+ value: shape = [num_tokens, num_kv_heads * head_size]
165
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
166
+ attn_metadata: Metadata for attention.
167
+ Returns:
168
+ shape = [num_tokens, num_heads * head_size]
169
+ """
170
+ num_tokens, hidden_size = query.shape
171
+ # Reshape the query, key, and value tensors.
172
+ query = query.view(-1, self.num_heads, self.head_size)
173
+ key = key.view(-1, self.num_kv_heads, self.head_size)
174
+ value = value.view(-1, self.num_kv_heads, self.head_size)
175
+
176
+ if kv_cache is not None:
177
+ key_cache, value_cache = PagedAttention.split_kv_cache(
178
+ kv_cache, self.num_kv_heads, self.head_size)
179
+
180
+ # Reshape the input keys and values and store them in the cache.
181
+ # If kv_cache is not provided, the new key and value tensors are
182
+ # not cached. This happens during the initial memory profiling run.
183
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
184
+ value_cache,
185
+ attn_metadata.slot_mapping,
186
+ attn_metadata.kv_cache_dtype)
187
+
188
+ if attn_metadata.is_prompt:
189
+ # Prompt run.
190
+ if kv_cache is None or attn_metadata.block_tables.numel() == 0:
191
+ # normal attention
192
+ # When block_tables are not filled, it means q and k are the
193
+ # prompt, and they have the same length.
194
+ output = flash_attn_varlen_func(
195
+ q=query,
196
+ k=key,
197
+ v=value,
198
+ cu_seqlens_q=attn_metadata.seq_start_loc,
199
+ cu_seqlens_k=attn_metadata.seq_start_loc,
200
+ max_seqlen_q=attn_metadata.max_prompt_len,
201
+ max_seqlen_k=attn_metadata.max_prompt_len,
202
+ softmax_scale=self.scale,
203
+ causal=True,
204
+ window_size=self.sliding_window,
205
+ alibi_slopes=self.alibi_slopes,
206
+ )
207
+ else:
208
+ # prefix-enabled attention
209
+ output = PagedAttention.forward_prefix(
210
+ query,
211
+ key,
212
+ value,
213
+ key_cache,
214
+ value_cache,
215
+ attn_metadata.block_tables,
216
+ attn_metadata.subquery_start_loc,
217
+ attn_metadata.prompt_lens_tensor,
218
+ attn_metadata.context_lens,
219
+ attn_metadata.max_subquery_len,
220
+ self.alibi_slopes,
221
+ )
222
+ else:
223
+ # Decoding run.
224
+ output = PagedAttention.forward_decode(
225
+ query,
226
+ key_cache,
227
+ value_cache,
228
+ attn_metadata.block_tables,
229
+ attn_metadata.context_lens,
230
+ attn_metadata.max_context_len,
231
+ attn_metadata.kv_cache_dtype,
232
+ self.num_kv_heads,
233
+ self.scale,
234
+ self.alibi_slopes,
235
+ )
236
+
237
+ # Reshape the output tensor.
238
+ return output.view(num_tokens, hidden_size)
vllm/attention/backends/xformers.py
renamed
+177
-55
@@ -1,19 +1,127 @@
1
"""Attention layer with xFormers and PagedAttention."""
2
import importlib
3
-from typing import List, Optional
3
+from dataclasses import dataclass
4
+from typing import Dict, List, Optional, Tuple, Type
5
6
import torch
7
from xformers import ops as xops
7
-from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8
+from xformers.ops.fmha.attn_bias import (AttentionBias,
9
+ BlockDiagonalCausalMask,
10
LowerTriangularMaskWithTensorBias)
11
10
-from vllm.model_executor.input_metadata import InputMetadata
11
-from vllm.model_executor.layers.attention.ops.paged_attn import (
12
- PagedAttentionImpl)
12
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
13
+ AttentionMetadata)
14
+from vllm.attention.ops.paged_attn import PagedAttention, PagedAttentionMetadata
15
+from vllm.logger import init_logger
16
from vllm.utils import is_hip
17
18
+logger = init_logger(__name__)
19
16
-class XFormersBackend:
20
+
21
+class XFormersBackend(AttentionBackend):
22
+
23
+ @staticmethod
24
+ def get_impl_cls() -> Type["XFormersImpl"]:
25
+ return XFormersImpl
26
+
27
+ @staticmethod
28
+ def make_metadata(*args, **kwargs) -> "XFormersMetadata":
29
+ return XFormersMetadata(*args, **kwargs)
30
+
31
+ @staticmethod
32
+ def get_kv_cache_shape(
33
+ num_blocks: int,
34
+ block_size: int,
35
+ num_kv_heads: int,
36
+ head_size: int,
37
+ ) -> Tuple[int, ...]:
38
+ return PagedAttention.get_kv_cache_shape(num_blocks, block_size,
39
+ num_kv_heads, head_size)
40
+
41
+ @staticmethod
42
+ def swap_blocks(
43
+ src_kv_cache: torch.Tensor,
44
+ dst_kv_cache: torch.Tensor,
45
+ src_to_dst: Dict[int, int],
46
+ ) -> None:
47
+ PagedAttention.swap_blocks(src_kv_cache, dst_kv_cache, src_to_dst)
48
+
49
+ @staticmethod
50
+ def copy_blocks(
51
+ kv_caches: List[torch.Tensor],
52
+ src_to_dists: Dict[int, List[int]],
53
+ ) -> None:
54
+ PagedAttention.copy_blocks(kv_caches, src_to_dists)
55
+
56
+
57
+@dataclass
58
+class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
59
+ """Metadata for XFormersbackend.
60
+
61
+ NOTE: Any python object stored here is not updated when it is
62
+ cuda-graph replayed. If you have values that need to be changed
63
+ dynamically, it should be stored in tensor. The tensor has to be
64
+ updated from `CUDAGraphRunner.forward` API.
65
+ """
66
+ # Currently, input sequences can only contain all prompts
67
+ # or all decoding. True if all sequences are prompts.
68
+ is_prompt: bool
69
+ # (num_tokens,). The indices of the token slots that input tokens will be
70
+ # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
71
+ # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
72
+ # in block 0, and 1st slot in block 1, respectively.
73
+ slot_mapping: torch.Tensor
74
+ # (batch_size,). The prompt length per sequence. None if it is a decoding.
75
+ prompt_lens: Optional[List[int]]
76
+ # prompt_lens stored as a tensor.
77
+ prompt_lens_tensor: Optional[torch.Tensor]
78
+ # The number of prompt tokens. Doesn't include padding.
79
+ num_prompt_tokens: int
80
+ # The number of generation tokens. Doesn't include padding.
81
+ num_generation_tokens: int
82
+
83
+ # NOTE(sang): Definition of context_len, subquery_len, and seqlen.
84
+ # |---------- N-1 iteration --------|
85
+ # |---------------- N iteration ---------------------|
86
+ # |- tokenA -|......................|-- newTokens ---|
87
+ # |---------- context_len ----------|
88
+ # |-------------------- seqlen ----------------------|
89
+ # |- subquery_len -|
90
+
91
+ # WARNING(sang): context_len has different definition depending on if it is
92
+ # prefill vs decoding. When it is prefill, it doesn't include new tokens.
93
+ # When it is for decoding, it includes a new token.
94
+
95
+ # Maximum subquery length in the batch.
96
+ max_subquery_len: Optional[int]
97
+ # FIXME: It is for flash attn.
98
+ # Maximum prompt length in the batch.
99
+ max_prompt_len: Optional[int]
100
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
101
+ # the batch, used to index into subquery. E.g., if the subquery length
102
+ # is [4, 6], it is [0, 4, 10].
103
+ subquery_start_loc: Optional[torch.Tensor]
104
+ # FIXME: It is for flash attn.
105
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
106
+ # the batch, used to index into sequence. E.g., if the sequence length is
107
+ # [4, 6], it is [0, 4, 10].
108
+ seq_start_loc: Optional[torch.Tensor]
109
+
110
+ # Whether or not if cuda graph is enabled.
111
+ # Cuda-graph is currently enabled for decoding only.
112
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
113
+ use_cuda_graph: bool
114
+
115
+ def __post_init__(self):
116
+ # Set during the execution of the first attention op.
117
+ # It is a list because it is needed to set per prompt
118
+ # when alibi slopes is used. It is because of the limitation
119
+ # from xformer API.
120
+ # will not appear in the __repr__ and __init__
121
+ self.attn_bias: Optional[List[AttentionBias]] = None
122
+
123
+
124
+class XFormersImpl(AttentionImpl):
125
"""
126
If the input tensors contain prompt tokens, the layout is as follows:
127
|<--------------- num_prompt_tokens --------------->|
@@ -50,22 +158,25 @@ class XFormersBackend:
158
159
assert self.num_heads % self.num_kv_heads == 0
160
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
53
- suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
161
+
162
+ suppored_head_sizes = PagedAttention.get_supported_head_sizes()
163
if head_size not in suppored_head_sizes:
164
raise ValueError(
165
f"Head size {head_size} is not supported by PagedAttention. "
166
f"Supported head sizes are: {suppored_head_sizes}.")
167
59
- self.use_ref_attention = _check_use_ref_attention()
168
+ # AMD Radeon 7900 series (gfx1100) currently does not support xFormers
169
+ # nor FlashAttention. As a temporary workaround, we use naive PyTorch
170
+ # implementation of attention.
171
+ self.use_naive_attention = _check_use_naive_attention()
172
173
def forward(
174
self,
175
query: torch.Tensor,
176
key: torch.Tensor,
177
value: torch.Tensor,
66
- key_cache: Optional[torch.Tensor],
67
- value_cache: Optional[torch.Tensor],
68
- input_metadata: InputMetadata,
178
+ kv_cache: Optional[torch.Tensor],
179
+ attn_metadata: XFormersMetadata,
180
) -> torch.Tensor:
181
"""Forward pass with xFormers and PagedAttention.
182
@@ -73,11 +184,8 @@ class XFormersBackend:
184
query: shape = [num_tokens, num_heads * head_size]
185
key: shape = [num_tokens, num_kv_heads * head_size]
186
value: shape = [num_tokens, num_kv_heads * head_size]
76
- key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
77
- block_size, x]
78
- value_cache: shape = [num_blocks, num_kv_heads, head_size,
79
- block_size]
80
- input_metadata: metadata for the inputs.
187
+ kv_cache = [2, num_blocks, block_size * num_kv_heads * head_size]
188
+ attn_metadata: Metadata for attention.
189
Returns:
190
shape = [num_tokens, num_heads * head_size]
191
"""
@@ -86,20 +194,24 @@ class XFormersBackend:
194
key = key.view(-1, self.num_kv_heads, self.head_size)
195
value = value.view(-1, self.num_kv_heads, self.head_size)
196
89
- # Reshape the keys and values and store them in the cache.
90
- # If key_cache and value_cache are not provided, the new key and value
91
- # vectors will not be cached. This happens during the initial memory
92
- # profiling run.
93
- if key_cache is not None and value_cache is not None:
94
- PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
95
- value_cache, input_metadata)
197
+ if kv_cache is not None:
198
+ key_cache, value_cache = PagedAttention.split_kv_cache(
199
+ kv_cache, self.num_kv_heads, self.head_size)
200
+
201
+ # Reshape the input keys and values and store them in the cache.
202
+ # If kv_cache is not provided, the new key and value tensors are
203
+ # not cached. This happens during the initial memory profiling run.
204
+ PagedAttention.write_to_paged_cache(key, value, key_cache,
205
+ value_cache,
206
+ attn_metadata.slot_mapping,
207
+ attn_metadata.kv_cache_dtype)
208
97
- if input_metadata.is_prompt:
209
+ if attn_metadata.is_prompt:
210
# Prompt run.
99
- # key_cache and value_cache are None when it is a profiling run.
100
- # block tables are empty if the prompt has never been computed.
101
- if (key_cache is None or value_cache is None
102
- or input_metadata.block_tables.numel() == 0):
211
+ if kv_cache is None or attn_metadata.block_tables.numel() == 0:
212
+ # normal attention.
213
+ # block tables are empty if the prompt does not have a cached
214
+ # prefix.
215
if self.num_kv_heads != self.num_heads:
216
# As of Nov 2023, xformers only supports MHA. For MQA/GQA,
217
# project the key and value tensors to the desired number of
@@ -118,13 +230,12 @@ class XFormersBackend:
230
self.num_queries_per_kv,
231
value.shape[-1])
232
121
- if self.use_ref_attention:
122
- print("ref attention used.")
233
+ if self.use_naive_attention:
234
output = torch.empty_like(query)
235
start = 0
125
- for _, prompt_len in enumerate(input_metadata.prompt_lens):
236
+ for _, prompt_len in enumerate(attn_metadata.prompt_lens):
237
end = start + prompt_len
127
- out = _ref_masked_attention(
238
+ out = _naive_masked_attention(
239
query[None, start:end],
240
key[None, start:end],
241
value[None, start:end],
@@ -143,26 +254,33 @@ class XFormersBackend:
254
# Use reshape instead.
255
return output.reshape(num_tokens, hidden_size)
256
146
- output = self._run_memory_efficient_xformer_forward(
147
- query, key, value, input_metadata)
257
+ output = self._run_memory_efficient_xformers_forward(
258
+ query, key, value, attn_metadata)
259
else:
260
# prefix-enabled attention
150
- output = PagedAttentionImpl.forward_prefix(
261
+ output = PagedAttention.forward_prefix(
262
query,
263
key,
264
value,
265
key_cache,
266
value_cache,
156
- input_metadata,
267
+ attn_metadata.block_tables,
268
+ attn_metadata.subquery_start_loc,
269
+ attn_metadata.prompt_lens_tensor,
270
+ attn_metadata.context_lens,
271
+ attn_metadata.max_subquery_len,
272
self.alibi_slopes,
273
)
274
else:
275
# Decoding run.
161
- output = PagedAttentionImpl.forward_decode(
276
+ output = PagedAttention.forward_decode(
277
query,
278
key_cache,
279
value_cache,
165
- input_metadata,
280
+ attn_metadata.block_tables,
281
+ attn_metadata.context_lens,
282
+ attn_metadata.max_context_len,
283
+ attn_metadata.kv_cache_dtype,
284
self.num_kv_heads,
285
self.scale,
286
self.alibi_slopes,
@@ -171,12 +289,12 @@ class XFormersBackend:
289
# Reshape the output tensor.
290
return output.view(-1, self.num_heads * self.head_size)
291
174
- def _run_memory_efficient_xformer_forward(
292
+ def _run_memory_efficient_xformers_forward(
293
self,
294
query: torch.Tensor,
295
key: torch.Tensor,
296
value: torch.Tensor,
179
- input_metadata: InputMetadata,
297
+ attn_metadata: XFormersMetadata,
298
) -> torch.Tensor:
299
"""Attention for 1D query of multiple prompts. Multiple prompt
300
tokens are flattened in to `query` input.
@@ -186,23 +304,23 @@ class XFormersBackend:
304
query: shape = [num_prompt_tokens, num_heads, head_size]
305
key: shape = [num_prompt_tokens, num_kv_heads, head_size]
306
value: shape = [num_prompt_tokens, num_kv_heads, head_size]
189
- input_metadata: metadata for paged attention.
307
+ attn_metadata: Metadata for attention.
308
"""
309
# Set attention bias if not provided. This typically happens at
310
# the very attention layer of every iteration.
311
# FIXME(woosuk): This is a hack.
194
- if input_metadata.attn_bias is None:
312
+ if attn_metadata.attn_bias is None:
313
if self.alibi_slopes is None:
314
attn_bias = BlockDiagonalCausalMask.from_seqlens(
197
- input_metadata.prompt_lens)
315
+ attn_metadata.prompt_lens)
316
if self.sliding_window is not None:
317
attn_bias = attn_bias.make_local_attention(
318
self.sliding_window)
201
- input_metadata.attn_bias = [attn_bias]
319
+ attn_metadata.attn_bias = [attn_bias]
320
else:
203
- input_metadata.attn_bias = _make_alibi_bias(
321
+ attn_metadata.attn_bias = _make_alibi_bias(
322
self.alibi_slopes, self.num_kv_heads, query.dtype,
205
- input_metadata)
323
+ attn_metadata.prompt_lens)
324
325
op = xops.fmha.MemoryEfficientAttentionFlashAttentionOp[0] if (
326
is_hip()) else None
@@ -217,7 +335,7 @@ class XFormersBackend:
335
query,
336
key,
337
value,
220
- attn_bias=input_metadata.attn_bias[0],
338
+ attn_bias=attn_metadata.attn_bias[0],
339
p=0.0,
340
scale=self.scale,
341
op=op)
@@ -230,13 +348,13 @@ class XFormersBackend:
348
# one. This is inefficient, especially when we have many short prompts.
349
output = torch.empty_like(query)
350
start = 0
233
- for i, prompt_len in enumerate(input_metadata.prompt_lens):
351
+ for i, prompt_len in enumerate(attn_metadata.prompt_lens):
352
end = start + prompt_len
353
out = xops.memory_efficient_attention_forward(
354
query[None, start:end],
355
key[None, start:end],
356
value[None, start:end],
239
- attn_bias=input_metadata.attn_bias[i],
357
+ attn_bias=attn_metadata.attn_bias[i],
358
p=0.0,
359
scale=self.scale,
360
op=op)
@@ -250,10 +368,10 @@ def _make_alibi_bias(
368
alibi_slopes: torch.Tensor,
369
num_kv_heads: int,
370
dtype: torch.dtype,
253
- input_metadata: InputMetadata,
371
+ prompt_lens: List[int],
372
) -> LowerTriangularMaskWithTensorBias:
373
attn_biases = []
256
- for prompt_len in input_metadata.prompt_lens:
374
+ for prompt_len in prompt_lens:
375
bias = torch.arange(prompt_len, dtype=dtype)
376
# NOTE(zhuohan): HF uses
377
# `bias = bias[None, :].repeat(prompt_len, 1)`
@@ -282,15 +400,19 @@ def _make_alibi_bias(
400
return attn_biases
401
402
285
-def _check_use_ref_attention() -> bool:
403
+def _check_use_naive_attention() -> bool:
404
if not is_hip():
405
return False
406
# For ROCm, check whether flash attention is installed or not.
289
- # if not, use_ref_attention needs to be True
290
- return importlib.util.find_spec("flash_attn") is None
407
+ has_flash_attn = importlib.util.find_spec("flash_attn") is None
408
+ if not has_flash_attn:
409
+ logger.warning("flash_attn is not installed. Using naive attention. "
410
+ "This will take significantly more GPU memory.")
411
+ return True
412
+ return False
413
414
293
-def _ref_masked_attention(
415
+def _naive_masked_attention(
416
query: torch.Tensor,
417
key: torch.Tensor,
418
value: torch.Tensor,
vllm/attention/layer.py
new
+46
@@ -0,0 +1,46 @@
1
+"""Attention layer."""
2
+from typing import List, Optional
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from vllm.attention.backends.abstract import AttentionMetadata
8
+from vllm.attention.selector import get_attn_backend
9
+
10
+
11
+class Attention(nn.Module):
12
+ """Attention layer.
13
+
14
+ This class takes query, key, and value tensors as input. The input tensors
15
+ can either contain prompt tokens or generation tokens.
16
+ The class does the following:
17
+
18
+ 1. Store the input key and value tensors in the KV cache.
19
+ 2. Perform (multi-head/multi-query/grouped-query) attention.
20
+ 3. Return the output tensor.
21
+ """
22
+
23
+ def __init__(
24
+ self,
25
+ num_heads: int,
26
+ head_size: int,
27
+ scale: float,
28
+ num_kv_heads: Optional[int] = None,
29
+ alibi_slopes: Optional[List[float]] = None,
30
+ sliding_window: Optional[int] = None,
31
+ ) -> None:
32
+ super().__init__()
33
+ self.backend = get_attn_backend(torch.get_default_dtype())
34
+ impl_cls = self.backend.get_impl_cls()
35
+ self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
36
+ alibi_slopes, sliding_window)
37
+
38
+ def forward(
39
+ self,
40
+ query: torch.Tensor,
41
+ key: torch.Tensor,
42
+ value: torch.Tensor,
43
+ kv_cache: Optional[torch.Tensor],
44
+ attn_metadata: AttentionMetadata,
45
+ ) -> torch.Tensor:
46
+ return self.impl.forward(query, key, value, kv_cache, attn_metadata)
vllm/attention/ops/__init__.py
renamed
vllm/attention/ops/paged_attn.py
new
+217
@@ -0,0 +1,217 @@
1
+from dataclasses import dataclass
2
+from typing import Dict, List, Optional, Tuple
3
+
4
+import torch
5
+
6
+from vllm._C import cache_ops
7
+from vllm._C import ops
8
+from vllm.attention.ops.prefix_prefill import context_attention_fwd
9
+
10
+# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
11
+_PARTITION_SIZE = 512
12
+
13
+
14
+@dataclass
15
+class PagedAttentionMetadata:
16
+ """Metadata for PagedAttention."""
17
+ # (num_tokens,). The indices of the token slots that input tokens will be
18
+ # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
19
+ # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
20
+ # in block 0, and 1st slot in block 1, respectively.
21
+ slot_mapping: torch.Tensor
22
+ # (batch_size,). The length of context (tokens stored in KV cache) per
23
+ # sequence. WARNING: When it is a prefill request, it doesn't include new
24
+ # tokens. When it is for decoding, it includes a new token.
25
+ context_lens: Optional[torch.Tensor]
26
+ # Maximum context length in the batch.
27
+ max_context_len: Optional[int]
28
+ # (batch_size, max_blocks_per_seq).
29
+ # Block addresses per sequence. (Seq id -> list of physical block)
30
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
31
+ # in the kv cache. Each block can contain up to block_size tokens.
32
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
33
+ # captured.
34
+ block_tables: Optional[torch.Tensor]
35
+ kv_cache_dtype: str
36
+
37
+
38
+class PagedAttention:
39
+
40
+ @staticmethod
41
+ def get_supported_head_sizes() -> List[int]:
42
+ return [64, 80, 96, 112, 128, 256]
43
+
44
+ @staticmethod
45
+ def get_kv_cache_shape(
46
+ num_blocks: int,
47
+ block_size: int,
48
+ num_kv_heads: int,
49
+ head_size: int,
50
+ ) -> Tuple[int, ...]:
51
+ return (2, num_blocks, block_size * num_kv_heads * head_size)
52
+
53
+ @staticmethod
54
+ def split_kv_cache(
55
+ kv_cache: torch.Tensor,
56
+ num_kv_heads: int,
57
+ head_size: int,
58
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
59
+ x = 16 // kv_cache.element_size()
60
+ num_blocks = kv_cache.shape[1]
61
+
62
+ key_cache = kv_cache[0]
63
+ key_cache = key_cache.view(num_blocks, num_kv_heads, head_size // x,
64
+ -1, x)
65
+ value_cache = kv_cache[1]
66
+ value_cache = value_cache.view(num_blocks, num_kv_heads, head_size, -1)
67
+ return key_cache, value_cache
68
+
69
+ @staticmethod
70
+ def write_to_paged_cache(
71
+ key: torch.Tensor,
72
+ value: torch.Tensor,
73
+ key_cache: torch.Tensor,
74
+ value_cache: torch.Tensor,
75
+ slot_mapping: torch.Tensor,
76
+ kv_cache_dtype: str,
77
+ ) -> None:
78
+ cache_ops.reshape_and_cache(
79
+ key,
80
+ value,
81
+ key_cache,
82
+ value_cache,
83
+ slot_mapping.flatten(),
84
+ kv_cache_dtype,
85
+ )
86
+
87
+ @staticmethod
88
+ def forward_decode(
89
+ query: torch.Tensor,
90
+ key_cache: torch.Tensor,
91
+ value_cache: torch.Tensor,
92
+ block_tables: torch.Tensor,
93
+ context_lens: torch.Tensor,
94
+ max_context_len: int,
95
+ kv_cache_dtype: str,
96
+ num_kv_heads: int,
97
+ scale: float,
98
+ alibi_slopes: Optional[torch.Tensor],
99
+ ) -> torch.Tensor:
100
+ output = torch.empty_like(query)
101
+
102
+ block_size = value_cache.shape[3]
103
+ num_seqs, num_heads, head_size = query.shape
104
+ max_num_partitions = ((max_context_len + _PARTITION_SIZE - 1) //
105
+ _PARTITION_SIZE)
106
+ # NOTE(woosuk): We use a simple heuristic to decide whether to use
107
+ # PagedAttention V1 or V2. If the number of partitions is 1, we use
108
+ # V1 to avoid the overhead of reduction. Also, if the number of
109
+ # sequences or heads is large, we use V1 since there is enough work
110
+ # to parallelize.
111
+ # TODO(woosuk): Tune this heuristic.
112
+ # For context len > 8192, use V2 kernel to avoid shared memory shortage.
113
+ use_v1 = (max_context_len <= 8192
114
+ and (max_num_partitions == 1 or num_seqs * num_heads > 512))
115
+ if use_v1:
116
+ # Run PagedAttention V1.
117
+ ops.paged_attention_v1(
118
+ output,
119
+ query,
120
+ key_cache,
121
+ value_cache,
122
+ num_kv_heads,
123
+ scale,
124
+ block_tables,
125
+ context_lens,
126
+ block_size,
127
+ max_context_len,
128
+ alibi_slopes,
129
+ kv_cache_dtype,
130
+ )
131
+ else:
132
+ # Run PagedAttention V2.
133
+ assert _PARTITION_SIZE % block_size == 0
134
+ tmp_output = torch.empty(
135
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
136
+ dtype=output.dtype,
137
+ device=output.device,
138
+ )
139
+ exp_sums = torch.empty(
140
+ size=(num_seqs, num_heads, max_num_partitions),
141
+ dtype=torch.float32,
142
+ device=output.device,
143
+ )
144
+ max_logits = torch.empty_like(exp_sums)
145
+ ops.paged_attention_v2(
146
+ output,
147
+ exp_sums,
148
+ max_logits,
149
+ tmp_output,
150
+ query,
151
+ key_cache,
152
+ value_cache,
153
+ num_kv_heads,
154
+ scale,
155
+ block_tables,
156
+ context_lens,
157
+ block_size,
158
+ max_context_len,
159
+ alibi_slopes,
160
+ kv_cache_dtype,
161
+ )
162
+ return output
163
+
164
+ @staticmethod
165
+ def forward_prefix(
166
+ query: torch.Tensor,
167
+ key: torch.Tensor,
168
+ value: torch.Tensor,
169
+ key_cache: torch.Tensor,
170
+ value_cache: torch.Tensor,
171
+ block_tables: torch.Tensor,
172
+ subquery_start_loc: torch.Tensor,
173
+ prompt_lens_tensor: torch.Tensor,
174
+ context_lens: torch.Tensor,
175
+ max_subquery_len: int,
176
+ alibi_slopes: Optional[torch.Tensor],
177
+ ) -> torch.Tensor:
178
+ output = torch.empty_like(query)
179
+ context_attention_fwd(
180
+ query,
181
+ key,
182
+ value,
183
+ output,
184
+ key_cache,
185
+ value_cache,
186
+ block_tables,
187
+ # subquery_start_loc is (batch_size + 1,)
188
+ subquery_start_loc[:-1],
189
+ prompt_lens_tensor,
190
+ context_lens,
191
+ max_subquery_len,
192
+ alibi_slopes,
193
+ )
194
+ return output
195
+
196
+ @staticmethod
197
+ def swap_blocks(
198
+ src_kv_cache: torch.Tensor,
199
+ dst_kv_cache: torch.Tensor,
200
+ src_to_dst: Dict[int, int],
201
+ ) -> None:
202
+ src_key_cache = src_kv_cache[0]
203
+ dst_key_cache = dst_kv_cache[0]
204
+ cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
205
+
206
+ src_value_cache = src_kv_cache[1]
207
+ dst_value_cache = dst_kv_cache[1]
208
+ cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
209
+
210
+ @staticmethod
211
+ def copy_blocks(
212
+ kv_caches: List[torch.Tensor],
213
+ src_to_dists: Dict[int, List[int]],
214
+ ) -> None:
215
+ key_caches = [kv_cache[0] for kv_cache in kv_caches]
216
+ value_caches = [kv_cache[1] for kv_cache in kv_caches]
217
+ cache_ops.copy_blocks(key_caches, value_caches, src_to_dists)
vllm/attention/ops/prefix_prefill.py
renamed
vllm/attention/selector.py
new
+44
@@ -0,0 +1,44 @@
1
+from functools import lru_cache
2
+
3
+import torch
4
+
5
+from vllm.attention.backends.abstract import AttentionBackend
6
+from vllm.logger import init_logger
7
+from vllm.utils import is_hip
8
+
9
+logger = init_logger(__name__)
10
+
11
+
12
+@lru_cache(maxsize=None)
13
+def get_attn_backend(dtype: torch.dtype) -> AttentionBackend:
14
+ if _can_use_flash_attn(dtype):
15
+ logger.info("Using FlashAttention backend.")
16
+ from vllm.attention.backends.flash_attn import FlashAttentionBackend # noqa: F401
17
+ return FlashAttentionBackend
18
+ else:
19
+ logger.info("Using XFormers backend.")
20
+ from vllm.attention.backends.xformers import XFormersBackend # noqa: F401
21
+ return XFormersBackend
22
+
23
+
24
+def _can_use_flash_attn(dtype: torch.dtype) -> bool:
25
+ if is_hip():
26
+ # AMD GPUs.
27
+ logger.info("Cannot use FlashAttention backend for AMD GPUs.")
28
+ return False
29
+ if torch.cuda.get_device_capability()[0] < 8:
30
+ # Volta and Turing NVIDIA GPUs.
31
+ logger.info("Cannot use FlashAttention backend for Volta and Turing "
32
+ "GPUs.")
33
+ return False
34
+ if dtype not in (torch.float16, torch.bfloat16):
35
+ logger.info("Cannot use FlashAttention backend for dtype other than "
36
+ "torch.float16 or torch.bfloat16.")
37
+ return False
38
+
39
+ try:
40
+ import flash_attn # noqa: F401
41
+ except ImportError:
42
+ logger.info("flash_attn is not found.")
43
+ return False
44
+ return True
vllm/model_executor/__init__.py
-2
@@ -1,9 +1,7 @@
1
-from vllm.model_executor.input_metadata import InputMetadata
1
from vllm.model_executor.sampling_metadata import SamplingMetadata
2
from vllm.model_executor.utils import set_random_seed
3
4
__all__ = [
6
- "InputMetadata",
5
"SamplingMetadata",
6
"set_random_seed",
7
]
vllm/model_executor/input_metadata.py
deleted
-99
@@ -1,99 +0,0 @@
1
-from dataclasses import dataclass, fields
2
-from typing import TYPE_CHECKING, Optional, List, Any, Dict
3
-
4
-import torch
5
-if TYPE_CHECKING:
6
- from xformers.ops.fmha.attn_bias import AttentionBias
7
-
8
-
9
-@dataclass
10
-class InputMetadata:
11
- """Metadata for input sequences. Used in PagedAttention.
12
-
13
- NOTE: Any python object stored here is not updated when it is
14
- cuda-graph replayed. If you have values that need to be changed
15
- dynamically, it should be stored in tensor. The tensor has to be
16
- updated from `CUDAGraphRunner.forward` API.
17
- """
18
- # Currently, input sequences can only contain all prompts
19
- # or all decoding. True if all sequences are prompts.
20
- is_prompt: bool
21
- # (num_tokens,). The indices of the token slots that input tokens will be
22
- # stored into. E.g., if `slot_mapping` is [35, 2, 17] and the block size
23
- # is 16, the three tokens are stored in the 3rd slot in block 2, 2nd slot
24
- # in block 0, and 1st slot in block 1, respectively.
25
- slot_mapping: torch.Tensor
26
- # (batch_size,). The prompt length per sequence. None if it is a decoding.
27
- prompt_lens: Optional[List[int]]
28
- # prompt_lens stored as a tensor.
29
- prompt_lens_tensor: Optional[torch.Tensor]
30
- # The number of prompt tokens. Doesn't include padding.
31
- num_prompt_tokens: int
32
- # The number of generation tokens. Doesn't include padding.
33
- num_generation_tokens: int
34
- """
35
- Definition of context_len, subquery_len, and seqlen.
36
- |---------- N-1 iteration --------|
37
- |---------------- N iteration ---------------------|
38
- |- tokenA -|......................|-- newTokens ---|
39
- |---------- context_len ----------|
40
- |-------------------- seqlen ----------------------|
41
- |- subquery_len -|
42
-
43
- WARNING: context_len has different definition depending on if it is
44
- prefill vs decoding. When it is prefill, it doesn't include new
45
- tokens. When it is for decoding, it includes a new token.
46
- """
47
-
48
- # Maximum subquery length in the batch.
49
- max_subquery_len: Optional[int]
50
- # Maximum context length in the batch.
51
- max_context_len: Optional[int]
52
- # FIXME: It is for flash attn.
53
- # Maximum sequence length in the batch.
54
- max_seq_len: Optional[int]
55
- # (batch_size + 1,). The cumulative subquery lengths of the sequences in
56
- # the batch, used to index into subquery. E.g., if the subquery length
57
- # is [4, 6], it is [0, 4, 10].
58
- subquery_start_loc: Optional[torch.Tensor]
59
- # FIXME: It is for flash attn.
60
- # (batch_size + 1,). The cumulative sequence lengths of the sequences in
61
- # the batch, used to index into sequence. E.g., if the sequence length is
62
- # [4, 6], it is [0, 4, 10].
63
- seq_start_loc: Optional[torch.Tensor]
64
- # (batch_size,). The length of context (tokens stored in KV cache) per
65
- # sequence. WARNING: When it is a prefill request, it doesn't include new
66
- # tokens. When it is for decoding, it includes a new token.
67
- context_lens: Optional[torch.Tensor]
68
- # (batch_size, max_blocks_per_seq).
69
- # Block addresses per sequence. (Seq id -> list of physical block)
70
- # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
71
- # in the kv cache. Each block can contain up to block_size tokens.
72
- # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
73
- # captured.
74
- block_tables: Optional[torch.Tensor]
75
- # Whether or not if cuda graph is enabled.
76
- # Cuda-graph is currently enabled for decoding only.
77
- use_cuda_graph: bool
78
- kv_cache_dtype: str
79
-
80
- def __post_init__(self):
81
- # Set during the execution of the first attention op.
82
- # It is a list because it is needed to set per prompt
83
- # when alibi slopes is used. It is because of the limitation
84
- # from xformer API.
85
- # will not appear in the __repr__ and __init__
86
- self.attn_bias: Optional[List["AttentionBias"]] = None
87
-
88
- # Cuda graph is only used for decoding now.
89
- if self.use_cuda_graph:
90
- assert self.num_prompt_tokens == 0
91
-
92
- def asdict_zerocopy(self) -> Dict[str, Any]:
93
- """Similar to dataclasses.asdict, but avoids deepcopying."""
94
- # Note that if we add dataclasses as fields, they will need
95
- # similar handling.
96
- return {
97
- field.name: getattr(self, field.name)
98
- for field in fields(self)
99
- }
vllm/model_executor/layers/attention/__init__.py
deleted
-5
@@ -1,5 +0,0 @@
1
-from vllm.model_executor.layers.attention.attention import Attention
2
-
3
-__all__ = [
4
- "Attention",
5
-]
vllm/model_executor/layers/attention/attention.py
deleted
-85
@@ -1,85 +0,0 @@
1
-"""Attention layer."""
2
-from functools import lru_cache
3
-from typing import List, Optional
4
-
5
-import torch
6
-import torch.nn as nn
7
-
8
-from vllm.logger import init_logger
9
-from vllm.model_executor.input_metadata import InputMetadata
10
-from vllm.utils import is_hip
11
-
12
-logger = init_logger(__name__)
13
-
14
-
15
-class Attention(nn.Module):
16
- """Attention layer.
17
-
18
- This class takes query, key, and value tensors as input. The input tensors
19
- can either contain prompt tokens or generation tokens.
20
-
21
- The class does the following:
22
-
23
- 1. Store the input key and value tensors in the KV cache.
24
- 2. Perform (multi-head/multi-query/grouped-query) attention.
25
- 3. Output the output tensor.
26
- """
27
-
28
- def __init__(
29
- self,
30
- num_heads: int,
31
- head_size: int,
32
- scale: float,
33
- num_kv_heads: Optional[int] = None,
34
- alibi_slopes: Optional[List[float]] = None,
35
- sliding_window: Optional[int] = None,
36
- ) -> None:
37
- super().__init__()
38
- if _use_flash_attn():
39
- from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend # noqa: E501
40
- self.backend = FlashAttentionBackend(num_heads, head_size, scale,
41
- num_kv_heads, alibi_slopes,
42
- sliding_window)
43
- else:
44
- from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend # noqa: E501
45
- self.backend = XFormersBackend(num_heads, head_size, scale,
46
- num_kv_heads, alibi_slopes,
47
- sliding_window)
48
-
49
- def forward(
50
- self,
51
- query: torch.Tensor,
52
- key: torch.Tensor,
53
- value: torch.Tensor,
54
- key_cache: Optional[torch.Tensor],
55
- value_cache: Optional[torch.Tensor],
56
- input_metadata: InputMetadata,
57
- ) -> torch.Tensor:
58
- return self.backend.forward(query, key, value, key_cache, value_cache,
59
- input_metadata)
60
-
61
-
62
-@lru_cache(maxsize=1)
63
-def _use_flash_attn() -> bool:
64
- try:
65
- import flash_attn # noqa: F401
66
- except ImportError:
67
- logger.info("flash_attn is not found. Using xformers backend.")
68
- return False
69
-
70
- if is_hip():
71
- # AMD GPUs.
72
- return False
73
- if torch.cuda.get_device_capability()[0] < 8:
74
- # Volta and Turing NVIDIA GPUs.
75
- logger.info("flash_attn is not supported on Turing or older GPUs. "
76
- "Using xformers backend.")
77
- return False
78
- if torch.get_default_dtype() not in (torch.float16, torch.bfloat16):
79
- logger.info(
80
- "flash_attn only supports torch.float16 or torch.bfloat16. "
81
- "Using xformers backend.")
82
- return False
83
-
84
- logger.info("Using flash_attn backend.")
85
- return True
vllm/model_executor/layers/attention/backends/flash_attn.py
deleted
-139
@@ -1,139 +0,0 @@
1
-"""Attention layer with Flash and PagedAttention."""
2
-from typing import List, Optional
3
-
4
-from flash_attn import flash_attn_varlen_func
5
-import torch
6
-
7
-from vllm.model_executor.input_metadata import InputMetadata
8
-from vllm.model_executor.layers.attention.ops.paged_attn import (
9
- PagedAttentionImpl)
10
-
11
-
12
-class FlashAttentionBackend:
13
- """
14
- If the input tensors contain prompt tokens, the layout is as follows:
15
- |<--------------- num_prompt_tokens -------------->|
16
- |<--prompt_0-->|<--prompt_1-->|...|<--prompt_N-1-->|
17
-
18
- Otherwise, the layout is as follows:
19
- |<------------------ num_generation_tokens (M) ----------------->|
20
- |<--generation_0-->|..........|<--generation_M-1-->|<--padding-->|
21
-
22
- Generation tokens can contain padding when cuda-graph is used.
23
- Currently, prompt tokens don't contain any padding.
24
-
25
- The prompts might have different lengths, while the generation tokens
26
- always have length 1.
27
- """
28
-
29
- def __init__(
30
- self,
31
- num_heads: int,
32
- head_size: int,
33
- scale: float,
34
- num_kv_heads: Optional[int] = None,
35
- alibi_slopes: Optional[List[float]] = None,
36
- sliding_window: Optional[int] = None,
37
- ) -> None:
38
- self.num_heads = num_heads
39
- self.head_size = head_size
40
- self.scale = float(scale)
41
- self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
42
- self.sliding_window = sliding_window
43
- if alibi_slopes is not None:
44
- alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
45
- self.alibi_slopes = alibi_slopes
46
-
47
- assert self.num_heads % self.num_kv_heads == 0
48
- self.num_queries_per_kv = self.num_heads // self.num_kv_heads
49
- suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
50
- if head_size not in suppored_head_sizes:
51
- raise ValueError(
52
- f"Head size {head_size} is not supported by PagedAttention. "
53
- f"Supported head sizes are: {suppored_head_sizes}.")
54
-
55
- self.sliding_window = ((self.sliding_window, self.sliding_window) if
56
- self.sliding_window is not None else (-1, -1))
57
-
58
- def forward(
59
- self,
60
- query: torch.Tensor,
61
- key: torch.Tensor,
62
- value: torch.Tensor,
63
- key_cache: Optional[torch.Tensor],
64
- value_cache: Optional[torch.Tensor],
65
- input_metadata: InputMetadata,
66
- ) -> torch.Tensor:
67
- """Forward pass with FlashAttention and PagedAttention.
68
-
69
- Args:
70
- query: shape = [num_tokens, num_heads * head_size]
71
- key: shape = [num_tokens, num_kv_heads * head_size]
72
- value: shape = [num_tokens, num_kv_heads * head_size]
73
- key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
74
- block_size, x]
75
- value_cache: shape = [num_blocks, num_kv_heads, head_size,
76
- block_size]
77
- input_metadata: metadata for the inputs.
78
- Returns:
79
- shape = [num_tokens, num_heads * head_size]
80
- """
81
- num_tokens, hidden_size = query.shape
82
- # Reshape the query, key, and value tensors.
83
- query = query.view(-1, self.num_heads, self.head_size)
84
- key = key.view(-1, self.num_kv_heads, self.head_size)
85
- value = value.view(-1, self.num_kv_heads, self.head_size)
86
-
87
- # Reshape the keys and values and store them in the cache.
88
- # If key_cache and value_cache are not provided, the new key and value
89
- # vectors will not be cached. This happens during the initial memory
90
- # profiling run.
91
- if key_cache is not None and value_cache is not None:
92
- PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
93
- value_cache, input_metadata)
94
-
95
- if input_metadata.is_prompt:
96
- # Prompt run.
97
- if (key_cache is None or value_cache is None
98
- or input_metadata.block_tables.numel() == 0):
99
- # normal attention
100
- # When block_tables are not filled, it means q and k are the
101
- # prompt, and they have the same length.
102
- output = flash_attn_varlen_func(
103
- q=query,
104
- k=key,
105
- v=value,
106
- cu_seqlens_q=input_metadata.seq_start_loc,
107
- cu_seqlens_k=input_metadata.seq_start_loc,
108
- max_seqlen_q=input_metadata.max_seq_len,
109
- max_seqlen_k=input_metadata.max_seq_len,
110
- softmax_scale=self.scale,
111
- causal=True,
112
- window_size=self.sliding_window,
113
- alibi_slopes=self.alibi_slopes,
114
- )
115
- else:
116
- # prefix-enabled attention
117
- output = PagedAttentionImpl.forward_prefix(
118
- query,
119
- key,
120
- value,
121
- key_cache,
122
- value_cache,
123
- input_metadata,
124
- self.alibi_slopes,
125
- )
126
- else:
127
- # Decoding run.
128
- output = PagedAttentionImpl.forward_decode(
129
- query,
130
- key_cache,
131
- value_cache,
132
- input_metadata,
133
- self.num_kv_heads,
134
- self.scale,
135
- self.alibi_slopes,
136
- )
137
-
138
- # Reshape the output tensor.
139
- return output.view(num_tokens, hidden_size)
vllm/model_executor/layers/attention/ops/paged_attn.py
deleted
-139
@@ -1,139 +0,0 @@
1
-from typing import List, Optional
2
-
3
-import torch
4
-
5
-from vllm._C import cache_ops
6
-from vllm._C import ops
7
-from vllm.model_executor.input_metadata import InputMetadata
8
-from vllm.model_executor.layers.attention.ops.prefix_prefill import (
9
- context_attention_fwd)
10
-
11
-# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
12
-_PARTITION_SIZE = 512
13
-
14
-
15
-class PagedAttentionImpl:
16
-
17
- @staticmethod
18
- def get_supported_head_sizes() -> List[int]:
19
- return [64, 80, 96, 112, 128, 256]
20
-
21
- @staticmethod
22
- def reshape_and_cache(
23
- key: torch.Tensor,
24
- value: torch.Tensor,
25
- key_cache: torch.Tensor,
26
- value_cache: torch.Tensor,
27
- input_metadata: InputMetadata,
28
- ) -> None:
29
- cache_ops.reshape_and_cache(
30
- key,
31
- value,
32
- key_cache,
33
- value_cache,
34
- input_metadata.slot_mapping.flatten(),
35
- input_metadata.kv_cache_dtype,
36
- )
37
-
38
- @staticmethod
39
- def forward_decode(
40
- query: torch.Tensor,
41
- key_cache: torch.Tensor,
42
- value_cache: torch.Tensor,
43
- input_metadata: InputMetadata,
44
- num_kv_heads: int,
45
- scale: float,
46
- alibi_slopes: Optional[torch.Tensor],
47
- ) -> torch.Tensor:
48
- output = torch.empty_like(query)
49
-
50
- block_size = value_cache.shape[3]
51
- num_seqs, num_heads, head_size = query.shape
52
- max_num_partitions = (
53
- (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
54
- _PARTITION_SIZE)
55
- # NOTE(woosuk): We use a simple heuristic to decide whether to use
56
- # PagedAttention V1 or V2. If the number of partitions is 1, we use
57
- # V1 to avoid the overhead of reduction. Also, if the number of
58
- # sequences or heads is large, we use V1 since there is enough work
59
- # to parallelize.
60
- # TODO(woosuk): Tune this heuristic.
61
- # For context len > 8192, use V2 kernel to avoid shared memory shortage.
62
- use_v1 = input_metadata.max_context_len <= 8192 and (
63
- max_num_partitions == 1 or num_seqs * num_heads > 512)
64
- if use_v1:
65
- # Run PagedAttention V1.
66
- ops.paged_attention_v1(
67
- output,
68
- query,
69
- key_cache,
70
- value_cache,
71
- num_kv_heads,
72
- scale,
73
- input_metadata.block_tables,
74
- input_metadata.context_lens,
75
- block_size,
76
- input_metadata.max_context_len,
77
- alibi_slopes,
78
- input_metadata.kv_cache_dtype,
79
- )
80
- else:
81
- # Run PagedAttention V2.
82
- assert _PARTITION_SIZE % block_size == 0
83
- tmp_output = torch.empty(
84
- size=(num_seqs, num_heads, max_num_partitions, head_size),
85
- dtype=output.dtype,
86
- device=output.device,
87
- )
88
- exp_sums = torch.empty(
89
- size=(num_seqs, num_heads, max_num_partitions),
90
- dtype=torch.float32,
91
- device=output.device,
92
- )
93
- max_logits = torch.empty_like(exp_sums)
94
- ops.paged_attention_v2(
95
- output,
96
- exp_sums,
97
- max_logits,
98
- tmp_output,
99
- query,
100
- key_cache,
101
- value_cache,
102
- num_kv_heads,
103
- scale,
104
- input_metadata.block_tables,
105
- input_metadata.context_lens,
106
- block_size,
107
- input_metadata.max_context_len,
108
- alibi_slopes,
109
- input_metadata.kv_cache_dtype,
110
- )
111
- return output
112
-
113
- @staticmethod
114
- def forward_prefix(
115
- query: torch.Tensor,
116
- key: torch.Tensor,
117
- value: torch.Tensor,
118
- key_cache: torch.Tensor,
119
- value_cache: torch.Tensor,
120
- input_metadata: InputMetadata,
121
- alibi_slopes: Optional[torch.Tensor],
122
- ) -> torch.Tensor:
123
- output = torch.empty_like(query)
124
- context_attention_fwd(
125
- query,
126
- key,
127
- value,
128
- output,
129
- key_cache,
130
- value_cache,
131
- input_metadata.block_tables,
132
- # subquery_start_loc is (batch_size + 1,)
133
- input_metadata.subquery_start_loc[:-1],
134
- input_metadata.prompt_lens_tensor,
135
- input_metadata.context_lens,
136
- input_metadata.max_subquery_len,
137
- alibi_slopes,
138
- )
139
- return output
vllm/model_executor/models/baichuan.py
+13
-17
@@ -25,9 +25,8 @@ import torch
25
from torch import nn
26
from transformers import PretrainedConfig
27
28
-from vllm.model_executor.input_metadata import InputMetadata
28
+from vllm.attention import Attention, AttentionMetadata
29
from vllm.model_executor.layers.activation import SiluAndMul
30
-from vllm.model_executor.layers.attention import Attention
30
from vllm.model_executor.layers.layernorm import RMSNorm
31
from vllm.model_executor.layers.linear import (LinearMethodBase,
32
MergedColumnParallelLinear,
@@ -45,8 +44,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
44
hf_model_weights_iterator)
45
from vllm.sequence import SamplerOutput
46
48
-KVCache = Tuple[torch.Tensor, torch.Tensor]
49
-
47
48
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
49
closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -170,15 +167,14 @@ class BaiChuanAttention(nn.Module):
167
self,
168
positions: torch.Tensor,
169
hidden_states: torch.Tensor,
173
- kv_cache: KVCache,
174
- input_metadata: InputMetadata,
170
+ kv_cache: torch.Tensor,
171
+ attn_metadata: AttentionMetadata,
172
) -> torch.Tensor:
173
qkv, _ = self.W_pack(hidden_states)
174
q, k, v = qkv.chunk(chunks=3, dim=-1)
175
if self.postion_embedding != "ALIBI":
176
q, k = self.rotary_emb(positions, q, k)
180
- k_cache, v_cache = kv_cache
181
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
177
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
178
output, _ = self.o_proj(attn_output)
179
return output
180
@@ -217,8 +213,8 @@ class BaiChuanDecoderLayer(nn.Module):
213
self,
214
positions: torch.Tensor,
215
hidden_states: torch.Tensor,
220
- kv_cache: KVCache,
221
- input_metadata: InputMetadata,
216
+ kv_cache: torch.Tensor,
217
+ attn_metadata: AttentionMetadata,
218
residual: Optional[torch.Tensor],
219
) -> Tuple[torch.Tensor, torch.Tensor]:
220
# Self Attention
@@ -232,7 +228,7 @@ class BaiChuanDecoderLayer(nn.Module):
228
positions=positions,
229
hidden_states=hidden_states,
230
kv_cache=kv_cache,
235
- input_metadata=input_metadata,
231
+ attn_metadata=attn_metadata,
232
)
233
234
# Fully Connected
@@ -267,8 +263,8 @@ class BaiChuanModel(nn.Module):
263
self,
264
input_ids: torch.Tensor,
265
positions: torch.Tensor,
270
- kv_caches: List[KVCache],
271
- input_metadata: InputMetadata,
266
+ kv_caches: List[torch.Tensor],
267
+ attn_metadata: AttentionMetadata,
268
) -> torch.Tensor:
269
hidden_states = self.embed_tokens(input_ids)
270
residual = None
@@ -278,7 +274,7 @@ class BaiChuanModel(nn.Module):
274
positions,
275
hidden_states,
276
kv_caches[i],
281
- input_metadata,
277
+ attn_metadata,
278
residual,
279
)
280
hidden_states, _ = self.norm(hidden_states, residual)
@@ -303,11 +299,11 @@ class BaiChuanBaseForCausalLM(nn.Module):
299
self,
300
input_ids: torch.Tensor,
301
positions: torch.Tensor,
306
- kv_caches: List[KVCache],
307
- input_metadata: InputMetadata,
302
+ kv_caches: List[torch.Tensor],
303
+ attn_metadata: AttentionMetadata,
304
) -> torch.Tensor:
305
hidden_states = self.model(input_ids, positions, kv_caches,
310
- input_metadata)
306
+ attn_metadata)
307
return hidden_states
308
309
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/bloom.py
+14
-18
@@ -17,15 +17,14 @@
17
# limitations under the License.
18
"""Inference-only BLOOM model compatible with HuggingFace weights."""
19
import math
20
-from typing import List, Optional, Tuple
20
+from typing import List, Optional
21
22
import torch
23
from torch import nn
24
from transformers import BloomConfig
25
26
-from vllm.model_executor.input_metadata import InputMetadata
26
+from vllm.attention import Attention, AttentionMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import Attention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
45
-
43
44
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
45
closest_power_of_2 = 2**math.floor(math.log2(total_num_heads))
@@ -117,14 +114,13 @@ class BloomAttention(nn.Module):
114
self,
115
position_ids: torch.Tensor,
116
hidden_states: torch.Tensor,
120
- kv_cache: KVCache,
121
- input_metadata: InputMetadata,
117
+ kv_cache: torch.Tensor,
118
+ attn_metadata: AttentionMetadata,
119
) -> torch.Tensor:
120
del position_ids # Unused.
121
qkv, _ = self.query_key_value(hidden_states)
122
q, k, v = qkv.chunk(chunks=3, dim=-1)
126
- k_cache, v_cache = kv_cache
127
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
123
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
124
output, _ = self.dense(attn_output)
125
return output
126
@@ -181,8 +177,8 @@ class BloomBlock(nn.Module):
177
self,
178
position_ids: torch.Tensor,
179
hidden_states: torch.Tensor,
184
- kv_cache: KVCache,
185
- input_metadata: InputMetadata,
180
+ kv_cache: torch.Tensor,
181
+ attn_metadata: AttentionMetadata,
182
) -> torch.Tensor:
183
# Layer norm at the beginning of the transformer layer.
184
layernorm_output = self.input_layernorm(hidden_states)
@@ -198,7 +194,7 @@ class BloomBlock(nn.Module):
194
position_ids=position_ids,
195
hidden_states=layernorm_output,
196
kv_cache=kv_cache,
201
- input_metadata=input_metadata,
197
+ attn_metadata=attn_metadata,
198
)
199
attention_output = attention_output + residual
200
layernorm_output = self.post_attention_layernorm(attention_output)
@@ -245,8 +241,8 @@ class BloomModel(nn.Module):
241
self,
242
input_ids: torch.Tensor,
243
position_ids: torch.Tensor,
248
- kv_caches: List[KVCache],
249
- input_metadata: InputMetadata,
244
+ kv_caches: List[torch.Tensor],
245
+ attn_metadata: AttentionMetadata,
246
) -> torch.Tensor:
247
hidden_states = self.word_embeddings(input_ids)
248
hidden_states = self.word_embeddings_layernorm(hidden_states)
@@ -256,7 +252,7 @@ class BloomModel(nn.Module):
252
position_ids,
253
hidden_states,
254
kv_caches[i],
259
- input_metadata,
255
+ attn_metadata,
256
)
257
hidden_states = self.ln_f(hidden_states)
258
return hidden_states
@@ -281,11 +277,11 @@ class BloomForCausalLM(nn.Module):
277
self,
278
input_ids: torch.Tensor,
279
positions: torch.Tensor,
284
- kv_caches: List[KVCache],
285
- input_metadata: InputMetadata,
280
+ kv_caches: List[torch.Tensor],
281
+ attn_metadata: AttentionMetadata,
282
) -> torch.Tensor:
283
hidden_states = self.transformer(input_ids, positions, kv_caches,
288
- input_metadata)
284
+ attn_metadata)
285
return hidden_states
286
287
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/chatglm.py
+18
-23
@@ -2,15 +2,14 @@
2
# Adapted from
3
# https://github.com/THUDM/ChatGLM2-6B
4
"""Inference-only ChatGLM model compatible with THUDM weights."""
5
-from typing import List, Optional, Tuple
5
+from typing import List, Optional
6
7
import torch
8
from torch import nn
9
from torch.nn import LayerNorm
10
11
-from vllm.model_executor.input_metadata import InputMetadata
11
+from vllm.attention import Attention, AttentionMetadata
12
from vllm.model_executor.layers.activation import SiluAndMul
13
-from vllm.model_executor.layers.attention import Attention
13
from vllm.model_executor.layers.layernorm import RMSNorm
14
from vllm.model_executor.layers.linear import (LinearMethodBase,
15
MergedColumnParallelLinear,
@@ -29,8 +28,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
28
from vllm.sequence import SamplerOutput
29
from vllm.transformers_utils.configs import ChatGLMConfig
30
32
-KVCache = Tuple[torch.Tensor, torch.Tensor]
33
-
31
32
class GLMAttention(nn.Module):
33
@@ -99,20 +96,18 @@ class GLMAttention(nn.Module):
96
self,
97
hidden_states: torch.Tensor,
98
position_ids: torch.Tensor,
102
- kv_cache: KVCache,
103
- input_metadata: InputMetadata,
99
+ kv_cache: torch.Tensor,
100
+ attn_metadata: AttentionMetadata,
101
) -> torch.Tensor:
102
qkv, _ = self.query_key_value(hidden_states)
103
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
104
q, k = self.rotary_emb(position_ids, q, k)
108
- key_cache, value_cache = kv_cache
105
context_layer = self.attn(
106
q,
107
k,
108
v,
113
- key_cache,
114
- value_cache,
115
- input_metadata,
109
+ kv_cache,
110
+ attn_metadata,
111
)
112
attn_output, _ = self.dense(context_layer)
113
return attn_output
@@ -200,8 +195,8 @@ class GLMBlock(nn.Module):
195
self,
196
hidden_states: torch.Tensor,
197
position_ids: torch.Tensor,
203
- kv_cache: KVCache,
204
- input_metadata: InputMetadata,
198
+ kv_cache: torch.Tensor,
199
+ attn_metadata: AttentionMetadata,
200
) -> torch.Tensor:
201
# hidden_states: [num_tokens, h]
202
# Layer norm at the beginning of the transformer layer.
@@ -211,7 +206,7 @@ class GLMBlock(nn.Module):
206
hidden_states=layernorm_output,
207
position_ids=position_ids,
208
kv_cache=kv_cache,
214
- input_metadata=input_metadata,
209
+ attn_metadata=attn_metadata,
210
)
211
212
# Residual connection.
@@ -264,8 +259,8 @@ class GLMTransformer(nn.Module):
259
self,
260
hidden_states: torch.Tensor,
261
position_ids: torch.Tensor,
267
- kv_caches: List[KVCache],
268
- input_metadata: InputMetadata,
262
+ kv_caches: List[torch.Tensor],
263
+ attn_metadata: AttentionMetadata,
264
) -> torch.Tensor:
265
for i in range(self.num_layers):
266
layer = self.layers[i]
@@ -273,7 +268,7 @@ class GLMTransformer(nn.Module):
268
hidden_states=hidden_states,
269
position_ids=position_ids,
270
kv_cache=kv_caches[i],
276
- input_metadata=input_metadata,
271
+ attn_metadata=attn_metadata,
272
)
273
# Final layer norm.
274
if self.post_layer_norm:
@@ -306,8 +301,8 @@ class ChatGLMModel(nn.Module):
301
self,
302
input_ids: torch.Tensor,
303
position_ids: torch.Tensor,
309
- kv_caches: List[KVCache],
310
- input_metadata: InputMetadata,
304
+ kv_caches: List[torch.Tensor],
305
+ attn_metadata: AttentionMetadata,
306
) -> torch.Tensor:
307
inputs_embeds = self.embedding(input_ids)
308
@@ -316,7 +311,7 @@ class ChatGLMModel(nn.Module):
311
hidden_states=inputs_embeds,
312
position_ids=position_ids,
313
kv_caches=kv_caches,
319
- input_metadata=input_metadata,
314
+ attn_metadata=attn_metadata,
315
)
316
return hidden_states
317
@@ -340,11 +335,11 @@ class ChatGLMForCausalLM(nn.Module):
335
self,
336
input_ids: torch.Tensor,
337
positions: torch.Tensor,
343
- kv_caches: List[KVCache],
344
- input_metadata: InputMetadata,
338
+ kv_caches: List[torch.Tensor],
339
+ attn_metadata: AttentionMetadata,
340
) -> torch.Tensor:
341
hidden_states = self.transformer(input_ids, positions, kv_caches,
347
- input_metadata)
342
+ attn_metadata)
343
return hidden_states
344
345
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/deepseek.py
+14
-18
@@ -21,15 +21,14 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Deepseek model."""
24
-from typing import Any, Dict, List, Optional, Tuple
24
+from typing import Any, Dict, List, Optional
25
26
import torch
27
from torch import nn
28
from transformers import PretrainedConfig
29
30
-from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.attention import Attention, AttentionMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import Attention
32
from vllm.model_executor.layers.fused_moe import fused_moe
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -51,8 +50,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
50
hf_model_weights_iterator)
51
from vllm.sequence import SamplerOutput
52
54
-KVCache = Tuple[torch.Tensor, torch.Tensor]
55
-
53
54
class DeepseekMLP(nn.Module):
55
@@ -239,14 +236,13 @@ class DeepseekAttention(nn.Module):
236
self,
237
positions: torch.Tensor,
238
hidden_states: torch.Tensor,
242
- kv_cache: KVCache,
243
- input_metadata: InputMetadata,
239
+ kv_cache: torch.Tensor,
240
+ attn_metadata: AttentionMetadata,
241
) -> torch.Tensor:
242
qkv, _ = self.qkv_proj(hidden_states)
243
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
244
q, k = self.rotary_emb(positions, q, k)
248
- k_cache, v_cache = kv_cache
249
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
245
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
246
output, _ = self.o_proj(attn_output)
247
return output
248
@@ -294,8 +290,8 @@ class DeepseekDecoderLayer(nn.Module):
290
self,
291
positions: torch.Tensor,
292
hidden_states: torch.Tensor,
297
- kv_cache: KVCache,
298
- input_metadata: InputMetadata,
293
+ kv_cache: torch.Tensor,
294
+ attn_metadata: AttentionMetadata,
295
residual: Optional[torch.Tensor],
296
) -> torch.Tensor:
297
# Self Attention
@@ -309,7 +305,7 @@ class DeepseekDecoderLayer(nn.Module):
305
positions=positions,
306
hidden_states=hidden_states,
307
kv_cache=kv_cache,
312
- input_metadata=input_metadata,
308
+ attn_metadata=attn_metadata,
309
)
310
311
# Fully Connected
@@ -346,15 +342,15 @@ class DeepseekModel(nn.Module):
342
self,
343
input_ids: torch.Tensor,
344
positions: torch.Tensor,
349
- kv_caches: List[KVCache],
350
- input_metadata: InputMetadata,
345
+ kv_caches: List[torch.Tensor],
346
+ attn_metadata: AttentionMetadata,
347
) -> torch.Tensor:
348
hidden_states = self.embed_tokens(input_ids)
349
residual = None
350
for i in range(len(self.layers)):
351
layer = self.layers[i]
352
hidden_states, residual = layer(positions, hidden_states,
357
- kv_caches[i], input_metadata,
353
+ kv_caches[i], attn_metadata,
354
residual)
355
hidden_states, _ = self.norm(hidden_states, residual)
356
return hidden_states
@@ -379,11 +375,11 @@ class DeepseekForCausalLM(nn.Module):
375
self,
376
input_ids: torch.Tensor,
377
positions: torch.Tensor,
382
- kv_caches: List[KVCache],
383
- input_metadata: InputMetadata,
378
+ kv_caches: List[torch.Tensor],
379
+ attn_metadata: AttentionMetadata,
380
) -> torch.Tensor:
381
hidden_states = self.model(input_ids, positions, kv_caches,
386
- input_metadata)
382
+ attn_metadata)
383
return hidden_states
384
385
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/falcon.py
+14
-17
@@ -19,16 +19,15 @@
19
"""PyTorch Falcon model."""
20
21
import math
22
-from typing import List, Optional, Tuple, Union
22
+from typing import List, Optional, Union
23
24
import torch
25
from torch import nn
26
from torch.nn import LayerNorm
27
from transformers import FalconConfig as HF_FalconConfig
28
29
-from vllm.model_executor.input_metadata import InputMetadata
29
+from vllm.attention import Attention, AttentionMetadata
30
from vllm.model_executor.layers.activation import get_act_fn
31
-from vllm.model_executor.layers.attention import Attention
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
32
LinearMethodBase,
33
QKVParallelLinear,
@@ -48,7 +47,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
47
from vllm.sequence import SamplerOutput
48
from vllm.transformers_utils.configs import RWConfig
49
51
-KVCache = Tuple[torch.Tensor, torch.Tensor]
50
FalconConfig = Union[HF_FalconConfig, RWConfig]
51
52
@@ -177,8 +175,8 @@ class FalconAttention(nn.Module):
175
self,
176
positions: torch.Tensor,
177
hidden_states: torch.Tensor,
180
- kv_cache: KVCache,
181
- input_metadata: InputMetadata,
178
+ kv_cache: torch.Tensor,
179
+ attn_metadata: AttentionMetadata,
180
) -> torch.Tensor:
181
qkv, bias = self.query_key_value(hidden_states)
182
if bias is not None:
@@ -186,8 +184,7 @@ class FalconAttention(nn.Module):
184
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
185
if self.use_rotary:
186
q, k = self.rotary_emb(positions, q, k)
189
- k_cache, v_cache = kv_cache
190
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
187
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
188
attn_output, bias = self.dense(attn_output)
189
return attn_output, bias
190
@@ -263,8 +260,8 @@ class FalconDecoderLayer(nn.Module):
260
self,
261
positions: torch.Tensor,
262
hidden_states: torch.Tensor,
266
- kv_cache: KVCache,
267
- input_metadata: InputMetadata,
263
+ kv_cache: torch.Tensor,
264
+ attn_metadata: AttentionMetadata,
265
) -> torch.Tensor:
266
residual = hidden_states
267
@@ -279,7 +276,7 @@ class FalconDecoderLayer(nn.Module):
276
positions=positions,
277
hidden_states=attention_layernorm_out,
278
kv_cache=kv_cache,
282
- input_metadata=input_metadata,
279
+ attn_metadata=attn_metadata,
280
)
281
if self.reduce_row_parallel_results and attention_bias is not None:
282
attention_output += attention_bias
@@ -343,8 +340,8 @@ class FalconModel(nn.Module):
340
self,
341
input_ids: torch.LongTensor,
342
positions: torch.Tensor,
346
- kv_caches: List[KVCache],
347
- input_metadata: InputMetadata,
343
+ kv_caches: List[torch.Tensor],
344
+ attn_metadata: AttentionMetadata,
345
) -> torch.Tensor:
346
hidden_states = self.word_embeddings(input_ids)
347
for i in range(len(self.h)):
@@ -353,7 +350,7 @@ class FalconModel(nn.Module):
350
positions,
351
hidden_states,
352
kv_caches[i],
356
- input_metadata,
353
+ attn_metadata,
354
)
355
hidden_states = self.ln_f(hidden_states)
356
return hidden_states
@@ -378,14 +375,14 @@ class FalconForCausalLM(nn.Module):
375
self,
376
input_ids: torch.LongTensor,
377
positions: torch.Tensor,
381
- kv_caches: List[KVCache],
382
- input_metadata: InputMetadata,
378
+ kv_caches: List[torch.Tensor],
379
+ attn_metadata: AttentionMetadata,
380
) -> torch.Tensor:
381
hidden_states = self.transformer(
382
input_ids,
383
positions,
384
kv_caches,
388
- input_metadata,
385
+ attn_metadata,
386
)
387
return hidden_states
388
vllm/model_executor/models/gemma.py
+13
-17
@@ -20,10 +20,9 @@ import torch
20
from torch import nn
21
from transformers import GemmaConfig
22
23
+from vllm.attention import Attention, AttentionMetadata
24
from vllm.config import LoRAConfig
24
-from vllm.model_executor.input_metadata import InputMetadata
25
from vllm.model_executor.layers.activation import GeluAndMul
26
-from vllm.model_executor.layers.attention import Attention
26
from vllm.model_executor.layers.layernorm import RMSNorm
27
from vllm.model_executor.layers.linear import (LinearMethodBase,
28
MergedColumnParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
45
-
43
44
class GemmaMLP(nn.Module):
45
@@ -133,14 +130,13 @@ class GemmaAttention(nn.Module):
130
self,
131
positions: torch.Tensor,
132
hidden_states: torch.Tensor,
136
- kv_cache: KVCache,
137
- input_metadata: InputMetadata,
133
+ kv_cache: torch.Tensor,
134
+ attn_metadata: AttentionMetadata,
135
) -> torch.Tensor:
136
qkv, _ = self.qkv_proj(hidden_states)
137
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
138
q, k = self.rotary_emb(positions, q, k)
142
- k_cache, v_cache = kv_cache
143
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
139
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
140
output, _ = self.o_proj(attn_output)
141
return output
142
@@ -177,8 +173,8 @@ class GemmaDecoderLayer(nn.Module):
173
self,
174
positions: torch.Tensor,
175
hidden_states: torch.Tensor,
180
- kv_cache: KVCache,
181
- input_metadata: InputMetadata,
176
+ kv_cache: torch.Tensor,
177
+ attn_metadata: AttentionMetadata,
178
residual: Optional[torch.Tensor],
179
) -> Tuple[torch.Tensor, torch.Tensor]:
180
# Self Attention
@@ -192,7 +188,7 @@ class GemmaDecoderLayer(nn.Module):
188
positions=positions,
189
hidden_states=hidden_states,
190
kv_cache=kv_cache,
195
- input_metadata=input_metadata,
191
+ attn_metadata=attn_metadata,
192
)
193
194
# Fully Connected
@@ -226,8 +222,8 @@ class GemmaModel(nn.Module):
222
self,
223
input_ids: torch.Tensor,
224
positions: torch.Tensor,
229
- kv_caches: List[KVCache],
230
- input_metadata: InputMetadata,
225
+ kv_caches: List[torch.Tensor],
226
+ attn_metadata: AttentionMetadata,
227
) -> torch.Tensor:
228
hidden_states = self.embed_tokens(input_ids)
229
# Normalize the embedding by sqrt(hidden_size)
@@ -240,7 +236,7 @@ class GemmaModel(nn.Module):
236
positions,
237
hidden_states,
238
kv_caches[i],
243
- input_metadata,
239
+ attn_metadata,
240
residual,
241
)
242
hidden_states, _ = self.norm(hidden_states, residual)
@@ -290,11 +286,11 @@ class GemmaForCausalLM(nn.Module):
286
self,
287
input_ids: torch.Tensor,
288
positions: torch.Tensor,
293
- kv_caches: List[KVCache],
294
- input_metadata: InputMetadata,
289
+ kv_caches: List[torch.Tensor],
290
+ attn_metadata: AttentionMetadata,
291
) -> torch.Tensor:
292
hidden_states = self.model(input_ids, positions, kv_caches,
297
- input_metadata)
293
+ attn_metadata)
294
return hidden_states
295
296
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt2.py
+14
-19
@@ -17,15 +17,14 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only GPT-2 model compatible with HuggingFace weights."""
20
-from typing import List, Optional, Tuple
20
+from typing import List, Optional
21
22
import torch
23
from torch import nn
24
from transformers import GPT2Config
25
26
-from vllm.model_executor.input_metadata import InputMetadata
26
+from vllm.attention import Attention, AttentionMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import Attention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
45
-
43
44
class GPT2Attention(nn.Module):
45
@@ -79,14 +76,12 @@ class GPT2Attention(nn.Module):
76
def forward(
77
self,
78
hidden_states: torch.Tensor,
82
- kv_cache: KVCache,
83
- input_metadata: InputMetadata,
79
+ kv_cache: torch.Tensor,
80
+ attn_metadata: AttentionMetadata,
81
) -> torch.Tensor:
82
qkv, _ = self.c_attn(hidden_states)
83
q, k, v = qkv.chunk(chunks=3, dim=-1)
87
- key_cache, value_cache = kv_cache
88
- attn_output = self.attn(q, k, v, key_cache, value_cache,
89
- input_metadata)
84
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
85
attn_output, _ = self.c_proj(attn_output)
86
return attn_output
87
@@ -144,15 +139,15 @@ class GPT2Block(nn.Module):
139
def forward(
140
self,
141
hidden_states: torch.Tensor,
147
- kv_cache: KVCache,
148
- input_metadata: InputMetadata,
142
+ kv_cache: torch.Tensor,
143
+ attn_metadata: AttentionMetadata,
144
) -> torch.Tensor:
145
residual = hidden_states
146
hidden_states = self.ln_1(hidden_states)
147
attn_output = self.attn(
148
hidden_states=hidden_states,
149
kv_cache=kv_cache,
155
- input_metadata=input_metadata,
150
+ attn_metadata=attn_metadata,
151
)
152
# residual connection
153
hidden_states = attn_output + residual
@@ -190,8 +185,8 @@ class GPT2Model(nn.Module):
185
self,
186
input_ids: torch.Tensor,
187
position_ids: torch.Tensor,
193
- kv_caches: List[KVCache],
194
- input_metadata: InputMetadata,
188
+ kv_caches: List[torch.Tensor],
189
+ attn_metadata: AttentionMetadata,
190
) -> torch.Tensor:
191
inputs_embeds = self.wte(input_ids)
192
position_embeds = self.wpe(position_ids)
@@ -199,7 +194,7 @@ class GPT2Model(nn.Module):
194
195
for i in range(len(self.h)):
196
layer = self.h[i]
202
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
197
+ hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
198
199
hidden_states = self.ln_f(hidden_states)
200
return hidden_states
@@ -224,11 +219,11 @@ class GPT2LMHeadModel(nn.Module):
219
self,
220
input_ids: torch.Tensor,
221
positions: torch.Tensor,
227
- kv_caches: List[KVCache],
228
- input_metadata: InputMetadata,
222
+ kv_caches: List[torch.Tensor],
223
+ attn_metadata: AttentionMetadata,
224
) -> torch.Tensor:
225
hidden_states = self.transformer(input_ids, positions, kv_caches,
231
- input_metadata)
226
+ attn_metadata)
227
return hidden_states
228
229
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_bigcode.py
+14
-19
@@ -18,15 +18,14 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
"""Inference-only GPTBigCode model compatible with HuggingFace weights."""
21
-from typing import List, Optional, Tuple
21
+from typing import List, Optional
22
23
import torch
24
from torch import nn
25
from transformers import GPTBigCodeConfig
26
27
-from vllm.model_executor.input_metadata import InputMetadata
27
+from vllm.attention import Attention, AttentionMetadata
28
from vllm.model_executor.layers.activation import get_act_fn
29
-from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
@@ -42,8 +41,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
43
45
-KVCache = Tuple[torch.Tensor, torch.Tensor]
46
-
44
45
class GPTBigCodeAttention(nn.Module):
46
@@ -94,8 +91,8 @@ class GPTBigCodeAttention(nn.Module):
91
def forward(
92
self,
93
hidden_states: torch.Tensor,
97
- kv_cache: KVCache,
98
- input_metadata: InputMetadata,
94
+ kv_cache: torch.Tensor,
95
+ attn_metadata: AttentionMetadata,
96
) -> torch.Tensor:
97
qkv, _ = self.c_attn(hidden_states)
98
q, k, v = qkv.split(
@@ -105,9 +102,7 @@ class GPTBigCodeAttention(nn.Module):
102
],
103
dim=-1,
104
)
108
- key_cache, value_cache = kv_cache
109
- attn_output = self.attn(q, k, v, key_cache, value_cache,
110
- input_metadata)
105
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
106
attn_output, _ = self.c_proj(attn_output)
107
return attn_output
108
@@ -165,15 +160,15 @@ class GPTBigCodeBlock(nn.Module):
160
def forward(
161
self,
162
hidden_states: torch.Tensor,
168
- kv_cache: KVCache,
169
- input_metadata: InputMetadata,
163
+ kv_cache: torch.Tensor,
164
+ attn_metadata: AttentionMetadata,
165
) -> torch.Tensor:
166
residual = hidden_states
167
hidden_states = self.ln_1(hidden_states)
168
attn_output = self.attn(
169
hidden_states=hidden_states,
170
kv_cache=kv_cache,
176
- input_metadata=input_metadata,
171
+ attn_metadata=attn_metadata,
172
)
173
# residual connection
174
hidden_states = attn_output + residual
@@ -211,8 +206,8 @@ class GPTBigCodeModel(nn.Module):
206
self,
207
input_ids: torch.Tensor,
208
position_ids: torch.Tensor,
214
- kv_caches: List[KVCache],
215
- input_metadata: InputMetadata,
209
+ kv_caches: List[torch.Tensor],
210
+ attn_metadata: AttentionMetadata,
211
) -> torch.Tensor:
212
inputs_embeds = self.wte(input_ids)
213
position_embeds = self.wpe(position_ids)
@@ -220,7 +215,7 @@ class GPTBigCodeModel(nn.Module):
215
216
for i in range(len(self.h)):
217
layer = self.h[i]
223
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
218
+ hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
219
220
hidden_states = self.ln_f(hidden_states)
221
return hidden_states
@@ -245,11 +240,11 @@ class GPTBigCodeForCausalLM(nn.Module):
240
self,
241
input_ids: torch.Tensor,
242
positions: torch.Tensor,
248
- kv_caches: List[KVCache],
249
- input_metadata: InputMetadata,
243
+ kv_caches: List[torch.Tensor],
244
+ attn_metadata: AttentionMetadata,
245
) -> torch.Tensor:
246
hidden_states = self.transformer(input_ids, positions, kv_caches,
252
- input_metadata)
247
+ attn_metadata)
248
return hidden_states
249
250
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_j.py
+14
-18
@@ -16,15 +16,14 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only GPT-J model compatible with HuggingFace weights."""
19
-from typing import List, Optional, Tuple
19
+from typing import List, Optional
20
21
import torch
22
from torch import nn
23
from transformers import GPTJConfig
24
25
-from vllm.model_executor.input_metadata import InputMetadata
25
+from vllm.attention import Attention, AttentionMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import Attention
27
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28
LinearMethodBase,
29
QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
45
-
43
44
class GPTJAttention(nn.Module):
45
@@ -93,14 +90,13 @@ class GPTJAttention(nn.Module):
90
self,
91
position_ids: torch.Tensor,
92
hidden_states: torch.Tensor,
96
- kv_cache: KVCache,
97
- input_metadata: InputMetadata,
93
+ kv_cache: torch.Tensor,
94
+ attn_metadata: AttentionMetadata,
95
) -> torch.Tensor:
96
qkv, _ = self.qkv_proj(hidden_states)
97
q, k, v = qkv.chunk(chunks=3, dim=-1)
98
q, k = self.rotary_emb(position_ids, q, k)
102
- k_cache, v_cache = kv_cache
103
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
99
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
100
attn_output, _ = self.out_proj(attn_output)
101
return attn_output
102
@@ -154,8 +150,8 @@ class GPTJBlock(nn.Module):
150
self,
151
position_ids: torch.Tensor,
152
hidden_states: torch.Tensor,
157
- kv_cache: KVCache,
158
- input_metadata: InputMetadata,
153
+ kv_cache: torch.Tensor,
154
+ attn_metadata: AttentionMetadata,
155
) -> torch.Tensor:
156
residual = hidden_states
157
hidden_states = self.ln_1(hidden_states)
@@ -163,7 +159,7 @@ class GPTJBlock(nn.Module):
159
position_ids=position_ids,
160
hidden_states=hidden_states,
161
kv_cache=kv_cache,
166
- input_metadata=input_metadata,
162
+ attn_metadata=attn_metadata,
163
)
164
mlp_output = self.mlp(hidden_states)
165
hidden_states = attn_output + mlp_output + residual
@@ -192,8 +188,8 @@ class GPTJModel(nn.Module):
188
self,
189
input_ids: torch.Tensor,
190
position_ids: torch.Tensor,
195
- kv_caches: List[KVCache],
196
- input_metadata: InputMetadata,
191
+ kv_caches: List[torch.Tensor],
192
+ attn_metadata: AttentionMetadata,
193
) -> torch.Tensor:
194
hidden_states = self.wte(input_ids)
195
for i in range(len(self.h)):
@@ -202,7 +198,7 @@ class GPTJModel(nn.Module):
198
position_ids,
199
hidden_states,
200
kv_caches[i],
205
- input_metadata,
201
+ attn_metadata,
202
)
203
hidden_states = self.ln_f(hidden_states)
204
return hidden_states
@@ -232,11 +228,11 @@ class GPTJForCausalLM(nn.Module):
228
self,
229
input_ids: torch.Tensor,
230
positions: torch.Tensor,
235
- kv_caches: List[KVCache],
236
- input_metadata: InputMetadata,
231
+ kv_caches: List[torch.Tensor],
232
+ attn_metadata: AttentionMetadata,
233
) -> torch.Tensor:
234
hidden_states = self.transformer(input_ids, positions, kv_caches,
239
- input_metadata)
235
+ attn_metadata)
236
return hidden_states
237
238
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/gpt_neox.py
+14
-18
@@ -16,15 +16,14 @@
16
# See the License for the specific language governing permissions and
17
# limitations under the License.
18
"""Inference-only GPT-NeoX model compatible with HuggingFace weights."""
19
-from typing import List, Optional, Tuple
19
+from typing import List, Optional
20
21
import torch
22
from torch import nn
23
from transformers import GPTNeoXConfig
24
25
-from vllm.model_executor.input_metadata import InputMetadata
25
+from vllm.attention import Attention, AttentionMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import Attention
27
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
28
LinearMethodBase,
29
QKVParallelLinear,
@@ -41,8 +40,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
40
hf_model_weights_iterator)
41
from vllm.sequence import SamplerOutput
42
44
-KVCache = Tuple[torch.Tensor, torch.Tensor]
45
-
43
44
class GPTNeoXAttention(nn.Module):
45
@@ -94,14 +91,13 @@ class GPTNeoXAttention(nn.Module):
91
self,
92
position_ids: torch.Tensor,
93
hidden_states: torch.Tensor,
97
- kv_cache: KVCache,
98
- input_metadata: InputMetadata,
94
+ kv_cache: torch.Tensor,
95
+ attn_metadata: AttentionMetadata,
96
) -> torch.Tensor:
97
qkv, _ = self.query_key_value(hidden_states)
98
q, k, v = qkv.chunk(chunks=3, dim=-1)
99
q, k = self.rotary_emb(position_ids, q, k)
103
- k_cache, v_cache = kv_cache
104
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
100
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
101
output, _ = self.dense(attn_output)
102
return output
103
@@ -155,15 +151,15 @@ class GPTNeoXLayer(nn.Module):
151
self,
152
position_ids: torch.Tensor,
153
hidden_states: torch.Tensor,
158
- kv_cache: KVCache,
159
- input_metadata: InputMetadata,
154
+ kv_cache: torch.Tensor,
155
+ attn_metadata: AttentionMetadata,
156
) -> torch.Tensor:
157
attn_input = self.input_layernorm(hidden_states)
158
attn_output = self.attention(
159
position_ids=position_ids,
160
hidden_states=attn_input,
161
kv_cache=kv_cache,
166
- input_metadata=input_metadata,
162
+ attn_metadata=attn_metadata,
163
)
164
165
if self.use_parallel_residual:
@@ -208,8 +204,8 @@ class GPTNeoXModel(nn.Module):
204
self,
205
input_ids: torch.Tensor,
206
position_ids: torch.Tensor,
211
- kv_caches: List[KVCache],
212
- input_metadata: InputMetadata,
207
+ kv_caches: List[torch.Tensor],
208
+ attn_metadata: AttentionMetadata,
209
) -> torch.Tensor:
210
hidden_states = self.embed_in(input_ids)
211
for i in range(len(self.layers)):
@@ -218,7 +214,7 @@ class GPTNeoXModel(nn.Module):
214
position_ids,
215
hidden_states,
216
kv_caches[i],
221
- input_metadata,
217
+ attn_metadata,
218
)
219
hidden_states = self.final_layer_norm(hidden_states)
220
return hidden_states
@@ -246,11 +242,11 @@ class GPTNeoXForCausalLM(nn.Module):
242
self,
243
input_ids: torch.Tensor,
244
positions: torch.Tensor,
249
- kv_caches: List[KVCache],
250
- input_metadata: InputMetadata,
245
+ kv_caches: List[torch.Tensor],
246
+ attn_metadata: AttentionMetadata,
247
) -> torch.Tensor:
248
hidden_states = self.gpt_neox(input_ids, positions, kv_caches,
253
- input_metadata)
249
+ attn_metadata)
250
return hidden_states
251
252
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/internlm2.py
+13
-17
@@ -5,9 +5,8 @@ import torch
5
from torch import nn
6
from transformers import PretrainedConfig
7
8
-from vllm.model_executor.input_metadata import InputMetadata
8
+from vllm.attention import Attention, AttentionMetadata
9
from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import Attention
10
from vllm.model_executor.layers.layernorm import RMSNorm
11
from vllm.model_executor.layers.linear import (LinearMethodBase,
12
MergedColumnParallelLinear,
@@ -25,8 +24,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
24
hf_model_weights_iterator)
25
from vllm.sequence import SamplerOutput
26
28
-KVCache = Tuple[torch.Tensor, torch.Tensor]
29
-
27
28
class InternLM2MLP(nn.Module):
29
@@ -124,14 +121,13 @@ class InternLM2Attention(nn.Module):
121
self,
122
positions: torch.Tensor,
123
hidden_states: torch.Tensor,
127
- kv_cache: KVCache,
128
- input_metadata: InputMetadata,
124
+ kv_cache: torch.Tensor,
125
+ attn_metadata: AttentionMetadata,
126
) -> torch.Tensor:
127
qkv, _ = self.wqkv(hidden_states)
128
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
129
q, k = self.rotary_emb(positions, q, k)
133
- k_cache, v_cache = kv_cache
134
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
130
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
131
output, _ = self.wo(attn_output)
132
return output
133
@@ -172,8 +168,8 @@ class InternLMDecoderLayer(nn.Module):
168
self,
169
positions: torch.Tensor,
170
hidden_states: torch.Tensor,
175
- kv_cache: KVCache,
176
- input_metadata: InputMetadata,
171
+ kv_cache: torch.Tensor,
172
+ attn_metadata: AttentionMetadata,
173
residual: Optional[torch.Tensor],
174
) -> Tuple[torch.Tensor, torch.Tensor]:
175
# Self Attention
@@ -187,7 +183,7 @@ class InternLMDecoderLayer(nn.Module):
183
positions=positions,
184
hidden_states=hidden_states,
185
kv_cache=kv_cache,
190
- input_metadata=input_metadata,
186
+ attn_metadata=attn_metadata,
187
)
188
189
# Fully Connected
@@ -221,8 +217,8 @@ class InternLM2Model(nn.Module):
217
self,
218
input_ids: torch.Tensor,
219
positions: torch.Tensor,
224
- kv_caches: List[KVCache],
225
- input_metadata: InputMetadata,
220
+ kv_caches: List[torch.Tensor],
221
+ attn_metadata: AttentionMetadata,
222
) -> torch.Tensor:
223
hidden_states = self.tok_embeddings(input_ids)
224
residual = None
@@ -232,7 +228,7 @@ class InternLM2Model(nn.Module):
228
positions,
229
hidden_states,
230
kv_caches[i],
235
- input_metadata,
231
+ attn_metadata,
232
residual,
233
)
234
hidden_states, _ = self.norm(hidden_states, residual)
@@ -258,11 +254,11 @@ class InternLM2ForCausalLM(nn.Module):
254
self,
255
input_ids: torch.Tensor,
256
positions: torch.Tensor,
261
- kv_caches: List[KVCache],
262
- input_metadata: InputMetadata,
257
+ kv_caches: List[torch.Tensor],
258
+ attn_metadata: AttentionMetadata,
259
) -> torch.Tensor:
260
hidden_states = self.model(input_ids, positions, kv_caches,
265
- input_metadata)
261
+ attn_metadata)
262
return hidden_states
263
264
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/jais.py
+15
-20
@@ -20,14 +20,13 @@
20
"""Inference-only Jais model compatible with HuggingFace weights."""
21
22
import math
23
-from typing import List, Optional, Tuple
23
+from typing import List, Optional
24
25
import torch
26
from torch import nn
27
from vllm.transformers_utils.configs import JAISConfig
28
29
-from vllm.model_executor.input_metadata import InputMetadata
30
-from vllm.model_executor.layers.attention import Attention
29
+from vllm.attention import Attention, AttentionMetadata
30
from vllm.model_executor.layers.linear import (
31
ColumnParallelLinear,
32
LinearMethodBase,
@@ -49,8 +48,6 @@ from vllm.model_executor.weight_utils import (
48
from vllm.sequence import SamplerOutput
49
from vllm.model_executor.sampling_metadata import SamplingMetadata
50
52
-KVCache = Tuple[torch.Tensor, torch.Tensor]
53
-
51
52
class SwiGLUActivation(nn.Module):
53
@@ -122,14 +119,12 @@ class JAISAttention(nn.Module):
119
def forward(
120
self,
121
hidden_states: torch.Tensor,
125
- kv_cache: KVCache,
126
- input_metadata: InputMetadata,
122
+ kv_cache: torch.Tensor,
123
+ attn_metadata: AttentionMetadata,
124
) -> torch.Tensor:
125
qkv, _ = self.c_attn(hidden_states)
126
q, k, v = qkv.chunk(chunks=3, dim=-1)
130
- key_cache, value_cache = kv_cache
131
- attn_output = self.attn(q, k, v, key_cache, value_cache,
132
- input_metadata)
127
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
128
attn_output, _ = self.c_proj(attn_output)
129
return attn_output
130
@@ -196,15 +191,15 @@ class JAISBlock(nn.Module):
191
def forward(
192
self,
193
hidden_states: torch.Tensor,
199
- kv_cache: KVCache,
200
- input_metadata: InputMetadata,
194
+ kv_cache: torch.Tensor,
195
+ attn_metadata: AttentionMetadata,
196
) -> torch.Tensor:
197
residual = hidden_states
198
hidden_states = self.ln_1(hidden_states)
199
attn_output = self.attn(
200
hidden_states=hidden_states,
201
kv_cache=kv_cache,
207
- input_metadata=input_metadata,
202
+ attn_metadata=attn_metadata,
203
)
204
# residual connection
205
hidden_states = attn_output + residual
@@ -248,8 +243,8 @@ class JAISModel(nn.Module):
243
self,
244
input_ids: torch.Tensor,
245
position_ids: torch.Tensor,
251
- kv_caches: List[KVCache],
252
- input_metadata: InputMetadata,
246
+ kv_caches: List[torch.Tensor],
247
+ attn_metadata: AttentionMetadata,
248
) -> torch.Tensor:
249
inputs_embeds = self.wte(input_ids)
250
if self.wpe is not None:
@@ -262,7 +257,7 @@ class JAISModel(nn.Module):
257
258
for i in range(len(self.h)):
259
layer = self.h[i]
265
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
260
+ hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
261
262
hidden_states = self.ln_f(hidden_states)
263
return hidden_states
@@ -293,11 +288,11 @@ class JAISLMHeadModel(nn.Module):
288
self,
289
input_ids: torch.Tensor,
290
positions: torch.Tensor,
296
- kv_caches: List[KVCache],
297
- input_metadata: InputMetadata,
291
+ kv_caches: List[torch.Tensor],
292
+ attn_metadata: AttentionMetadata,
293
) -> torch.Tensor:
294
hidden_states = self.transformer(input_ids, positions, kv_caches,
300
- input_metadata)
295
+ attn_metadata)
296
return hidden_states
297
298
def compute_logits(self, hidden_states: torch.Tensor,
@@ -348,4 +343,4 @@ class JAISLMHeadModel(nn.Module):
343
loaded_weight = loaded_weight.t()
344
weight_loader = getattr(param, "weight_loader",
345
default_weight_loader)
351
- weight_loader(param, loaded_weight)
\ No newline at end of file
346
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+13
-17
@@ -27,10 +27,9 @@ import torch
27
from torch import nn
28
from transformers import LlamaConfig
29
30
+from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import LoRAConfig
31
-from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.attention import Attention
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
@@ -48,8 +47,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
49
51
-KVCache = Tuple[torch.Tensor, torch.Tensor]
52
-
50
51
class LlamaMLP(nn.Module):
52
@@ -150,14 +147,13 @@ class LlamaAttention(nn.Module):
147
self,
148
positions: torch.Tensor,
149
hidden_states: torch.Tensor,
153
- kv_cache: KVCache,
154
- input_metadata: InputMetadata,
150
+ kv_cache: torch.Tensor,
151
+ attn_metadata: AttentionMetadata,
152
) -> torch.Tensor:
153
qkv, _ = self.qkv_proj(hidden_states)
154
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
155
q, k = self.rotary_emb(positions, q, k)
159
- k_cache, v_cache = kv_cache
160
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
156
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
157
output, _ = self.o_proj(attn_output)
158
return output
159
@@ -203,8 +199,8 @@ class LlamaDecoderLayer(nn.Module):
199
self,
200
positions: torch.Tensor,
201
hidden_states: torch.Tensor,
206
- kv_cache: KVCache,
207
- input_metadata: InputMetadata,
202
+ kv_cache: torch.Tensor,
203
+ attn_metadata: AttentionMetadata,
204
residual: Optional[torch.Tensor],
205
) -> Tuple[torch.Tensor, torch.Tensor]:
206
# Self Attention
@@ -218,7 +214,7 @@ class LlamaDecoderLayer(nn.Module):
214
positions=positions,
215
hidden_states=hidden_states,
216
kv_cache=kv_cache,
221
- input_metadata=input_metadata,
217
+ attn_metadata=attn_metadata,
218
)
219
220
# Fully Connected
@@ -258,8 +254,8 @@ class LlamaModel(nn.Module):
254
self,
255
input_ids: torch.Tensor,
256
positions: torch.Tensor,
261
- kv_caches: List[KVCache],
262
- input_metadata: InputMetadata,
257
+ kv_caches: List[torch.Tensor],
258
+ attn_metadata: AttentionMetadata,
259
) -> torch.Tensor:
260
hidden_states = self.embed_tokens(input_ids)
261
residual = None
@@ -269,7 +265,7 @@ class LlamaModel(nn.Module):
265
positions,
266
hidden_states,
267
kv_caches[i],
272
- input_metadata,
268
+ attn_metadata,
269
residual,
270
)
271
hidden_states, _ = self.norm(hidden_states, residual)
@@ -336,11 +332,11 @@ class LlamaForCausalLM(nn.Module):
332
self,
333
input_ids: torch.Tensor,
334
positions: torch.Tensor,
339
- kv_caches: List[KVCache],
340
- input_metadata: InputMetadata,
335
+ kv_caches: List[torch.Tensor],
336
+ attn_metadata: AttentionMetadata,
337
) -> torch.Tensor:
338
hidden_states = self.model(input_ids, positions, kv_caches,
343
- input_metadata)
339
+ attn_metadata)
340
return hidden_states
341
342
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mixtral.py
+14
-18
@@ -21,15 +21,14 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Mixtral model."""
24
-from typing import List, Optional, Tuple
24
+from typing import List, Optional
25
26
import torch
27
from torch import nn
28
from transformers import MixtralConfig
29
30
+from vllm.attention import Attention, AttentionMetadata
31
from vllm.config import LoRAConfig
31
-from vllm.model_executor.input_metadata import InputMetadata
32
-from vllm.model_executor.layers.attention import Attention
32
from vllm.model_executor.layers.fused_moe import fused_moe
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -51,8 +50,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
50
hf_model_weights_iterator)
51
from vllm.sequence import SamplerOutput
52
54
-KVCache = Tuple[torch.Tensor, torch.Tensor]
55
-
53
54
class MixtralMoE(nn.Module):
55
"""A tensor-parallel MoE implementation for Mixtral that shards each expert
@@ -209,14 +206,13 @@ class MixtralAttention(nn.Module):
206
self,
207
positions: torch.Tensor,
208
hidden_states: torch.Tensor,
212
- kv_cache: KVCache,
213
- input_metadata: InputMetadata,
209
+ kv_cache: torch.Tensor,
210
+ attn_metadata: AttentionMetadata,
211
) -> torch.Tensor:
212
qkv, _ = self.qkv_proj(hidden_states)
213
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
214
q, k = self.rotary_emb(positions, q, k)
218
- k_cache, v_cache = kv_cache
219
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
215
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
216
output, _ = self.o_proj(attn_output)
217
return output
218
@@ -254,8 +250,8 @@ class MixtralDecoderLayer(nn.Module):
250
self,
251
positions: torch.Tensor,
252
hidden_states: torch.Tensor,
257
- kv_cache: KVCache,
258
- input_metadata: InputMetadata,
253
+ kv_cache: torch.Tensor,
254
+ attn_metadata: AttentionMetadata,
255
residual: Optional[torch.Tensor],
256
) -> torch.Tensor:
257
# Self Attention
@@ -269,7 +265,7 @@ class MixtralDecoderLayer(nn.Module):
265
positions=positions,
266
hidden_states=hidden_states,
267
kv_cache=kv_cache,
272
- input_metadata=input_metadata,
268
+ attn_metadata=attn_metadata,
269
)
270
271
# Fully Connected
@@ -309,15 +305,15 @@ class MixtralModel(nn.Module):
305
self,
306
input_ids: torch.Tensor,
307
positions: torch.Tensor,
312
- kv_caches: List[KVCache],
313
- input_metadata: InputMetadata,
308
+ kv_caches: List[torch.Tensor],
309
+ attn_metadata: AttentionMetadata,
310
) -> torch.Tensor:
311
hidden_states = self.embed_tokens(input_ids)
312
residual = None
313
for i in range(len(self.layers)):
314
layer = self.layers[i]
315
hidden_states, residual = layer(positions, hidden_states,
320
- kv_caches[i], input_metadata,
316
+ kv_caches[i], attn_metadata,
317
residual)
318
hidden_states, _ = self.norm(hidden_states, residual)
319
return hidden_states
@@ -377,11 +373,11 @@ class MixtralForCausalLM(nn.Module):
373
self,
374
input_ids: torch.Tensor,
375
positions: torch.Tensor,
380
- kv_caches: List[KVCache],
381
- input_metadata: InputMetadata,
376
+ kv_caches: List[torch.Tensor],
377
+ attn_metadata: AttentionMetadata,
378
) -> torch.Tensor:
379
hidden_states = self.model(input_ids, positions, kv_caches,
384
- input_metadata)
380
+ attn_metadata)
381
return hidden_states
382
383
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mixtral_quant.py
+14
-18
@@ -21,7 +21,7 @@
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
"""Inference-only Mixtral model."""
24
-from typing import List, Optional, Tuple
24
+from typing import List, Optional
25
26
import numpy as np
27
@@ -31,8 +31,7 @@ import torch.nn.functional as F
31
from torch import nn
32
from transformers import MixtralConfig
33
34
-from vllm.model_executor.input_metadata import InputMetadata
35
-from vllm.model_executor.layers.attention import Attention
34
+from vllm.attention import Attention, AttentionMetadata
35
from vllm.model_executor.layers.layernorm import RMSNorm
36
from vllm.model_executor.layers.linear import (LinearMethodBase,
37
ReplicatedLinear,
@@ -52,8 +51,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
51
hf_model_weights_iterator)
52
from vllm.sequence import SamplerOutput
53
55
-KVCache = Tuple[torch.Tensor, torch.Tensor]
56
-
54
55
class MixtralMLP(nn.Module):
56
@@ -227,14 +224,13 @@ class MixtralAttention(nn.Module):
224
self,
225
positions: torch.Tensor,
226
hidden_states: torch.Tensor,
230
- kv_cache: KVCache,
231
- input_metadata: InputMetadata,
227
+ kv_cache: torch.Tensor,
228
+ attn_metadata: AttentionMetadata,
229
) -> torch.Tensor:
230
qkv, _ = self.qkv_proj(hidden_states)
231
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
232
q, k = self.rotary_emb(positions, q, k)
236
- k_cache, v_cache = kv_cache
237
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
233
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
234
output, _ = self.o_proj(attn_output)
235
return output
236
@@ -269,8 +265,8 @@ class MixtralDecoderLayer(nn.Module):
265
self,
266
positions: torch.Tensor,
267
hidden_states: torch.Tensor,
272
- kv_cache: KVCache,
273
- input_metadata: InputMetadata,
268
+ kv_cache: torch.Tensor,
269
+ attn_metadata: AttentionMetadata,
270
residual: Optional[torch.Tensor],
271
) -> torch.Tensor:
272
# Self Attention
@@ -284,7 +280,7 @@ class MixtralDecoderLayer(nn.Module):
280
positions=positions,
281
hidden_states=hidden_states,
282
kv_cache=kv_cache,
287
- input_metadata=input_metadata,
283
+ attn_metadata=attn_metadata,
284
)
285
286
# Fully Connected
@@ -319,15 +315,15 @@ class MixtralModel(nn.Module):
315
self,
316
input_ids: torch.Tensor,
317
positions: torch.Tensor,
322
- kv_caches: List[KVCache],
323
- input_metadata: InputMetadata,
318
+ kv_caches: List[torch.Tensor],
319
+ attn_metadata: AttentionMetadata,
320
) -> torch.Tensor:
321
hidden_states = self.embed_tokens(input_ids)
322
residual = None
323
for i in range(len(self.layers)):
324
layer = self.layers[i]
325
hidden_states, residual = layer(positions, hidden_states,
330
- kv_caches[i], input_metadata,
326
+ kv_caches[i], attn_metadata,
327
residual)
328
hidden_states, _ = self.norm(hidden_states, residual)
329
return hidden_states
@@ -352,11 +348,11 @@ class MixtralForCausalLM(nn.Module):
348
self,
349
input_ids: torch.Tensor,
350
positions: torch.Tensor,
355
- kv_caches: List[KVCache],
356
- input_metadata: InputMetadata,
351
+ kv_caches: List[torch.Tensor],
352
+ attn_metadata: AttentionMetadata,
353
) -> torch.Tensor:
354
hidden_states = self.model(input_ids, positions, kv_caches,
359
- input_metadata)
355
+ attn_metadata)
356
return hidden_states
357
358
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/mpt.py
+14
-18
@@ -1,14 +1,13 @@
1
# coding=utf-8
2
# Adapted from https://huggingface.co/mosaicml/mpt-7b/tree/main
3
import math
4
-from typing import List, Optional, Tuple
4
+from typing import List, Optional
5
6
import torch
7
import torch.nn as nn
8
9
-from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.attention import Attention, AttentionMetadata
10
from vllm.model_executor.layers.activation import get_act_fn
11
-from vllm.model_executor.layers.attention import Attention
11
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
12
LinearMethodBase,
13
QKVParallelLinear,
@@ -25,8 +24,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
24
from vllm.sequence import SamplerOutput
25
from vllm.transformers_utils.configs.mpt import MPTConfig
26
28
-KVCache = Tuple[torch.Tensor, torch.Tensor]
29
-
27
28
def _get_alibi_slopes(
29
total_num_heads: int,
@@ -116,8 +113,8 @@ class MPTAttention(nn.Module):
113
self,
114
position_ids: torch.Tensor,
115
hidden_states: torch.Tensor,
119
- kv_cache: KVCache,
120
- input_metadata: InputMetadata,
116
+ kv_cache: torch.Tensor,
117
+ attn_metadata: AttentionMetadata,
118
) -> torch.Tensor:
119
del position_ids # unused.
120
qkv, _ = self.Wqkv(hidden_states)
@@ -127,8 +124,7 @@ class MPTAttention(nn.Module):
124
if self.qk_ln:
125
q = self.q_ln(q)
126
k = self.k_ln(k)
130
- k_cache, v_cache = kv_cache
131
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
127
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
128
output, _ = self.out_proj(attn_output)
129
return output
130
@@ -184,15 +180,15 @@ class MPTBlock(nn.Module):
180
self,
181
position_ids: torch.Tensor,
182
hidden_states: torch.Tensor,
187
- kv_cache: KVCache,
188
- input_metadata: InputMetadata,
183
+ kv_cache: torch.Tensor,
184
+ attn_metadata: AttentionMetadata,
185
) -> torch.Tensor:
186
x = self.norm_1(hidden_states)
187
x = self.attn(
188
position_ids=position_ids,
189
hidden_states=x,
190
kv_cache=kv_cache,
195
- input_metadata=input_metadata,
191
+ attn_metadata=attn_metadata,
192
)
193
hidden_states = hidden_states + x
194
x = self.norm_2(hidden_states)
@@ -230,8 +226,8 @@ class MPTModel(nn.Module):
226
self,
227
input_ids: torch.Tensor,
228
position_ids: torch.Tensor,
233
- kv_caches: List[KVCache],
234
- input_metadata: InputMetadata,
229
+ kv_caches: List[torch.Tensor],
230
+ attn_metadata: AttentionMetadata,
231
) -> torch.Tensor:
232
hidden_states = self.wte(input_ids)
233
for i in range(len(self.blocks)):
@@ -240,7 +236,7 @@ class MPTModel(nn.Module):
236
position_ids,
237
hidden_states,
238
kv_caches[i],
243
- input_metadata,
239
+ attn_metadata,
240
)
241
hidden_states = self.norm_f(hidden_states)
242
return hidden_states
@@ -267,11 +263,11 @@ class MPTForCausalLM(nn.Module):
263
self,
264
input_ids: torch.Tensor,
265
positions: torch.Tensor,
270
- kv_caches: List[KVCache],
271
- input_metadata: InputMetadata,
266
+ kv_caches: List[torch.Tensor],
267
+ attn_metadata: AttentionMetadata,
268
) -> torch.Tensor:
269
hidden_states = self.transformer(input_ids, positions, kv_caches,
274
- input_metadata)
270
+ attn_metadata)
271
return hidden_states
272
273
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/olmo.py
+13
-17
@@ -42,8 +42,7 @@ import torch
42
import torch.nn.functional as F
43
from torch import nn
44
45
-from vllm.model_executor.input_metadata import InputMetadata
46
-from vllm.model_executor.layers.attention import Attention
45
+from vllm.attention import Attention, AttentionMetadata
46
from vllm.model_executor.layers.linear import (
47
ColumnParallelLinear,
48
LinearMethodBase,
@@ -67,8 +66,6 @@ from vllm.sequence import SamplerOutput
66
# this model must need this dependency
67
from hf_olmo import OLMoConfig
68
70
-KVCache = Tuple[torch.Tensor, torch.Tensor]
71
-
69
70
class SwiGLU(nn.Module):
71
@@ -146,16 +143,15 @@ class OlmoAttention(nn.Module):
143
self,
144
positions: torch.Tensor,
145
hidden_states: torch.Tensor,
149
- kv_cache: KVCache,
150
- input_metadata: InputMetadata,
146
+ kv_cache: torch.Tensor,
147
+ attn_metadata: AttentionMetadata,
148
) -> torch.Tensor:
149
hidden_states = self.attn_norm(hidden_states)
150
qkv, _ = self.att_proj(hidden_states)
151
q, k, v = qkv.chunk(chunks=3, dim=-1)
152
if self.config.rope:
153
q, k = self.rotary_emb(positions, q, k)
157
- k_cache, v_cache = kv_cache
158
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
154
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
155
output, _ = self.attn_out(attn_output)
156
return output
157
@@ -241,12 +237,12 @@ class OlmoBlock(nn.Module):
237
self,
238
positions: torch.Tensor,
239
hidden_states: torch.Tensor,
244
- kv_cache: KVCache,
245
- input_metadata: InputMetadata,
240
+ kv_cache: torch.Tensor,
241
+ attn_metadata: AttentionMetadata,
242
) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
243
# Attention block.
244
og_x = hidden_states
249
- x = self.attn(positions, hidden_states, kv_cache, input_metadata)
245
+ x = self.attn(positions, hidden_states, kv_cache, attn_metadata)
246
x = x + og_x
247
248
# MLP block.
@@ -296,8 +292,8 @@ class OlmoModel(nn.Module):
292
self,
293
input_ids: torch.Tensor,
294
positions: torch.Tensor,
299
- kv_caches: List[KVCache],
300
- input_metadata: InputMetadata,
295
+ kv_caches: List[torch.Tensor],
296
+ attn_metadata: AttentionMetadata,
297
) -> torch.Tensor:
298
"""
299
:param input_ids: A tensor of shape `(batch_size, seq_len)`.
@@ -313,7 +309,7 @@ class OlmoModel(nn.Module):
309
positions,
310
x,
311
kv_caches[block_idx],
316
- input_metadata,
312
+ attn_metadata,
313
)
314
315
# Apply final layer norm.
@@ -344,14 +340,14 @@ class OLMoForCausalLM(nn.Module):
340
self,
341
input_ids: torch.Tensor,
342
positions: torch.Tensor,
347
- kv_caches: List[KVCache],
348
- input_metadata: InputMetadata,
343
+ kv_caches: List[torch.Tensor],
344
+ attn_metadata: AttentionMetadata,
345
) -> torch.Tensor:
346
hidden_states = self.model(
347
input_ids=input_ids,
348
positions=positions,
349
kv_caches=kv_caches,
354
- input_metadata=input_metadata,
350
+ attn_metadata=attn_metadata,
351
)
352
return hidden_states
353
vllm/model_executor/models/opt.py
+17
-22
@@ -17,15 +17,14 @@
17
# See the License for the specific language governing permissions and
18
# limitations under the License.
19
"""Inference-only OPT model compatible with HuggingFace weights."""
20
-from typing import List, Optional, Tuple
20
+from typing import List, Optional
21
22
import torch
23
from torch import nn
24
from transformers import OPTConfig
25
26
-from vllm.model_executor.input_metadata import InputMetadata
26
+from vllm.attention import Attention, AttentionMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import Attention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
@@ -42,8 +41,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
41
hf_model_weights_iterator)
42
from vllm.sequence import SamplerOutput
43
45
-KVCache = Tuple[torch.Tensor, torch.Tensor]
46
-
44
45
class OPTLearnedPositionalEmbedding(nn.Embedding):
46
@@ -97,14 +94,12 @@ class OPTAttention(nn.Module):
94
def forward(
95
self,
96
hidden_states: torch.Tensor,
100
- kv_cache: KVCache,
101
- input_metadata: InputMetadata,
97
+ kv_cache: torch.Tensor,
98
+ attn_metadata: AttentionMetadata,
99
) -> torch.Tensor:
100
qkv, _ = self.qkv_proj(hidden_states)
101
q, k, v = qkv.chunk(chunks=3, dim=-1)
105
- key_cache, value_cache = kv_cache
106
- attn_output = self.attn(q, k, v, key_cache, value_cache,
107
- input_metadata)
102
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
103
output, _ = self.out_proj(attn_output)
104
return output
105
@@ -152,8 +147,8 @@ class OPTDecoderLayer(nn.Module):
147
def forward(
148
self,
149
hidden_states: torch.Tensor,
155
- kv_cache: KVCache,
156
- input_metadata: InputMetadata,
150
+ kv_cache: torch.Tensor,
151
+ attn_metadata: AttentionMetadata,
152
) -> torch.Tensor:
153
# Self Attention
154
residual = hidden_states
@@ -162,7 +157,7 @@ class OPTDecoderLayer(nn.Module):
157
hidden_states = self.self_attn_layer_norm(hidden_states)
158
hidden_states = self.self_attn(hidden_states=hidden_states,
159
kv_cache=kv_cache,
165
- input_metadata=input_metadata)
160
+ attn_metadata=attn_metadata)
161
hidden_states = residual + hidden_states
162
# 350m applies layer norm AFTER attention
163
if not self.do_layer_norm_before:
@@ -241,8 +236,8 @@ class OPTDecoder(nn.Module):
236
self,
237
input_ids: torch.Tensor,
238
positions: torch.Tensor,
244
- kv_caches: List[KVCache],
245
- input_metadata: InputMetadata,
239
+ kv_caches: List[torch.Tensor],
240
+ attn_metadata: AttentionMetadata,
241
) -> torch.Tensor:
242
inputs_embeds = self.embed_tokens(input_ids)
243
pos_embeds = self.embed_positions(positions)
@@ -252,7 +247,7 @@ class OPTDecoder(nn.Module):
247
248
for i in range(len(self.layers)):
249
layer = self.layers[i]
255
- hidden_states = layer(hidden_states, kv_caches[i], input_metadata)
250
+ hidden_states = layer(hidden_states, kv_caches[i], attn_metadata)
251
252
if self.final_layer_norm is not None:
253
hidden_states = self.final_layer_norm(hidden_states)
@@ -275,10 +270,10 @@ class OPTModel(nn.Module):
270
self,
271
input_ids: torch.Tensor,
272
positions: torch.Tensor,
278
- kv_caches: List[KVCache],
279
- input_metadata: InputMetadata,
273
+ kv_caches: List[torch.Tensor],
274
+ attn_metadata: AttentionMetadata,
275
) -> torch.Tensor:
281
- return self.decoder(input_ids, positions, kv_caches, input_metadata)
276
+ return self.decoder(input_ids, positions, kv_caches, attn_metadata)
277
278
279
class OPTForCausalLM(nn.Module):
@@ -300,11 +295,11 @@ class OPTForCausalLM(nn.Module):
295
self,
296
input_ids: torch.Tensor,
297
positions: torch.Tensor,
303
- kv_caches: List[KVCache],
304
- input_metadata: InputMetadata,
298
+ kv_caches: List[torch.Tensor],
299
+ attn_metadata: AttentionMetadata,
300
) -> torch.Tensor:
301
hidden_states = self.model(input_ids, positions, kv_caches,
307
- input_metadata)
302
+ attn_metadata)
303
return hidden_states
304
305
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/orion.py
+13
-17
@@ -10,9 +10,8 @@ import torch
10
from torch import nn
11
from transformers import PretrainedConfig
12
13
-from vllm.model_executor.input_metadata import InputMetadata
13
+from vllm.attention import Attention, AttentionMetadata
14
from vllm.model_executor.layers.activation import SiluAndMul
15
-from vllm.model_executor.layers.attention import Attention
15
from vllm.model_executor.layers.linear import (LinearMethodBase,
16
MergedColumnParallelLinear,
17
QKVParallelLinear,
@@ -29,8 +28,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
28
hf_model_weights_iterator)
29
from vllm.sequence import SamplerOutput
30
32
-KVCache = Tuple[torch.Tensor, torch.Tensor]
33
-
31
32
class OrionMLP(nn.Module):
33
@@ -128,14 +125,13 @@ class OrionAttention(nn.Module):
125
self,
126
positions: torch.Tensor,
127
hidden_states: torch.Tensor,
131
- kv_cache: KVCache,
132
- input_metadata: InputMetadata,
128
+ kv_cache: torch.Tensor,
129
+ attn_metadata: AttentionMetadata,
130
) -> torch.Tensor:
131
qkv, _ = self.qkv_proj(hidden_states)
132
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
133
q, k = self.rotary_emb(positions, q, k)
137
- k_cache, v_cache = kv_cache
138
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
135
output, _ = self.o_proj(attn_output)
136
return output
137
@@ -178,8 +174,8 @@ class OrionDecoderLayer(nn.Module):
174
self,
175
positions: torch.Tensor,
176
hidden_states: torch.Tensor,
181
- kv_cache: KVCache,
182
- input_metadata: InputMetadata,
177
+ kv_cache: torch.Tensor,
178
+ attn_metadata: AttentionMetadata,
179
residual: Optional[torch.Tensor],
180
) -> Tuple[torch.Tensor, torch.Tensor]:
181
# Self Attention
@@ -189,7 +185,7 @@ class OrionDecoderLayer(nn.Module):
185
positions=positions,
186
hidden_states=hidden_states,
187
kv_cache=kv_cache,
192
- input_metadata=input_metadata,
188
+ attn_metadata=attn_metadata,
189
)
190
191
hidden_states = residual + hidden_states
@@ -227,8 +223,8 @@ class OrionModel(nn.Module):
223
self,
224
input_ids: torch.Tensor,
225
positions: torch.Tensor,
230
- kv_caches: List[KVCache],
231
- input_metadata: InputMetadata,
226
+ kv_caches: List[torch.Tensor],
227
+ attn_metadata: AttentionMetadata,
228
) -> torch.Tensor:
229
hidden_states = self.embed_tokens(input_ids)
230
residual = None
@@ -238,7 +234,7 @@ class OrionModel(nn.Module):
234
positions,
235
hidden_states,
236
kv_caches[i],
241
- input_metadata,
237
+ attn_metadata,
238
residual,
239
)
240
hidden_states = self.norm(hidden_states)
@@ -264,11 +260,11 @@ class OrionForCausalLM(nn.Module):
260
self,
261
input_ids: torch.Tensor,
262
positions: torch.Tensor,
267
- kv_caches: List[KVCache],
268
- input_metadata: InputMetadata,
263
+ kv_caches: List[torch.Tensor],
264
+ attn_metadata: AttentionMetadata,
265
) -> torch.Tensor:
266
hidden_states = self.model(input_ids, positions, kv_caches,
271
- input_metadata)
267
+ attn_metadata)
268
return hidden_states
269
270
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/phi.py
+14
-18
@@ -35,15 +35,14 @@
35
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37
"""Inference-only Phi-1.5 model compatible with HuggingFace weights."""
38
-from typing import List, Optional, Tuple
38
+from typing import List, Optional
39
40
import torch
41
from torch import nn
42
from transformers import PretrainedConfig
43
44
-from vllm.model_executor.input_metadata import InputMetadata
44
+from vllm.attention import Attention, AttentionMetadata
45
from vllm.model_executor.layers.activation import get_act_fn
46
-from vllm.model_executor.layers.attention import Attention
46
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
47
LinearMethodBase,
48
QKVParallelLinear,
@@ -60,8 +59,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
59
hf_model_weights_iterator)
60
from vllm.sequence import SamplerOutput
61
63
-KVCache = Tuple[torch.Tensor, torch.Tensor]
64
-
62
63
class PhiAttention(nn.Module):
64
@@ -115,14 +112,13 @@ class PhiAttention(nn.Module):
112
self,
113
position_ids: torch.Tensor,
114
hidden_states: torch.Tensor,
118
- kv_cache: KVCache,
119
- input_metadata: InputMetadata,
115
+ kv_cache: torch.Tensor,
116
+ attn_metadata: AttentionMetadata,
117
) -> torch.Tensor:
118
qkv, _ = self.qkv_proj(hidden_states)
119
q, k, v = qkv.chunk(chunks=3, dim=-1)
120
q, k = self.rotary_emb(position_ids, q, k)
124
- k_cache, v_cache = kv_cache
125
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
121
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
122
output, _ = self.dense(attn_output)
123
return output
124
@@ -172,8 +168,8 @@ class PhiLayer(nn.Module):
168
self,
169
position_ids: torch.Tensor,
170
hidden_states: torch.Tensor,
175
- kv_cache: KVCache,
176
- input_metadata: InputMetadata,
171
+ kv_cache: torch.Tensor,
172
+ attn_metadata: AttentionMetadata,
173
) -> torch.Tensor:
174
residual = hidden_states
175
hidden_states = self.input_layernorm(hidden_states)
@@ -181,7 +177,7 @@ class PhiLayer(nn.Module):
177
position_ids=position_ids,
178
hidden_states=hidden_states,
179
kv_cache=kv_cache,
184
- input_metadata=input_metadata,
180
+ attn_metadata=attn_metadata,
181
)
182
feed_forward_hidden_states = self.mlp(hidden_states)
183
hidden_states = attn_outputs + feed_forward_hidden_states + residual
@@ -209,8 +205,8 @@ class PhiModel(nn.Module):
205
self,
206
input_ids: torch.Tensor,
207
positions: torch.Tensor,
212
- kv_caches: List[KVCache],
213
- input_metadata: InputMetadata,
208
+ kv_caches: List[torch.Tensor],
209
+ attn_metadata: AttentionMetadata,
210
) -> torch.Tensor:
211
hidden_states = self.embed_tokens(input_ids)
212
for i in range(self.config.num_hidden_layers):
@@ -219,7 +215,7 @@ class PhiModel(nn.Module):
215
positions,
216
hidden_states,
217
kv_caches[i],
222
- input_metadata,
218
+ attn_metadata,
219
)
220
221
hidden_states = self.final_layernorm(hidden_states)
@@ -248,11 +244,11 @@ class PhiForCausalLM(nn.Module):
244
self,
245
input_ids: torch.Tensor,
246
positions: torch.Tensor,
251
- kv_caches: List[KVCache],
252
- input_metadata: InputMetadata,
247
+ kv_caches: List[torch.Tensor],
248
+ attn_metadata: AttentionMetadata,
249
) -> torch.Tensor:
250
hidden_states = self.model(input_ids, positions, kv_caches,
255
- input_metadata)
251
+ attn_metadata)
252
253
return hidden_states
254
vllm/model_executor/models/qwen.py
+13
-18
@@ -10,9 +10,8 @@ import torch
10
from torch import nn
11
from transformers import PretrainedConfig
12
13
-from vllm.model_executor.input_metadata import InputMetadata
13
+from vllm.attention import Attention, AttentionMetadata
14
from vllm.model_executor.layers.activation import SiluAndMul
15
-from vllm.model_executor.layers.attention import Attention
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (LinearMethodBase,
17
MergedColumnParallelLinear,
@@ -30,8 +29,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
29
hf_model_weights_iterator)
30
from vllm.sequence import SamplerOutput
31
33
-KVCache = Tuple[torch.Tensor, torch.Tensor]
34
-
32
33
class QWenMLP(nn.Module):
34
@@ -111,15 +108,13 @@ class QWenAttention(nn.Module):
108
self,
109
positions: torch.Tensor,
110
hidden_states: torch.Tensor,
114
- kv_cache: KVCache,
115
- input_metadata: InputMetadata,
111
+ kv_cache: torch.Tensor,
112
+ attn_metadata: AttentionMetadata,
113
) -> torch.Tensor:
114
qkv, _ = self.c_attn(hidden_states)
115
q, k, v = qkv.chunk(chunks=3, dim=-1)
116
q, k = self.rotary_emb(positions, q, k)
120
- k_cache, v_cache = kv_cache
121
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
122
-
117
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
118
output, _ = self.c_proj(attn_output)
119
return output
120
@@ -153,8 +148,8 @@ class QWenBlock(nn.Module):
148
self,
149
positions: torch.Tensor,
150
hidden_states: torch.Tensor,
156
- kv_cache: KVCache,
157
- input_metadata: InputMetadata,
151
+ kv_cache: torch.Tensor,
152
+ attn_metadata: AttentionMetadata,
153
residual: Optional[torch.Tensor],
154
) -> Tuple[torch.Tensor, torch.Tensor]:
155
# Self Attention
@@ -167,7 +162,7 @@ class QWenBlock(nn.Module):
162
positions=positions,
163
hidden_states=hidden_states,
164
kv_cache=kv_cache,
170
- input_metadata=input_metadata,
165
+ attn_metadata=attn_metadata,
166
)
167
168
# Fully Connected
@@ -201,8 +196,8 @@ class QWenModel(nn.Module):
196
self,
197
input_ids: torch.Tensor,
198
positions: torch.Tensor,
204
- kv_caches: List[KVCache],
205
- input_metadata: InputMetadata,
199
+ kv_caches: List[torch.Tensor],
200
+ attn_metadata: AttentionMetadata,
201
) -> torch.Tensor:
202
hidden_states = self.wte(input_ids)
203
residual = None
@@ -212,7 +207,7 @@ class QWenModel(nn.Module):
207
positions,
208
hidden_states,
209
kv_caches[i],
215
- input_metadata,
210
+ attn_metadata,
211
residual,
212
)
213
hidden_states, _ = self.ln_f(hidden_states, residual)
@@ -238,11 +233,11 @@ class QWenLMHeadModel(nn.Module):
233
self,
234
input_ids: torch.Tensor,
235
positions: torch.Tensor,
241
- kv_caches: List[KVCache],
242
- input_metadata: InputMetadata,
236
+ kv_caches: List[torch.Tensor],
237
+ attn_metadata: AttentionMetadata,
238
) -> torch.Tensor:
239
hidden_states = self.transformer(input_ids, positions, kv_caches,
245
- input_metadata)
240
+ attn_metadata)
241
return hidden_states
242
243
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/qwen2.py
+13
-17
@@ -28,9 +28,8 @@ import torch
28
from torch import nn
29
from transformers import Qwen2Config
30
31
-from vllm.model_executor.input_metadata import InputMetadata
31
+from vllm.attention import Attention, AttentionMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.attention import Attention
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
@@ -49,8 +48,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
48
from vllm.sequence import SamplerOutput
49
from vllm.config import LoRAConfig
50
52
-KVCache = Tuple[torch.Tensor, torch.Tensor]
53
-
51
52
class Qwen2MLP(nn.Module):
53
@@ -147,14 +144,13 @@ class Qwen2Attention(nn.Module):
144
self,
145
positions: torch.Tensor,
146
hidden_states: torch.Tensor,
150
- kv_cache: KVCache,
151
- input_metadata: InputMetadata,
147
+ kv_cache: torch.Tensor,
148
+ attn_metadata: AttentionMetadata,
149
) -> torch.Tensor:
150
qkv, _ = self.qkv_proj(hidden_states)
151
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
q, k = self.rotary_emb(positions, q, k)
156
- k_cache, v_cache = kv_cache
157
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
153
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
154
output, _ = self.o_proj(attn_output)
155
return output
156
@@ -197,8 +193,8 @@ class Qwen2DecoderLayer(nn.Module):
193
self,
194
positions: torch.Tensor,
195
hidden_states: torch.Tensor,
200
- kv_cache: KVCache,
201
- input_metadata: InputMetadata,
196
+ kv_cache: torch.Tensor,
197
+ attn_metadata: AttentionMetadata,
198
residual: Optional[torch.Tensor],
199
) -> Tuple[torch.Tensor, torch.Tensor]:
200
# Self Attention
@@ -212,7 +208,7 @@ class Qwen2DecoderLayer(nn.Module):
208
positions=positions,
209
hidden_states=hidden_states,
210
kv_cache=kv_cache,
215
- input_metadata=input_metadata,
211
+ attn_metadata=attn_metadata,
212
)
213
214
# Fully Connected
@@ -248,8 +244,8 @@ class Qwen2Model(nn.Module):
244
self,
245
input_ids: torch.Tensor,
246
positions: torch.Tensor,
251
- kv_caches: List[KVCache],
252
- input_metadata: InputMetadata,
247
+ kv_caches: List[torch.Tensor],
248
+ attn_metadata: AttentionMetadata,
249
) -> torch.Tensor:
250
hidden_states = self.embed_tokens(input_ids)
251
residual = None
@@ -259,7 +255,7 @@ class Qwen2Model(nn.Module):
255
positions,
256
hidden_states,
257
kv_caches[i],
262
- input_metadata,
258
+ attn_metadata,
259
residual,
260
)
261
hidden_states, _ = self.norm(hidden_states, residual)
@@ -315,11 +311,11 @@ class Qwen2ForCausalLM(nn.Module):
311
self,
312
input_ids: torch.Tensor,
313
positions: torch.Tensor,
318
- kv_caches: List[KVCache],
319
- input_metadata: InputMetadata,
314
+ kv_caches: List[torch.Tensor],
315
+ attn_metadata: AttentionMetadata,
316
) -> torch.Tensor:
317
hidden_states = self.model(input_ids, positions, kv_caches,
322
- input_metadata)
318
+ attn_metadata)
319
return hidden_states
320
321
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/stablelm.py
+13
-17
@@ -25,9 +25,8 @@ import torch
25
from torch import nn
26
from transformers import PretrainedConfig
27
28
-from vllm.model_executor.input_metadata import InputMetadata
28
+from vllm.attention import Attention, AttentionMetadata
29
from vllm.model_executor.layers.activation import SiluAndMul
30
-from vllm.model_executor.layers.attention import Attention
30
from vllm.model_executor.layers.linear import (LinearMethodBase,
31
MergedColumnParallelLinear,
32
QKVParallelLinear,
@@ -44,8 +43,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
43
hf_model_weights_iterator)
44
from vllm.sequence import SamplerOutput
45
47
-KVCache = Tuple[torch.Tensor, torch.Tensor]
48
-
46
47
class StablelmMLP(nn.Module):
48
@@ -134,14 +131,13 @@ class StablelmAttention(nn.Module):
131
self,
132
positions: torch.Tensor,
133
hidden_states: torch.Tensor,
137
- kv_cache: KVCache,
138
- input_metadata: InputMetadata,
134
+ kv_cache: torch.Tensor,
135
+ attn_metadata: AttentionMetadata,
136
) -> torch.Tensor:
137
qkv, _ = self.qkv_proj(hidden_states)
138
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
139
q, k = self.rotary_emb(positions, q, k)
143
- k_cache, v_cache = kv_cache
144
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
140
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
141
output, _ = self.o_proj(attn_output)
142
return output
143
@@ -166,8 +162,8 @@ class StablelmDecoderLayer(nn.Module):
162
self,
163
positions: torch.Tensor,
164
hidden_states: torch.Tensor,
169
- kv_cache: KVCache,
170
- input_metadata: InputMetadata,
165
+ kv_cache: torch.Tensor,
166
+ attn_metadata: AttentionMetadata,
167
) -> Tuple[torch.Tensor, torch.Tensor]:
168
# Self Attention
169
residual = hidden_states
@@ -176,7 +172,7 @@ class StablelmDecoderLayer(nn.Module):
172
positions=positions,
173
hidden_states=hidden_states,
174
kv_cache=kv_cache,
179
- input_metadata=input_metadata,
175
+ attn_metadata=attn_metadata,
176
)
177
hidden_states = residual + hidden_states
178
@@ -211,8 +207,8 @@ class StableLMEpochModel(nn.Module):
207
self,
208
input_ids: torch.Tensor,
209
positions: torch.Tensor,
214
- kv_caches: List[KVCache],
215
- input_metadata: InputMetadata,
210
+ kv_caches: List[torch.Tensor],
211
+ attn_metadata: AttentionMetadata,
212
) -> torch.Tensor:
213
hidden_states = self.embed_tokens(input_ids)
214
for i in range(len(self.layers)):
@@ -221,7 +217,7 @@ class StableLMEpochModel(nn.Module):
217
positions,
218
hidden_states,
219
kv_caches[i],
224
- input_metadata,
220
+ attn_metadata,
221
)
222
hidden_states = self.norm(hidden_states)
223
return hidden_states
@@ -246,11 +242,11 @@ class StablelmForCausalLM(nn.Module):
242
self,
243
input_ids: torch.Tensor,
244
positions: torch.Tensor,
249
- kv_caches: List[KVCache],
250
- input_metadata: InputMetadata,
245
+ kv_caches: List[torch.Tensor],
246
+ attn_metadata: AttentionMetadata,
247
) -> torch.Tensor:
248
hidden_states = self.model(input_ids, positions, kv_caches,
253
- input_metadata)
249
+ attn_metadata)
250
return hidden_states
251
252
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/starcoder2.py
+14
-18
@@ -18,15 +18,14 @@
18
# See the License for the specific language governing permissions and
19
# limitations under the License.
20
""" PyTorch Starcoder2 model."""
21
-from typing import List, Optional, Tuple
21
+from typing import List, Optional
22
23
import torch
24
from torch import nn
25
from transformers import Starcoder2Config
26
27
-from vllm.model_executor.input_metadata import InputMetadata
27
+from vllm.attention import Attention, AttentionMetadata
28
from vllm.model_executor.sampling_metadata import SamplingMetadata
29
-from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.rotary_embedding import get_rope
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -43,8 +42,6 @@ from vllm.model_executor.weight_utils import (default_weight_loader,
42
hf_model_weights_iterator)
43
from vllm.sequence import SamplerOutput
44
46
-KVCache = Tuple[torch.Tensor, torch.Tensor]
47
-
45
46
class Starcoder2Attention(nn.Module):
47
@@ -111,14 +108,13 @@ class Starcoder2Attention(nn.Module):
108
self,
109
positions: torch.Tensor,
110
hidden_states: torch.Tensor,
114
- kv_cache: KVCache,
115
- input_metadata: InputMetadata,
111
+ kv_cache: torch.Tensor,
112
+ attn_metadata: AttentionMetadata,
113
) -> torch.Tensor:
114
qkv, _ = self.qkv_proj(hidden_states)
115
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
116
q, k = self.rotary_emb(positions, q, k)
120
- k_cache, v_cache = kv_cache
121
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
117
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
118
output, _ = self.o_proj(attn_output)
119
return output
120
@@ -171,8 +167,8 @@ class Starcoder2DecoderLayer(nn.Module):
167
self,
168
positions: torch.Tensor,
169
hidden_states: torch.Tensor,
174
- kv_cache: KVCache,
175
- input_metadata: InputMetadata,
170
+ kv_cache: torch.Tensor,
171
+ attn_metadata: AttentionMetadata,
172
) -> torch.Tensor:
173
# Self Attention
174
residual = hidden_states
@@ -181,7 +177,7 @@ class Starcoder2DecoderLayer(nn.Module):
177
positions=positions,
178
hidden_states=hidden_states,
179
kv_cache=kv_cache,
184
- input_metadata=input_metadata,
180
+ attn_metadata=attn_metadata,
181
)
182
hidden_states = residual + hidden_states
183
@@ -217,14 +213,14 @@ class Starcoder2Model(nn.Module):
213
self,
214
input_ids: torch.Tensor,
215
positions: torch.Tensor,
220
- kv_caches: List[KVCache],
221
- input_metadata: InputMetadata,
216
+ kv_caches: List[torch.Tensor],
217
+ attn_metadata: AttentionMetadata,
218
) -> torch.Tensor:
219
hidden_states = self.embed_tokens(input_ids)
220
for i in range(len(self.layers)):
221
layer = self.layers[i]
222
hidden_states = layer(positions, hidden_states, kv_caches[i],
227
- input_metadata)
223
+ attn_metadata)
224
hidden_states = self.norm(hidden_states)
225
return hidden_states
226
@@ -258,11 +254,11 @@ class Starcoder2ForCausalLM(nn.Module):
254
self,
255
input_ids: torch.Tensor,
256
positions: torch.Tensor,
261
- kv_caches: List[KVCache],
262
- input_metadata: InputMetadata,
257
+ kv_caches: List[torch.Tensor],
258
+ attn_metadata: AttentionMetadata,
259
) -> torch.Tensor:
260
hidden_states = self.model(input_ids, positions, kv_caches,
265
- input_metadata)
261
+ attn_metadata)
262
return hidden_states
263
264
def compute_logits(self, hidden_states: torch.Tensor,
vllm/sequence.py
+1
-1
@@ -431,7 +431,7 @@ class SequenceGroup:
431
432
433
class SequenceGroupMetadata:
434
- """Metadata for a sequence group. Used to create `InputMetadata`.
434
+ """Metadata for a sequence group. Used to create `AttentionMetadata`.
435
436
Args:
437
request_id: The ID of the request.
vllm/worker/cache_engine.py
+31
-84
@@ -1,16 +1,15 @@
1
"""CacheEngine class for managing the KV cache."""
2
-from typing import Dict, List, Tuple
2
+from typing import Dict, List
3
4
import torch
5
6
+from vllm.attention import get_attn_backend
7
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
from vllm.utils import is_pin_memory_available, STR_DTYPE_TO_TORCH_DTYPE
10
11
logger = init_logger(__name__)
12
12
-KVCache = Tuple[torch.Tensor, torch.Tensor]
13
-
13
14
class CacheEngine:
15
"""Manages the KV cache.
@@ -43,95 +42,43 @@ class CacheEngine:
42
else:
43
self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
44
45
+ # Get attention backend.
46
+ self.attn_backend = get_attn_backend(model_config.dtype)
47
+
48
# Initialize the cache.
47
- self.gpu_cache = self.allocate_gpu_cache()
48
- self.cpu_cache = self.allocate_cpu_cache()
49
-
50
- def get_key_block_shape(self) -> Tuple[int, int, int, int]:
51
- element_size = torch.tensor([], dtype=self.dtype).element_size()
52
- x = 16 // element_size
53
- return (
54
- self.num_heads,
55
- self.head_size // x,
56
- self.block_size,
57
- x,
58
- )
59
-
60
- def get_value_block_shape(self) -> Tuple[int, int, int]:
61
- return (
62
- self.num_heads,
63
- self.head_size,
64
- self.block_size,
65
- )
66
-
67
- def allocate_gpu_cache(self) -> List[KVCache]:
68
- gpu_cache: List[KVCache] = []
69
- key_block_shape = self.get_key_block_shape()
70
- value_block_shape = self.get_value_block_shape()
71
- for _ in range(self.num_layers):
72
- key_blocks = torch.empty(
73
- size=(self.num_gpu_blocks, *key_block_shape),
74
- dtype=self.dtype,
75
- device="cuda",
76
- )
77
- value_blocks = torch.empty(
78
- size=(self.num_gpu_blocks, *value_block_shape),
79
- dtype=self.dtype,
80
- device="cuda",
81
- )
82
- gpu_cache.append((key_blocks, value_blocks))
83
- return gpu_cache
84
-
85
- def allocate_cpu_cache(self) -> List[KVCache]:
86
- cpu_cache: List[KVCache] = []
87
- key_block_shape = self.get_key_block_shape()
88
- value_block_shape = self.get_value_block_shape()
89
- pin_memory = is_pin_memory_available()
90
- for _ in range(self.num_layers):
91
- key_blocks = torch.empty(
92
- size=(self.num_cpu_blocks, *key_block_shape),
93
- dtype=self.dtype,
94
- pin_memory=pin_memory,
95
- device="cpu",
96
- )
97
- value_blocks = torch.empty(
98
- size=(self.num_cpu_blocks, *value_block_shape),
99
- dtype=self.dtype,
100
- pin_memory=pin_memory,
101
- device="cpu",
102
- )
103
- cpu_cache.append((key_blocks, value_blocks))
104
- return cpu_cache
105
-
106
- def _swap(
107
- self,
108
- src: List[KVCache],
109
- dst: List[KVCache],
110
- src_to_dst: Dict[int, int],
111
- ) -> None:
112
- from vllm._C import cache_ops
49
+ self.gpu_cache = self._allocate_kv_cache(self.num_gpu_blocks, "cuda")
50
+ self.cpu_cache = self._allocate_kv_cache(self.num_cpu_blocks, "cpu")
51
114
- for i in range(self.num_layers):
115
- src_key_cache, src_value_cache = src[i]
116
- dst_key_cache, dst_value_cache = dst[i]
117
- # Copy the key blocks.
118
- cache_ops.swap_blocks(src_key_cache, dst_key_cache, src_to_dst)
119
- # Copy the value blocks.
120
- cache_ops.swap_blocks(src_value_cache, dst_value_cache, src_to_dst)
52
+ def _allocate_kv_cache(
53
+ self,
54
+ num_blocks: int,
55
+ device: str,
56
+ ) -> List[torch.Tensor]:
57
+ """Allocates KV cache on the specified device."""
58
+ kv_cache_shape = self.attn_backend.get_kv_cache_shape(
59
+ num_blocks, self.block_size, self.num_heads, self.head_size)
60
+ pin_memory = is_pin_memory_available() if device == "cpu" else False
61
+ kv_cache: List[torch.Tensor] = []
62
+ for _ in range(self.num_layers):
63
+ kv_cache.append(
64
+ torch.empty(kv_cache_shape,
65
+ dtype=self.dtype,
66
+ pin_memory=pin_memory,
67
+ device=device))
68
+ return kv_cache
69
70
def swap_in(self, src_to_dst: Dict[int, int]) -> None:
123
- self._swap(self.cpu_cache, self.gpu_cache, src_to_dst)
71
+ for i in range(self.num_layers):
72
+ self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
73
+ src_to_dst)
74
75
def swap_out(self, src_to_dst: Dict[int, int]) -> None:
126
- self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
76
+ for i in range(self.num_layers):
77
+ self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
78
+ src_to_dst)
79
80
def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
129
- from vllm._C import cache_ops
130
-
131
- key_caches = [key_cache for key_cache, _ in self.gpu_cache]
132
- value_caches = [value_cache for _, value_cache in self.gpu_cache]
133
- # NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
134
- cache_ops.copy_blocks(key_caches, value_caches, src_to_dsts)
81
+ self.attn_backend.copy_blocks(self.gpu_cache, src_to_dsts)
82
83
@staticmethod
84
def get_cache_block_size(
vllm/worker/model_runner.py
+44
-41
@@ -6,10 +6,11 @@ import numpy as np
6
import torch
7
import torch.nn as nn
8
9
+from vllm.attention import AttentionMetadata, get_attn_backend
10
from vllm.config import (DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig,
11
SchedulerConfig)
12
from vllm.logger import init_logger
12
-from vllm.model_executor import InputMetadata, SamplingMetadata
13
+from vllm.model_executor import SamplingMetadata
14
from vllm.model_executor.model_loader import get_model
15
from vllm.model_executor.parallel_utils import cupy_utils
16
from vllm.model_executor.parallel_utils.communication_op import (
@@ -28,7 +29,6 @@ from vllm.utils import (async_tensor_h2d, CudaMemoryProfiler,
29
30
logger = init_logger(__name__)
31
31
-KVCache = Tuple[torch.Tensor, torch.Tensor]
32
_PAD_SLOT_ID = -1
33
LORA_WARMUP_RANK = 8
34
_BATCH_SIZE_ALIGNMENT = 8
@@ -85,6 +85,9 @@ class ModelRunner:
85
self.pin_memory = is_pin_memory_available()
86
self.kv_cache_dtype = kv_cache_dtype
87
88
+ self.attn_backend = get_attn_backend(
89
+ self.model_config.dtype if model_config is not None else None)
90
+
91
def load_model(self) -> None:
92
with CudaMemoryProfiler() as m:
93
self.model = get_model(self.model_config,
@@ -127,8 +130,8 @@ class ModelRunner:
130
def _prepare_prompt(
131
self,
132
seq_group_metadata_list: List[SequenceGroupMetadata],
130
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
131
- List[int], List[int], Set[LoRARequest]]:
133
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
134
+ List[int], List[int], List[int], Set[LoRARequest]]:
135
assert len(seq_group_metadata_list) > 0
136
input_tokens: List[int] = []
137
input_positions: List[int] = []
@@ -216,7 +219,7 @@ class ModelRunner:
219
slot_mapping.append(slot)
220
221
max_subquery_len = max(subquery_lens)
219
- max_seq_len = max(prompt_lens)
222
+ max_prompt_len = max(prompt_lens)
223
num_prompt_tokens = len(input_tokens)
224
assert max_subquery_len > 0
225
@@ -270,7 +273,7 @@ class ModelRunner:
273
dtype=seq_start_loc.dtype,
274
out=seq_start_loc[1:])
275
273
- input_metadata = InputMetadata(
276
+ attn_metadata = self.attn_backend.make_metadata(
277
is_prompt=True,
278
slot_mapping=slot_mapping,
279
prompt_lens=prompt_lens,
@@ -279,7 +282,7 @@ class ModelRunner:
282
num_generation_tokens=0,
283
max_subquery_len=max_subquery_len,
284
max_context_len=None,
282
- max_seq_len=max_seq_len,
285
+ max_prompt_len=max_prompt_len,
286
subquery_start_loc=subquery_start_loc,
287
seq_start_loc=seq_start_loc,
288
context_lens=context_lens_tensor,
@@ -287,15 +290,15 @@ class ModelRunner:
290
use_cuda_graph=False,
291
kv_cache_dtype=self.kv_cache_dtype,
292
)
290
- return (input_tokens, input_positions, input_metadata, prompt_lens,
293
+ return (input_tokens, input_positions, attn_metadata, prompt_lens,
294
subquery_lens, lora_index_mapping, lora_prompt_mapping,
295
lora_requests)
296
297
def _prepare_decode(
298
self,
299
seq_group_metadata_list: List[SequenceGroupMetadata],
297
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, List[int], List[int],
298
- Set[LoRARequest]]:
300
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, List[int],
301
+ List[int], Set[LoRARequest]]:
302
assert len(seq_group_metadata_list) > 0
303
input_tokens: List[int] = []
304
input_positions: List[int] = []
@@ -401,7 +404,7 @@ class ModelRunner:
404
device=self.device,
405
)
406
404
- input_metadata = InputMetadata(
407
+ attn_metadata = self.attn_backend.make_metadata(
408
is_prompt=False,
409
slot_mapping=slot_mapping,
410
prompt_lens=None,
@@ -410,7 +413,7 @@ class ModelRunner:
413
num_generation_tokens=len(input_tokens),
414
max_subquery_len=None,
415
max_context_len=max_context_len,
413
- max_seq_len=None,
416
+ max_prompt_len=None,
417
subquery_start_loc=None,
418
seq_start_loc=None,
419
context_lens=context_lens,
@@ -418,7 +421,7 @@ class ModelRunner:
421
use_cuda_graph=use_captured_graph,
422
kv_cache_dtype=self.kv_cache_dtype,
423
)
421
- return (input_tokens, input_positions, input_metadata,
424
+ return (input_tokens, input_positions, attn_metadata,
425
lora_index_mapping, lora_prompt_mapping, lora_requests)
426
427
def _prepare_sample(
@@ -522,7 +525,7 @@ class ModelRunner:
525
def prepare_input_tensors(
526
self,
527
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
525
- ) -> Tuple[torch.Tensor, torch.Tensor, InputMetadata, SamplingMetadata,
528
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, SamplingMetadata,
529
Set[int], LoRAMapping]:
530
if self.is_driver_worker:
531
# NOTE: We assume that all sequences in the group are all prompts or
@@ -530,11 +533,11 @@ class ModelRunner:
533
is_prompt = seq_group_metadata_list[0].is_prompt
534
# Prepare input tensors.
535
if is_prompt:
533
- (input_tokens, input_positions, input_metadata, prompt_lens,
536
+ (input_tokens, input_positions, attn_metadata, prompt_lens,
537
subquery_lens, lora_index_mapping, lora_prompt_mapping,
538
lora_requests) = self._prepare_prompt(seq_group_metadata_list)
539
else:
537
- (input_tokens, input_positions, input_metadata,
540
+ (input_tokens, input_positions, attn_metadata,
541
lora_index_mapping, lora_prompt_mapping,
542
lora_requests) = self._prepare_decode(seq_group_metadata_list)
543
prompt_lens = []
@@ -560,7 +563,7 @@ class ModelRunner:
563
"lora_requests": lora_requests,
564
"lora_mapping": lora_mapping,
565
}
563
- metadata_dict.update(input_metadata.asdict_zerocopy())
566
+ metadata_dict.update(attn_metadata.asdict_zerocopy())
567
broadcast_tensor_dict(metadata_dict, src=0)
568
else:
569
metadata_dict = broadcast_tensor_dict(src=0)
@@ -570,7 +573,7 @@ class ModelRunner:
573
"selected_token_indices")
574
lora_mapping = metadata_dict.pop("lora_mapping")
575
lora_requests = metadata_dict.pop("lora_requests")
573
- input_metadata = InputMetadata(**metadata_dict)
576
+ attn_metadata = self.attn_backend.make_metadata(**metadata_dict)
577
sampling_metadata = SamplingMetadata(
578
seq_groups=None,
579
seq_data=None,
@@ -581,16 +584,16 @@ class ModelRunner:
584
perform_sampling=False,
585
)
586
584
- return (input_tokens, input_positions, input_metadata,
587
+ return (input_tokens, input_positions, attn_metadata,
588
sampling_metadata, lora_requests, lora_mapping)
589
590
@torch.inference_mode()
591
def execute_model(
592
self,
593
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
591
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
594
+ kv_caches: List[torch.Tensor],
595
) -> Optional[SamplerOutput]:
593
- (input_tokens, input_positions, input_metadata, sampling_metadata,
596
+ (input_tokens, input_positions, attn_metadata, sampling_metadata,
597
lora_requests,
598
lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
599
@@ -598,7 +601,7 @@ class ModelRunner:
601
self.set_active_loras(lora_requests, lora_mapping)
602
603
# Execute the model.
601
- if input_metadata.use_cuda_graph:
604
+ if attn_metadata.use_cuda_graph:
605
graph_batch_size = input_tokens.shape[0]
606
model_executable = self.graph_runners[graph_batch_size]
607
else:
@@ -607,7 +610,7 @@ class ModelRunner:
610
input_ids=input_tokens,
611
positions=input_positions,
612
kv_caches=kv_caches,
610
- input_metadata=input_metadata,
613
+ attn_metadata=attn_metadata,
614
)
615
616
# Compute the logits.
@@ -673,7 +676,7 @@ class ModelRunner:
676
677
# Run the model with the dummy inputs.
678
num_layers = self.model_config.get_num_layers(self.parallel_config)
676
- kv_caches = [(None, None)] * num_layers
679
+ kv_caches = [None] * num_layers
680
self.execute_model(seqs, kv_caches)
681
torch.cuda.synchronize()
682
return
@@ -705,7 +708,7 @@ class ModelRunner:
708
return self.lora_manager.list_loras()
709
710
@torch.inference_mode()
708
- def capture_model(self, kv_caches: List[KVCache]) -> None:
711
+ def capture_model(self, kv_caches: List[torch.Tensor]) -> None:
712
"""Cuda graph capture a model.
713
714
Note that CUDA graph's performance gain is negligible if number
@@ -759,8 +762,8 @@ class ModelRunner:
762
# NOTE: Capturing the largest batch size first may help reduce the
763
# memory usage of CUDA graph.
764
for batch_size in reversed(batch_size_capture_list):
762
- # Create dummy input_metadata.
763
- input_metadata = InputMetadata(
765
+ # Create dummy attn_metadata.
766
+ attn_metadata = self.attn_backend.make_metadata(
767
is_prompt=False,
768
slot_mapping=slot_mapping[:batch_size],
769
prompt_lens=None,
@@ -769,7 +772,7 @@ class ModelRunner:
772
num_generation_tokens=batch_size,
773
max_subquery_len=None,
774
max_context_len=self.max_context_len_to_capture,
772
- max_seq_len=None,
775
+ max_prompt_len=None,
776
subquery_start_loc=None,
777
seq_start_loc=None,
778
context_lens=context_lens[:batch_size],
@@ -790,7 +793,7 @@ class ModelRunner:
793
input_tokens[:batch_size],
794
input_positions[:batch_size],
795
kv_caches,
793
- input_metadata,
796
+ attn_metadata,
797
memory_pool=self.graph_memory_pool,
798
)
799
self.graph_memory_pool = graph_runner.graph.pool()
@@ -826,8 +829,8 @@ class CUDAGraphRunner:
829
self,
830
input_ids: torch.Tensor,
831
positions: torch.Tensor,
829
- kv_caches: List[KVCache],
830
- input_metadata: InputMetadata,
832
+ kv_caches: List[torch.Tensor],
833
+ attn_metadata: AttentionMetadata,
834
memory_pool,
835
) -> None:
836
assert self.graph is None
@@ -839,7 +842,7 @@ class CUDAGraphRunner:
842
input_ids,
843
positions,
844
kv_caches,
842
- input_metadata,
845
+ attn_metadata,
846
)
847
torch.cuda.synchronize()
848
@@ -853,7 +856,7 @@ class CUDAGraphRunner:
856
input_ids,
857
positions,
858
kv_caches,
856
- input_metadata,
859
+ attn_metadata,
860
)
861
torch.cuda.synchronize()
862
@@ -862,9 +865,9 @@ class CUDAGraphRunner:
865
"input_ids": input_ids,
866
"positions": positions,
867
"kv_caches": kv_caches,
865
- "slot_mapping": input_metadata.slot_mapping,
866
- "context_lens": input_metadata.context_lens,
867
- "block_tables": input_metadata.block_tables,
868
+ "slot_mapping": attn_metadata.slot_mapping,
869
+ "context_lens": attn_metadata.context_lens,
870
+ "block_tables": attn_metadata.block_tables,
871
}
872
self.output_buffers = {"hidden_states": hidden_states}
873
return
@@ -873,8 +876,8 @@ class CUDAGraphRunner:
876
self,
877
input_ids: torch.Tensor,
878
positions: torch.Tensor,
876
- kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
877
- input_metadata: InputMetadata,
879
+ kv_caches: List[torch.Tensor],
880
+ attn_metadata: AttentionMetadata,
881
) -> torch.Tensor:
882
# KV caches are fixed tensors, so we don't need to copy them.
883
del kv_caches
@@ -882,11 +885,11 @@ class CUDAGraphRunner:
885
# Copy the input tensors to the input buffers.
886
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
887
self.input_buffers["positions"].copy_(positions, non_blocking=True)
885
- self.input_buffers["slot_mapping"].copy_(input_metadata.slot_mapping,
888
+ self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
889
non_blocking=True)
887
- self.input_buffers["context_lens"].copy_(input_metadata.context_lens,
890
+ self.input_buffers["context_lens"].copy_(attn_metadata.context_lens,
891
non_blocking=True)
889
- self.input_buffers["block_tables"].copy_(input_metadata.block_tables,
892
+ self.input_buffers["block_tables"].copy_(attn_metadata.block_tables,
893
non_blocking=True)
894
# Run the graph.
895
self.graph.replay()
vllm/worker/worker.py
+3
@@ -128,6 +128,9 @@ class Worker:
128
# NOTE(woosuk): Here we assume that the other processes using the same
129
# GPU did not change their memory usage during the profiling.
130
peak_memory = self.init_gpu_memory - free_gpu_memory
131
+ assert peak_memory > 0, (
132
+ "Error in memory profiling. This happens when the GPU memory was "
133
+ "not properly cleaned up before initializing the vLLM instance.")
134
135
cache_block_size = self.get_cache_block_size_bytes(
136
block_size, cache_dtype)