A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Separate attention backends (#3005)
Woosuk Kwon committed
Mar 7, 2024 at 01:45 UTC
2daf23ab0cf00da157b1255faddcf0a269283d36
35 files changed
+558
-268
.gitignore
+3
@@ -184,3 +184,6 @@ _build/
184
185
# Benchmark dataset
186
*.json
187
+
188
+# Third-party Python packages.
189
+vllm/thirdparty_files/
setup.py
+45
-3
@@ -3,6 +3,7 @@ import io
3
import os
4
import re
5
import subprocess
6
+import sys
7
import warnings
8
from pathlib import Path
9
from typing import List, Set
@@ -14,6 +15,8 @@ import torch.utils.cpp_extension as torch_cpp_ext
15
from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
16
17
ROOT_DIR = os.path.dirname(__file__)
18
+# This is a temporary directory to store third-party packages.
19
+THIRDPARTY_SUBDIR = "vllm/thirdparty_files"
20
21
# If you are developing the C++ backend of vLLM, consider building vLLM with
22
# `python setup.py develop` since it will give you incremental builds.
@@ -324,8 +327,46 @@ if _is_cuda():
327
"nvcc": NVCC_FLAGS_PUNICA,
328
},
329
))
327
-elif _is_neuron():
328
- neuronxcc_version = get_neuronxcc_version()
330
+
331
+ # Download the FlashAttention package.
332
+ # Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/setup.py#L518-L530
333
+ flash_attn_version = "2.5.6"
334
+ install_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
335
+ subprocess.check_call(
336
+ [
337
+ sys.executable,
338
+ "-m",
339
+ "pip",
340
+ "install",
341
+ "-q",
342
+ f"--target={install_dir}",
343
+ "einops", # Dependency of flash-attn.
344
+ f"flash-attn=={flash_attn_version}",
345
+ "--no-dependencies", # Required to avoid re-installing torch.
346
+ ],
347
+ env=dict(os.environ, CC="gcc"),
348
+ )
349
+
350
+ # Copy the FlashAttention package into the vLLM package after build.
351
+ class build_ext(BuildExtension):
352
+
353
+ def run(self):
354
+ super().run()
355
+ target_dir = os.path.join(self.build_lib, THIRDPARTY_SUBDIR)
356
+ if not os.path.exists(target_dir):
357
+ os.makedirs(target_dir)
358
+ self.copy_tree(install_dir, target_dir)
359
+
360
+ class BinaryDistribution(setuptools.Distribution):
361
+
362
+ def has_ext_modules(self):
363
+ return True
364
+
365
+else:
366
+ build_ext = BuildExtension
367
+ BinaryDistribution = setuptools.Distribution
368
+ if _is_neuron():
369
+ neuronxcc_version = get_neuronxcc_version()
370
371
vllm_extension_sources = [
372
"csrc/cache_kernels.cu",
@@ -468,6 +509,7 @@ setuptools.setup(
509
python_requires=">=3.8",
510
install_requires=get_requirements(),
511
ext_modules=ext_modules,
471
- cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
512
+ cmdclass={"build_ext": build_ext} if not _is_neuron() else {},
513
+ distclass=BinaryDistribution,
514
package_data=package_data,
515
)
tests/kernels/test_prefix_prefill.py
+1
-1
@@ -3,7 +3,7 @@ import pytest
3
import time
4
5
import torch
6
-from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
6
+from vllm.model_executor.layers.attention.ops.prefix_prefill import (
7
context_attention_fwd)
8
from xformers import ops as xops
9
from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
vllm/__init__.py
+23
-7
@@ -1,12 +1,28 @@
1
"""vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2
3
-from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4
-from vllm.engine.async_llm_engine import AsyncLLMEngine
5
-from vllm.engine.llm_engine import LLMEngine
6
-from vllm.engine.ray_utils import initialize_cluster
7
-from vllm.entrypoints.llm import LLM
8
-from vllm.outputs import CompletionOutput, RequestOutput
9
-from vllm.sampling_params import SamplingParams
3
+
4
+# Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/ray/__init__.py#L11
5
+def _configure_system():
6
+ import os
7
+ import sys
8
+
9
+ # Importing flash-attn.
10
+ thirdparty_files = os.path.join(os.path.abspath(os.path.dirname(__file__)),
11
+ "thirdparty_files")
12
+ sys.path.insert(0, thirdparty_files)
13
+
14
+
15
+_configure_system()
16
+# Delete configuration function.
17
+del _configure_system
18
+
19
+from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs # noqa: E402
20
+from vllm.engine.async_llm_engine import AsyncLLMEngine # noqa: E402
21
+from vllm.engine.llm_engine import LLMEngine # noqa: E402
22
+from vllm.engine.ray_utils import initialize_cluster # noqa: E402
23
+from vllm.entrypoints.llm import LLM # noqa: E402
24
+from vllm.outputs import CompletionOutput, RequestOutput # noqa: E402
25
+from vllm.sampling_params import SamplingParams # noqa: E402
26
27
__version__ = "0.3.3"
28
vllm/model_executor/layers/attention/__init__.py
new
+5
@@ -0,0 +1,5 @@
1
+from vllm.model_executor.layers.attention.attention import Attention
2
+
3
+__all__ = [
4
+ "Attention",
5
+]
vllm/model_executor/layers/attention/attention.py
new
+59
@@ -0,0 +1,59 @@
1
+"""Attention layer."""
2
+from typing import List, Optional
3
+
4
+import torch
5
+import torch.nn as nn
6
+
7
+from vllm.model_executor.input_metadata import InputMetadata
8
+from vllm.utils import is_hip
9
+
10
+
11
+class Attention(nn.Module):
12
+ """Attention layer.
13
+
14
+ This class takes query, key, and value tensors as input. The input tensors
15
+ can either contain prompt tokens or generation tokens.
16
+ The class does the following:
17
+
18
+ 1. Store the input key and value tensors in the KV cache.
19
+ 2. Perform (multi-head/multi-query/grouped-query) attention.
20
+ 3. Return the output tensor.
21
+ """
22
+
23
+ def __init__(
24
+ self,
25
+ num_heads: int,
26
+ head_size: int,
27
+ scale: float,
28
+ num_kv_heads: Optional[int] = None,
29
+ alibi_slopes: Optional[List[float]] = None,
30
+ sliding_window: Optional[int] = None,
31
+ ) -> None:
32
+ super().__init__()
33
+ if (not is_hip() and torch.cuda.get_device_capability()[0] >= 8 and
34
+ torch.get_default_dtype() in (torch.float16, torch.bfloat16)):
35
+ # Ampere or later NVIDIA GPUs.
36
+ # NOTE(woosuk): FlashAttention does not support FP32.
37
+ from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend
38
+ self.backend = FlashAttentionBackend(num_heads, head_size, scale,
39
+ num_kv_heads, alibi_slopes,
40
+ sliding_window)
41
+ else:
42
+ # Turing and Volta NVIDIA GPUs or AMD GPUs.
43
+ # Or FP32 on any GPU.
44
+ from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend
45
+ self.backend = XFormersBackend(num_heads, head_size, scale,
46
+ num_kv_heads, alibi_slopes,
47
+ sliding_window)
48
+
49
+ def forward(
50
+ self,
51
+ query: torch.Tensor,
52
+ key: torch.Tensor,
53
+ value: torch.Tensor,
54
+ key_cache: Optional[torch.Tensor],
55
+ value_cache: Optional[torch.Tensor],
56
+ input_metadata: InputMetadata,
57
+ ) -> torch.Tensor:
58
+ return self.backend.forward(query, key, value, key_cache, value_cache,
59
+ input_metadata)
vllm/model_executor/layers/attention/backends/__init__.py
renamed
vllm/model_executor/layers/attention/backends/flash_attn.py
new
+124
@@ -0,0 +1,124 @@
1
+"""Attention layer with Flash and PagedAttention."""
2
+from typing import List, Optional
3
+
4
+# NOTE(woosuk): This imports flash_attn under vllm/thirdparty_files/.
5
+from flash_attn import flash_attn_func
6
+import torch
7
+
8
+from vllm.model_executor.input_metadata import InputMetadata
9
+from vllm.model_executor.layers.attention.ops.paged_attn import (
10
+ PagedAttentionImpl)
11
+
12
+
13
+class FlashAttentionBackend:
14
+
15
+ def __init__(
16
+ self,
17
+ num_heads: int,
18
+ head_size: int,
19
+ scale: float,
20
+ num_kv_heads: Optional[int] = None,
21
+ alibi_slopes: Optional[List[float]] = None,
22
+ sliding_window: Optional[int] = None,
23
+ ) -> None:
24
+ self.num_heads = num_heads
25
+ self.head_size = head_size
26
+ self.scale = float(scale)
27
+ self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
28
+ self.sliding_window = sliding_window
29
+ if alibi_slopes is not None:
30
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
31
+ self.alibi_slopes = alibi_slopes
32
+
33
+ assert self.num_heads % self.num_kv_heads == 0
34
+ self.num_queries_per_kv = self.num_heads // self.num_kv_heads
35
+ suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
36
+ if head_size not in suppored_head_sizes:
37
+ raise ValueError(
38
+ f"Head size {head_size} is not supported by PagedAttention. "
39
+ f"Supported head sizes are: {suppored_head_sizes}.")
40
+
41
+ self.sliding_window = ((self.sliding_window, self.sliding_window) if
42
+ self.sliding_window is not None else (-1, -1))
43
+
44
+ def forward(
45
+ self,
46
+ query: torch.Tensor,
47
+ key: torch.Tensor,
48
+ value: torch.Tensor,
49
+ key_cache: Optional[torch.Tensor],
50
+ value_cache: Optional[torch.Tensor],
51
+ input_metadata: InputMetadata,
52
+ ) -> torch.Tensor:
53
+ """Forward pass with FlashAttention and PagedAttention.
54
+
55
+ Args:
56
+ query: shape = [batch_size, seq_len, num_heads * head_size]
57
+ key: shape = [batch_size, seq_len, num_kv_heads * head_size]
58
+ value: shape = [batch_size, seq_len, num_kv_heads * head_size]
59
+ key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
60
+ block_size, x]
61
+ value_cache: shape = [num_blocks, num_kv_heads, head_size,
62
+ block_size]
63
+ input_metadata: metadata for the inputs.
64
+ Returns:
65
+ shape = [batch_size, seq_len, num_heads * head_size]
66
+ """
67
+ batch_size, seq_len, hidden_size = query.shape
68
+ # Reshape the query, key, and value tensors.
69
+ query = query.view(-1, self.num_heads, self.head_size)
70
+ key = key.view(-1, self.num_kv_heads, self.head_size)
71
+ value = value.view(-1, self.num_kv_heads, self.head_size)
72
+
73
+ # Reshape the keys and values and store them in the cache.
74
+ # If key_cache and value_cache are not provided, the new key and value
75
+ # vectors will not be cached. This happens during the initial memory
76
+ # profiling run.
77
+ if key_cache is not None and value_cache is not None:
78
+ PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
79
+ value_cache, input_metadata)
80
+
81
+ if input_metadata.is_prompt:
82
+ # Prompt run.
83
+ if (key_cache is None or value_cache is None
84
+ or input_metadata.block_tables.numel() == 0):
85
+ # normal attention
86
+ query = query.unflatten(0, (batch_size, seq_len))
87
+ key = key.unflatten(0, (batch_size, seq_len))
88
+ value = value.unflatten(0, (batch_size, seq_len))
89
+ output = flash_attn_func(
90
+ query,
91
+ key,
92
+ value,
93
+ softmax_scale=self.scale,
94
+ causal=True,
95
+ window_size=self.sliding_window,
96
+ alibi_slopes=self.alibi_slopes,
97
+ )
98
+ else:
99
+ # prefix-enabled attention
100
+ output = PagedAttentionImpl.forward_prefix(
101
+ query,
102
+ key,
103
+ value,
104
+ key_cache,
105
+ value_cache,
106
+ input_metadata,
107
+ self.num_heads,
108
+ self.num_kv_heads,
109
+ self.alibi_slopes,
110
+ )
111
+ else:
112
+ # Decoding run.
113
+ output = PagedAttentionImpl.forward_decode(
114
+ query,
115
+ key_cache,
116
+ value_cache,
117
+ input_metadata,
118
+ self.num_kv_heads,
119
+ self.scale,
120
+ self.alibi_slopes,
121
+ )
122
+
123
+ # Reshape the output tensor.
124
+ return output.view(batch_size, seq_len, hidden_size)
vllm/model_executor/layers/attention/backends/xformers.py
renamed
+61
-155
@@ -1,37 +1,19 @@
1
-"""Multi-head attention."""
1
+"""Attention layer with xFormers and PagedAttention."""
2
+import importlib
3
from typing import List, Optional
4
4
-import importlib
5
import torch
6
-import torch.nn as nn
6
from xformers import ops as xops
7
from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8
LowerTriangularMaskWithTensorBias)
9
11
-from vllm._C import ops
12
-from vllm._C import cache_ops
10
from vllm.model_executor.input_metadata import InputMetadata
14
-from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
15
- context_attention_fwd)
11
+from vllm.model_executor.layers.attention.ops.paged_attn import (
12
+ PagedAttentionImpl)
13
from vllm.utils import is_hip
14
18
-_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19
-# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
20
-_PARTITION_SIZE = 512
21
-
22
-
23
-class PagedAttention(nn.Module):
24
- """MHA/MQA/GQA layer with PagedAttention.
15
26
- This class takes query, key, and value tensors as input. The input tensors
27
- can either contain prompt tokens or generation tokens.
28
- The class does the following:
29
-
30
- 1. Reshape and store the input key and value tensors in the KV cache.
31
- 2. Perform (multi-head/multi-query/grouped-query) attention using either
32
- xformers or the PagedAttention custom op.
33
- 3. Return the output tensor.
34
- """
16
+class XFormersBackend:
17
18
def __init__(
19
self,
@@ -42,7 +24,6 @@ class PagedAttention(nn.Module):
24
alibi_slopes: Optional[List[float]] = None,
25
sliding_window: Optional[int] = None,
26
) -> None:
45
- super().__init__()
27
self.num_heads = num_heads
28
self.head_size = head_size
29
self.scale = float(scale)
@@ -50,48 +31,17 @@ class PagedAttention(nn.Module):
31
self.sliding_window = sliding_window
32
if alibi_slopes is not None:
33
alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
53
- self.register_buffer("alibi_slopes", alibi_slopes, persistent=False)
34
+ self.alibi_slopes = alibi_slopes
35
36
assert self.num_heads % self.num_kv_heads == 0
37
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
38
+ suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
39
+ if head_size not in suppored_head_sizes:
40
+ raise ValueError(
41
+ f"Head size {head_size} is not supported by PagedAttention. "
42
+ f"Supported head sizes are: {suppored_head_sizes}.")
43
58
- if self.head_size not in _SUPPORTED_HEAD_SIZES:
59
- raise ValueError(f"head_size ({self.head_size}) is not supported. "
60
- f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
61
-
62
- self.use_ref_attention = self.check_use_ref_attention()
63
-
64
- def check_use_ref_attention(self) -> bool:
65
- if not is_hip():
66
- return False
67
- # For ROCm, check whether flash attention is installed or not.
68
- # if not, use_ref_attention needs to be True
69
- return importlib.util.find_spec("flash_attn") is None
70
-
71
- def ref_masked_attention(
72
- self,
73
- query: torch.Tensor,
74
- key: torch.Tensor,
75
- value: torch.Tensor,
76
- ) -> torch.Tensor:
77
- query = query.view(-1, self.num_heads, self.head_size)
78
- key = key.view(-1, self.num_kv_heads, self.head_size)
79
- value = value.view(-1, self.num_kv_heads, self.head_size)
80
-
81
- seq_len, _, _ = query.shape
82
- attn_mask = torch.triu(torch.ones(seq_len,
83
- seq_len,
84
- dtype=query.dtype,
85
- device=query.device),
86
- diagonal=1)
87
- attn_mask = attn_mask * torch.finfo(query.dtype).min
88
-
89
- attn_weights = self.scale * torch.einsum("qhd,khd->hqk", query,
90
- key).float()
91
- attn_weights = attn_weights + attn_mask.float()
92
- attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
93
- out = torch.einsum("hqk,khd->qhd", attn_weights, value)
94
- return out
44
+ self.use_ref_attention = _check_use_ref_attention()
45
46
def forward(
47
self,
@@ -102,7 +52,7 @@ class PagedAttention(nn.Module):
52
value_cache: Optional[torch.Tensor],
53
input_metadata: InputMetadata,
54
) -> torch.Tensor:
105
- """PagedAttention forward pass.
55
+ """Forward pass with xFormers and PagedAttention.
56
57
Args:
58
query: shape = [batch_size, seq_len, num_heads * head_size]
@@ -127,19 +77,14 @@ class PagedAttention(nn.Module):
77
# vectors will not be cached. This happens during the initial memory
78
# profiling run.
79
if key_cache is not None and value_cache is not None:
130
- cache_ops.reshape_and_cache(
131
- key,
132
- value,
133
- key_cache,
134
- value_cache,
135
- input_metadata.slot_mapping.flatten(),
136
- input_metadata.kv_cache_dtype,
137
- )
80
+ PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
81
+ value_cache, input_metadata)
82
83
if input_metadata.is_prompt:
140
- # normal attention
84
+ # Prompt run.
85
if (key_cache is None or value_cache is None
86
or input_metadata.block_tables.numel() == 0):
87
+ # normal attention
88
if self.num_kv_heads != self.num_heads:
89
# As of Nov 2023, xformers only supports MHA. For MQA/GQA,
90
# project the key and value tensors to the desired number of
@@ -175,13 +120,19 @@ class PagedAttention(nn.Module):
120
seq_len, query.dtype)
121
122
if self.use_ref_attention:
178
- output = self.ref_masked_attention(
123
+ output = _ref_masked_attention(
124
query,
125
key,
126
value,
127
+ self.num_heads,
128
+ self.num_kv_heads,
129
+ self.head_size,
130
+ self.scale,
131
)
183
- # Using view got RuntimeError: view size is not compatible with input tensor's size and stride
184
- # (at least one dimension spans across two contiguous subspaces). Use reshape instead
132
+ # Using view got RuntimeError: view size is not compatible
133
+ # with input tensor's size and stride (at least one
134
+ # dimension spans across two contiguous subspaces).
135
+ # Use reshape instead.
136
return output.reshape(batch_size, seq_len, hidden_size)
137
138
# TODO(woosuk): Too many view operations. Let's try to reduce
@@ -206,27 +157,21 @@ class PagedAttention(nn.Module):
157
(is_hip()) else None,
158
)
159
output = out.view_as(query)
160
+
161
else:
162
# prefix-enabled attention
211
- output = torch.empty_like(query)
212
- context_attention_fwd(
163
+ output = PagedAttentionImpl.forward_prefix(
164
query,
165
key,
166
value,
216
- output,
167
key_cache,
168
value_cache,
219
- input_metadata.block_tables, # [BS, max_block_per_request]
220
- input_metadata.start_loc,
221
- input_metadata.prompt_lens,
222
- input_metadata.context_lens,
223
- input_metadata.max_seq_len,
224
- getattr(self, "alibi_slopes", None),
169
+ input_metadata,
170
+ self.alibi_slopes,
171
)
226
-
172
else:
173
# Decoding run.
229
- output = _paged_attention(
174
+ output = PagedAttentionImpl.forward_decode(
175
query,
176
key_cache,
177
value_cache,
@@ -274,76 +219,37 @@ def _make_alibi_bias(
219
return attn_bias
220
221
277
-def _paged_attention(
222
+def _check_use_ref_attention() -> bool:
223
+ if not is_hip():
224
+ return False
225
+ # For ROCm, check whether flash attention is installed or not.
226
+ # if not, use_ref_attention needs to be True
227
+ return importlib.util.find_spec("flash_attn") is None
228
+
229
+
230
+def _ref_masked_attention(
231
query: torch.Tensor,
279
- key_cache: torch.Tensor,
280
- value_cache: torch.Tensor,
281
- input_metadata: InputMetadata,
232
+ key: torch.Tensor,
233
+ value: torch.Tensor,
234
+ num_heads: int,
235
num_kv_heads: int,
236
+ head_size: int,
237
scale: float,
284
- alibi_slopes: Optional[torch.Tensor],
238
) -> torch.Tensor:
286
- output = torch.empty_like(query)
287
-
288
- block_size = value_cache.shape[3]
289
- num_seqs, num_heads, head_size = query.shape
290
- max_num_partitions = (
291
- (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
292
- _PARTITION_SIZE)
293
- # NOTE(woosuk): We use a simple heuristic to decide whether to use
294
- # PagedAttention V1 or V2. If the number of partitions is 1, we use
295
- # V1 to avoid the overhead of reduction. Also, if the number of
296
- # sequences or heads is large, we use V1 since there is enough work
297
- # to parallelize.
298
- # TODO(woosuk): Tune this heuristic.
299
- # For context len > 8192, use V2 kernel to avoid shared memory shortage.
300
- use_v1 = input_metadata.max_context_len <= 8192 and (
301
- max_num_partitions == 1 or num_seqs * num_heads > 512)
302
- if use_v1:
303
- # Run PagedAttention V1.
304
- ops.paged_attention_v1(
305
- output,
306
- query,
307
- key_cache,
308
- value_cache,
309
- num_kv_heads,
310
- scale,
311
- input_metadata.block_tables,
312
- input_metadata.context_lens,
313
- block_size,
314
- input_metadata.max_context_len,
315
- alibi_slopes,
316
- input_metadata.kv_cache_dtype,
317
- )
318
- else:
319
- # Run PagedAttention V2.
320
- assert _PARTITION_SIZE % block_size == 0
321
- tmp_output = torch.empty(
322
- size=(num_seqs, num_heads, max_num_partitions, head_size),
323
- dtype=output.dtype,
324
- device=output.device,
325
- )
326
- exp_sums = torch.empty(
327
- size=(num_seqs, num_heads, max_num_partitions),
328
- dtype=torch.float32,
329
- device=output.device,
330
- )
331
- max_logits = torch.empty_like(exp_sums)
332
- ops.paged_attention_v2(
333
- output,
334
- exp_sums,
335
- max_logits,
336
- tmp_output,
337
- query,
338
- key_cache,
339
- value_cache,
340
- num_kv_heads,
341
- scale,
342
- input_metadata.block_tables,
343
- input_metadata.context_lens,
344
- block_size,
345
- input_metadata.max_context_len,
346
- alibi_slopes,
347
- input_metadata.kv_cache_dtype,
348
- )
349
- return output
239
+ query = query.view(-1, num_heads, head_size)
240
+ key = key.view(-1, num_kv_heads, head_size)
241
+ value = value.view(-1, num_kv_heads, head_size)
242
+
243
+ seq_len, _, _ = query.shape
244
+ attn_mask = torch.triu(torch.ones(seq_len,
245
+ seq_len,
246
+ dtype=query.dtype,
247
+ device=query.device),
248
+ diagonal=1)
249
+ attn_mask = attn_mask * torch.finfo(query.dtype).min
250
+
251
+ attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
252
+ attn_weights = attn_weights + attn_mask.float()
253
+ attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
254
+ out = torch.einsum("hqk,khd->qhd", attn_weights, value)
255
+ return out
vllm/model_executor/layers/attention/ops/__init__.py
vllm/model_executor/layers/attention/ops/paged_attn.py
new
+138
@@ -0,0 +1,138 @@
1
+from typing import List, Optional
2
+
3
+import torch
4
+
5
+from vllm._C import cache_ops
6
+from vllm._C import ops
7
+from vllm.model_executor.input_metadata import InputMetadata
8
+from vllm.model_executor.layers.attention.ops.prefix_prefill import (
9
+ context_attention_fwd)
10
+
11
+# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
12
+_PARTITION_SIZE = 512
13
+
14
+
15
+class PagedAttentionImpl:
16
+
17
+ @staticmethod
18
+ def get_supported_head_sizes() -> List[int]:
19
+ return [64, 80, 96, 112, 128, 256]
20
+
21
+ @staticmethod
22
+ def reshape_and_cache(
23
+ key: torch.Tensor,
24
+ value: torch.Tensor,
25
+ key_cache: torch.Tensor,
26
+ value_cache: torch.Tensor,
27
+ input_metadata: InputMetadata,
28
+ ) -> None:
29
+ cache_ops.reshape_and_cache(
30
+ key,
31
+ value,
32
+ key_cache,
33
+ value_cache,
34
+ input_metadata.slot_mapping.flatten(),
35
+ input_metadata.kv_cache_dtype,
36
+ )
37
+
38
+ @staticmethod
39
+ def forward_decode(
40
+ query: torch.Tensor,
41
+ key_cache: torch.Tensor,
42
+ value_cache: torch.Tensor,
43
+ input_metadata: InputMetadata,
44
+ num_kv_heads: int,
45
+ scale: float,
46
+ alibi_slopes: Optional[torch.Tensor],
47
+ ) -> torch.Tensor:
48
+ output = torch.empty_like(query)
49
+
50
+ block_size = value_cache.shape[3]
51
+ num_seqs, num_heads, head_size = query.shape
52
+ max_num_partitions = (
53
+ (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
54
+ _PARTITION_SIZE)
55
+ # NOTE(woosuk): We use a simple heuristic to decide whether to use
56
+ # PagedAttention V1 or V2. If the number of partitions is 1, we use
57
+ # V1 to avoid the overhead of reduction. Also, if the number of
58
+ # sequences or heads is large, we use V1 since there is enough work
59
+ # to parallelize.
60
+ # TODO(woosuk): Tune this heuristic.
61
+ # For context len > 8192, use V2 kernel to avoid shared memory shortage.
62
+ use_v1 = input_metadata.max_context_len <= 8192 and (
63
+ max_num_partitions == 1 or num_seqs * num_heads > 512)
64
+ if use_v1:
65
+ # Run PagedAttention V1.
66
+ ops.paged_attention_v1(
67
+ output,
68
+ query,
69
+ key_cache,
70
+ value_cache,
71
+ num_kv_heads,
72
+ scale,
73
+ input_metadata.block_tables,
74
+ input_metadata.context_lens,
75
+ block_size,
76
+ input_metadata.max_context_len,
77
+ alibi_slopes,
78
+ input_metadata.kv_cache_dtype,
79
+ )
80
+ else:
81
+ # Run PagedAttention V2.
82
+ assert _PARTITION_SIZE % block_size == 0
83
+ tmp_output = torch.empty(
84
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
85
+ dtype=output.dtype,
86
+ device=output.device,
87
+ )
88
+ exp_sums = torch.empty(
89
+ size=(num_seqs, num_heads, max_num_partitions),
90
+ dtype=torch.float32,
91
+ device=output.device,
92
+ )
93
+ max_logits = torch.empty_like(exp_sums)
94
+ ops.paged_attention_v2(
95
+ output,
96
+ exp_sums,
97
+ max_logits,
98
+ tmp_output,
99
+ query,
100
+ key_cache,
101
+ value_cache,
102
+ num_kv_heads,
103
+ scale,
104
+ input_metadata.block_tables,
105
+ input_metadata.context_lens,
106
+ block_size,
107
+ input_metadata.max_context_len,
108
+ alibi_slopes,
109
+ input_metadata.kv_cache_dtype,
110
+ )
111
+ return output
112
+
113
+ @staticmethod
114
+ def forward_prefix(
115
+ query: torch.Tensor,
116
+ key: torch.Tensor,
117
+ value: torch.Tensor,
118
+ key_cache: torch.Tensor,
119
+ value_cache: torch.Tensor,
120
+ input_metadata: InputMetadata,
121
+ alibi_slopes: Optional[torch.Tensor],
122
+ ) -> torch.Tensor:
123
+ output = torch.empty_like(query)
124
+ context_attention_fwd(
125
+ query,
126
+ key,
127
+ value,
128
+ output,
129
+ key_cache,
130
+ value_cache,
131
+ input_metadata.block_tables, # [BS, max_block_per_request]
132
+ input_metadata.start_loc,
133
+ input_metadata.prompt_lens,
134
+ input_metadata.context_lens,
135
+ input_metadata.max_seq_len,
136
+ alibi_slopes,
137
+ )
138
+ return output
vllm/model_executor/layers/attention/ops/prefix_prefill.py
renamed
vllm/model_executor/models/baichuan.py
+6
-7
@@ -27,7 +27,7 @@ from transformers import PretrainedConfig
27
28
from vllm.model_executor.input_metadata import InputMetadata
29
from vllm.model_executor.layers.activation import SiluAndMul
30
-from vllm.model_executor.layers.attention import PagedAttention
30
+from vllm.model_executor.layers.attention import Attention
31
from vllm.model_executor.layers.layernorm import RMSNorm
32
from vllm.model_executor.layers.linear import (LinearMethodBase,
33
MergedColumnParallelLinear,
@@ -151,10 +151,10 @@ class BaiChuanAttention(nn.Module):
151
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152
153
scaling = self.head_dim**-0.5
154
- self.attn = PagedAttention(self.num_heads,
155
- self.head_dim,
156
- scaling,
157
- alibi_slopes=alibi_slopes)
154
+ self.attn = Attention(self.num_heads,
155
+ self.head_dim,
156
+ scaling,
157
+ alibi_slopes=alibi_slopes)
158
else:
159
self.rotary_emb = get_rope(
160
self.head_dim,
@@ -163,8 +163,7 @@ class BaiChuanAttention(nn.Module):
163
base=self.rope_theta,
164
)
165
self.scaling = self.head_dim**-0.5
166
- self.attn = PagedAttention(self.num_heads, self.head_dim,
167
- self.scaling)
166
+ self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
167
168
def forward(
169
self,
vllm/model_executor/models/bloom.py
+5
-5
@@ -25,7 +25,7 @@ from transformers import BloomConfig
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import PagedAttention
28
+from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
@@ -107,10 +107,10 @@ class BloomAttention(nn.Module):
107
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
108
109
scaling = self.head_dim**-0.5
110
- self.attn = PagedAttention(self.num_heads,
111
- self.head_dim,
112
- scaling,
113
- alibi_slopes=alibi_slopes)
110
+ self.attn = Attention(self.num_heads,
111
+ self.head_dim,
112
+ scaling,
113
+ alibi_slopes=alibi_slopes)
114
115
def forward(
116
self,
vllm/model_executor/models/chatglm.py
+2
-2
@@ -10,7 +10,7 @@ from torch.nn import LayerNorm
10
11
from vllm.model_executor.input_metadata import InputMetadata
12
from vllm.model_executor.layers.activation import SiluAndMul
13
-from vllm.model_executor.layers.attention import PagedAttention
13
+from vllm.model_executor.layers.attention import Attention
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import (LinearMethodBase,
16
MergedColumnParallelLinear,
@@ -87,7 +87,7 @@ class GLMAttention(nn.Module):
87
base=10000 * rope_ratio,
88
is_neox_style=False,
89
)
90
- self.attn = PagedAttention(
90
+ self.attn = Attention(
91
self.num_heads,
92
self.head_dim,
93
self.scaling,
vllm/model_executor/models/deepseek.py
+5
-5
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttention
32
+from vllm.model_executor.layers.attention import Attention
33
from vllm.model_executor.layers.fused_moe import fused_moe
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -229,10 +229,10 @@ class DeepseekAttention(nn.Module):
229
base=rope_theta,
230
rope_scaling=rope_scaling,
231
)
232
- self.attn = PagedAttention(self.num_heads,
233
- self.head_dim,
234
- self.scaling,
235
- num_kv_heads=self.num_kv_heads)
232
+ self.attn = Attention(self.num_heads,
233
+ self.head_dim,
234
+ self.scaling,
235
+ num_kv_heads=self.num_kv_heads)
236
237
def forward(
238
self,
vllm/model_executor/models/falcon.py
+14
-14
@@ -28,7 +28,7 @@ from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.activation import get_act_fn
31
-from vllm.model_executor.layers.attention import PagedAttention
31
+from vllm.model_executor.layers.attention import Attention
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
LinearMethodBase,
34
QKVParallelLinear,
@@ -150,10 +150,10 @@ class FalconAttention(nn.Module):
150
max_position=max_position_embeddings,
151
base=rope_theta,
152
)
153
- self.attn = PagedAttention(self.num_heads,
154
- self.head_dim,
155
- self.inv_norm_factor,
156
- num_kv_heads=self.num_kv_heads)
153
+ self.attn = Attention(self.num_heads,
154
+ self.head_dim,
155
+ self.inv_norm_factor,
156
+ num_kv_heads=self.num_kv_heads)
157
elif self.use_alibi:
158
tp_rank = get_tensor_model_parallel_rank()
159
head_start = tp_rank * self.num_heads
@@ -161,16 +161,16 @@ class FalconAttention(nn.Module):
161
alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
162
self.inv_norm_factor)
163
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
164
- self.attn = PagedAttention(self.num_heads,
165
- self.head_dim,
166
- self.inv_norm_factor,
167
- num_kv_heads=self.num_kv_heads,
168
- alibi_slopes=alibi_slopes)
164
+ self.attn = Attention(self.num_heads,
165
+ self.head_dim,
166
+ self.inv_norm_factor,
167
+ num_kv_heads=self.num_kv_heads,
168
+ alibi_slopes=alibi_slopes)
169
else:
170
- self.attn = PagedAttention(self.num_heads,
171
- self.head_dim,
172
- scale=self.inv_norm_factor,
173
- num_kv_heads=self.num_kv_heads)
170
+ self.attn = Attention(self.num_heads,
171
+ self.head_dim,
172
+ scale=self.inv_norm_factor,
173
+ num_kv_heads=self.num_kv_heads)
174
175
def forward(
176
self,
vllm/model_executor/models/gemma.py
+5
-5
@@ -23,7 +23,7 @@ from transformers import GemmaConfig
23
from vllm.config import LoRAConfig
24
from vllm.model_executor.input_metadata import InputMetadata
25
from vllm.model_executor.layers.activation import GeluAndMul
26
-from vllm.model_executor.layers.attention import PagedAttention
26
+from vllm.model_executor.layers.attention import Attention
27
from vllm.model_executor.layers.layernorm import RMSNorm
28
from vllm.model_executor.layers.linear import (LinearMethodBase,
29
MergedColumnParallelLinear,
@@ -123,10 +123,10 @@ class GemmaAttention(nn.Module):
123
base=self.rope_theta,
124
is_neox_style=True,
125
)
126
- self.attn = PagedAttention(self.num_heads,
127
- self.head_dim,
128
- self.scaling,
129
- num_kv_heads=self.num_kv_heads)
126
+ self.attn = Attention(self.num_heads,
127
+ self.head_dim,
128
+ self.scaling,
129
+ num_kv_heads=self.num_kv_heads)
130
131
def forward(
132
self,
vllm/model_executor/models/gpt2.py
+2
-4
@@ -25,7 +25,7 @@ from transformers import GPT2Config
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import PagedAttention
28
+from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
@@ -73,9 +73,7 @@ class GPT2Attention(nn.Module):
73
bias=True,
74
linear_method=linear_method,
75
)
76
- self.attn = PagedAttention(self.num_heads,
77
- self.head_dim,
78
- scale=self.scale)
76
+ self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
77
78
def forward(
79
self,
vllm/model_executor/models/gpt_bigcode.py
+5
-5
@@ -26,7 +26,7 @@ from transformers import GPTBigCodeConfig
26
27
from vllm.model_executor.input_metadata import InputMetadata
28
from vllm.model_executor.layers.activation import get_act_fn
29
-from vllm.model_executor.layers.attention import PagedAttention
29
+from vllm.model_executor.layers.attention import Attention
30
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
LinearMethodBase,
32
QKVParallelLinear,
@@ -85,10 +85,10 @@ class GPTBigCodeAttention(nn.Module):
85
bias=True,
86
linear_method=linear_method,
87
)
88
- self.attn = PagedAttention(self.num_heads,
89
- self.head_dim,
90
- scale=self.scale,
91
- num_kv_heads=self.num_kv_heads)
88
+ self.attn = Attention(self.num_heads,
89
+ self.head_dim,
90
+ scale=self.scale,
91
+ num_kv_heads=self.num_kv_heads)
92
93
def forward(
94
self,
vllm/model_executor/models/gpt_j.py
+2
-2
@@ -24,7 +24,7 @@ from transformers import GPTJConfig
24
25
from vllm.model_executor.input_metadata import InputMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import PagedAttention
27
+from vllm.model_executor.layers.attention import Attention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
@@ -86,7 +86,7 @@ class GPTJAttention(nn.Module):
86
base=rope_theta,
87
is_neox_style=False,
88
)
89
- self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
89
+ self.attn = Attention(self.num_heads, self.head_size, scaling)
90
91
def forward(
92
self,
vllm/model_executor/models/gpt_neox.py
+2
-2
@@ -24,7 +24,7 @@ from transformers import GPTNeoXConfig
24
25
from vllm.model_executor.input_metadata import InputMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import PagedAttention
27
+from vllm.model_executor.layers.attention import Attention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
@@ -87,7 +87,7 @@ class GPTNeoXAttention(nn.Module):
87
max_position=max_position_embeddings,
88
base=rope_theta,
89
)
90
- self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
90
+ self.attn = Attention(self.num_heads, self.head_size, scaling)
91
92
def forward(
93
self,
vllm/model_executor/models/internlm2.py
+5
-5
@@ -7,7 +7,7 @@ from transformers import PretrainedConfig
7
8
from vllm.model_executor.input_metadata import InputMetadata
9
from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import PagedAttention
10
+from vllm.model_executor.layers.attention import Attention
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
13
MergedColumnParallelLinear,
@@ -114,10 +114,10 @@ class InternLM2Attention(nn.Module):
114
base=rope_theta,
115
rope_scaling=rope_scaling,
116
)
117
- self.attn = PagedAttention(self.num_heads,
118
- self.head_dim,
119
- self.scaling,
120
- num_kv_heads=self.num_kv_heads)
117
+ self.attn = Attention(self.num_heads,
118
+ self.head_dim,
119
+ self.scaling,
120
+ num_kv_heads=self.num_kv_heads)
121
122
def forward(
123
self,
vllm/model_executor/models/llama.py
+6
-6
@@ -30,7 +30,7 @@ from transformers import LlamaConfig
30
from vllm.config import LoRAConfig
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.attention import Attention
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
36
MergedColumnParallelLinear,
@@ -139,11 +139,11 @@ class LlamaAttention(nn.Module):
139
base=rope_theta,
140
rope_scaling=rope_scaling,
141
)
142
- self.attn = PagedAttention(self.num_heads,
143
- self.head_dim,
144
- self.scaling,
145
- num_kv_heads=self.num_kv_heads,
146
- sliding_window=sliding_window)
142
+ self.attn = Attention(self.num_heads,
143
+ self.head_dim,
144
+ self.scaling,
145
+ num_kv_heads=self.num_kv_heads,
146
+ sliding_window=sliding_window)
147
148
def forward(
149
self,
vllm/model_executor/models/mixtral.py
+2
-2
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30
from vllm.config import LoRAConfig
31
from vllm.model_executor.input_metadata import InputMetadata
32
-from vllm.model_executor.layers.attention import PagedAttention
32
+from vllm.model_executor.layers.attention import Attention
33
from vllm.model_executor.layers.fused_moe import fused_moe
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -197,7 +197,7 @@ class MixtralAttention(nn.Module):
197
base=int(self.rope_theta),
198
is_neox_style=True,
199
)
200
- self.attn = PagedAttention(
200
+ self.attn = Attention(
201
self.num_heads,
202
self.head_dim,
203
self.scaling,
vllm/model_executor/models/mixtral_quant.py
+2
-2
@@ -32,7 +32,7 @@ from torch import nn
32
from transformers import MixtralConfig
33
34
from vllm.model_executor.input_metadata import InputMetadata
35
-from vllm.model_executor.layers.attention import PagedAttention
35
+from vllm.model_executor.layers.attention import Attention
36
from vllm.model_executor.layers.layernorm import RMSNorm
37
from vllm.model_executor.layers.linear import (LinearMethodBase,
38
ReplicatedLinear,
@@ -214,7 +214,7 @@ class MixtralAttention(nn.Module):
214
base=int(self.rope_theta),
215
is_neox_style=True,
216
)
217
- self.attn = PagedAttention(
217
+ self.attn = Attention(
218
self.num_heads,
219
self.head_dim,
220
self.scaling,
vllm/model_executor/models/mpt.py
+6
-6
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9
from vllm.model_executor.input_metadata import InputMetadata
10
from vllm.model_executor.layers.activation import get_act_fn
11
-from vllm.model_executor.layers.attention import PagedAttention
11
+from vllm.model_executor.layers.attention import Attention
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
LinearMethodBase,
14
QKVParallelLinear,
@@ -105,11 +105,11 @@ class MPTAttention(nn.Module):
105
106
self.head_dim = self.d_model // self.total_num_heads
107
scaling = self.head_dim**-0.5
108
- self.attn = PagedAttention(self.num_heads,
109
- self.head_dim,
110
- scaling,
111
- alibi_slopes=alibi_slopes,
112
- num_kv_heads=self.num_kv_heads)
108
+ self.attn = Attention(self.num_heads,
109
+ self.head_dim,
110
+ scaling,
111
+ alibi_slopes=alibi_slopes,
112
+ num_kv_heads=self.num_kv_heads)
113
114
def forward(
115
self,
vllm/model_executor/models/olmo.py
+4
-4
@@ -43,7 +43,7 @@ import torch.nn.functional as F
43
from torch import nn
44
45
from vllm.model_executor.input_metadata import InputMetadata
46
-from vllm.model_executor.layers.attention import PagedAttention
46
+from vllm.model_executor.layers.attention import Attention
47
from vllm.model_executor.layers.linear import (
48
ColumnParallelLinear,
49
LinearMethodBase,
@@ -126,9 +126,9 @@ class OlmoAttention(nn.Module):
126
base=rope_theta,
127
)
128
self.scaling = self.head_dim**-0.5
129
- self.attn = PagedAttention(self.num_heads,
130
- self.head_dim,
131
- scale=self.scaling)
129
+ self.attn = Attention(self.num_heads,
130
+ self.head_dim,
131
+ scale=self.scaling)
132
133
# Attention output projection.
134
self.attn_out = RowParallelLinear(
vllm/model_executor/models/opt.py
+4
-4
@@ -25,7 +25,7 @@ from transformers import OPTConfig
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import PagedAttention
28
+from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
@@ -89,9 +89,9 @@ class OPTAttention(nn.Module):
89
bias=bias,
90
linear_method=linear_method,
91
)
92
- self.attn = PagedAttention(self.num_heads,
93
- self.head_dim,
94
- scale=self.scaling)
92
+ self.attn = Attention(self.num_heads,
93
+ self.head_dim,
94
+ scale=self.scaling)
95
96
def forward(
97
self,
vllm/model_executor/models/orion.py
+5
-5
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
12
13
from vllm.model_executor.input_metadata import InputMetadata
14
from vllm.model_executor.layers.activation import SiluAndMul
15
-from vllm.model_executor.layers.attention import PagedAttention
15
+from vllm.model_executor.layers.attention import Attention
16
from vllm.model_executor.layers.linear import (LinearMethodBase,
17
MergedColumnParallelLinear,
18
QKVParallelLinear,
@@ -118,10 +118,10 @@ class OrionAttention(nn.Module):
118
base=rope_theta,
119
rope_scaling=rope_scaling,
120
)
121
- self.attn = PagedAttention(self.num_heads,
122
- self.head_dim,
123
- self.scaling,
124
- num_kv_heads=self.num_kv_heads)
121
+ self.attn = Attention(self.num_heads,
122
+ self.head_dim,
123
+ self.scaling,
124
+ num_kv_heads=self.num_kv_heads)
125
126
def forward(
127
self,
vllm/model_executor/models/phi.py
+2
-2
@@ -43,7 +43,7 @@ from transformers import PretrainedConfig
43
44
from vllm.model_executor.input_metadata import InputMetadata
45
from vllm.model_executor.layers.activation import get_act_fn
46
-from vllm.model_executor.layers.attention import PagedAttention
46
+from vllm.model_executor.layers.attention import Attention
47
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48
LinearMethodBase,
49
QKVParallelLinear,
@@ -108,7 +108,7 @@ class PhiAttention(nn.Module):
108
max_position=max_position_embeddings,
109
base=rope_theta,
110
)
111
- self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
111
+ self.attn = Attention(self.num_heads, self.head_size, scaling)
112
113
def forward(
114
self,
vllm/model_executor/models/qwen.py
+2
-2
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
12
13
from vllm.model_executor.input_metadata import InputMetadata
14
from vllm.model_executor.layers.activation import SiluAndMul
15
-from vllm.model_executor.layers.attention import PagedAttention
15
+from vllm.model_executor.layers.attention import Attention
16
from vllm.model_executor.layers.layernorm import RMSNorm
17
from vllm.model_executor.layers.linear import (LinearMethodBase,
18
MergedColumnParallelLinear,
@@ -104,7 +104,7 @@ class QWenAttention(nn.Module):
104
base=rope_theta,
105
rope_scaling=rope_scaling,
106
)
107
- self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
107
+ self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
108
109
def forward(
110
self,
vllm/model_executor/models/qwen2.py
+6
-6
@@ -30,7 +30,7 @@ from transformers import Qwen2Config
30
31
from vllm.model_executor.input_metadata import InputMetadata
32
from vllm.model_executor.layers.activation import SiluAndMul
33
-from vllm.model_executor.layers.attention import PagedAttention
33
+from vllm.model_executor.layers.attention import Attention
34
from vllm.model_executor.layers.layernorm import RMSNorm
35
from vllm.model_executor.layers.linear import (LinearMethodBase,
36
MergedColumnParallelLinear,
@@ -135,11 +135,11 @@ class Qwen2Attention(nn.Module):
135
max_position=max_position,
136
base=self.rope_theta,
137
)
138
- self.attn = PagedAttention(self.num_heads,
139
- self.head_dim,
140
- self.scaling,
141
- num_kv_heads=self.num_kv_heads,
142
- sliding_window=self.sliding_window)
138
+ self.attn = Attention(self.num_heads,
139
+ self.head_dim,
140
+ self.scaling,
141
+ num_kv_heads=self.num_kv_heads,
142
+ sliding_window=self.sliding_window)
143
144
def forward(
145
self,
vllm/model_executor/models/stablelm.py
+5
-5
@@ -25,7 +25,7 @@ from transformers import PretrainedConfig
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.layers.activation import SiluAndMul
28
-from vllm.model_executor.layers.attention import PagedAttention
28
+from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.linear import (LinearMethodBase,
30
MergedColumnParallelLinear,
31
QKVParallelLinear,
@@ -122,10 +122,10 @@ class StablelmAttention(nn.Module):
122
max_position=self.config.max_position_embeddings,
123
base=self.config.rope_theta,
124
)
125
- self.attn = PagedAttention(self.num_heads,
126
- self.head_dim,
127
- self.scaling,
128
- num_kv_heads=self.num_key_value_heads)
125
+ self.attn = Attention(self.num_heads,
126
+ self.head_dim,
127
+ self.scaling,
128
+ num_kv_heads=self.num_key_value_heads)
129
130
def forward(
131
self,
vllm/model_executor/models/starcoder2.py
+2
-2
@@ -25,7 +25,7 @@ from torch import nn
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
-from vllm.model_executor.layers.attention import PagedAttention
28
+from vllm.model_executor.layers.attention import Attention
29
from vllm.model_executor.layers.activation import get_act_fn
30
from vllm.model_executor.layers.rotary_embedding import get_rope
31
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -103,7 +103,7 @@ class Starcoder2Attention(nn.Module):
103
base=int(self.rope_theta),
104
is_neox_style=True,
105
)
106
- self.attn = PagedAttention(
106
+ self.attn = Attention(
107
self.num_heads,
108
self.head_dim,
109
self.scaling,