@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2daf23ab0c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Separate attention backends (#3005)

Woosuk Kwon committed Mar 7, 2024 at 01:45 UTC 2daf23ab0cf00da157b1255faddcf0a269283d36
35 files changed +558 -268
.gitignore
+3
@@ -184,3 +184,6 @@ _build/
184
185 # Benchmark dataset
186 *.json
187 +
188 +# Third-party Python packages.
189 +vllm/thirdparty_files/
setup.py
+45 -3
@@ -3,6 +3,7 @@ import io
3 import os
4 import re
5 import subprocess
6 +import sys
7 import warnings
8 from pathlib import Path
9 from typing import List, Set
@@ -14,6 +15,8 @@ import torch.utils.cpp_extension as torch_cpp_ext
15 from torch.utils.cpp_extension import BuildExtension, CUDAExtension, CUDA_HOME, ROCM_HOME
16
17 ROOT_DIR = os.path.dirname(__file__)
18 +# This is a temporary directory to store third-party packages.
19 +THIRDPARTY_SUBDIR = "vllm/thirdparty_files"
20
21 # If you are developing the C++ backend of vLLM, consider building vLLM with
22 # `python setup.py develop` since it will give you incremental builds.
@@ -324,8 +327,46 @@ if _is_cuda():
327 "nvcc": NVCC_FLAGS_PUNICA,
328 },
329 ))
327 -elif _is_neuron():
328 - neuronxcc_version = get_neuronxcc_version()
330 +
331 + # Download the FlashAttention package.
332 + # Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/setup.py#L518-L530
333 + flash_attn_version = "2.5.6"
334 + install_dir = os.path.join(ROOT_DIR, THIRDPARTY_SUBDIR)
335 + subprocess.check_call(
336 + [
337 + sys.executable,
338 + "-m",
339 + "pip",
340 + "install",
341 + "-q",
342 + f"--target={install_dir}",
343 + "einops", # Dependency of flash-attn.
344 + f"flash-attn=={flash_attn_version}",
345 + "--no-dependencies", # Required to avoid re-installing torch.
346 + ],
347 + env=dict(os.environ, CC="gcc"),
348 + )
349 +
350 + # Copy the FlashAttention package into the vLLM package after build.
351 + class build_ext(BuildExtension):
352 +
353 + def run(self):
354 + super().run()
355 + target_dir = os.path.join(self.build_lib, THIRDPARTY_SUBDIR)
356 + if not os.path.exists(target_dir):
357 + os.makedirs(target_dir)
358 + self.copy_tree(install_dir, target_dir)
359 +
360 + class BinaryDistribution(setuptools.Distribution):
361 +
362 + def has_ext_modules(self):
363 + return True
364 +
365 +else:
366 + build_ext = BuildExtension
367 + BinaryDistribution = setuptools.Distribution
368 + if _is_neuron():
369 + neuronxcc_version = get_neuronxcc_version()
370
371 vllm_extension_sources = [
372 "csrc/cache_kernels.cu",
@@ -468,6 +509,7 @@ setuptools.setup(
509 python_requires=">=3.8",
510 install_requires=get_requirements(),
511 ext_modules=ext_modules,
471 - cmdclass={"build_ext": BuildExtension} if not _is_neuron() else {},
512 + cmdclass={"build_ext": build_ext} if not _is_neuron() else {},
513 + distclass=BinaryDistribution,
514 package_data=package_data,
515 )
tests/kernels/test_prefix_prefill.py
+1 -1
@@ -3,7 +3,7 @@ import pytest
3 import time
4
5 import torch
6 -from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
6 +from vllm.model_executor.layers.attention.ops.prefix_prefill import (
7 context_attention_fwd)
8 from xformers import ops as xops
9 from xformers.ops.fmha.attn_bias import BlockDiagonalCausalFromBottomRightMask
vllm/__init__.py
+23 -7
@@ -1,12 +1,28 @@
1 """vLLM: a high-throughput and memory-efficient inference engine for LLMs"""
2
3 -from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4 -from vllm.engine.async_llm_engine import AsyncLLMEngine
5 -from vllm.engine.llm_engine import LLMEngine
6 -from vllm.engine.ray_utils import initialize_cluster
7 -from vllm.entrypoints.llm import LLM
8 -from vllm.outputs import CompletionOutput, RequestOutput
9 -from vllm.sampling_params import SamplingParams
3 +
4 +# Adapted from https://github.com/ray-project/ray/blob/f92928c9cfcbbf80c3a8534ca4911de1b44069c0/python/ray/__init__.py#L11
5 +def _configure_system():
6 + import os
7 + import sys
8 +
9 + # Importing flash-attn.
10 + thirdparty_files = os.path.join(os.path.abspath(os.path.dirname(__file__)),
11 + "thirdparty_files")
12 + sys.path.insert(0, thirdparty_files)
13 +
14 +
15 +_configure_system()
16 +# Delete configuration function.
17 +del _configure_system
18 +
19 +from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs # noqa: E402
20 +from vllm.engine.async_llm_engine import AsyncLLMEngine # noqa: E402
21 +from vllm.engine.llm_engine import LLMEngine # noqa: E402
22 +from vllm.engine.ray_utils import initialize_cluster # noqa: E402
23 +from vllm.entrypoints.llm import LLM # noqa: E402
24 +from vllm.outputs import CompletionOutput, RequestOutput # noqa: E402
25 +from vllm.sampling_params import SamplingParams # noqa: E402
26
27 __version__ = "0.3.3"
28
vllm/model_executor/layers/attention/__init__.py new
+5
@@ -0,0 +1,5 @@
1 +from vllm.model_executor.layers.attention.attention import Attention
2 +
3 +__all__ = [
4 + "Attention",
5 +]
vllm/model_executor/layers/attention/attention.py new
+59
@@ -0,0 +1,59 @@
1 +"""Attention layer."""
2 +from typing import List, Optional
3 +
4 +import torch
5 +import torch.nn as nn
6 +
7 +from vllm.model_executor.input_metadata import InputMetadata
8 +from vllm.utils import is_hip
9 +
10 +
11 +class Attention(nn.Module):
12 + """Attention layer.
13 +
14 + This class takes query, key, and value tensors as input. The input tensors
15 + can either contain prompt tokens or generation tokens.
16 + The class does the following:
17 +
18 + 1. Store the input key and value tensors in the KV cache.
19 + 2. Perform (multi-head/multi-query/grouped-query) attention.
20 + 3. Return the output tensor.
21 + """
22 +
23 + def __init__(
24 + self,
25 + num_heads: int,
26 + head_size: int,
27 + scale: float,
28 + num_kv_heads: Optional[int] = None,
29 + alibi_slopes: Optional[List[float]] = None,
30 + sliding_window: Optional[int] = None,
31 + ) -> None:
32 + super().__init__()
33 + if (not is_hip() and torch.cuda.get_device_capability()[0] >= 8 and
34 + torch.get_default_dtype() in (torch.float16, torch.bfloat16)):
35 + # Ampere or later NVIDIA GPUs.
36 + # NOTE(woosuk): FlashAttention does not support FP32.
37 + from vllm.model_executor.layers.attention.backends.flash_attn import FlashAttentionBackend
38 + self.backend = FlashAttentionBackend(num_heads, head_size, scale,
39 + num_kv_heads, alibi_slopes,
40 + sliding_window)
41 + else:
42 + # Turing and Volta NVIDIA GPUs or AMD GPUs.
43 + # Or FP32 on any GPU.
44 + from vllm.model_executor.layers.attention.backends.xformers import XFormersBackend
45 + self.backend = XFormersBackend(num_heads, head_size, scale,
46 + num_kv_heads, alibi_slopes,
47 + sliding_window)
48 +
49 + def forward(
50 + self,
51 + query: torch.Tensor,
52 + key: torch.Tensor,
53 + value: torch.Tensor,
54 + key_cache: Optional[torch.Tensor],
55 + value_cache: Optional[torch.Tensor],
56 + input_metadata: InputMetadata,
57 + ) -> torch.Tensor:
58 + return self.backend.forward(query, key, value, key_cache, value_cache,
59 + input_metadata)
vllm/model_executor/layers/attention/backends/__init__.py renamed
vllm/model_executor/layers/attention/backends/flash_attn.py new
+124
@@ -0,0 +1,124 @@
1 +"""Attention layer with Flash and PagedAttention."""
2 +from typing import List, Optional
3 +
4 +# NOTE(woosuk): This imports flash_attn under vllm/thirdparty_files/.
5 +from flash_attn import flash_attn_func
6 +import torch
7 +
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.attention.ops.paged_attn import (
10 + PagedAttentionImpl)
11 +
12 +
13 +class FlashAttentionBackend:
14 +
15 + def __init__(
16 + self,
17 + num_heads: int,
18 + head_size: int,
19 + scale: float,
20 + num_kv_heads: Optional[int] = None,
21 + alibi_slopes: Optional[List[float]] = None,
22 + sliding_window: Optional[int] = None,
23 + ) -> None:
24 + self.num_heads = num_heads
25 + self.head_size = head_size
26 + self.scale = float(scale)
27 + self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
28 + self.sliding_window = sliding_window
29 + if alibi_slopes is not None:
30 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
31 + self.alibi_slopes = alibi_slopes
32 +
33 + assert self.num_heads % self.num_kv_heads == 0
34 + self.num_queries_per_kv = self.num_heads // self.num_kv_heads
35 + suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
36 + if head_size not in suppored_head_sizes:
37 + raise ValueError(
38 + f"Head size {head_size} is not supported by PagedAttention. "
39 + f"Supported head sizes are: {suppored_head_sizes}.")
40 +
41 + self.sliding_window = ((self.sliding_window, self.sliding_window) if
42 + self.sliding_window is not None else (-1, -1))
43 +
44 + def forward(
45 + self,
46 + query: torch.Tensor,
47 + key: torch.Tensor,
48 + value: torch.Tensor,
49 + key_cache: Optional[torch.Tensor],
50 + value_cache: Optional[torch.Tensor],
51 + input_metadata: InputMetadata,
52 + ) -> torch.Tensor:
53 + """Forward pass with FlashAttention and PagedAttention.
54 +
55 + Args:
56 + query: shape = [batch_size, seq_len, num_heads * head_size]
57 + key: shape = [batch_size, seq_len, num_kv_heads * head_size]
58 + value: shape = [batch_size, seq_len, num_kv_heads * head_size]
59 + key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
60 + block_size, x]
61 + value_cache: shape = [num_blocks, num_kv_heads, head_size,
62 + block_size]
63 + input_metadata: metadata for the inputs.
64 + Returns:
65 + shape = [batch_size, seq_len, num_heads * head_size]
66 + """
67 + batch_size, seq_len, hidden_size = query.shape
68 + # Reshape the query, key, and value tensors.
69 + query = query.view(-1, self.num_heads, self.head_size)
70 + key = key.view(-1, self.num_kv_heads, self.head_size)
71 + value = value.view(-1, self.num_kv_heads, self.head_size)
72 +
73 + # Reshape the keys and values and store them in the cache.
74 + # If key_cache and value_cache are not provided, the new key and value
75 + # vectors will not be cached. This happens during the initial memory
76 + # profiling run.
77 + if key_cache is not None and value_cache is not None:
78 + PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
79 + value_cache, input_metadata)
80 +
81 + if input_metadata.is_prompt:
82 + # Prompt run.
83 + if (key_cache is None or value_cache is None
84 + or input_metadata.block_tables.numel() == 0):
85 + # normal attention
86 + query = query.unflatten(0, (batch_size, seq_len))
87 + key = key.unflatten(0, (batch_size, seq_len))
88 + value = value.unflatten(0, (batch_size, seq_len))
89 + output = flash_attn_func(
90 + query,
91 + key,
92 + value,
93 + softmax_scale=self.scale,
94 + causal=True,
95 + window_size=self.sliding_window,
96 + alibi_slopes=self.alibi_slopes,
97 + )
98 + else:
99 + # prefix-enabled attention
100 + output = PagedAttentionImpl.forward_prefix(
101 + query,
102 + key,
103 + value,
104 + key_cache,
105 + value_cache,
106 + input_metadata,
107 + self.num_heads,
108 + self.num_kv_heads,
109 + self.alibi_slopes,
110 + )
111 + else:
112 + # Decoding run.
113 + output = PagedAttentionImpl.forward_decode(
114 + query,
115 + key_cache,
116 + value_cache,
117 + input_metadata,
118 + self.num_kv_heads,
119 + self.scale,
120 + self.alibi_slopes,
121 + )
122 +
123 + # Reshape the output tensor.
124 + return output.view(batch_size, seq_len, hidden_size)
vllm/model_executor/layers/attention/backends/xformers.py renamed
+61 -155
@@ -1,37 +1,19 @@
1 -"""Multi-head attention."""
1 +"""Attention layer with xFormers and PagedAttention."""
2 +import importlib
3 from typing import List, Optional
4
4 -import importlib
5 import torch
6 -import torch.nn as nn
6 from xformers import ops as xops
7 from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
8 LowerTriangularMaskWithTensorBias)
9
11 -from vllm._C import ops
12 -from vllm._C import cache_ops
10 from vllm.model_executor.input_metadata import InputMetadata
14 -from vllm.model_executor.layers.triton_kernel.prefix_prefill import (
15 - context_attention_fwd)
11 +from vllm.model_executor.layers.attention.ops.paged_attn import (
12 + PagedAttentionImpl)
13 from vllm.utils import is_hip
14
18 -_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
19 -# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
20 -_PARTITION_SIZE = 512
21 -
22 -
23 -class PagedAttention(nn.Module):
24 - """MHA/MQA/GQA layer with PagedAttention.
15
26 - This class takes query, key, and value tensors as input. The input tensors
27 - can either contain prompt tokens or generation tokens.
28 - The class does the following:
29 -
30 - 1. Reshape and store the input key and value tensors in the KV cache.
31 - 2. Perform (multi-head/multi-query/grouped-query) attention using either
32 - xformers or the PagedAttention custom op.
33 - 3. Return the output tensor.
34 - """
16 +class XFormersBackend:
17
18 def __init__(
19 self,
@@ -42,7 +24,6 @@ class PagedAttention(nn.Module):
24 alibi_slopes: Optional[List[float]] = None,
25 sliding_window: Optional[int] = None,
26 ) -> None:
45 - super().__init__()
27 self.num_heads = num_heads
28 self.head_size = head_size
29 self.scale = float(scale)
@@ -50,48 +31,17 @@ class PagedAttention(nn.Module):
31 self.sliding_window = sliding_window
32 if alibi_slopes is not None:
33 alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
53 - self.register_buffer("alibi_slopes", alibi_slopes, persistent=False)
34 + self.alibi_slopes = alibi_slopes
35
36 assert self.num_heads % self.num_kv_heads == 0
37 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
38 + suppored_head_sizes = PagedAttentionImpl.get_supported_head_sizes()
39 + if head_size not in suppored_head_sizes:
40 + raise ValueError(
41 + f"Head size {head_size} is not supported by PagedAttention. "
42 + f"Supported head sizes are: {suppored_head_sizes}.")
43
58 - if self.head_size not in _SUPPORTED_HEAD_SIZES:
59 - raise ValueError(f"head_size ({self.head_size}) is not supported. "
60 - f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
61 -
62 - self.use_ref_attention = self.check_use_ref_attention()
63 -
64 - def check_use_ref_attention(self) -> bool:
65 - if not is_hip():
66 - return False
67 - # For ROCm, check whether flash attention is installed or not.
68 - # if not, use_ref_attention needs to be True
69 - return importlib.util.find_spec("flash_attn") is None
70 -
71 - def ref_masked_attention(
72 - self,
73 - query: torch.Tensor,
74 - key: torch.Tensor,
75 - value: torch.Tensor,
76 - ) -> torch.Tensor:
77 - query = query.view(-1, self.num_heads, self.head_size)
78 - key = key.view(-1, self.num_kv_heads, self.head_size)
79 - value = value.view(-1, self.num_kv_heads, self.head_size)
80 -
81 - seq_len, _, _ = query.shape
82 - attn_mask = torch.triu(torch.ones(seq_len,
83 - seq_len,
84 - dtype=query.dtype,
85 - device=query.device),
86 - diagonal=1)
87 - attn_mask = attn_mask * torch.finfo(query.dtype).min
88 -
89 - attn_weights = self.scale * torch.einsum("qhd,khd->hqk", query,
90 - key).float()
91 - attn_weights = attn_weights + attn_mask.float()
92 - attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
93 - out = torch.einsum("hqk,khd->qhd", attn_weights, value)
94 - return out
44 + self.use_ref_attention = _check_use_ref_attention()
45
46 def forward(
47 self,
@@ -102,7 +52,7 @@ class PagedAttention(nn.Module):
52 value_cache: Optional[torch.Tensor],
53 input_metadata: InputMetadata,
54 ) -> torch.Tensor:
105 - """PagedAttention forward pass.
55 + """Forward pass with xFormers and PagedAttention.
56
57 Args:
58 query: shape = [batch_size, seq_len, num_heads * head_size]
@@ -127,19 +77,14 @@ class PagedAttention(nn.Module):
77 # vectors will not be cached. This happens during the initial memory
78 # profiling run.
79 if key_cache is not None and value_cache is not None:
130 - cache_ops.reshape_and_cache(
131 - key,
132 - value,
133 - key_cache,
134 - value_cache,
135 - input_metadata.slot_mapping.flatten(),
136 - input_metadata.kv_cache_dtype,
137 - )
80 + PagedAttentionImpl.reshape_and_cache(key, value, key_cache,
81 + value_cache, input_metadata)
82
83 if input_metadata.is_prompt:
140 - # normal attention
84 + # Prompt run.
85 if (key_cache is None or value_cache is None
86 or input_metadata.block_tables.numel() == 0):
87 + # normal attention
88 if self.num_kv_heads != self.num_heads:
89 # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
90 # project the key and value tensors to the desired number of
@@ -175,13 +120,19 @@ class PagedAttention(nn.Module):
120 seq_len, query.dtype)
121
122 if self.use_ref_attention:
178 - output = self.ref_masked_attention(
123 + output = _ref_masked_attention(
124 query,
125 key,
126 value,
127 + self.num_heads,
128 + self.num_kv_heads,
129 + self.head_size,
130 + self.scale,
131 )
183 - # Using view got RuntimeError: view size is not compatible with input tensor's size and stride
184 - # (at least one dimension spans across two contiguous subspaces). Use reshape instead
132 + # Using view got RuntimeError: view size is not compatible
133 + # with input tensor's size and stride (at least one
134 + # dimension spans across two contiguous subspaces).
135 + # Use reshape instead.
136 return output.reshape(batch_size, seq_len, hidden_size)
137
138 # TODO(woosuk): Too many view operations. Let's try to reduce
@@ -206,27 +157,21 @@ class PagedAttention(nn.Module):
157 (is_hip()) else None,
158 )
159 output = out.view_as(query)
160 +
161 else:
162 # prefix-enabled attention
211 - output = torch.empty_like(query)
212 - context_attention_fwd(
163 + output = PagedAttentionImpl.forward_prefix(
164 query,
165 key,
166 value,
216 - output,
167 key_cache,
168 value_cache,
219 - input_metadata.block_tables, # [BS, max_block_per_request]
220 - input_metadata.start_loc,
221 - input_metadata.prompt_lens,
222 - input_metadata.context_lens,
223 - input_metadata.max_seq_len,
224 - getattr(self, "alibi_slopes", None),
169 + input_metadata,
170 + self.alibi_slopes,
171 )
226 -
172 else:
173 # Decoding run.
229 - output = _paged_attention(
174 + output = PagedAttentionImpl.forward_decode(
175 query,
176 key_cache,
177 value_cache,
@@ -274,76 +219,37 @@ def _make_alibi_bias(
219 return attn_bias
220
221
277 -def _paged_attention(
222 +def _check_use_ref_attention() -> bool:
223 + if not is_hip():
224 + return False
225 + # For ROCm, check whether flash attention is installed or not.
226 + # if not, use_ref_attention needs to be True
227 + return importlib.util.find_spec("flash_attn") is None
228 +
229 +
230 +def _ref_masked_attention(
231 query: torch.Tensor,
279 - key_cache: torch.Tensor,
280 - value_cache: torch.Tensor,
281 - input_metadata: InputMetadata,
232 + key: torch.Tensor,
233 + value: torch.Tensor,
234 + num_heads: int,
235 num_kv_heads: int,
236 + head_size: int,
237 scale: float,
284 - alibi_slopes: Optional[torch.Tensor],
238 ) -> torch.Tensor:
286 - output = torch.empty_like(query)
287 -
288 - block_size = value_cache.shape[3]
289 - num_seqs, num_heads, head_size = query.shape
290 - max_num_partitions = (
291 - (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
292 - _PARTITION_SIZE)
293 - # NOTE(woosuk): We use a simple heuristic to decide whether to use
294 - # PagedAttention V1 or V2. If the number of partitions is 1, we use
295 - # V1 to avoid the overhead of reduction. Also, if the number of
296 - # sequences or heads is large, we use V1 since there is enough work
297 - # to parallelize.
298 - # TODO(woosuk): Tune this heuristic.
299 - # For context len > 8192, use V2 kernel to avoid shared memory shortage.
300 - use_v1 = input_metadata.max_context_len <= 8192 and (
301 - max_num_partitions == 1 or num_seqs * num_heads > 512)
302 - if use_v1:
303 - # Run PagedAttention V1.
304 - ops.paged_attention_v1(
305 - output,
306 - query,
307 - key_cache,
308 - value_cache,
309 - num_kv_heads,
310 - scale,
311 - input_metadata.block_tables,
312 - input_metadata.context_lens,
313 - block_size,
314 - input_metadata.max_context_len,
315 - alibi_slopes,
316 - input_metadata.kv_cache_dtype,
317 - )
318 - else:
319 - # Run PagedAttention V2.
320 - assert _PARTITION_SIZE % block_size == 0
321 - tmp_output = torch.empty(
322 - size=(num_seqs, num_heads, max_num_partitions, head_size),
323 - dtype=output.dtype,
324 - device=output.device,
325 - )
326 - exp_sums = torch.empty(
327 - size=(num_seqs, num_heads, max_num_partitions),
328 - dtype=torch.float32,
329 - device=output.device,
330 - )
331 - max_logits = torch.empty_like(exp_sums)
332 - ops.paged_attention_v2(
333 - output,
334 - exp_sums,
335 - max_logits,
336 - tmp_output,
337 - query,
338 - key_cache,
339 - value_cache,
340 - num_kv_heads,
341 - scale,
342 - input_metadata.block_tables,
343 - input_metadata.context_lens,
344 - block_size,
345 - input_metadata.max_context_len,
346 - alibi_slopes,
347 - input_metadata.kv_cache_dtype,
348 - )
349 - return output
239 + query = query.view(-1, num_heads, head_size)
240 + key = key.view(-1, num_kv_heads, head_size)
241 + value = value.view(-1, num_kv_heads, head_size)
242 +
243 + seq_len, _, _ = query.shape
244 + attn_mask = torch.triu(torch.ones(seq_len,
245 + seq_len,
246 + dtype=query.dtype,
247 + device=query.device),
248 + diagonal=1)
249 + attn_mask = attn_mask * torch.finfo(query.dtype).min
250 +
251 + attn_weights = scale * torch.einsum("qhd,khd->hqk", query, key).float()
252 + attn_weights = attn_weights + attn_mask.float()
253 + attn_weights = torch.softmax(attn_weights, dim=-1).to(value.dtype)
254 + out = torch.einsum("hqk,khd->qhd", attn_weights, value)
255 + return out
vllm/model_executor/layers/attention/ops/__init__.py
vllm/model_executor/layers/attention/ops/paged_attn.py new
+138
@@ -0,0 +1,138 @@
1 +from typing import List, Optional
2 +
3 +import torch
4 +
5 +from vllm._C import cache_ops
6 +from vllm._C import ops
7 +from vllm.model_executor.input_metadata import InputMetadata
8 +from vllm.model_executor.layers.attention.ops.prefix_prefill import (
9 + context_attention_fwd)
10 +
11 +# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
12 +_PARTITION_SIZE = 512
13 +
14 +
15 +class PagedAttentionImpl:
16 +
17 + @staticmethod
18 + def get_supported_head_sizes() -> List[int]:
19 + return [64, 80, 96, 112, 128, 256]
20 +
21 + @staticmethod
22 + def reshape_and_cache(
23 + key: torch.Tensor,
24 + value: torch.Tensor,
25 + key_cache: torch.Tensor,
26 + value_cache: torch.Tensor,
27 + input_metadata: InputMetadata,
28 + ) -> None:
29 + cache_ops.reshape_and_cache(
30 + key,
31 + value,
32 + key_cache,
33 + value_cache,
34 + input_metadata.slot_mapping.flatten(),
35 + input_metadata.kv_cache_dtype,
36 + )
37 +
38 + @staticmethod
39 + def forward_decode(
40 + query: torch.Tensor,
41 + key_cache: torch.Tensor,
42 + value_cache: torch.Tensor,
43 + input_metadata: InputMetadata,
44 + num_kv_heads: int,
45 + scale: float,
46 + alibi_slopes: Optional[torch.Tensor],
47 + ) -> torch.Tensor:
48 + output = torch.empty_like(query)
49 +
50 + block_size = value_cache.shape[3]
51 + num_seqs, num_heads, head_size = query.shape
52 + max_num_partitions = (
53 + (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
54 + _PARTITION_SIZE)
55 + # NOTE(woosuk): We use a simple heuristic to decide whether to use
56 + # PagedAttention V1 or V2. If the number of partitions is 1, we use
57 + # V1 to avoid the overhead of reduction. Also, if the number of
58 + # sequences or heads is large, we use V1 since there is enough work
59 + # to parallelize.
60 + # TODO(woosuk): Tune this heuristic.
61 + # For context len > 8192, use V2 kernel to avoid shared memory shortage.
62 + use_v1 = input_metadata.max_context_len <= 8192 and (
63 + max_num_partitions == 1 or num_seqs * num_heads > 512)
64 + if use_v1:
65 + # Run PagedAttention V1.
66 + ops.paged_attention_v1(
67 + output,
68 + query,
69 + key_cache,
70 + value_cache,
71 + num_kv_heads,
72 + scale,
73 + input_metadata.block_tables,
74 + input_metadata.context_lens,
75 + block_size,
76 + input_metadata.max_context_len,
77 + alibi_slopes,
78 + input_metadata.kv_cache_dtype,
79 + )
80 + else:
81 + # Run PagedAttention V2.
82 + assert _PARTITION_SIZE % block_size == 0
83 + tmp_output = torch.empty(
84 + size=(num_seqs, num_heads, max_num_partitions, head_size),
85 + dtype=output.dtype,
86 + device=output.device,
87 + )
88 + exp_sums = torch.empty(
89 + size=(num_seqs, num_heads, max_num_partitions),
90 + dtype=torch.float32,
91 + device=output.device,
92 + )
93 + max_logits = torch.empty_like(exp_sums)
94 + ops.paged_attention_v2(
95 + output,
96 + exp_sums,
97 + max_logits,
98 + tmp_output,
99 + query,
100 + key_cache,
101 + value_cache,
102 + num_kv_heads,
103 + scale,
104 + input_metadata.block_tables,
105 + input_metadata.context_lens,
106 + block_size,
107 + input_metadata.max_context_len,
108 + alibi_slopes,
109 + input_metadata.kv_cache_dtype,
110 + )
111 + return output
112 +
113 + @staticmethod
114 + def forward_prefix(
115 + query: torch.Tensor,
116 + key: torch.Tensor,
117 + value: torch.Tensor,
118 + key_cache: torch.Tensor,
119 + value_cache: torch.Tensor,
120 + input_metadata: InputMetadata,
121 + alibi_slopes: Optional[torch.Tensor],
122 + ) -> torch.Tensor:
123 + output = torch.empty_like(query)
124 + context_attention_fwd(
125 + query,
126 + key,
127 + value,
128 + output,
129 + key_cache,
130 + value_cache,
131 + input_metadata.block_tables, # [BS, max_block_per_request]
132 + input_metadata.start_loc,
133 + input_metadata.prompt_lens,
134 + input_metadata.context_lens,
135 + input_metadata.max_seq_len,
136 + alibi_slopes,
137 + )
138 + return output
vllm/model_executor/layers/attention/ops/prefix_prefill.py renamed
vllm/model_executor/models/baichuan.py
+6 -7
@@ -27,7 +27,7 @@ from transformers import PretrainedConfig
27
28 from vllm.model_executor.input_metadata import InputMetadata
29 from vllm.model_executor.layers.activation import SiluAndMul
30 -from vllm.model_executor.layers.attention import PagedAttention
30 +from vllm.model_executor.layers.attention import Attention
31 from vllm.model_executor.layers.layernorm import RMSNorm
32 from vllm.model_executor.layers.linear import (LinearMethodBase,
33 MergedColumnParallelLinear,
@@ -151,10 +151,10 @@ class BaiChuanAttention(nn.Module):
151 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
152
153 scaling = self.head_dim**-0.5
154 - self.attn = PagedAttention(self.num_heads,
155 - self.head_dim,
156 - scaling,
157 - alibi_slopes=alibi_slopes)
154 + self.attn = Attention(self.num_heads,
155 + self.head_dim,
156 + scaling,
157 + alibi_slopes=alibi_slopes)
158 else:
159 self.rotary_emb = get_rope(
160 self.head_dim,
@@ -163,8 +163,7 @@ class BaiChuanAttention(nn.Module):
163 base=self.rope_theta,
164 )
165 self.scaling = self.head_dim**-0.5
166 - self.attn = PagedAttention(self.num_heads, self.head_dim,
167 - self.scaling)
166 + self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
167
168 def forward(
169 self,
vllm/model_executor/models/bloom.py
+5 -5
@@ -25,7 +25,7 @@ from transformers import BloomConfig
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import PagedAttention
28 +from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
@@ -107,10 +107,10 @@ class BloomAttention(nn.Module):
107 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
108
109 scaling = self.head_dim**-0.5
110 - self.attn = PagedAttention(self.num_heads,
111 - self.head_dim,
112 - scaling,
113 - alibi_slopes=alibi_slopes)
110 + self.attn = Attention(self.num_heads,
111 + self.head_dim,
112 + scaling,
113 + alibi_slopes=alibi_slopes)
114
115 def forward(
116 self,
vllm/model_executor/models/chatglm.py
+2 -2
@@ -10,7 +10,7 @@ from torch.nn import LayerNorm
10
11 from vllm.model_executor.input_metadata import InputMetadata
12 from vllm.model_executor.layers.activation import SiluAndMul
13 -from vllm.model_executor.layers.attention import PagedAttention
13 +from vllm.model_executor.layers.attention import Attention
14 from vllm.model_executor.layers.layernorm import RMSNorm
15 from vllm.model_executor.layers.linear import (LinearMethodBase,
16 MergedColumnParallelLinear,
@@ -87,7 +87,7 @@ class GLMAttention(nn.Module):
87 base=10000 * rope_ratio,
88 is_neox_style=False,
89 )
90 - self.attn = PagedAttention(
90 + self.attn = Attention(
91 self.num_heads,
92 self.head_dim,
93 self.scaling,
vllm/model_executor/models/deepseek.py
+5 -5
@@ -29,7 +29,7 @@ from transformers import PretrainedConfig
29
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttention
32 +from vllm.model_executor.layers.attention import Attention
33 from vllm.model_executor.layers.fused_moe import fused_moe
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -229,10 +229,10 @@ class DeepseekAttention(nn.Module):
229 base=rope_theta,
230 rope_scaling=rope_scaling,
231 )
232 - self.attn = PagedAttention(self.num_heads,
233 - self.head_dim,
234 - self.scaling,
235 - num_kv_heads=self.num_kv_heads)
232 + self.attn = Attention(self.num_heads,
233 + self.head_dim,
234 + self.scaling,
235 + num_kv_heads=self.num_kv_heads)
236
237 def forward(
238 self,
vllm/model_executor/models/falcon.py
+14 -14
@@ -28,7 +28,7 @@ from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.activation import get_act_fn
31 -from vllm.model_executor.layers.attention import PagedAttention
31 +from vllm.model_executor.layers.attention import Attention
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 LinearMethodBase,
34 QKVParallelLinear,
@@ -150,10 +150,10 @@ class FalconAttention(nn.Module):
150 max_position=max_position_embeddings,
151 base=rope_theta,
152 )
153 - self.attn = PagedAttention(self.num_heads,
154 - self.head_dim,
155 - self.inv_norm_factor,
156 - num_kv_heads=self.num_kv_heads)
153 + self.attn = Attention(self.num_heads,
154 + self.head_dim,
155 + self.inv_norm_factor,
156 + num_kv_heads=self.num_kv_heads)
157 elif self.use_alibi:
158 tp_rank = get_tensor_model_parallel_rank()
159 head_start = tp_rank * self.num_heads
@@ -161,16 +161,16 @@ class FalconAttention(nn.Module):
161 alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
162 self.inv_norm_factor)
163 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
164 - self.attn = PagedAttention(self.num_heads,
165 - self.head_dim,
166 - self.inv_norm_factor,
167 - num_kv_heads=self.num_kv_heads,
168 - alibi_slopes=alibi_slopes)
164 + self.attn = Attention(self.num_heads,
165 + self.head_dim,
166 + self.inv_norm_factor,
167 + num_kv_heads=self.num_kv_heads,
168 + alibi_slopes=alibi_slopes)
169 else:
170 - self.attn = PagedAttention(self.num_heads,
171 - self.head_dim,
172 - scale=self.inv_norm_factor,
173 - num_kv_heads=self.num_kv_heads)
170 + self.attn = Attention(self.num_heads,
171 + self.head_dim,
172 + scale=self.inv_norm_factor,
173 + num_kv_heads=self.num_kv_heads)
174
175 def forward(
176 self,
vllm/model_executor/models/gemma.py
+5 -5
@@ -23,7 +23,7 @@ from transformers import GemmaConfig
23 from vllm.config import LoRAConfig
24 from vllm.model_executor.input_metadata import InputMetadata
25 from vllm.model_executor.layers.activation import GeluAndMul
26 -from vllm.model_executor.layers.attention import PagedAttention
26 +from vllm.model_executor.layers.attention import Attention
27 from vllm.model_executor.layers.layernorm import RMSNorm
28 from vllm.model_executor.layers.linear import (LinearMethodBase,
29 MergedColumnParallelLinear,
@@ -123,10 +123,10 @@ class GemmaAttention(nn.Module):
123 base=self.rope_theta,
124 is_neox_style=True,
125 )
126 - self.attn = PagedAttention(self.num_heads,
127 - self.head_dim,
128 - self.scaling,
129 - num_kv_heads=self.num_kv_heads)
126 + self.attn = Attention(self.num_heads,
127 + self.head_dim,
128 + self.scaling,
129 + num_kv_heads=self.num_kv_heads)
130
131 def forward(
132 self,
vllm/model_executor/models/gpt2.py
+2 -4
@@ -25,7 +25,7 @@ from transformers import GPT2Config
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import PagedAttention
28 +from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
@@ -73,9 +73,7 @@ class GPT2Attention(nn.Module):
73 bias=True,
74 linear_method=linear_method,
75 )
76 - self.attn = PagedAttention(self.num_heads,
77 - self.head_dim,
78 - scale=self.scale)
76 + self.attn = Attention(self.num_heads, self.head_dim, scale=self.scale)
77
78 def forward(
79 self,
vllm/model_executor/models/gpt_bigcode.py
+5 -5
@@ -26,7 +26,7 @@ from transformers import GPTBigCodeConfig
26
27 from vllm.model_executor.input_metadata import InputMetadata
28 from vllm.model_executor.layers.activation import get_act_fn
29 -from vllm.model_executor.layers.attention import PagedAttention
29 +from vllm.model_executor.layers.attention import Attention
30 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 LinearMethodBase,
32 QKVParallelLinear,
@@ -85,10 +85,10 @@ class GPTBigCodeAttention(nn.Module):
85 bias=True,
86 linear_method=linear_method,
87 )
88 - self.attn = PagedAttention(self.num_heads,
89 - self.head_dim,
90 - scale=self.scale,
91 - num_kv_heads=self.num_kv_heads)
88 + self.attn = Attention(self.num_heads,
89 + self.head_dim,
90 + scale=self.scale,
91 + num_kv_heads=self.num_kv_heads)
92
93 def forward(
94 self,
vllm/model_executor/models/gpt_j.py
+2 -2
@@ -24,7 +24,7 @@ from transformers import GPTJConfig
24
25 from vllm.model_executor.input_metadata import InputMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import PagedAttention
27 +from vllm.model_executor.layers.attention import Attention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
@@ -86,7 +86,7 @@ class GPTJAttention(nn.Module):
86 base=rope_theta,
87 is_neox_style=False,
88 )
89 - self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
89 + self.attn = Attention(self.num_heads, self.head_size, scaling)
90
91 def forward(
92 self,
vllm/model_executor/models/gpt_neox.py
+2 -2
@@ -24,7 +24,7 @@ from transformers import GPTNeoXConfig
24
25 from vllm.model_executor.input_metadata import InputMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import PagedAttention
27 +from vllm.model_executor.layers.attention import Attention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
@@ -87,7 +87,7 @@ class GPTNeoXAttention(nn.Module):
87 max_position=max_position_embeddings,
88 base=rope_theta,
89 )
90 - self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
90 + self.attn = Attention(self.num_heads, self.head_size, scaling)
91
92 def forward(
93 self,
vllm/model_executor/models/internlm2.py
+5 -5
@@ -7,7 +7,7 @@ from transformers import PretrainedConfig
7
8 from vllm.model_executor.input_metadata import InputMetadata
9 from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import PagedAttention
10 +from vllm.model_executor.layers.attention import Attention
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 MergedColumnParallelLinear,
@@ -114,10 +114,10 @@ class InternLM2Attention(nn.Module):
114 base=rope_theta,
115 rope_scaling=rope_scaling,
116 )
117 - self.attn = PagedAttention(self.num_heads,
118 - self.head_dim,
119 - self.scaling,
120 - num_kv_heads=self.num_kv_heads)
117 + self.attn = Attention(self.num_heads,
118 + self.head_dim,
119 + self.scaling,
120 + num_kv_heads=self.num_kv_heads)
121
122 def forward(
123 self,
vllm/model_executor/models/llama.py
+6 -6
@@ -30,7 +30,7 @@ from transformers import LlamaConfig
30 from vllm.config import LoRAConfig
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.attention import Attention
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
36 MergedColumnParallelLinear,
@@ -139,11 +139,11 @@ class LlamaAttention(nn.Module):
139 base=rope_theta,
140 rope_scaling=rope_scaling,
141 )
142 - self.attn = PagedAttention(self.num_heads,
143 - self.head_dim,
144 - self.scaling,
145 - num_kv_heads=self.num_kv_heads,
146 - sliding_window=sliding_window)
142 + self.attn = Attention(self.num_heads,
143 + self.head_dim,
144 + self.scaling,
145 + num_kv_heads=self.num_kv_heads,
146 + sliding_window=sliding_window)
147
148 def forward(
149 self,
vllm/model_executor/models/mixtral.py
+2 -2
@@ -29,7 +29,7 @@ from transformers import MixtralConfig
29
30 from vllm.config import LoRAConfig
31 from vllm.model_executor.input_metadata import InputMetadata
32 -from vllm.model_executor.layers.attention import PagedAttention
32 +from vllm.model_executor.layers.attention import Attention
33 from vllm.model_executor.layers.fused_moe import fused_moe
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
@@ -197,7 +197,7 @@ class MixtralAttention(nn.Module):
197 base=int(self.rope_theta),
198 is_neox_style=True,
199 )
200 - self.attn = PagedAttention(
200 + self.attn = Attention(
201 self.num_heads,
202 self.head_dim,
203 self.scaling,
vllm/model_executor/models/mixtral_quant.py
+2 -2
@@ -32,7 +32,7 @@ from torch import nn
32 from transformers import MixtralConfig
33
34 from vllm.model_executor.input_metadata import InputMetadata
35 -from vllm.model_executor.layers.attention import PagedAttention
35 +from vllm.model_executor.layers.attention import Attention
36 from vllm.model_executor.layers.layernorm import RMSNorm
37 from vllm.model_executor.layers.linear import (LinearMethodBase,
38 ReplicatedLinear,
@@ -214,7 +214,7 @@ class MixtralAttention(nn.Module):
214 base=int(self.rope_theta),
215 is_neox_style=True,
216 )
217 - self.attn = PagedAttention(
217 + self.attn = Attention(
218 self.num_heads,
219 self.head_dim,
220 self.scaling,
vllm/model_executor/models/mpt.py
+6 -6
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9 from vllm.model_executor.input_metadata import InputMetadata
10 from vllm.model_executor.layers.activation import get_act_fn
11 -from vllm.model_executor.layers.attention import PagedAttention
11 +from vllm.model_executor.layers.attention import Attention
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 LinearMethodBase,
14 QKVParallelLinear,
@@ -105,11 +105,11 @@ class MPTAttention(nn.Module):
105
106 self.head_dim = self.d_model // self.total_num_heads
107 scaling = self.head_dim**-0.5
108 - self.attn = PagedAttention(self.num_heads,
109 - self.head_dim,
110 - scaling,
111 - alibi_slopes=alibi_slopes,
112 - num_kv_heads=self.num_kv_heads)
108 + self.attn = Attention(self.num_heads,
109 + self.head_dim,
110 + scaling,
111 + alibi_slopes=alibi_slopes,
112 + num_kv_heads=self.num_kv_heads)
113
114 def forward(
115 self,
vllm/model_executor/models/olmo.py
+4 -4
@@ -43,7 +43,7 @@ import torch.nn.functional as F
43 from torch import nn
44
45 from vllm.model_executor.input_metadata import InputMetadata
46 -from vllm.model_executor.layers.attention import PagedAttention
46 +from vllm.model_executor.layers.attention import Attention
47 from vllm.model_executor.layers.linear import (
48 ColumnParallelLinear,
49 LinearMethodBase,
@@ -126,9 +126,9 @@ class OlmoAttention(nn.Module):
126 base=rope_theta,
127 )
128 self.scaling = self.head_dim**-0.5
129 - self.attn = PagedAttention(self.num_heads,
130 - self.head_dim,
131 - scale=self.scaling)
129 + self.attn = Attention(self.num_heads,
130 + self.head_dim,
131 + scale=self.scaling)
132
133 # Attention output projection.
134 self.attn_out = RowParallelLinear(
vllm/model_executor/models/opt.py
+4 -4
@@ -25,7 +25,7 @@ from transformers import OPTConfig
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import PagedAttention
28 +from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
@@ -89,9 +89,9 @@ class OPTAttention(nn.Module):
89 bias=bias,
90 linear_method=linear_method,
91 )
92 - self.attn = PagedAttention(self.num_heads,
93 - self.head_dim,
94 - scale=self.scaling)
92 + self.attn = Attention(self.num_heads,
93 + self.head_dim,
94 + scale=self.scaling)
95
96 def forward(
97 self,
vllm/model_executor/models/orion.py
+5 -5
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
12
13 from vllm.model_executor.input_metadata import InputMetadata
14 from vllm.model_executor.layers.activation import SiluAndMul
15 -from vllm.model_executor.layers.attention import PagedAttention
15 +from vllm.model_executor.layers.attention import Attention
16 from vllm.model_executor.layers.linear import (LinearMethodBase,
17 MergedColumnParallelLinear,
18 QKVParallelLinear,
@@ -118,10 +118,10 @@ class OrionAttention(nn.Module):
118 base=rope_theta,
119 rope_scaling=rope_scaling,
120 )
121 - self.attn = PagedAttention(self.num_heads,
122 - self.head_dim,
123 - self.scaling,
124 - num_kv_heads=self.num_kv_heads)
121 + self.attn = Attention(self.num_heads,
122 + self.head_dim,
123 + self.scaling,
124 + num_kv_heads=self.num_kv_heads)
125
126 def forward(
127 self,
vllm/model_executor/models/phi.py
+2 -2
@@ -43,7 +43,7 @@ from transformers import PretrainedConfig
43
44 from vllm.model_executor.input_metadata import InputMetadata
45 from vllm.model_executor.layers.activation import get_act_fn
46 -from vllm.model_executor.layers.attention import PagedAttention
46 +from vllm.model_executor.layers.attention import Attention
47 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48 LinearMethodBase,
49 QKVParallelLinear,
@@ -108,7 +108,7 @@ class PhiAttention(nn.Module):
108 max_position=max_position_embeddings,
109 base=rope_theta,
110 )
111 - self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
111 + self.attn = Attention(self.num_heads, self.head_size, scaling)
112
113 def forward(
114 self,
vllm/model_executor/models/qwen.py
+2 -2
@@ -12,7 +12,7 @@ from transformers import PretrainedConfig
12
13 from vllm.model_executor.input_metadata import InputMetadata
14 from vllm.model_executor.layers.activation import SiluAndMul
15 -from vllm.model_executor.layers.attention import PagedAttention
15 +from vllm.model_executor.layers.attention import Attention
16 from vllm.model_executor.layers.layernorm import RMSNorm
17 from vllm.model_executor.layers.linear import (LinearMethodBase,
18 MergedColumnParallelLinear,
@@ -104,7 +104,7 @@ class QWenAttention(nn.Module):
104 base=rope_theta,
105 rope_scaling=rope_scaling,
106 )
107 - self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
107 + self.attn = Attention(self.num_heads, self.head_dim, self.scaling)
108
109 def forward(
110 self,
vllm/model_executor/models/qwen2.py
+6 -6
@@ -30,7 +30,7 @@ from transformers import Qwen2Config
30
31 from vllm.model_executor.input_metadata import InputMetadata
32 from vllm.model_executor.layers.activation import SiluAndMul
33 -from vllm.model_executor.layers.attention import PagedAttention
33 +from vllm.model_executor.layers.attention import Attention
34 from vllm.model_executor.layers.layernorm import RMSNorm
35 from vllm.model_executor.layers.linear import (LinearMethodBase,
36 MergedColumnParallelLinear,
@@ -135,11 +135,11 @@ class Qwen2Attention(nn.Module):
135 max_position=max_position,
136 base=self.rope_theta,
137 )
138 - self.attn = PagedAttention(self.num_heads,
139 - self.head_dim,
140 - self.scaling,
141 - num_kv_heads=self.num_kv_heads,
142 - sliding_window=self.sliding_window)
138 + self.attn = Attention(self.num_heads,
139 + self.head_dim,
140 + self.scaling,
141 + num_kv_heads=self.num_kv_heads,
142 + sliding_window=self.sliding_window)
143
144 def forward(
145 self,
vllm/model_executor/models/stablelm.py
+5 -5
@@ -25,7 +25,7 @@ from transformers import PretrainedConfig
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.layers.activation import SiluAndMul
28 -from vllm.model_executor.layers.attention import PagedAttention
28 +from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.linear import (LinearMethodBase,
30 MergedColumnParallelLinear,
31 QKVParallelLinear,
@@ -122,10 +122,10 @@ class StablelmAttention(nn.Module):
122 max_position=self.config.max_position_embeddings,
123 base=self.config.rope_theta,
124 )
125 - self.attn = PagedAttention(self.num_heads,
126 - self.head_dim,
127 - self.scaling,
128 - num_kv_heads=self.num_key_value_heads)
125 + self.attn = Attention(self.num_heads,
126 + self.head_dim,
127 + self.scaling,
128 + num_kv_heads=self.num_key_value_heads)
129
130 def forward(
131 self,
vllm/model_executor/models/starcoder2.py
+2 -2
@@ -25,7 +25,7 @@ from torch import nn
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 -from vllm.model_executor.layers.attention import PagedAttention
28 +from vllm.model_executor.layers.attention import Attention
29 from vllm.model_executor.layers.activation import get_act_fn
30 from vllm.model_executor.layers.rotary_embedding import get_rope
31 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -103,7 +103,7 @@ class Starcoder2Attention(nn.Module):
103 base=int(self.rope_theta),
104 is_neox_style=True,
105 )
106 - self.attn = PagedAttention(
106 + self.attn = Attention(
107 self.num_heads,
108 self.head_dim,
109 self.scaling,