A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Refactor Attention (#1840)
Woosuk Kwon committed
Nov 29, 2023 at 15:37 UTC
a9e4574261a20d4ada213d26671da7dc7633580b
16 files changed
+360
-498
vllm/model_executor/layers/attention.py
+191
-361
@@ -1,5 +1,5 @@
1
"""Multi-head attention."""
2
-from typing import Any, Dict, List, Optional
2
+from typing import List, Optional
3
4
import torch
5
import torch.nn as nn
@@ -10,7 +10,6 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10
from vllm._C import ops
11
from vllm._C import cache_ops
12
from vllm.model_executor.input_metadata import InputMetadata
13
-from vllm.model_executor.layers.rotary_embedding import get_rope
13
14
_SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
15
# Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -18,37 +17,39 @@ _PARTITION_SIZE = 512
17
18
19
class PagedAttention(nn.Module):
21
- """GPT-style multi-head PagedAttention.
20
+ """MHA/MQA/GQA layer with PagedAttention.
21
22
This class takes query, key, and value tensors as input. The input tensors
24
- can either contain prompt tokens or generation tokens, in addition to
25
- paddings.
26
-
23
+ can either contain prompt tokens or generation tokens.
24
The class does the following:
28
- 1. Perform multi_query_kv_attention for the prompts. This operation does
29
- not use the KV cache.
30
- 2. Wait for the cache operations (e.g., swap, copy) to finish. The cache
25
+
26
+ 1. Wait for the cache operations (e.g., swap, copy) to finish. The cache
27
operations are issued by the cache engine before executing the forward
28
pass of the model, and they are executed asynchronously.
33
- 3. Reshape and store the input key and value tensors in the KV cache.
34
- 4. Perform single_query_cached_kv_attention for the generation tokens.
35
- This operation reads the previous key and value tensors from the KV
36
- cache.
37
- 5. Return the output tensor.
29
+ 2. Reshape and store the input key and value tensors in the KV cache.
30
+ 3. Perform (multi-head/multi-query/grouped-query) attention using either
31
+ xformers or the PagedAttention custom op.
32
+ 4. Return the output tensor.
33
"""
34
40
- def __init__(self,
41
- num_heads: int,
42
- head_size: int,
43
- scale: float,
44
- num_kv_heads: Optional[int] = None,
45
- sliding_window: Optional[int] = None) -> None:
35
+ def __init__(
36
+ self,
37
+ num_heads: int,
38
+ head_size: int,
39
+ scale: float,
40
+ num_kv_heads: Optional[int] = None,
41
+ alibi_slopes: Optional[List[float]] = None,
42
+ sliding_window: Optional[int] = None,
43
+ ) -> None:
44
super().__init__()
45
self.num_heads = num_heads
46
self.head_size = head_size
47
self.scale = float(scale)
48
self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
49
self.sliding_window = sliding_window
50
+ if alibi_slopes is not None:
51
+ alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
52
+ self.register_buffer("alibi_slopes", alibi_slopes, persistent=False)
53
54
assert self.num_heads % self.num_kv_heads == 0
55
self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -60,153 +61,6 @@ class PagedAttention(nn.Module):
61
raise ValueError(f"head_size ({self.head_size}) is not supported. "
62
f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
63
63
- def set_attn_bias(
64
- self,
65
- input_metadata: InputMetadata,
66
- dtype: torch.dtype,
67
- ) -> None:
68
- del dtype # Unused.
69
- if input_metadata.attn_bias is not None:
70
- # Already set by a previous layer.
71
- return
72
- prompt_lens = [input_metadata.max_prompt_len
73
- ] * input_metadata.num_prompts
74
- attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
75
- if self.sliding_window is not None:
76
- attn_bias = attn_bias.make_local_attention(self.sliding_window)
77
- input_metadata.attn_bias = attn_bias
78
-
79
- def multi_query_kv_attention(
80
- self,
81
- output: torch.Tensor,
82
- query: torch.Tensor,
83
- key: torch.Tensor,
84
- value: torch.Tensor,
85
- input_metadata: InputMetadata,
86
- ) -> torch.Tensor:
87
- """Normal attention for the prompt tokens.
88
-
89
- Args:
90
- output: shape = [num_prompt_tokens, num_heads, head_size]
91
- query: shape = [num_prompt_tokens, num_heads, head_size]
92
- key: shape = [num_prompt_tokens, num_kv_heads, head_size]
93
- value: shape = [num_prompt_tokens, num_kv_heads, head_size]
94
- input_metadata: metadata for paged attention.
95
- """
96
- if self.num_kv_heads != self.num_heads:
97
- # Project the key and value tensors to the desired number of heads.
98
- query = query.view(query.shape[0], self.num_kv_heads,
99
- self.num_queries_per_kv, query.shape[-1])
100
- key = key[:, :,
101
- None, :].expand(key.shape[0], self.num_kv_heads,
102
- self.num_queries_per_kv, key.shape[-1])
103
- value = value[:, :,
104
- None, :].expand(value.shape[0], self.num_kv_heads,
105
- self.num_queries_per_kv,
106
- value.shape[-1])
107
-
108
- # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
109
- out = xops.memory_efficient_attention_forward(
110
- query.unsqueeze(0),
111
- key.unsqueeze(0),
112
- value.unsqueeze(0),
113
- attn_bias=input_metadata.attn_bias,
114
- p=0.0,
115
- scale=self.scale,
116
- )
117
- # TODO(woosuk): Unnecessary copy. Optimize.
118
- output.copy_(out.view_as(output))
119
- return output
120
-
121
- def get_alibi_slopes(self) -> Optional[torch.Tensor]:
122
- """Returns the slopes for the alibi attention bias.
123
-
124
- Returns:
125
- slopes: shape = [num_heads]
126
- """
127
- return None
128
-
129
- def single_query_cached_kv_attention(
130
- self,
131
- output: torch.Tensor,
132
- query: torch.Tensor,
133
- key_cache: torch.Tensor,
134
- value_cache: torch.Tensor,
135
- input_metadata: InputMetadata,
136
- alibi_slopes: Optional[torch.Tensor],
137
- ) -> None:
138
- """PagedAttention for the generation tokens.
139
-
140
- Args:
141
- output: shape = [num_generation_tokens, num_heads, head_size]
142
- query: shape = [num_generation_tokens, num_heads, head_size]
143
- key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
144
- block_size, x]
145
- value_cache: shape = [num_blocks, num_kv_heads, head_size,
146
- block_size]
147
- input_metadata: metadata for paged attention.
148
- alibi_slopes: shape = [num_heads]
149
- """
150
- block_size = value_cache.shape[3]
151
- num_seqs, num_heads, head_size = query.shape
152
- max_num_partitions = (
153
- (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
154
- _PARTITION_SIZE)
155
- # NOTE(woosuk): We use a simple heuristic to decide whether to use
156
- # PagedAttention V1 or V2. If the number of partitions is 1, we use
157
- # V1 to avoid the overhead of reduction. Also, if the number of
158
- # sequences or heads is large, we use V1 since there is enough work
159
- # to parallelize.
160
- # TODO(woosuk): Tune this heuristic.
161
- # For context len > 8192, use V2 kernel to avoid shared memory shortage.
162
- use_v1 = input_metadata.max_context_len <= 8192 and (
163
- max_num_partitions == 1 or num_seqs * num_heads > 512)
164
- if use_v1:
165
- # Run PagedAttention V1.
166
- ops.paged_attention_v1(
167
- output,
168
- query,
169
- key_cache,
170
- value_cache,
171
- self.head_mapping,
172
- self.scale,
173
- input_metadata.block_tables,
174
- input_metadata.context_lens,
175
- block_size,
176
- input_metadata.max_context_len,
177
- alibi_slopes,
178
- )
179
- else:
180
- # Run PagedAttention V2.
181
- assert _PARTITION_SIZE % block_size == 0
182
- tmp_output = torch.empty(
183
- size=(num_seqs, num_heads, max_num_partitions, head_size),
184
- dtype=output.dtype,
185
- device=output.device,
186
- )
187
- exp_sums = torch.empty(
188
- size=(num_seqs, num_heads, max_num_partitions),
189
- dtype=torch.float32,
190
- device=output.device,
191
- )
192
- max_logits = torch.empty_like(exp_sums)
193
- ops.paged_attention_v2(
194
- output,
195
- exp_sums,
196
- max_logits,
197
- tmp_output,
198
- query,
199
- key_cache,
200
- value_cache,
201
- self.head_mapping,
202
- self.scale,
203
- input_metadata.block_tables,
204
- input_metadata.context_lens,
205
- block_size,
206
- input_metadata.max_context_len,
207
- alibi_slopes,
208
- )
209
-
64
def forward(
65
self,
66
query: torch.Tensor,
@@ -219,9 +73,6 @@ class PagedAttention(nn.Module):
73
) -> torch.Tensor:
74
"""PagedAttention forward pass.
75
222
- NOTE: The query, key, and value tensors must be sliced from a qkv
223
- tensor of shape [batch_size, seq_len, 3 * num_heads * head_size].
224
-
76
Args:
77
query: shape = [batch_size, seq_len, num_heads * head_size]
78
key: shape = [batch_size, seq_len, num_kv_heads * head_size]
@@ -230,46 +81,28 @@ class PagedAttention(nn.Module):
81
block_size, x]
82
value_cache: shape = [num_blocks, num_kv_heads, head_size,
83
block_size]
233
- input_metadata: metadata for paged attention.
84
+ input_metadata: metadata for the inputs.
85
cache_event: event to wait for the cache operations to finish.
235
-
86
Returns:
87
shape = [batch_size, seq_len, num_heads * head_size]
88
"""
239
- batch_size, seq_len, _ = query.shape
89
+ batch_size, seq_len, hidden_size = query.shape
90
# Reshape the query, key, and value tensors.
91
query = query.view(-1, self.num_heads, self.head_size)
92
key = key.view(-1, self.num_kv_heads, self.head_size)
93
value = value.view(-1, self.num_kv_heads, self.head_size)
94
+ slot_mapping = input_metadata.slot_mapping.flatten()
95
245
- # Pre-allocate the output tensor.
246
- output = torch.empty_like(query)
247
-
248
- # Compute the attention op for prompts.
249
- num_prompt_tokens = input_metadata.num_prompt_tokens
250
- if num_prompt_tokens > 0:
251
- # Prompt run.
252
- assert input_metadata.num_generation_tokens == 0
253
- self.set_attn_bias(input_metadata, dtype=query.dtype)
254
- self.multi_query_kv_attention(
255
- output,
256
- query,
257
- key,
258
- value,
259
- input_metadata,
260
- )
261
-
262
- # Wait until the cache op is done.
96
if cache_event is not None:
97
cache_event.wait()
98
99
# Reshape the keys and values and store them in the cache.
267
- # When key_cache and value_cache are not provided, the new key
268
- # and value vectors will not be cached.
100
+ # If key_cache and value_cache are not provided, the new key and value
101
+ # vectors will not be cached. This happens during the initial memory
102
+ # profiling run.
103
if key_cache is not None and value_cache is not None:
104
key_to_cache = key
105
value_to_cache = value
272
- slot_mapping = input_metadata.slot_mapping.view(-1)
106
if input_metadata.to_cache is not None:
107
key_to_cache = key_to_cache[input_metadata.to_cache]
108
value_to_cache = value_to_cache[input_metadata.to_cache]
@@ -283,178 +116,175 @@ class PagedAttention(nn.Module):
116
slot_mapping,
117
)
118
286
- if input_metadata.num_generation_tokens > 0:
119
+ is_prompt = len(input_metadata.prompt_lens) > 0
120
+ if is_prompt:
121
+ # Prompt run.
122
+ if self.num_kv_heads != self.num_heads:
123
+ # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
124
+ # project the key and value tensors to the desired number of
125
+ # heads.
126
+ # TODO(woosuk): Use MQA/GQA kernels for higher performance.
127
+ query = query.view(query.shape[0], self.num_kv_heads,
128
+ self.num_queries_per_kv, query.shape[-1])
129
+ key = key[:, :,
130
+ None, :].expand(key.shape[0], self.num_kv_heads,
131
+ self.num_queries_per_kv,
132
+ key.shape[-1])
133
+ value = value[:, :, None, :].expand(value.shape[0],
134
+ self.num_kv_heads,
135
+ self.num_queries_per_kv,
136
+ value.shape[-1])
137
+
138
+ # Set attention bias if not provided. This typically happens at the
139
+ # very attention layer of every iteration.
140
+ # FIXME(woosuk): This is a hack.
141
+ if input_metadata.attn_bias is None:
142
+ if self.alibi_slopes is None:
143
+ attn_bias = BlockDiagonalCausalMask.from_seqlens(
144
+ [seq_len] * batch_size)
145
+ if self.sliding_window is not None:
146
+ attn_bias = attn_bias.make_local_attention(
147
+ self.sliding_window)
148
+ input_metadata.attn_bias = attn_bias
149
+ else:
150
+ input_metadata.attn_bias = _make_alibi_bias(
151
+ self.alibi_slopes, batch_size, seq_len, query.dtype)
152
+
153
+ # TODO(woosuk): Too many view operations. Let's try to reduce them
154
+ # in the future for code readability.
155
+ if self.alibi_slopes is None:
156
+ query = query.unsqueeze(0)
157
+ key = key.unsqueeze(0)
158
+ value = value.unsqueeze(0)
159
+ else:
160
+ query = query.unflatten(0, (batch_size, seq_len))
161
+ key = key.unflatten(0, (batch_size, seq_len))
162
+ value = value.unflatten(0, (batch_size, seq_len))
163
+
164
+ out = xops.memory_efficient_attention_forward(
165
+ query,
166
+ key,
167
+ value,
168
+ attn_bias=input_metadata.attn_bias,
169
+ p=0.0,
170
+ scale=self.scale,
171
+ )
172
+ output = out.view_as(query)
173
+ else:
174
# Decoding run.
288
- assert input_metadata.num_prompt_tokens == 0
289
- assert key_cache is not None and value_cache is not None, (
290
- "key_cache and value_cache must be provided when "
291
- "generating tokens.")
292
- # Compute the attention op for generation tokens.
293
- self.single_query_cached_kv_attention(output, query, key_cache,
294
- value_cache, input_metadata,
295
- self.get_alibi_slopes())
175
+ output = _paged_attention(
176
+ query,
177
+ key_cache,
178
+ value_cache,
179
+ input_metadata,
180
+ self.head_mapping,
181
+ self.scale,
182
+ self.alibi_slopes,
183
+ )
184
185
# Reshape the output tensor.
298
- # NOTE(woosuk): The output tensor may include paddings.
299
- return output.view(batch_size, seq_len,
300
- self.num_heads * self.head_size)
301
-
302
-
303
-class PagedAttentionWithRoPE(PagedAttention):
304
- """PagedAttention with rotary positional embedding."""
305
-
306
- def __init__(
307
- self,
308
- num_heads: int,
309
- head_size: int,
310
- scale: float,
311
- rotary_dim: int,
312
- max_position: int = 8192,
313
- base: int = 10000,
314
- num_kv_heads: Optional[int] = None,
315
- is_neox_style: bool = True,
316
- rope_scaling: Optional[Dict[str, Any]] = None,
317
- sliding_window: Optional[int] = None,
318
- ) -> None:
319
- super().__init__(num_heads,
320
- head_size,
321
- scale,
322
- num_kv_heads,
323
- sliding_window=sliding_window)
324
- self.rotary_emb = get_rope(head_size, rotary_dim, max_position, base,
325
- is_neox_style, rope_scaling)
326
-
327
- def forward(
328
- self,
329
- positions: torch.Tensor,
330
- query: torch.Tensor,
331
- key: torch.Tensor,
332
- value: torch.Tensor,
333
- key_cache: torch.Tensor,
334
- value_cache: torch.Tensor,
335
- input_metadata: InputMetadata,
336
- cache_event: Optional[torch.cuda.Event],
337
- ) -> torch.Tensor:
338
- """ PagedAttention forward pass with rotary embedding.
339
-
340
- Args:
341
- positions: shape = [batch_size, seq_len]
342
- query: shape = [batch_size, seq_len, num_heads * head_size]
343
- key: shape = [batch_size, seq_len, num_kv_heads * head_size]
344
- value: shape = [batch_size, seq_len, num_kv_heads * head_size]
345
- key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
346
- block_size, x]
347
- value_cache: shape = [num_blocks, num_kv_heads, head_size,
348
- block_size]
349
- input_metadata: metadata for paged attention.
350
- cache_event: event to wait for the cache operations to finish.
351
-
352
- Returns:
353
- shape = [batch_size, seq_len, num_heads * head_size]
354
- """
355
-
356
- # Apply rotary embedding to the query and key before passing them
357
- # to the attention op.
358
- query, key = self.rotary_emb(positions, query, key)
359
- return super().forward(
186
+ return output.view(batch_size, seq_len, hidden_size)
187
+
188
+
189
+def _make_alibi_bias(
190
+ alibi_slopes: torch.Tensor,
191
+ batch_size: int,
192
+ seq_len: int,
193
+ dtype: torch.dtype,
194
+) -> LowerTriangularMaskWithTensorBias:
195
+ bias = torch.arange(seq_len, dtype=dtype)
196
+ # NOTE(zhuohan): HF uses
197
+ # `bias = bias[None, :].repeat(prompt_len, 1)`
198
+ # here. We find that both biases give the same results, but
199
+ # the bias below more accurately follows the original ALiBi
200
+ # paper.
201
+ bias = bias[None, :] - bias[:, None]
202
+ bias = bias.to(alibi_slopes.device)
203
+
204
+ # When using custom attention bias, xformers requires the bias to
205
+ # be sliced from a tensor whose length is a multiple of 8.
206
+ padded_len = (seq_len + 7) // 8 * 8
207
+ bias = torch.empty(
208
+ batch_size,
209
+ alibi_slopes.shape[0],
210
+ seq_len,
211
+ padded_len,
212
+ device=alibi_slopes.device,
213
+ dtype=dtype,
214
+ )[:, :, :, :seq_len].copy_(bias)
215
+ bias.mul_(alibi_slopes[:, None, None])
216
+ attn_bias = LowerTriangularMaskWithTensorBias(bias)
217
+ return attn_bias
218
+
219
+
220
+def _paged_attention(
221
+ query: torch.Tensor,
222
+ key_cache: torch.Tensor,
223
+ value_cache: torch.Tensor,
224
+ input_metadata: InputMetadata,
225
+ head_mapping: torch.Tensor,
226
+ scale: float,
227
+ alibi_slopes: Optional[torch.Tensor],
228
+) -> torch.Tensor:
229
+ output = torch.empty_like(query)
230
+
231
+ block_size = value_cache.shape[3]
232
+ num_seqs, num_heads, head_size = query.shape
233
+ max_num_partitions = (
234
+ (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
235
+ _PARTITION_SIZE)
236
+ # NOTE(woosuk): We use a simple heuristic to decide whether to use
237
+ # PagedAttention V1 or V2. If the number of partitions is 1, we use
238
+ # V1 to avoid the overhead of reduction. Also, if the number of
239
+ # sequences or heads is large, we use V1 since there is enough work
240
+ # to parallelize.
241
+ # TODO(woosuk): Tune this heuristic.
242
+ # For context len > 8192, use V2 kernel to avoid shared memory shortage.
243
+ use_v1 = input_metadata.max_context_len <= 8192 and (
244
+ max_num_partitions == 1 or num_seqs * num_heads > 512)
245
+ if use_v1:
246
+ # Run PagedAttention V1.
247
+ ops.paged_attention_v1(
248
+ output,
249
query,
361
- key,
362
- value,
250
key_cache,
251
value_cache,
365
- input_metadata,
366
- cache_event,
252
+ head_mapping,
253
+ scale,
254
+ input_metadata.block_tables,
255
+ input_metadata.context_lens,
256
+ block_size,
257
+ input_metadata.max_context_len,
258
+ alibi_slopes,
259
)
368
-
369
-
370
-class PagedAttentionWithALiBi(PagedAttention):
371
- """PagedAttention with ALiBi attention bias."""
372
-
373
- def __init__(self,
374
- num_heads: int,
375
- head_size: int,
376
- scale: float,
377
- slopes: List[float],
378
- num_kv_heads: Optional[int] = None) -> None:
379
- super().__init__(num_heads, head_size, scale, num_kv_heads)
380
- assert len(slopes) == num_heads
381
-
382
- slopes = torch.tensor(slopes, dtype=torch.float32)
383
- self.register_buffer("alibi_slopes", slopes, persistent=False)
384
-
385
- def set_attn_bias(self, input_metadata: InputMetadata,
386
- dtype: torch.dtype) -> None:
387
- if input_metadata.attn_bias is not None:
388
- # Already set by a previous layer.
389
- return
390
- # Generates ALiBi mask based on the max prompt length.
391
- max_prompt_len = input_metadata.max_prompt_len
392
- bias = torch.arange(max_prompt_len, dtype=dtype)
393
- # NOTE(zhuohan): HF uses
394
- # `bias = bias[None, :].repeat(prompt_len, 1)`
395
- # here. We find that both biases give the same results, but
396
- # the bias below more accurately follows the original ALiBi
397
- # paper.
398
- bias = bias[None, :] - bias[:, None]
399
- bias = bias.to(self.alibi_slopes.device)
400
-
401
- # When using custom attention bias, xformers requires the bias to
402
- # be sliced from a tensor whose length is a multiple of 8.
403
- padded_len = (max_prompt_len + 7) // 8 * 8
404
- bias = torch.empty(
405
- input_metadata.num_prompts,
406
- self.num_heads,
407
- max_prompt_len,
408
- padded_len,
409
- device=self.alibi_slopes.device,
410
- dtype=dtype,
411
- )[:, :, :, :max_prompt_len].copy_(bias)
412
- bias.mul_(self.alibi_slopes[:, None, None])
413
- attn_bias = LowerTriangularMaskWithTensorBias(bias)
414
- input_metadata.attn_bias = attn_bias
415
-
416
- def multi_query_kv_attention(
417
- self,
418
- output: torch.Tensor,
419
- query: torch.Tensor,
420
- key: torch.Tensor,
421
- value: torch.Tensor,
422
- input_metadata: InputMetadata,
423
- ) -> torch.Tensor:
424
- """Attention with ALiBi bias for the prompt tokens.
425
-
426
- Args:
427
- output: shape = [num_prompt_tokens, num_heads, head_size]
428
- query: shape = [num_prompt_tokens, num_heads, head_size]
429
- key: shape = [num_prompt_tokens, num_kv_heads, head_size]
430
- value: shape = [num_prompt_tokens, num_kv_heads, head_size]
431
- input_metadata: metadata for paged attention.
432
- """
433
- if self.num_kv_heads != self.num_heads:
434
- # Project the key and value tensors to the desired number of heads.
435
- query = query.view(query.shape[0], self.num_kv_heads,
436
- self.num_queries_per_kv, query.shape[-1])
437
- key = key[:, :,
438
- None, :].expand(key.shape[0], self.num_kv_heads,
439
- self.num_queries_per_kv, key.shape[-1])
440
- value = value[:, :,
441
- None, :].expand(value.shape[0], self.num_kv_heads,
442
- self.num_queries_per_kv,
443
- value.shape[-1])
444
- batch_size = input_metadata.num_prompts
445
- seq_len = input_metadata.max_prompt_len
446
-
447
- out = xops.memory_efficient_attention_forward(
448
- query.view(batch_size, seq_len, self.num_heads, self.head_size),
449
- key.view(batch_size, seq_len, self.num_heads, self.head_size),
450
- value.view(batch_size, seq_len, self.num_heads, self.head_size),
451
- attn_bias=input_metadata.attn_bias,
452
- p=0.0,
453
- scale=self.scale,
260
+ else:
261
+ # Run PagedAttention V2.
262
+ assert _PARTITION_SIZE % block_size == 0
263
+ tmp_output = torch.empty(
264
+ size=(num_seqs, num_heads, max_num_partitions, head_size),
265
+ dtype=output.dtype,
266
+ device=output.device,
267
)
455
- # TODO(woosuk): Unnecessary copy. Optimize.
456
- output.copy_(out.view_as(output))
457
- return output
458
-
459
- def get_alibi_slopes(self) -> Optional[torch.Tensor]:
460
- return self.alibi_slopes
268
+ exp_sums = torch.empty(
269
+ size=(num_seqs, num_heads, max_num_partitions),
270
+ dtype=torch.float32,
271
+ device=output.device,
272
+ )
273
+ max_logits = torch.empty_like(exp_sums)
274
+ ops.paged_attention_v2(
275
+ output,
276
+ exp_sums,
277
+ max_logits,
278
+ tmp_output,
279
+ query,
280
+ key_cache,
281
+ value_cache,
282
+ head_mapping,
283
+ scale,
284
+ input_metadata.block_tables,
285
+ input_metadata.context_lens,
286
+ block_size,
287
+ input_metadata.max_context_len,
288
+ alibi_slopes,
289
+ )
290
+ return output
vllm/model_executor/layers/rotary_embedding.py
+2
-2
@@ -277,8 +277,8 @@ def get_rope(
277
rotary_dim: int,
278
max_position: int,
279
base: int,
280
- is_neox_style: bool,
281
- rope_scaling: Optional[Dict[str, Any]],
280
+ is_neox_style: bool = True,
281
+ rope_scaling: Optional[Dict[str, Any]] = None,
282
) -> RotaryEmbedding:
283
if rope_scaling is None:
284
rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
vllm/model_executor/models/aquila.py
+14
-10
@@ -28,11 +28,12 @@ from torch import nn
28
29
from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.activation import SiluAndMul
31
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
31
+from vllm.model_executor.layers.attention import PagedAttention
32
from vllm.model_executor.layers.linear import (LinearMethodBase,
33
MergedColumnParallelLinear,
34
QKVParallelLinear,
35
RowParallelLinear)
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding, ParallelLMHead)
@@ -138,15 +139,17 @@ class AquilaAttention(nn.Module):
139
bias=False,
140
linear_method=linear_method,
141
)
141
- self.attn = PagedAttentionWithRoPE(
142
- self.num_heads,
142
+ self.rotary_emb = get_rope(
143
self.head_dim,
144
- self.scaling,
145
- base=self.rope_theta,
146
- max_position=self.max_position_embeddings,
144
rotary_dim=self.head_dim,
148
- num_kv_heads=self.num_kv_heads,
149
- rope_scaling=rope_scaling)
145
+ max_position=self.max_position_embeddings,
146
+ base=self.rope_theta,
147
+ rope_scaling=rope_scaling,
148
+ )
149
+ self.attn = PagedAttention(self.num_heads,
150
+ self.head_dim,
151
+ self.scaling,
152
+ num_kv_heads=self.num_kv_heads)
153
154
def forward(
155
self,
@@ -158,9 +161,10 @@ class AquilaAttention(nn.Module):
161
) -> torch.Tensor:
162
qkv, _ = self.qkv_proj(hidden_states)
163
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164
+ q, k = self.rotary_emb(positions, q, k)
165
k_cache, v_cache = kv_cache
162
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
163
- input_metadata, cache_event)
166
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
167
+ cache_event)
168
output, _ = self.o_proj(attn_output)
169
return output
170
vllm/model_executor/models/baichuan.py
+16
-16
@@ -26,13 +26,13 @@ from torch import nn
26
27
from vllm.model_executor.input_metadata import InputMetadata
28
from vllm.model_executor.layers.activation import SiluAndMul
29
-from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
30
- PagedAttentionWithALiBi)
29
+from vllm.model_executor.layers.attention import PagedAttention
30
from vllm.model_executor.layers.layernorm import RMSNorm
31
from vllm.model_executor.layers.linear import (LinearMethodBase,
32
MergedColumnParallelLinear,
33
QKVParallelLinear,
34
RowParallelLinear)
35
+from vllm.model_executor.layers.rotary_embedding import get_rope
36
from vllm.model_executor.layers.sampler import Sampler
37
from vllm.model_executor.layers.vocab_parallel_embedding import (
38
VocabParallelEmbedding, ParallelLMHead)
@@ -150,17 +150,20 @@ class BaiChuanAttention(nn.Module):
150
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
151
152
scaling = self.head_dim**-0.5
153
- self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
154
- scaling, alibi_slopes)
153
+ self.attn = PagedAttention(self.num_heads,
154
+ self.head_dim,
155
+ scaling,
156
+ alibi_slopes=alibi_slopes)
157
else:
156
- self.scaling = self.head_dim**-0.5
157
- self.attn = PagedAttentionWithRoPE(
158
- self.num_heads,
158
+ self.rotary_emb = get_rope(
159
self.head_dim,
160
- self.scaling,
160
rotary_dim=self.head_dim,
161
+ max_position=self.max_position_embeddings,
162
base=self.rope_theta,
163
- max_position=self.max_position_embeddings)
163
+ )
164
+ self.scaling = self.head_dim**-0.5
165
+ self.attn = PagedAttention(self.num_heads, self.head_dim,
166
+ self.scaling)
167
168
def forward(
169
self,
@@ -172,14 +175,11 @@ class BaiChuanAttention(nn.Module):
175
) -> torch.Tensor:
176
qkv, _ = self.W_pack(hidden_states)
177
q, k, v = qkv.chunk(chunks=3, dim=-1)
178
+ if self.postion_embedding != "ALIBI":
179
+ q, k = self.rotary_emb(positions, q, k)
180
k_cache, v_cache = kv_cache
176
- if self.postion_embedding == "ALIBI":
177
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
178
- cache_event)
179
- else:
180
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
181
- input_metadata, cache_event)
182
-
181
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
182
+ cache_event)
183
output, _ = self.o_proj(attn_output)
184
return output
185
vllm/model_executor/models/bloom.py
+5
-3
@@ -25,7 +25,7 @@ from transformers import BloomConfig
25
26
from vllm.model_executor.input_metadata import InputMetadata
27
from vllm.model_executor.layers.activation import get_act_fn
28
-from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
28
+from vllm.model_executor.layers.attention import PagedAttention
29
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30
LinearMethodBase,
31
QKVParallelLinear,
@@ -106,8 +106,10 @@ class BloomAttention(nn.Module):
106
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
107
108
scaling = self.head_dim**-0.5
109
- self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
110
- scaling, alibi_slopes)
109
+ self.attn = PagedAttention(self.num_heads,
110
+ self.head_dim,
111
+ scaling,
112
+ alibi_slopes=alibi_slopes)
113
114
def forward(
115
self,
vllm/model_executor/models/chatglm.py
+10
-9
@@ -10,12 +10,13 @@ from torch.nn import LayerNorm
10
11
from vllm.model_executor.input_metadata import InputMetadata
12
from vllm.model_executor.layers.activation import SiluAndMul
13
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
13
+from vllm.model_executor.layers.attention import PagedAttention
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import (LinearMethodBase,
16
MergedColumnParallelLinear,
17
QKVParallelLinear,
18
RowParallelLinear)
19
+from vllm.model_executor.layers.rotary_embedding import get_rope
20
from vllm.model_executor.layers.sampler import Sampler
21
from vllm.model_executor.layers.vocab_parallel_embedding import (
22
VocabParallelEmbedding, ParallelLMHead)
@@ -78,16 +79,19 @@ class GLMAttention(nn.Module):
79
# https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
80
rope_ratio = getattr(config, "rope_ratio", 1.0)
81
max_positions = getattr(config, "seq_length", 8192)
81
- self.attn = PagedAttentionWithRoPE(
82
- self.num_heads,
82
+ self.rotary_emb = get_rope(
83
self.head_dim,
84
- self.scaling,
84
rotary_dim=self.head_dim // 2,
86
- num_kv_heads=self.num_kv_heads,
85
max_position=max_positions,
86
base=10000 * rope_ratio,
87
is_neox_style=False,
88
)
89
+ self.attn = PagedAttention(
90
+ self.num_heads,
91
+ self.head_dim,
92
+ self.scaling,
93
+ num_kv_heads=self.num_kv_heads,
94
+ )
95
96
def forward(
97
self,
@@ -99,10 +103,9 @@ class GLMAttention(nn.Module):
103
) -> torch.Tensor:
104
qkv, _ = self.query_key_value(hidden_states)
105
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
106
+ q, k = self.rotary_emb(position_ids, q, k)
107
key_cache, value_cache = kv_cache
103
-
108
context_layer = self.attn(
105
- position_ids,
109
q,
110
k,
111
v,
@@ -111,9 +114,7 @@ class GLMAttention(nn.Module):
114
input_metadata,
115
cache_event,
116
)
114
-
117
attn_output, _ = self.dense(context_layer)
116
-
118
return attn_output
119
120
vllm/model_executor/models/falcon.py
+19
-20
@@ -28,13 +28,12 @@ from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.model_executor.input_metadata import InputMetadata
30
from vllm.model_executor.layers.activation import get_act_fn
31
-from vllm.model_executor.layers.attention import (PagedAttention,
32
- PagedAttentionWithALiBi,
33
- PagedAttentionWithRoPE)
31
+from vllm.model_executor.layers.attention import PagedAttention
32
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33
LinearMethodBase,
34
QKVParallelLinear,
35
RowParallelLinear)
36
+from vllm.model_executor.layers.rotary_embedding import get_rope
37
from vllm.model_executor.layers.sampler import Sampler
38
from vllm.model_executor.layers.vocab_parallel_embedding import (
39
VocabParallelEmbedding, ParallelLMHead)
@@ -144,14 +143,16 @@ class FalconAttention(nn.Module):
143
rope_theta = getattr(config, "rope_theta", 10000)
144
max_position_embeddings = getattr(config,
145
"max_position_embeddings", 8192)
147
- self.attn = PagedAttentionWithRoPE(
148
- self.num_heads,
146
+ self.rotary_emb = get_rope(
147
self.head_dim,
150
- self.inv_norm_factor,
151
- base=rope_theta,
152
- max_position=max_position_embeddings,
148
rotary_dim=self.head_dim,
154
- num_kv_heads=self.num_kv_heads)
149
+ max_position=max_position_embeddings,
150
+ base=rope_theta,
151
+ )
152
+ self.attn = PagedAttention(self.num_heads,
153
+ self.head_dim,
154
+ self.inv_norm_factor,
155
+ num_kv_heads=self.num_kv_heads)
156
elif self.use_alibi:
157
tp_rank = get_tensor_model_parallel_rank()
158
head_start = tp_rank * self.num_heads
@@ -159,11 +160,11 @@ class FalconAttention(nn.Module):
160
alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
161
self.inv_norm_factor)
162
alibi_slopes = alibi_slopes[head_start:head_end].tolist()
162
- self.attn = PagedAttentionWithALiBi(self.num_heads,
163
- self.head_dim,
164
- self.inv_norm_factor,
165
- alibi_slopes,
166
- num_kv_heads=self.num_kv_heads)
163
+ self.attn = PagedAttention(self.num_heads,
164
+ self.head_dim,
165
+ self.inv_norm_factor,
166
+ num_kv_heads=self.num_kv_heads,
167
+ alibi_slopes=alibi_slopes)
168
else:
169
self.attn = PagedAttention(self.num_heads,
170
self.head_dim,
@@ -182,13 +183,11 @@ class FalconAttention(nn.Module):
183
if bias is not None:
184
qkv += bias
185
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
185
- k_cache, v_cache = kv_cache
186
if self.use_rotary:
187
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
188
- input_metadata, cache_event)
189
- else:
190
- attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
191
- cache_event)
187
+ q, k = self.rotary_emb(positions, q, k)
188
+ k_cache, v_cache = kv_cache
189
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
190
+ cache_event)
191
attn_output, bias = self.dense(attn_output)
192
return attn_output, bias
193
vllm/model_executor/models/gpt_j.py
+11
-10
@@ -24,11 +24,12 @@ from transformers import GPTJConfig
24
25
from vllm.model_executor.input_metadata import InputMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
27
+from vllm.model_executor.layers.attention import PagedAttention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
31
RowParallelLinear)
32
+from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding, ParallelLMHead)
@@ -77,15 +78,14 @@ class GPTJAttention(nn.Module):
78
rope_theta = getattr(config, "rope_theta", 10000)
79
max_position_embeddings = getattr(config, "max_position_embeddings",
80
8192)
80
- self.attn = PagedAttentionWithRoPE(
81
- self.num_heads,
81
+ self.rotary_emb = get_rope(
82
self.head_size,
83
- scaling,
84
- config.rotary_dim,
85
- base=rope_theta,
83
+ rotary_dim=config.rotary_dim,
84
max_position=max_position_embeddings,
87
- is_neox_style=False)
88
- self.warmup = False
85
+ base=rope_theta,
86
+ is_neox_style=False,
87
+ )
88
+ self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
89
90
def forward(
91
self,
@@ -97,9 +97,10 @@ class GPTJAttention(nn.Module):
97
) -> torch.Tensor:
98
qkv, _ = self.qkv_proj(hidden_states)
99
q, k, v = qkv.chunk(chunks=3, dim=-1)
100
+ q, k = self.rotary_emb(position_ids, q, k)
101
k_cache, v_cache = kv_cache
101
- attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
102
- input_metadata, cache_event)
102
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
103
+ cache_event)
104
attn_output, _ = self.out_proj(attn_output)
105
return attn_output
106
vllm/model_executor/models/gpt_neox.py
+10
-8
@@ -24,11 +24,12 @@ from transformers import GPTNeoXConfig
24
25
from vllm.model_executor.input_metadata import InputMetadata
26
from vllm.model_executor.layers.activation import get_act_fn
27
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
27
+from vllm.model_executor.layers.attention import PagedAttention
28
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29
LinearMethodBase,
30
QKVParallelLinear,
31
RowParallelLinear)
32
+from vllm.model_executor.layers.rotary_embedding import get_rope
33
from vllm.model_executor.layers.sampler import Sampler
34
from vllm.model_executor.layers.vocab_parallel_embedding import (
35
VocabParallelEmbedding, ParallelLMHead)
@@ -77,13 +78,13 @@ class GPTNeoXAttention(nn.Module):
78
rope_theta = getattr(config, "rope_theta", 10000)
79
max_position_embeddings = getattr(config, "max_position_embeddings",
80
8192)
80
- self.attn = PagedAttentionWithRoPE(
81
- self.num_heads,
81
+ self.rotary_emb = get_rope(
82
self.head_size,
83
- scaling,
84
- rotary_dim,
83
+ rotary_dim=rotary_dim,
84
+ max_position=max_position_embeddings,
85
base=rope_theta,
86
- max_position=max_position_embeddings)
86
+ )
87
+ self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
88
89
def forward(
90
self,
@@ -95,9 +96,10 @@ class GPTNeoXAttention(nn.Module):
96
) -> torch.Tensor:
97
qkv, _ = self.query_key_value(hidden_states)
98
q, k, v = qkv.chunk(chunks=3, dim=-1)
99
+ q, k = self.rotary_emb(position_ids, q, k)
100
k_cache, v_cache = kv_cache
99
- attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
100
- input_metadata, cache_event)
101
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
102
+ cache_event)
103
output, _ = self.dense(attn_output)
104
return output
105
vllm/model_executor/models/internlm.py
+10
-8
@@ -7,12 +7,13 @@ from transformers import LlamaConfig
7
8
from vllm.model_executor.input_metadata import InputMetadata
9
from vllm.model_executor.layers.activation import SiluAndMul
10
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
10
+from vllm.model_executor.layers.attention import PagedAttention
11
from vllm.model_executor.layers.layernorm import RMSNorm
12
from vllm.model_executor.layers.linear import (LinearMethodBase,
13
MergedColumnParallelLinear,
14
QKVParallelLinear,
15
RowParallelLinear)
16
+from vllm.model_executor.layers.rotary_embedding import get_rope
17
from vllm.model_executor.layers.sampler import Sampler
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
VocabParallelEmbedding, ParallelLMHead)
@@ -92,13 +93,13 @@ class InternLMAttention(nn.Module):
93
bias=bias,
94
linear_method=linear_method,
95
)
95
- self.attn = PagedAttentionWithRoPE(
96
- self.num_heads,
96
+ self.rotary_emb = get_rope(
97
self.head_dim,
98
- self.scaling,
99
- base=self.rope_theta,
98
+ rotary_dim=self.head_dim,
99
max_position=self.max_position_embeddings,
101
- rotary_dim=self.head_dim)
100
+ base=self.rope_theta,
101
+ )
102
+ self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
103
104
def forward(
105
self,
@@ -110,9 +111,10 @@ class InternLMAttention(nn.Module):
111
) -> torch.Tensor:
112
qkv, _ = self.qkv_proj(hidden_states)
113
q, k, v = qkv.chunk(chunks=3, dim=-1)
114
+ q, k = self.rotary_emb(positions, q, k)
115
k_cache, v_cache = kv_cache
114
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
115
- input_metadata, cache_event)
116
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
117
+ cache_event)
118
output, _ = self.o_proj(attn_output)
119
return output
120
vllm/model_executor/models/llama.py
+15
-10
@@ -29,12 +29,13 @@ from transformers import LlamaConfig
29
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32
+from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
36
QKVParallelLinear,
37
RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
@@ -126,15 +127,18 @@ class LlamaAttention(nn.Module):
127
bias=False,
128
linear_method=linear_method,
129
)
129
- self.attn = PagedAttentionWithRoPE(
130
- self.num_heads,
130
+
131
+ self.rotary_emb = get_rope(
132
self.head_dim,
132
- self.scaling,
133
- base=self.rope_theta,
134
- max_position=self.max_position_embeddings,
133
rotary_dim=self.head_dim,
136
- num_kv_heads=self.num_kv_heads,
137
- rope_scaling=rope_scaling)
134
+ max_position=max_position_embeddings,
135
+ base=rope_theta,
136
+ rope_scaling=rope_scaling,
137
+ )
138
+ self.attn = PagedAttention(self.num_heads,
139
+ self.head_dim,
140
+ self.scaling,
141
+ num_kv_heads=self.num_kv_heads)
142
143
def forward(
144
self,
@@ -146,9 +150,10 @@ class LlamaAttention(nn.Module):
150
) -> torch.Tensor:
151
qkv, _ = self.qkv_proj(hidden_states)
152
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
153
+ q, k = self.rotary_emb(positions, q, k)
154
k_cache, v_cache = kv_cache
150
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
151
- input_metadata, cache_event)
155
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
156
+ cache_event)
157
output, _ = self.o_proj(attn_output)
158
return output
159
vllm/model_executor/models/mistral.py
+17
-11
@@ -29,12 +29,13 @@ from transformers import MistralConfig
29
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32
+from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
36
QKVParallelLinear,
37
RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
@@ -124,14 +125,18 @@ class MistralAttention(nn.Module):
125
bias=False,
126
linear_method=linear_method,
127
)
127
- self.attn = PagedAttentionWithRoPE(self.num_heads,
128
- self.head_dim,
129
- self.scaling,
130
- base=self.rope_theta,
131
- max_position=max_position,
132
- rotary_dim=self.head_dim,
133
- num_kv_heads=self.num_kv_heads,
134
- sliding_window=self.sliding_window)
128
+
129
+ self.rotary_emb = get_rope(
130
+ self.head_dim,
131
+ rotary_dim=self.head_dim,
132
+ max_position=max_position,
133
+ base=self.rope_theta,
134
+ )
135
+ self.attn = PagedAttention(self.num_heads,
136
+ self.head_dim,
137
+ self.scaling,
138
+ num_kv_heads=self.num_kv_heads,
139
+ sliding_window=self.sliding_window)
140
141
def forward(
142
self,
@@ -143,9 +148,10 @@ class MistralAttention(nn.Module):
148
) -> torch.Tensor:
149
qkv, _ = self.qkv_proj(hidden_states)
150
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
151
+ q, k = self.rotary_emb(positions, q, k)
152
k_cache, v_cache = kv_cache
147
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
148
- input_metadata, cache_event)
153
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
154
+ cache_event)
155
output, _ = self.o_proj(attn_output)
156
return output
157
vllm/model_executor/models/mpt.py
+5
-3
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9
from vllm.model_executor.input_metadata import InputMetadata
10
from vllm.model_executor.layers.activation import get_act_fn
11
-from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
11
+from vllm.model_executor.layers.attention import PagedAttention
12
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13
LinearMethodBase,
14
QKVParallelLinear,
@@ -87,8 +87,10 @@ class MPTAttention(nn.Module):
87
88
self.head_dim = self.d_model // self.total_num_heads
89
scaling = self.head_dim**-0.5
90
- self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
91
- scaling, alibi_slopes)
90
+ self.attn = PagedAttention(self.num_heads,
91
+ self.head_dim,
92
+ scaling,
93
+ alibi_slopes=alibi_slopes)
94
95
def forward(
96
self,
vllm/model_executor/models/phi_1_5.py
+10
-8
@@ -43,11 +43,12 @@ from transformers import PretrainedConfig
43
44
from vllm.model_executor.input_metadata import InputMetadata
45
from vllm.model_executor.layers.activation import get_act_fn
46
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
46
+from vllm.model_executor.layers.attention import PagedAttention
47
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48
LinearMethodBase,
49
QKVParallelLinear,
50
RowParallelLinear)
51
+from vllm.model_executor.layers.rotary_embedding import get_rope
52
from vllm.model_executor.layers.sampler import Sampler
53
from vllm.model_executor.layers.vocab_parallel_embedding import (
54
VocabParallelEmbedding, ParallelLMHead)
@@ -119,13 +120,13 @@ class PhiAttention(nn.Module):
120
# https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
121
rope_theta = 10000
122
max_position_embeddings = getattr(config, "n_positions", 2048)
122
- self.attn = PagedAttentionWithRoPE(
123
- self.num_heads,
123
+ self.rotary_emb = get_rope(
124
self.head_size,
125
- scaling,
126
- rotary_dim,
125
+ rotary_dim=rotary_dim,
126
+ max_position=max_position_embeddings,
127
base=rope_theta,
128
- max_position=max_position_embeddings)
128
+ )
129
+ self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
130
131
def forward(
132
self,
@@ -137,9 +138,10 @@ class PhiAttention(nn.Module):
138
) -> torch.Tensor:
139
qkv, _ = self.Wqkv(hidden_states)
140
q, k, v = qkv.chunk(chunks=3, dim=-1)
141
+ q, k = self.rotary_emb(position_ids, q, k)
142
k_cache, v_cache = kv_cache
141
- attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
142
- input_metadata, cache_event)
143
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
144
+ cache_event)
145
output, _ = self.out_proj(attn_output)
146
return output
147
vllm/model_executor/models/qwen.py
+11
-9
@@ -11,12 +11,13 @@ from torch import nn
11
12
from vllm.model_executor.input_metadata import InputMetadata
13
from vllm.model_executor.layers.activation import SiluAndMul
14
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
14
+from vllm.model_executor.layers.attention import PagedAttention
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (LinearMethodBase,
17
MergedColumnParallelLinear,
18
QKVParallelLinear,
19
RowParallelLinear)
20
+from vllm.model_executor.layers.rotary_embedding import get_rope
21
from vllm.model_executor.layers.sampler import Sampler
22
from vllm.model_executor.layers.vocab_parallel_embedding import (
23
VocabParallelEmbedding, ParallelLMHead)
@@ -95,14 +96,15 @@ class QWenAttention(nn.Module):
96
linear_method=linear_method,
97
)
98
self.scaling = self.head_dim**-0.5
98
- self.attn = PagedAttentionWithRoPE(
99
- self.num_heads,
99
+
100
+ self.rotary_emb = get_rope(
101
self.head_dim,
101
- self.scaling,
102
rotary_dim=self.head_dim,
103
- base=rope_theta,
103
max_position=max_position_embeddings,
105
- rope_scaling=rope_scaling)
104
+ base=rope_theta,
105
+ rope_scaling=rope_scaling,
106
+ )
107
+ self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108
109
def forward(
110
self,
@@ -114,10 +116,10 @@ class QWenAttention(nn.Module):
116
) -> torch.Tensor:
117
qkv, _ = self.c_attn(hidden_states)
118
q, k, v = qkv.chunk(chunks=3, dim=-1)
117
-
119
+ q, k = self.rotary_emb(positions, q, k)
120
k_cache, v_cache = kv_cache
119
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
120
- input_metadata, cache_event)
121
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
122
+ cache_event)
123
124
output, _ = self.c_proj(attn_output)
125
return output
vllm/model_executor/models/yi.py
+14
-10
@@ -29,12 +29,13 @@ from vllm.transformers_utils.configs.yi import YiConfig
29
30
from vllm.model_executor.input_metadata import InputMetadata
31
from vllm.model_executor.layers.activation import SiluAndMul
32
-from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32
+from vllm.model_executor.layers.attention import PagedAttention
33
from vllm.model_executor.layers.layernorm import RMSNorm
34
from vllm.model_executor.layers.linear import (LinearMethodBase,
35
MergedColumnParallelLinear,
36
QKVParallelLinear,
37
RowParallelLinear)
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding, ParallelLMHead)
@@ -126,15 +127,17 @@ class YiAttention(nn.Module):
127
bias=False,
128
linear_method=linear_method,
129
)
129
- self.attn = PagedAttentionWithRoPE(
130
- self.num_heads,
130
+ self.rotary_emb = get_rope(
131
self.head_dim,
132
- self.scaling,
133
- base=self.rope_theta,
134
- max_position=self.max_position_embeddings,
132
rotary_dim=self.head_dim,
136
- num_kv_heads=self.num_kv_heads,
137
- rope_scaling=rope_scaling)
133
+ max_position=max_position_embeddings,
134
+ base=self.rope_theta,
135
+ rope_scaling=rope_scaling,
136
+ )
137
+ self.attn = PagedAttention(self.num_heads,
138
+ self.head_dim,
139
+ self.scaling,
140
+ num_kv_heads=self.num_kv_heads)
141
142
def forward(
143
self,
@@ -146,9 +149,10 @@ class YiAttention(nn.Module):
149
) -> torch.Tensor:
150
qkv, _ = self.qkv_proj(hidden_states)
151
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152
+ q, k = self.rotary_emb(positions, q, k)
153
k_cache, v_cache = kv_cache
150
- attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
151
- input_metadata, cache_event)
154
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
155
+ cache_event)
156
output, _ = self.o_proj(attn_output)
157
return output
158