@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a9e4574261
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Refactor Attention (#1840)

Woosuk Kwon committed Nov 29, 2023 at 15:37 UTC a9e4574261a20d4ada213d26671da7dc7633580b
16 files changed +360 -498
vllm/model_executor/layers/attention.py
+191 -361
@@ -1,5 +1,5 @@
1 """Multi-head attention."""
2 -from typing import Any, Dict, List, Optional
2 +from typing import List, Optional
3
4 import torch
5 import torch.nn as nn
@@ -10,7 +10,6 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
10 from vllm._C import ops
11 from vllm._C import cache_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 -from vllm.model_executor.layers.rotary_embedding import get_rope
13
14 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
15 # Should be the same as PARTITION_SIZE in `paged_attention_v2_launcher`.
@@ -18,37 +17,39 @@ _PARTITION_SIZE = 512
17
18
19 class PagedAttention(nn.Module):
21 - """GPT-style multi-head PagedAttention.
20 + """MHA/MQA/GQA layer with PagedAttention.
21
22 This class takes query, key, and value tensors as input. The input tensors
24 - can either contain prompt tokens or generation tokens, in addition to
25 - paddings.
26 -
23 + can either contain prompt tokens or generation tokens.
24 The class does the following:
28 - 1. Perform multi_query_kv_attention for the prompts. This operation does
29 - not use the KV cache.
30 - 2. Wait for the cache operations (e.g., swap, copy) to finish. The cache
25 +
26 + 1. Wait for the cache operations (e.g., swap, copy) to finish. The cache
27 operations are issued by the cache engine before executing the forward
28 pass of the model, and they are executed asynchronously.
33 - 3. Reshape and store the input key and value tensors in the KV cache.
34 - 4. Perform single_query_cached_kv_attention for the generation tokens.
35 - This operation reads the previous key and value tensors from the KV
36 - cache.
37 - 5. Return the output tensor.
29 + 2. Reshape and store the input key and value tensors in the KV cache.
30 + 3. Perform (multi-head/multi-query/grouped-query) attention using either
31 + xformers or the PagedAttention custom op.
32 + 4. Return the output tensor.
33 """
34
40 - def __init__(self,
41 - num_heads: int,
42 - head_size: int,
43 - scale: float,
44 - num_kv_heads: Optional[int] = None,
45 - sliding_window: Optional[int] = None) -> None:
35 + def __init__(
36 + self,
37 + num_heads: int,
38 + head_size: int,
39 + scale: float,
40 + num_kv_heads: Optional[int] = None,
41 + alibi_slopes: Optional[List[float]] = None,
42 + sliding_window: Optional[int] = None,
43 + ) -> None:
44 super().__init__()
45 self.num_heads = num_heads
46 self.head_size = head_size
47 self.scale = float(scale)
48 self.num_kv_heads = num_heads if num_kv_heads is None else num_kv_heads
49 self.sliding_window = sliding_window
50 + if alibi_slopes is not None:
51 + alibi_slopes = torch.tensor(alibi_slopes, dtype=torch.float32)
52 + self.register_buffer("alibi_slopes", alibi_slopes, persistent=False)
53
54 assert self.num_heads % self.num_kv_heads == 0
55 self.num_queries_per_kv = self.num_heads // self.num_kv_heads
@@ -60,153 +61,6 @@ class PagedAttention(nn.Module):
61 raise ValueError(f"head_size ({self.head_size}) is not supported. "
62 f"Supported head sizes: {_SUPPORTED_HEAD_SIZES}.")
63
63 - def set_attn_bias(
64 - self,
65 - input_metadata: InputMetadata,
66 - dtype: torch.dtype,
67 - ) -> None:
68 - del dtype # Unused.
69 - if input_metadata.attn_bias is not None:
70 - # Already set by a previous layer.
71 - return
72 - prompt_lens = [input_metadata.max_prompt_len
73 - ] * input_metadata.num_prompts
74 - attn_bias = BlockDiagonalCausalMask.from_seqlens(prompt_lens)
75 - if self.sliding_window is not None:
76 - attn_bias = attn_bias.make_local_attention(self.sliding_window)
77 - input_metadata.attn_bias = attn_bias
78 -
79 - def multi_query_kv_attention(
80 - self,
81 - output: torch.Tensor,
82 - query: torch.Tensor,
83 - key: torch.Tensor,
84 - value: torch.Tensor,
85 - input_metadata: InputMetadata,
86 - ) -> torch.Tensor:
87 - """Normal attention for the prompt tokens.
88 -
89 - Args:
90 - output: shape = [num_prompt_tokens, num_heads, head_size]
91 - query: shape = [num_prompt_tokens, num_heads, head_size]
92 - key: shape = [num_prompt_tokens, num_kv_heads, head_size]
93 - value: shape = [num_prompt_tokens, num_kv_heads, head_size]
94 - input_metadata: metadata for paged attention.
95 - """
96 - if self.num_kv_heads != self.num_heads:
97 - # Project the key and value tensors to the desired number of heads.
98 - query = query.view(query.shape[0], self.num_kv_heads,
99 - self.num_queries_per_kv, query.shape[-1])
100 - key = key[:, :,
101 - None, :].expand(key.shape[0], self.num_kv_heads,
102 - self.num_queries_per_kv, key.shape[-1])
103 - value = value[:, :,
104 - None, :].expand(value.shape[0], self.num_kv_heads,
105 - self.num_queries_per_kv,
106 - value.shape[-1])
107 -
108 - # TODO(woosuk): The unsqueeze op may incur some CPU overhead. Optimize.
109 - out = xops.memory_efficient_attention_forward(
110 - query.unsqueeze(0),
111 - key.unsqueeze(0),
112 - value.unsqueeze(0),
113 - attn_bias=input_metadata.attn_bias,
114 - p=0.0,
115 - scale=self.scale,
116 - )
117 - # TODO(woosuk): Unnecessary copy. Optimize.
118 - output.copy_(out.view_as(output))
119 - return output
120 -
121 - def get_alibi_slopes(self) -> Optional[torch.Tensor]:
122 - """Returns the slopes for the alibi attention bias.
123 -
124 - Returns:
125 - slopes: shape = [num_heads]
126 - """
127 - return None
128 -
129 - def single_query_cached_kv_attention(
130 - self,
131 - output: torch.Tensor,
132 - query: torch.Tensor,
133 - key_cache: torch.Tensor,
134 - value_cache: torch.Tensor,
135 - input_metadata: InputMetadata,
136 - alibi_slopes: Optional[torch.Tensor],
137 - ) -> None:
138 - """PagedAttention for the generation tokens.
139 -
140 - Args:
141 - output: shape = [num_generation_tokens, num_heads, head_size]
142 - query: shape = [num_generation_tokens, num_heads, head_size]
143 - key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
144 - block_size, x]
145 - value_cache: shape = [num_blocks, num_kv_heads, head_size,
146 - block_size]
147 - input_metadata: metadata for paged attention.
148 - alibi_slopes: shape = [num_heads]
149 - """
150 - block_size = value_cache.shape[3]
151 - num_seqs, num_heads, head_size = query.shape
152 - max_num_partitions = (
153 - (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
154 - _PARTITION_SIZE)
155 - # NOTE(woosuk): We use a simple heuristic to decide whether to use
156 - # PagedAttention V1 or V2. If the number of partitions is 1, we use
157 - # V1 to avoid the overhead of reduction. Also, if the number of
158 - # sequences or heads is large, we use V1 since there is enough work
159 - # to parallelize.
160 - # TODO(woosuk): Tune this heuristic.
161 - # For context len > 8192, use V2 kernel to avoid shared memory shortage.
162 - use_v1 = input_metadata.max_context_len <= 8192 and (
163 - max_num_partitions == 1 or num_seqs * num_heads > 512)
164 - if use_v1:
165 - # Run PagedAttention V1.
166 - ops.paged_attention_v1(
167 - output,
168 - query,
169 - key_cache,
170 - value_cache,
171 - self.head_mapping,
172 - self.scale,
173 - input_metadata.block_tables,
174 - input_metadata.context_lens,
175 - block_size,
176 - input_metadata.max_context_len,
177 - alibi_slopes,
178 - )
179 - else:
180 - # Run PagedAttention V2.
181 - assert _PARTITION_SIZE % block_size == 0
182 - tmp_output = torch.empty(
183 - size=(num_seqs, num_heads, max_num_partitions, head_size),
184 - dtype=output.dtype,
185 - device=output.device,
186 - )
187 - exp_sums = torch.empty(
188 - size=(num_seqs, num_heads, max_num_partitions),
189 - dtype=torch.float32,
190 - device=output.device,
191 - )
192 - max_logits = torch.empty_like(exp_sums)
193 - ops.paged_attention_v2(
194 - output,
195 - exp_sums,
196 - max_logits,
197 - tmp_output,
198 - query,
199 - key_cache,
200 - value_cache,
201 - self.head_mapping,
202 - self.scale,
203 - input_metadata.block_tables,
204 - input_metadata.context_lens,
205 - block_size,
206 - input_metadata.max_context_len,
207 - alibi_slopes,
208 - )
209 -
64 def forward(
65 self,
66 query: torch.Tensor,
@@ -219,9 +73,6 @@ class PagedAttention(nn.Module):
73 ) -> torch.Tensor:
74 """PagedAttention forward pass.
75
222 - NOTE: The query, key, and value tensors must be sliced from a qkv
223 - tensor of shape [batch_size, seq_len, 3 * num_heads * head_size].
224 -
76 Args:
77 query: shape = [batch_size, seq_len, num_heads * head_size]
78 key: shape = [batch_size, seq_len, num_kv_heads * head_size]
@@ -230,46 +81,28 @@ class PagedAttention(nn.Module):
81 block_size, x]
82 value_cache: shape = [num_blocks, num_kv_heads, head_size,
83 block_size]
233 - input_metadata: metadata for paged attention.
84 + input_metadata: metadata for the inputs.
85 cache_event: event to wait for the cache operations to finish.
235 -
86 Returns:
87 shape = [batch_size, seq_len, num_heads * head_size]
88 """
239 - batch_size, seq_len, _ = query.shape
89 + batch_size, seq_len, hidden_size = query.shape
90 # Reshape the query, key, and value tensors.
91 query = query.view(-1, self.num_heads, self.head_size)
92 key = key.view(-1, self.num_kv_heads, self.head_size)
93 value = value.view(-1, self.num_kv_heads, self.head_size)
94 + slot_mapping = input_metadata.slot_mapping.flatten()
95
245 - # Pre-allocate the output tensor.
246 - output = torch.empty_like(query)
247 -
248 - # Compute the attention op for prompts.
249 - num_prompt_tokens = input_metadata.num_prompt_tokens
250 - if num_prompt_tokens > 0:
251 - # Prompt run.
252 - assert input_metadata.num_generation_tokens == 0
253 - self.set_attn_bias(input_metadata, dtype=query.dtype)
254 - self.multi_query_kv_attention(
255 - output,
256 - query,
257 - key,
258 - value,
259 - input_metadata,
260 - )
261 -
262 - # Wait until the cache op is done.
96 if cache_event is not None:
97 cache_event.wait()
98
99 # Reshape the keys and values and store them in the cache.
267 - # When key_cache and value_cache are not provided, the new key
268 - # and value vectors will not be cached.
100 + # If key_cache and value_cache are not provided, the new key and value
101 + # vectors will not be cached. This happens during the initial memory
102 + # profiling run.
103 if key_cache is not None and value_cache is not None:
104 key_to_cache = key
105 value_to_cache = value
272 - slot_mapping = input_metadata.slot_mapping.view(-1)
106 if input_metadata.to_cache is not None:
107 key_to_cache = key_to_cache[input_metadata.to_cache]
108 value_to_cache = value_to_cache[input_metadata.to_cache]
@@ -283,178 +116,175 @@ class PagedAttention(nn.Module):
116 slot_mapping,
117 )
118
286 - if input_metadata.num_generation_tokens > 0:
119 + is_prompt = len(input_metadata.prompt_lens) > 0
120 + if is_prompt:
121 + # Prompt run.
122 + if self.num_kv_heads != self.num_heads:
123 + # As of Nov 2023, xformers only supports MHA. For MQA/GQA,
124 + # project the key and value tensors to the desired number of
125 + # heads.
126 + # TODO(woosuk): Use MQA/GQA kernels for higher performance.
127 + query = query.view(query.shape[0], self.num_kv_heads,
128 + self.num_queries_per_kv, query.shape[-1])
129 + key = key[:, :,
130 + None, :].expand(key.shape[0], self.num_kv_heads,
131 + self.num_queries_per_kv,
132 + key.shape[-1])
133 + value = value[:, :, None, :].expand(value.shape[0],
134 + self.num_kv_heads,
135 + self.num_queries_per_kv,
136 + value.shape[-1])
137 +
138 + # Set attention bias if not provided. This typically happens at the
139 + # very attention layer of every iteration.
140 + # FIXME(woosuk): This is a hack.
141 + if input_metadata.attn_bias is None:
142 + if self.alibi_slopes is None:
143 + attn_bias = BlockDiagonalCausalMask.from_seqlens(
144 + [seq_len] * batch_size)
145 + if self.sliding_window is not None:
146 + attn_bias = attn_bias.make_local_attention(
147 + self.sliding_window)
148 + input_metadata.attn_bias = attn_bias
149 + else:
150 + input_metadata.attn_bias = _make_alibi_bias(
151 + self.alibi_slopes, batch_size, seq_len, query.dtype)
152 +
153 + # TODO(woosuk): Too many view operations. Let's try to reduce them
154 + # in the future for code readability.
155 + if self.alibi_slopes is None:
156 + query = query.unsqueeze(0)
157 + key = key.unsqueeze(0)
158 + value = value.unsqueeze(0)
159 + else:
160 + query = query.unflatten(0, (batch_size, seq_len))
161 + key = key.unflatten(0, (batch_size, seq_len))
162 + value = value.unflatten(0, (batch_size, seq_len))
163 +
164 + out = xops.memory_efficient_attention_forward(
165 + query,
166 + key,
167 + value,
168 + attn_bias=input_metadata.attn_bias,
169 + p=0.0,
170 + scale=self.scale,
171 + )
172 + output = out.view_as(query)
173 + else:
174 # Decoding run.
288 - assert input_metadata.num_prompt_tokens == 0
289 - assert key_cache is not None and value_cache is not None, (
290 - "key_cache and value_cache must be provided when "
291 - "generating tokens.")
292 - # Compute the attention op for generation tokens.
293 - self.single_query_cached_kv_attention(output, query, key_cache,
294 - value_cache, input_metadata,
295 - self.get_alibi_slopes())
175 + output = _paged_attention(
176 + query,
177 + key_cache,
178 + value_cache,
179 + input_metadata,
180 + self.head_mapping,
181 + self.scale,
182 + self.alibi_slopes,
183 + )
184
185 # Reshape the output tensor.
298 - # NOTE(woosuk): The output tensor may include paddings.
299 - return output.view(batch_size, seq_len,
300 - self.num_heads * self.head_size)
301 -
302 -
303 -class PagedAttentionWithRoPE(PagedAttention):
304 - """PagedAttention with rotary positional embedding."""
305 -
306 - def __init__(
307 - self,
308 - num_heads: int,
309 - head_size: int,
310 - scale: float,
311 - rotary_dim: int,
312 - max_position: int = 8192,
313 - base: int = 10000,
314 - num_kv_heads: Optional[int] = None,
315 - is_neox_style: bool = True,
316 - rope_scaling: Optional[Dict[str, Any]] = None,
317 - sliding_window: Optional[int] = None,
318 - ) -> None:
319 - super().__init__(num_heads,
320 - head_size,
321 - scale,
322 - num_kv_heads,
323 - sliding_window=sliding_window)
324 - self.rotary_emb = get_rope(head_size, rotary_dim, max_position, base,
325 - is_neox_style, rope_scaling)
326 -
327 - def forward(
328 - self,
329 - positions: torch.Tensor,
330 - query: torch.Tensor,
331 - key: torch.Tensor,
332 - value: torch.Tensor,
333 - key_cache: torch.Tensor,
334 - value_cache: torch.Tensor,
335 - input_metadata: InputMetadata,
336 - cache_event: Optional[torch.cuda.Event],
337 - ) -> torch.Tensor:
338 - """ PagedAttention forward pass with rotary embedding.
339 -
340 - Args:
341 - positions: shape = [batch_size, seq_len]
342 - query: shape = [batch_size, seq_len, num_heads * head_size]
343 - key: shape = [batch_size, seq_len, num_kv_heads * head_size]
344 - value: shape = [batch_size, seq_len, num_kv_heads * head_size]
345 - key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
346 - block_size, x]
347 - value_cache: shape = [num_blocks, num_kv_heads, head_size,
348 - block_size]
349 - input_metadata: metadata for paged attention.
350 - cache_event: event to wait for the cache operations to finish.
351 -
352 - Returns:
353 - shape = [batch_size, seq_len, num_heads * head_size]
354 - """
355 -
356 - # Apply rotary embedding to the query and key before passing them
357 - # to the attention op.
358 - query, key = self.rotary_emb(positions, query, key)
359 - return super().forward(
186 + return output.view(batch_size, seq_len, hidden_size)
187 +
188 +
189 +def _make_alibi_bias(
190 + alibi_slopes: torch.Tensor,
191 + batch_size: int,
192 + seq_len: int,
193 + dtype: torch.dtype,
194 +) -> LowerTriangularMaskWithTensorBias:
195 + bias = torch.arange(seq_len, dtype=dtype)
196 + # NOTE(zhuohan): HF uses
197 + # `bias = bias[None, :].repeat(prompt_len, 1)`
198 + # here. We find that both biases give the same results, but
199 + # the bias below more accurately follows the original ALiBi
200 + # paper.
201 + bias = bias[None, :] - bias[:, None]
202 + bias = bias.to(alibi_slopes.device)
203 +
204 + # When using custom attention bias, xformers requires the bias to
205 + # be sliced from a tensor whose length is a multiple of 8.
206 + padded_len = (seq_len + 7) // 8 * 8
207 + bias = torch.empty(
208 + batch_size,
209 + alibi_slopes.shape[0],
210 + seq_len,
211 + padded_len,
212 + device=alibi_slopes.device,
213 + dtype=dtype,
214 + )[:, :, :, :seq_len].copy_(bias)
215 + bias.mul_(alibi_slopes[:, None, None])
216 + attn_bias = LowerTriangularMaskWithTensorBias(bias)
217 + return attn_bias
218 +
219 +
220 +def _paged_attention(
221 + query: torch.Tensor,
222 + key_cache: torch.Tensor,
223 + value_cache: torch.Tensor,
224 + input_metadata: InputMetadata,
225 + head_mapping: torch.Tensor,
226 + scale: float,
227 + alibi_slopes: Optional[torch.Tensor],
228 +) -> torch.Tensor:
229 + output = torch.empty_like(query)
230 +
231 + block_size = value_cache.shape[3]
232 + num_seqs, num_heads, head_size = query.shape
233 + max_num_partitions = (
234 + (input_metadata.max_context_len + _PARTITION_SIZE - 1) //
235 + _PARTITION_SIZE)
236 + # NOTE(woosuk): We use a simple heuristic to decide whether to use
237 + # PagedAttention V1 or V2. If the number of partitions is 1, we use
238 + # V1 to avoid the overhead of reduction. Also, if the number of
239 + # sequences or heads is large, we use V1 since there is enough work
240 + # to parallelize.
241 + # TODO(woosuk): Tune this heuristic.
242 + # For context len > 8192, use V2 kernel to avoid shared memory shortage.
243 + use_v1 = input_metadata.max_context_len <= 8192 and (
244 + max_num_partitions == 1 or num_seqs * num_heads > 512)
245 + if use_v1:
246 + # Run PagedAttention V1.
247 + ops.paged_attention_v1(
248 + output,
249 query,
361 - key,
362 - value,
250 key_cache,
251 value_cache,
365 - input_metadata,
366 - cache_event,
252 + head_mapping,
253 + scale,
254 + input_metadata.block_tables,
255 + input_metadata.context_lens,
256 + block_size,
257 + input_metadata.max_context_len,
258 + alibi_slopes,
259 )
368 -
369 -
370 -class PagedAttentionWithALiBi(PagedAttention):
371 - """PagedAttention with ALiBi attention bias."""
372 -
373 - def __init__(self,
374 - num_heads: int,
375 - head_size: int,
376 - scale: float,
377 - slopes: List[float],
378 - num_kv_heads: Optional[int] = None) -> None:
379 - super().__init__(num_heads, head_size, scale, num_kv_heads)
380 - assert len(slopes) == num_heads
381 -
382 - slopes = torch.tensor(slopes, dtype=torch.float32)
383 - self.register_buffer("alibi_slopes", slopes, persistent=False)
384 -
385 - def set_attn_bias(self, input_metadata: InputMetadata,
386 - dtype: torch.dtype) -> None:
387 - if input_metadata.attn_bias is not None:
388 - # Already set by a previous layer.
389 - return
390 - # Generates ALiBi mask based on the max prompt length.
391 - max_prompt_len = input_metadata.max_prompt_len
392 - bias = torch.arange(max_prompt_len, dtype=dtype)
393 - # NOTE(zhuohan): HF uses
394 - # `bias = bias[None, :].repeat(prompt_len, 1)`
395 - # here. We find that both biases give the same results, but
396 - # the bias below more accurately follows the original ALiBi
397 - # paper.
398 - bias = bias[None, :] - bias[:, None]
399 - bias = bias.to(self.alibi_slopes.device)
400 -
401 - # When using custom attention bias, xformers requires the bias to
402 - # be sliced from a tensor whose length is a multiple of 8.
403 - padded_len = (max_prompt_len + 7) // 8 * 8
404 - bias = torch.empty(
405 - input_metadata.num_prompts,
406 - self.num_heads,
407 - max_prompt_len,
408 - padded_len,
409 - device=self.alibi_slopes.device,
410 - dtype=dtype,
411 - )[:, :, :, :max_prompt_len].copy_(bias)
412 - bias.mul_(self.alibi_slopes[:, None, None])
413 - attn_bias = LowerTriangularMaskWithTensorBias(bias)
414 - input_metadata.attn_bias = attn_bias
415 -
416 - def multi_query_kv_attention(
417 - self,
418 - output: torch.Tensor,
419 - query: torch.Tensor,
420 - key: torch.Tensor,
421 - value: torch.Tensor,
422 - input_metadata: InputMetadata,
423 - ) -> torch.Tensor:
424 - """Attention with ALiBi bias for the prompt tokens.
425 -
426 - Args:
427 - output: shape = [num_prompt_tokens, num_heads, head_size]
428 - query: shape = [num_prompt_tokens, num_heads, head_size]
429 - key: shape = [num_prompt_tokens, num_kv_heads, head_size]
430 - value: shape = [num_prompt_tokens, num_kv_heads, head_size]
431 - input_metadata: metadata for paged attention.
432 - """
433 - if self.num_kv_heads != self.num_heads:
434 - # Project the key and value tensors to the desired number of heads.
435 - query = query.view(query.shape[0], self.num_kv_heads,
436 - self.num_queries_per_kv, query.shape[-1])
437 - key = key[:, :,
438 - None, :].expand(key.shape[0], self.num_kv_heads,
439 - self.num_queries_per_kv, key.shape[-1])
440 - value = value[:, :,
441 - None, :].expand(value.shape[0], self.num_kv_heads,
442 - self.num_queries_per_kv,
443 - value.shape[-1])
444 - batch_size = input_metadata.num_prompts
445 - seq_len = input_metadata.max_prompt_len
446 -
447 - out = xops.memory_efficient_attention_forward(
448 - query.view(batch_size, seq_len, self.num_heads, self.head_size),
449 - key.view(batch_size, seq_len, self.num_heads, self.head_size),
450 - value.view(batch_size, seq_len, self.num_heads, self.head_size),
451 - attn_bias=input_metadata.attn_bias,
452 - p=0.0,
453 - scale=self.scale,
260 + else:
261 + # Run PagedAttention V2.
262 + assert _PARTITION_SIZE % block_size == 0
263 + tmp_output = torch.empty(
264 + size=(num_seqs, num_heads, max_num_partitions, head_size),
265 + dtype=output.dtype,
266 + device=output.device,
267 )
455 - # TODO(woosuk): Unnecessary copy. Optimize.
456 - output.copy_(out.view_as(output))
457 - return output
458 -
459 - def get_alibi_slopes(self) -> Optional[torch.Tensor]:
460 - return self.alibi_slopes
268 + exp_sums = torch.empty(
269 + size=(num_seqs, num_heads, max_num_partitions),
270 + dtype=torch.float32,
271 + device=output.device,
272 + )
273 + max_logits = torch.empty_like(exp_sums)
274 + ops.paged_attention_v2(
275 + output,
276 + exp_sums,
277 + max_logits,
278 + tmp_output,
279 + query,
280 + key_cache,
281 + value_cache,
282 + head_mapping,
283 + scale,
284 + input_metadata.block_tables,
285 + input_metadata.context_lens,
286 + block_size,
287 + input_metadata.max_context_len,
288 + alibi_slopes,
289 + )
290 + return output
vllm/model_executor/layers/rotary_embedding.py
+2 -2
@@ -277,8 +277,8 @@ def get_rope(
277 rotary_dim: int,
278 max_position: int,
279 base: int,
280 - is_neox_style: bool,
281 - rope_scaling: Optional[Dict[str, Any]],
280 + is_neox_style: bool = True,
281 + rope_scaling: Optional[Dict[str, Any]] = None,
282 ) -> RotaryEmbedding:
283 if rope_scaling is None:
284 rotary_emb = RotaryEmbedding(head_size, rotary_dim, max_position, base,
vllm/model_executor/models/aquila.py
+14 -10
@@ -28,11 +28,12 @@ from torch import nn
28
29 from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.activation import SiluAndMul
31 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
31 +from vllm.model_executor.layers.attention import PagedAttention
32 from vllm.model_executor.layers.linear import (LinearMethodBase,
33 MergedColumnParallelLinear,
34 QKVParallelLinear,
35 RowParallelLinear)
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding, ParallelLMHead)
@@ -138,15 +139,17 @@ class AquilaAttention(nn.Module):
139 bias=False,
140 linear_method=linear_method,
141 )
141 - self.attn = PagedAttentionWithRoPE(
142 - self.num_heads,
142 + self.rotary_emb = get_rope(
143 self.head_dim,
144 - self.scaling,
145 - base=self.rope_theta,
146 - max_position=self.max_position_embeddings,
144 rotary_dim=self.head_dim,
148 - num_kv_heads=self.num_kv_heads,
149 - rope_scaling=rope_scaling)
145 + max_position=self.max_position_embeddings,
146 + base=self.rope_theta,
147 + rope_scaling=rope_scaling,
148 + )
149 + self.attn = PagedAttention(self.num_heads,
150 + self.head_dim,
151 + self.scaling,
152 + num_kv_heads=self.num_kv_heads)
153
154 def forward(
155 self,
@@ -158,9 +161,10 @@ class AquilaAttention(nn.Module):
161 ) -> torch.Tensor:
162 qkv, _ = self.qkv_proj(hidden_states)
163 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
164 + q, k = self.rotary_emb(positions, q, k)
165 k_cache, v_cache = kv_cache
162 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
163 - input_metadata, cache_event)
166 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
167 + cache_event)
168 output, _ = self.o_proj(attn_output)
169 return output
170
vllm/model_executor/models/baichuan.py
+16 -16
@@ -26,13 +26,13 @@ from torch import nn
26
27 from vllm.model_executor.input_metadata import InputMetadata
28 from vllm.model_executor.layers.activation import SiluAndMul
29 -from vllm.model_executor.layers.attention import (PagedAttentionWithRoPE,
30 - PagedAttentionWithALiBi)
29 +from vllm.model_executor.layers.attention import PagedAttention
30 from vllm.model_executor.layers.layernorm import RMSNorm
31 from vllm.model_executor.layers.linear import (LinearMethodBase,
32 MergedColumnParallelLinear,
33 QKVParallelLinear,
34 RowParallelLinear)
35 +from vllm.model_executor.layers.rotary_embedding import get_rope
36 from vllm.model_executor.layers.sampler import Sampler
37 from vllm.model_executor.layers.vocab_parallel_embedding import (
38 VocabParallelEmbedding, ParallelLMHead)
@@ -150,17 +150,20 @@ class BaiChuanAttention(nn.Module):
150 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
151
152 scaling = self.head_dim**-0.5
153 - self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
154 - scaling, alibi_slopes)
153 + self.attn = PagedAttention(self.num_heads,
154 + self.head_dim,
155 + scaling,
156 + alibi_slopes=alibi_slopes)
157 else:
156 - self.scaling = self.head_dim**-0.5
157 - self.attn = PagedAttentionWithRoPE(
158 - self.num_heads,
158 + self.rotary_emb = get_rope(
159 self.head_dim,
160 - self.scaling,
160 rotary_dim=self.head_dim,
161 + max_position=self.max_position_embeddings,
162 base=self.rope_theta,
163 - max_position=self.max_position_embeddings)
163 + )
164 + self.scaling = self.head_dim**-0.5
165 + self.attn = PagedAttention(self.num_heads, self.head_dim,
166 + self.scaling)
167
168 def forward(
169 self,
@@ -172,14 +175,11 @@ class BaiChuanAttention(nn.Module):
175 ) -> torch.Tensor:
176 qkv, _ = self.W_pack(hidden_states)
177 q, k, v = qkv.chunk(chunks=3, dim=-1)
178 + if self.postion_embedding != "ALIBI":
179 + q, k = self.rotary_emb(positions, q, k)
180 k_cache, v_cache = kv_cache
176 - if self.postion_embedding == "ALIBI":
177 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
178 - cache_event)
179 - else:
180 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
181 - input_metadata, cache_event)
182 -
181 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
182 + cache_event)
183 output, _ = self.o_proj(attn_output)
184 return output
185
vllm/model_executor/models/bloom.py
+5 -3
@@ -25,7 +25,7 @@ from transformers import BloomConfig
25
26 from vllm.model_executor.input_metadata import InputMetadata
27 from vllm.model_executor.layers.activation import get_act_fn
28 -from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
28 +from vllm.model_executor.layers.attention import PagedAttention
29 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
30 LinearMethodBase,
31 QKVParallelLinear,
@@ -106,8 +106,10 @@ class BloomAttention(nn.Module):
106 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
107
108 scaling = self.head_dim**-0.5
109 - self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
110 - scaling, alibi_slopes)
109 + self.attn = PagedAttention(self.num_heads,
110 + self.head_dim,
111 + scaling,
112 + alibi_slopes=alibi_slopes)
113
114 def forward(
115 self,
vllm/model_executor/models/chatglm.py
+10 -9
@@ -10,12 +10,13 @@ from torch.nn import LayerNorm
10
11 from vllm.model_executor.input_metadata import InputMetadata
12 from vllm.model_executor.layers.activation import SiluAndMul
13 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
13 +from vllm.model_executor.layers.attention import PagedAttention
14 from vllm.model_executor.layers.layernorm import RMSNorm
15 from vllm.model_executor.layers.linear import (LinearMethodBase,
16 MergedColumnParallelLinear,
17 QKVParallelLinear,
18 RowParallelLinear)
19 +from vllm.model_executor.layers.rotary_embedding import get_rope
20 from vllm.model_executor.layers.sampler import Sampler
21 from vllm.model_executor.layers.vocab_parallel_embedding import (
22 VocabParallelEmbedding, ParallelLMHead)
@@ -78,16 +79,19 @@ class GLMAttention(nn.Module):
79 # https://huggingface.co/THUDM/chatglm3-6b-32k/blob/e210410255278dd9d74463cf396ba559c0ef801c/modeling_chatglm.py#L141
80 rope_ratio = getattr(config, "rope_ratio", 1.0)
81 max_positions = getattr(config, "seq_length", 8192)
81 - self.attn = PagedAttentionWithRoPE(
82 - self.num_heads,
82 + self.rotary_emb = get_rope(
83 self.head_dim,
84 - self.scaling,
84 rotary_dim=self.head_dim // 2,
86 - num_kv_heads=self.num_kv_heads,
85 max_position=max_positions,
86 base=10000 * rope_ratio,
87 is_neox_style=False,
88 )
89 + self.attn = PagedAttention(
90 + self.num_heads,
91 + self.head_dim,
92 + self.scaling,
93 + num_kv_heads=self.num_kv_heads,
94 + )
95
96 def forward(
97 self,
@@ -99,10 +103,9 @@ class GLMAttention(nn.Module):
103 ) -> torch.Tensor:
104 qkv, _ = self.query_key_value(hidden_states)
105 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
106 + q, k = self.rotary_emb(position_ids, q, k)
107 key_cache, value_cache = kv_cache
103 -
108 context_layer = self.attn(
105 - position_ids,
109 q,
110 k,
111 v,
@@ -111,9 +114,7 @@ class GLMAttention(nn.Module):
114 input_metadata,
115 cache_event,
116 )
114 -
117 attn_output, _ = self.dense(context_layer)
116 -
118 return attn_output
119
120
vllm/model_executor/models/falcon.py
+19 -20
@@ -28,13 +28,12 @@ from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.model_executor.input_metadata import InputMetadata
30 from vllm.model_executor.layers.activation import get_act_fn
31 -from vllm.model_executor.layers.attention import (PagedAttention,
32 - PagedAttentionWithALiBi,
33 - PagedAttentionWithRoPE)
31 +from vllm.model_executor.layers.attention import PagedAttention
32 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
33 LinearMethodBase,
34 QKVParallelLinear,
35 RowParallelLinear)
36 +from vllm.model_executor.layers.rotary_embedding import get_rope
37 from vllm.model_executor.layers.sampler import Sampler
38 from vllm.model_executor.layers.vocab_parallel_embedding import (
39 VocabParallelEmbedding, ParallelLMHead)
@@ -144,14 +143,16 @@ class FalconAttention(nn.Module):
143 rope_theta = getattr(config, "rope_theta", 10000)
144 max_position_embeddings = getattr(config,
145 "max_position_embeddings", 8192)
147 - self.attn = PagedAttentionWithRoPE(
148 - self.num_heads,
146 + self.rotary_emb = get_rope(
147 self.head_dim,
150 - self.inv_norm_factor,
151 - base=rope_theta,
152 - max_position=max_position_embeddings,
148 rotary_dim=self.head_dim,
154 - num_kv_heads=self.num_kv_heads)
149 + max_position=max_position_embeddings,
150 + base=rope_theta,
151 + )
152 + self.attn = PagedAttention(self.num_heads,
153 + self.head_dim,
154 + self.inv_norm_factor,
155 + num_kv_heads=self.num_kv_heads)
156 elif self.use_alibi:
157 tp_rank = get_tensor_model_parallel_rank()
158 head_start = tp_rank * self.num_heads
@@ -159,11 +160,11 @@ class FalconAttention(nn.Module):
160 alibi_slopes = (_get_alibi_slopes(self.total_num_heads) *
161 self.inv_norm_factor)
162 alibi_slopes = alibi_slopes[head_start:head_end].tolist()
162 - self.attn = PagedAttentionWithALiBi(self.num_heads,
163 - self.head_dim,
164 - self.inv_norm_factor,
165 - alibi_slopes,
166 - num_kv_heads=self.num_kv_heads)
163 + self.attn = PagedAttention(self.num_heads,
164 + self.head_dim,
165 + self.inv_norm_factor,
166 + num_kv_heads=self.num_kv_heads,
167 + alibi_slopes=alibi_slopes)
168 else:
169 self.attn = PagedAttention(self.num_heads,
170 self.head_dim,
@@ -182,13 +183,11 @@ class FalconAttention(nn.Module):
183 if bias is not None:
184 qkv += bias
185 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
185 - k_cache, v_cache = kv_cache
186 if self.use_rotary:
187 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
188 - input_metadata, cache_event)
189 - else:
190 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
191 - cache_event)
187 + q, k = self.rotary_emb(positions, q, k)
188 + k_cache, v_cache = kv_cache
189 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
190 + cache_event)
191 attn_output, bias = self.dense(attn_output)
192 return attn_output, bias
193
vllm/model_executor/models/gpt_j.py
+11 -10
@@ -24,11 +24,12 @@ from transformers import GPTJConfig
24
25 from vllm.model_executor.input_metadata import InputMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
27 +from vllm.model_executor.layers.attention import PagedAttention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
31 RowParallelLinear)
32 +from vllm.model_executor.layers.rotary_embedding import get_rope
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding, ParallelLMHead)
@@ -77,15 +78,14 @@ class GPTJAttention(nn.Module):
78 rope_theta = getattr(config, "rope_theta", 10000)
79 max_position_embeddings = getattr(config, "max_position_embeddings",
80 8192)
80 - self.attn = PagedAttentionWithRoPE(
81 - self.num_heads,
81 + self.rotary_emb = get_rope(
82 self.head_size,
83 - scaling,
84 - config.rotary_dim,
85 - base=rope_theta,
83 + rotary_dim=config.rotary_dim,
84 max_position=max_position_embeddings,
87 - is_neox_style=False)
88 - self.warmup = False
85 + base=rope_theta,
86 + is_neox_style=False,
87 + )
88 + self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
89
90 def forward(
91 self,
@@ -97,9 +97,10 @@ class GPTJAttention(nn.Module):
97 ) -> torch.Tensor:
98 qkv, _ = self.qkv_proj(hidden_states)
99 q, k, v = qkv.chunk(chunks=3, dim=-1)
100 + q, k = self.rotary_emb(position_ids, q, k)
101 k_cache, v_cache = kv_cache
101 - attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
102 - input_metadata, cache_event)
102 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
103 + cache_event)
104 attn_output, _ = self.out_proj(attn_output)
105 return attn_output
106
vllm/model_executor/models/gpt_neox.py
+10 -8
@@ -24,11 +24,12 @@ from transformers import GPTNeoXConfig
24
25 from vllm.model_executor.input_metadata import InputMetadata
26 from vllm.model_executor.layers.activation import get_act_fn
27 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
27 +from vllm.model_executor.layers.attention import PagedAttention
28 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
29 LinearMethodBase,
30 QKVParallelLinear,
31 RowParallelLinear)
32 +from vllm.model_executor.layers.rotary_embedding import get_rope
33 from vllm.model_executor.layers.sampler import Sampler
34 from vllm.model_executor.layers.vocab_parallel_embedding import (
35 VocabParallelEmbedding, ParallelLMHead)
@@ -77,13 +78,13 @@ class GPTNeoXAttention(nn.Module):
78 rope_theta = getattr(config, "rope_theta", 10000)
79 max_position_embeddings = getattr(config, "max_position_embeddings",
80 8192)
80 - self.attn = PagedAttentionWithRoPE(
81 - self.num_heads,
81 + self.rotary_emb = get_rope(
82 self.head_size,
83 - scaling,
84 - rotary_dim,
83 + rotary_dim=rotary_dim,
84 + max_position=max_position_embeddings,
85 base=rope_theta,
86 - max_position=max_position_embeddings)
86 + )
87 + self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
88
89 def forward(
90 self,
@@ -95,9 +96,10 @@ class GPTNeoXAttention(nn.Module):
96 ) -> torch.Tensor:
97 qkv, _ = self.query_key_value(hidden_states)
98 q, k, v = qkv.chunk(chunks=3, dim=-1)
99 + q, k = self.rotary_emb(position_ids, q, k)
100 k_cache, v_cache = kv_cache
99 - attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
100 - input_metadata, cache_event)
101 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
102 + cache_event)
103 output, _ = self.dense(attn_output)
104 return output
105
vllm/model_executor/models/internlm.py
+10 -8
@@ -7,12 +7,13 @@ from transformers import LlamaConfig
7
8 from vllm.model_executor.input_metadata import InputMetadata
9 from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
10 +from vllm.model_executor.layers.attention import PagedAttention
11 from vllm.model_executor.layers.layernorm import RMSNorm
12 from vllm.model_executor.layers.linear import (LinearMethodBase,
13 MergedColumnParallelLinear,
14 QKVParallelLinear,
15 RowParallelLinear)
16 +from vllm.model_executor.layers.rotary_embedding import get_rope
17 from vllm.model_executor.layers.sampler import Sampler
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 VocabParallelEmbedding, ParallelLMHead)
@@ -92,13 +93,13 @@ class InternLMAttention(nn.Module):
93 bias=bias,
94 linear_method=linear_method,
95 )
95 - self.attn = PagedAttentionWithRoPE(
96 - self.num_heads,
96 + self.rotary_emb = get_rope(
97 self.head_dim,
98 - self.scaling,
99 - base=self.rope_theta,
98 + rotary_dim=self.head_dim,
99 max_position=self.max_position_embeddings,
101 - rotary_dim=self.head_dim)
100 + base=self.rope_theta,
101 + )
102 + self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
103
104 def forward(
105 self,
@@ -110,9 +111,10 @@ class InternLMAttention(nn.Module):
111 ) -> torch.Tensor:
112 qkv, _ = self.qkv_proj(hidden_states)
113 q, k, v = qkv.chunk(chunks=3, dim=-1)
114 + q, k = self.rotary_emb(positions, q, k)
115 k_cache, v_cache = kv_cache
114 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
115 - input_metadata, cache_event)
116 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
117 + cache_event)
118 output, _ = self.o_proj(attn_output)
119 return output
120
vllm/model_executor/models/llama.py
+15 -10
@@ -29,12 +29,13 @@ from transformers import LlamaConfig
29
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32 +from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
36 QKVParallelLinear,
37 RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
@@ -126,15 +127,18 @@ class LlamaAttention(nn.Module):
127 bias=False,
128 linear_method=linear_method,
129 )
129 - self.attn = PagedAttentionWithRoPE(
130 - self.num_heads,
130 +
131 + self.rotary_emb = get_rope(
132 self.head_dim,
132 - self.scaling,
133 - base=self.rope_theta,
134 - max_position=self.max_position_embeddings,
133 rotary_dim=self.head_dim,
136 - num_kv_heads=self.num_kv_heads,
137 - rope_scaling=rope_scaling)
134 + max_position=max_position_embeddings,
135 + base=rope_theta,
136 + rope_scaling=rope_scaling,
137 + )
138 + self.attn = PagedAttention(self.num_heads,
139 + self.head_dim,
140 + self.scaling,
141 + num_kv_heads=self.num_kv_heads)
142
143 def forward(
144 self,
@@ -146,9 +150,10 @@ class LlamaAttention(nn.Module):
150 ) -> torch.Tensor:
151 qkv, _ = self.qkv_proj(hidden_states)
152 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
153 + q, k = self.rotary_emb(positions, q, k)
154 k_cache, v_cache = kv_cache
150 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
151 - input_metadata, cache_event)
155 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
156 + cache_event)
157 output, _ = self.o_proj(attn_output)
158 return output
159
vllm/model_executor/models/mistral.py
+17 -11
@@ -29,12 +29,13 @@ from transformers import MistralConfig
29
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32 +from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
36 QKVParallelLinear,
37 RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
@@ -124,14 +125,18 @@ class MistralAttention(nn.Module):
125 bias=False,
126 linear_method=linear_method,
127 )
127 - self.attn = PagedAttentionWithRoPE(self.num_heads,
128 - self.head_dim,
129 - self.scaling,
130 - base=self.rope_theta,
131 - max_position=max_position,
132 - rotary_dim=self.head_dim,
133 - num_kv_heads=self.num_kv_heads,
134 - sliding_window=self.sliding_window)
128 +
129 + self.rotary_emb = get_rope(
130 + self.head_dim,
131 + rotary_dim=self.head_dim,
132 + max_position=max_position,
133 + base=self.rope_theta,
134 + )
135 + self.attn = PagedAttention(self.num_heads,
136 + self.head_dim,
137 + self.scaling,
138 + num_kv_heads=self.num_kv_heads,
139 + sliding_window=self.sliding_window)
140
141 def forward(
142 self,
@@ -143,9 +148,10 @@ class MistralAttention(nn.Module):
148 ) -> torch.Tensor:
149 qkv, _ = self.qkv_proj(hidden_states)
150 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
151 + q, k = self.rotary_emb(positions, q, k)
152 k_cache, v_cache = kv_cache
147 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
148 - input_metadata, cache_event)
153 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
154 + cache_event)
155 output, _ = self.o_proj(attn_output)
156 return output
157
vllm/model_executor/models/mpt.py
+5 -3
@@ -8,7 +8,7 @@ import torch.nn as nn
8
9 from vllm.model_executor.input_metadata import InputMetadata
10 from vllm.model_executor.layers.activation import get_act_fn
11 -from vllm.model_executor.layers.attention import PagedAttentionWithALiBi
11 +from vllm.model_executor.layers.attention import PagedAttention
12 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
13 LinearMethodBase,
14 QKVParallelLinear,
@@ -87,8 +87,10 @@ class MPTAttention(nn.Module):
87
88 self.head_dim = self.d_model // self.total_num_heads
89 scaling = self.head_dim**-0.5
90 - self.attn = PagedAttentionWithALiBi(self.num_heads, self.head_dim,
91 - scaling, alibi_slopes)
90 + self.attn = PagedAttention(self.num_heads,
91 + self.head_dim,
92 + scaling,
93 + alibi_slopes=alibi_slopes)
94
95 def forward(
96 self,
vllm/model_executor/models/phi_1_5.py
+10 -8
@@ -43,11 +43,12 @@ from transformers import PretrainedConfig
43
44 from vllm.model_executor.input_metadata import InputMetadata
45 from vllm.model_executor.layers.activation import get_act_fn
46 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
46 +from vllm.model_executor.layers.attention import PagedAttention
47 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
48 LinearMethodBase,
49 QKVParallelLinear,
50 RowParallelLinear)
51 +from vllm.model_executor.layers.rotary_embedding import get_rope
52 from vllm.model_executor.layers.sampler import Sampler
53 from vllm.model_executor.layers.vocab_parallel_embedding import (
54 VocabParallelEmbedding, ParallelLMHead)
@@ -119,13 +120,13 @@ class PhiAttention(nn.Module):
120 # https://huggingface.co/microsoft/phi-1_5/blob/d212a789620c380ff32ca1d1ee9943a777360987/modeling_phi.py#L518
121 rope_theta = 10000
122 max_position_embeddings = getattr(config, "n_positions", 2048)
122 - self.attn = PagedAttentionWithRoPE(
123 - self.num_heads,
123 + self.rotary_emb = get_rope(
124 self.head_size,
125 - scaling,
126 - rotary_dim,
125 + rotary_dim=rotary_dim,
126 + max_position=max_position_embeddings,
127 base=rope_theta,
128 - max_position=max_position_embeddings)
128 + )
129 + self.attn = PagedAttention(self.num_heads, self.head_size, scaling)
130
131 def forward(
132 self,
@@ -137,9 +138,10 @@ class PhiAttention(nn.Module):
138 ) -> torch.Tensor:
139 qkv, _ = self.Wqkv(hidden_states)
140 q, k, v = qkv.chunk(chunks=3, dim=-1)
141 + q, k = self.rotary_emb(position_ids, q, k)
142 k_cache, v_cache = kv_cache
141 - attn_output = self.attn(position_ids, q, k, v, k_cache, v_cache,
142 - input_metadata, cache_event)
143 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
144 + cache_event)
145 output, _ = self.out_proj(attn_output)
146 return output
147
vllm/model_executor/models/qwen.py
+11 -9
@@ -11,12 +11,13 @@ from torch import nn
11
12 from vllm.model_executor.input_metadata import InputMetadata
13 from vllm.model_executor.layers.activation import SiluAndMul
14 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
14 +from vllm.model_executor.layers.attention import PagedAttention
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 from vllm.model_executor.layers.linear import (LinearMethodBase,
17 MergedColumnParallelLinear,
18 QKVParallelLinear,
19 RowParallelLinear)
20 +from vllm.model_executor.layers.rotary_embedding import get_rope
21 from vllm.model_executor.layers.sampler import Sampler
22 from vllm.model_executor.layers.vocab_parallel_embedding import (
23 VocabParallelEmbedding, ParallelLMHead)
@@ -95,14 +96,15 @@ class QWenAttention(nn.Module):
96 linear_method=linear_method,
97 )
98 self.scaling = self.head_dim**-0.5
98 - self.attn = PagedAttentionWithRoPE(
99 - self.num_heads,
99 +
100 + self.rotary_emb = get_rope(
101 self.head_dim,
101 - self.scaling,
102 rotary_dim=self.head_dim,
103 - base=rope_theta,
103 max_position=max_position_embeddings,
105 - rope_scaling=rope_scaling)
104 + base=rope_theta,
105 + rope_scaling=rope_scaling,
106 + )
107 + self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
108
109 def forward(
110 self,
@@ -114,10 +116,10 @@ class QWenAttention(nn.Module):
116 ) -> torch.Tensor:
117 qkv, _ = self.c_attn(hidden_states)
118 q, k, v = qkv.chunk(chunks=3, dim=-1)
117 -
119 + q, k = self.rotary_emb(positions, q, k)
120 k_cache, v_cache = kv_cache
119 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
120 - input_metadata, cache_event)
121 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
122 + cache_event)
123
124 output, _ = self.c_proj(attn_output)
125 return output
vllm/model_executor/models/yi.py
+14 -10
@@ -29,12 +29,13 @@ from vllm.transformers_utils.configs.yi import YiConfig
29
30 from vllm.model_executor.input_metadata import InputMetadata
31 from vllm.model_executor.layers.activation import SiluAndMul
32 -from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
32 +from vllm.model_executor.layers.attention import PagedAttention
33 from vllm.model_executor.layers.layernorm import RMSNorm
34 from vllm.model_executor.layers.linear import (LinearMethodBase,
35 MergedColumnParallelLinear,
36 QKVParallelLinear,
37 RowParallelLinear)
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 VocabParallelEmbedding, ParallelLMHead)
@@ -126,15 +127,17 @@ class YiAttention(nn.Module):
127 bias=False,
128 linear_method=linear_method,
129 )
129 - self.attn = PagedAttentionWithRoPE(
130 - self.num_heads,
130 + self.rotary_emb = get_rope(
131 self.head_dim,
132 - self.scaling,
133 - base=self.rope_theta,
134 - max_position=self.max_position_embeddings,
132 rotary_dim=self.head_dim,
136 - num_kv_heads=self.num_kv_heads,
137 - rope_scaling=rope_scaling)
133 + max_position=max_position_embeddings,
134 + base=self.rope_theta,
135 + rope_scaling=rope_scaling,
136 + )
137 + self.attn = PagedAttention(self.num_heads,
138 + self.head_dim,
139 + self.scaling,
140 + num_kv_heads=self.num_kv_heads)
141
142 def forward(
143 self,
@@ -146,9 +149,10 @@ class YiAttention(nn.Module):
149 ) -> torch.Tensor:
150 qkv, _ = self.qkv_proj(hidden_states)
151 q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
152 + q, k = self.rotary_emb(positions, q, k)
153 k_cache, v_cache = kv_cache
150 - attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
151 - input_metadata, cache_event)
154 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata,
155 + cache_event)
156 output, _ = self.o_proj(attn_output)
157 return output
158