@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21877b0d75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Support Longchat and RoPE scaling (#555)

Co-authored-by: Wing Lian <wing.lian@gmail.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Lily Liu committed Sep 27, 2023 at 03:36 UTC 21877b0d7523a1202ed279c046d9268baaee3d5c
4 files changed +211 -40
vllm/config.py
+11
@@ -351,6 +351,17 @@ def _get_and_verify_max_len(
351 if max_len_key is not None:
352 derived_max_model_len = min(derived_max_model_len, max_len_key)
353
354 + rope_scaling = getattr(hf_config, "rope_scaling", None)
355 + if rope_scaling is not None:
356 + if derived_max_model_len == float("inf"):
357 + raise ValueError(
358 + "When using rope_scaling, the model's config.json must "
359 + "contain one of the following keys to determine the original "
360 + f"maximum length of the model: {possible_keys}")
361 + assert "factor" in rope_scaling
362 + scaling_factor = rope_scaling["factor"]
363 + derived_max_model_len *= scaling_factor
364 +
365 if max_model_len is None:
366 max_model_len = derived_max_model_len
367 elif max_model_len > derived_max_model_len:
vllm/model_executor/layers/attention.py
+25 -38
@@ -1,5 +1,5 @@
1 """Multi-head attention."""
2 -from typing import List, Optional
2 +from typing import Any, Dict, List, Optional
3
4 import torch
5 import torch.nn as nn
@@ -9,8 +9,10 @@ from xformers.ops.fmha.attn_bias import (BlockDiagonalCausalMask,
9
10 from vllm import attention_ops
11 from vllm import cache_ops
12 -from vllm import pos_encoding_ops
12 from vllm.model_executor.input_metadata import InputMetadata
13 +from vllm.model_executor.layers.rotary_embedding import (
14 + DynamicNTKScalingRotaryEmbedding, LinearScalingRotaryEmbedding,
15 + RotaryEmbedding)
16
17 _SUPPORTED_HEAD_SIZES = [64, 80, 96, 112, 128, 256]
18
@@ -247,7 +249,7 @@ class PagedAttention(nn.Module):
249
250
251 class PagedAttentionWithRoPE(PagedAttention):
250 - """PagedAttention with rotary embedding."""
252 + """PagedAttention with rotary positional embedding."""
253
254 def __init__(
255 self,
@@ -259,34 +261,26 @@ class PagedAttentionWithRoPE(PagedAttention):
261 base: int = 10000,
262 num_kv_heads: Optional[int] = None,
263 is_neox_style: bool = True,
264 + rope_scaling: Optional[Dict[str, Any]] = None,
265 ) -> None:
266 super().__init__(num_heads, head_size, scale, num_kv_heads)
264 - self.is_neox_style = is_neox_style
265 -
266 - # Create the cos and sin cache.
267 - # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
268 - # However, we use `torch.arange(..., dtype=torch.float)` instead to
269 - # avoid numerical issues with large base values (e.g., 10000000).
270 - # This may cause a slight numerical difference between the HF
271 - # implementation and ours.
272 - # NOTE(woosuk): To exactly match the HF implementation, we need to
273 - # use CPU to compute the cache and then move it to GPU. However, we
274 - # create the cache on GPU for faster initialization. This may cause
275 - # a slight numerical difference between the HF implementation and ours.
276 - inv_freq = 1.0 / (base**(torch.arange(
277 - 0, rotary_dim, 2, dtype=torch.float, device="cuda") / rotary_dim))
278 - t = torch.arange(max_position, dtype=torch.float, device="cuda")
279 - freqs = torch.einsum("i,j -> ij", t, inv_freq)
280 - cos = freqs.cos()
281 - sin = freqs.sin()
282 - cache = torch.cat((cos, sin), dim=-1)
283 -
284 - # FIXME(woosuk): This assumes that we configure the default dtype when
285 - # initializing the model.
286 - torch_dtype = torch.get_default_dtype()
287 - cache = cache.to(torch_dtype)
288 - # Embedding size: [max_position, rotary_dim]
289 - self.register_buffer("cos_sin_cache", cache, persistent=False)
267 + if rope_scaling is None:
268 + self.rotary_emb = RotaryEmbedding(head_size, rotary_dim,
269 + max_position, base,
270 + is_neox_style)
271 + else:
272 + scaling_type = rope_scaling["type"]
273 + scaling_factor = rope_scaling["factor"]
274 + if scaling_type == "linear":
275 + self.rotary_emb = LinearScalingRotaryEmbedding(
276 + head_size, rotary_dim, max_position, base, is_neox_style,
277 + scaling_factor)
278 + elif scaling_type == "dynamic":
279 + self.rotary_emb = DynamicNTKScalingRotaryEmbedding(
280 + head_size, rotary_dim, max_position, base, is_neox_style,
281 + scaling_factor)
282 + else:
283 + raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
284
285 def forward(
286 self,
@@ -303,7 +297,7 @@ class PagedAttentionWithRoPE(PagedAttention):
297
298 Args:
299 positions: shape = [num_tokens]
306 - query: shape = [num_tokens, num_heads * head_size]
300 + query: shape = [num_tokens, num_heads * head_size]
301 key: shape = [num_tokens, num_kv_heads * head_size]
302 value: shape = [num_tokens, num_kv_heads * head_size]
303 key_cache: shape = [num_blocks, num_kv_heads, head_size/x,
@@ -319,14 +313,7 @@ class PagedAttentionWithRoPE(PagedAttention):
313
314 # Apply rotary embedding to the query and key before passing them
315 # to the attention op.
322 - pos_encoding_ops.rotary_embedding(
323 - positions,
324 - query,
325 - key,
326 - self.head_size,
327 - self.cos_sin_cache,
328 - self.is_neox_style,
329 - )
316 + query, key = self.rotary_emb(positions, query, key)
317 return super().forward(
318 query,
319 key,
vllm/model_executor/layers/rotary_embedding.py new
+169
@@ -0,0 +1,169 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.33.2/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Rotary Positional Embeddings."""
24 +from typing import Tuple, Union
25 +
26 +import torch
27 +import torch.nn as nn
28 +
29 +from vllm import pos_encoding_ops
30 +
31 +
32 +class RotaryEmbedding(nn.Module):
33 + """Original rotary positional embedding."""
34 +
35 + def __init__(
36 + self,
37 + head_size: int,
38 + rotary_dim: int,
39 + max_position_embeddings: int,
40 + base: int,
41 + is_neox_style: bool,
42 + ) -> None:
43 + super().__init__()
44 + self.head_size = head_size
45 + self.rotary_dim = rotary_dim
46 + self.max_position_embeddings = max_position_embeddings
47 + self.base = base
48 + self.is_neox_style = is_neox_style
49 +
50 + cache = self._compute_cos_sin_cache()
51 + cache = cache.to(torch.get_default_dtype())
52 + self.register_buffer("cos_sin_cache", cache, persistent=False)
53 +
54 + def _compute_inv_freq(self, base: Union[int, float]) -> torch.Tensor:
55 + """Compute the inverse frequency."""
56 + # NOTE(woosuk): The HF implementation uses `torch.arange(...).float()`.
57 + # However, we use `torch.arange(..., dtype=torch.float)` instead to
58 + # avoid numerical issues with large base values (e.g., 10000000).
59 + # This may cause a slight numerical difference between the HF
60 + # implementation and ours.
61 + # NOTE(woosuk): To exactly match the HF implementation, we need to
62 + # use CPU to compute the cache and then move it to GPU. However, we
63 + # create the cache on GPU for faster initialization. This may cause
64 + # a slight numerical difference between the HF implementation and ours.
65 + inv_freq = 1.0 / (base**(torch.arange(
66 + 0, self.rotary_dim, 2, dtype=torch.float, device="cuda") /
67 + self.rotary_dim))
68 + return inv_freq
69 +
70 + def _compute_cos_sin_cache(self) -> torch.Tensor:
71 + """Compute the cos and sin cache."""
72 + inv_freq = self._compute_inv_freq(self.base)
73 + t = torch.arange(self.max_position_embeddings,
74 + dtype=torch.float,
75 + device="cuda")
76 +
77 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
78 + cos = freqs.cos()
79 + sin = freqs.sin()
80 + cache = torch.cat((cos, sin), dim=-1)
81 + return cache
82 +
83 + def forward(
84 + self,
85 + positions: torch.Tensor,
86 + query: torch.Tensor,
87 + key: torch.Tensor,
88 + ) -> Tuple[torch.Tensor, torch.Tensor]:
89 + # pos_encoding_ops.rotary_embedding() is an in-place operation that
90 + # updates the query and key tensors.
91 + pos_encoding_ops.rotary_embedding(positions, query, key,
92 + self.head_size, self.cos_sin_cache,
93 + self.is_neox_style)
94 + return query, key
95 +
96 +
97 +class LinearScalingRotaryEmbedding(RotaryEmbedding):
98 + """RotaryEmbedding extended with linear scaling.
99 +
100 + Credits to the Reddit user /u/kaiokendev
101 + """
102 +
103 + def __init__(
104 + self,
105 + head_size: int,
106 + rotary_dim: int,
107 + max_position_embeddings: int,
108 + base: int,
109 + is_neox_style: bool,
110 + scaling_factor: float,
111 + ) -> None:
112 + self.scaling_factor = scaling_factor
113 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
114 + is_neox_style)
115 +
116 + def _compute_cos_sin_cache(self) -> torch.Tensor:
117 + inv_freq = self._compute_inv_freq(self.base)
118 + # NOTE(woosuk): self.max_position_embeddings is the original
119 + # maximum length before applying the rope scaling.
120 + # Thus, the maximum length after applying the rope scaling is
121 + # self.max_position_embeddings * self.scaling_factor.
122 + max_len = self.max_position_embeddings * self.scaling_factor
123 + t = torch.arange(max_len, dtype=torch.float, device="cuda")
124 + t = t / self.scaling_factor
125 +
126 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
127 + cos = freqs.cos()
128 + sin = freqs.sin()
129 + cache = torch.cat((cos, sin), dim=-1)
130 + return cache
131 +
132 +
133 +class DynamicNTKScalingRotaryEmbedding(RotaryEmbedding):
134 + """RotaryEmbedding extended with Dynamic NTK scaling.
135 +
136 + Credits to the Reddit users /u/bloc97 and /u/emozilla
137 + """
138 +
139 + def __init__(
140 + self,
141 + head_size: int,
142 + rotary_dim: int,
143 + max_position_embeddings: int,
144 + base: int,
145 + is_neox_style: bool,
146 + scaling_factor: float,
147 + ) -> None:
148 + self.scaling_factor = scaling_factor
149 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
150 + is_neox_style)
151 +
152 + def _compute_cos_sin_cache(self) -> torch.Tensor:
153 + # NOTE(woosuk): self.max_position_embeddings is the original
154 + # maximum length before applying the rope scaling.
155 + # Thus, the maximum length after applying the rope scaling is
156 + # self.max_position_embeddings * self.scaling_factor.
157 + max_len = self.max_position_embeddings * self.scaling_factor
158 + base = self.base * (
159 + (self.scaling_factor * max_len / self.max_position_embeddings) -
160 + (self.scaling_factor - 1))**(self.rotary_dim /
161 + (self.rotary_dim - 2))
162 + inv_freq = self._compute_inv_freq(base)
163 + t = torch.arange(max_len, dtype=torch.float, device="cuda")
164 +
165 + freqs = torch.einsum("i,j -> ij", t, inv_freq)
166 + cos = freqs.cos()
167 + sin = freqs.sin()
168 + cache = torch.cat((cos, sin), dim=-1)
169 + return cache
vllm/model_executor/models/llama.py
+6 -2
@@ -25,7 +25,7 @@
25 The input of the model is flattened to a 1D tensor of tokens. The model uses
26 InputMetadata to extract the original 2D shape of the input.
27 """
28 -from typing import List, Optional, Tuple
28 +from typing import Any, Dict, List, Optional, Tuple
29
30 import torch
31 from torch import nn
@@ -92,6 +92,7 @@ class LlamaAttention(nn.Module):
92 num_heads: int,
93 num_kv_heads: int,
94 rope_theta: float = 10000,
95 + rope_scaling: Optional[Dict[str, Any]] = None,
96 max_position_embeddings: int = 8192,
97 quant_config: Optional[QuantizationConfig] = None,
98 ) -> None:
@@ -135,7 +136,8 @@ class LlamaAttention(nn.Module):
136 base=self.rope_theta,
137 max_position=self.max_position_embeddings,
138 rotary_dim=self.head_dim,
138 - num_kv_heads=self.num_kv_heads)
139 + num_kv_heads=self.num_kv_heads,
140 + rope_scaling=rope_scaling)
141
142 def forward(
143 self,
@@ -165,6 +167,7 @@ class LlamaDecoderLayer(nn.Module):
167 self.hidden_size = config.hidden_size
168 # Requires transformers > 4.32.0
169 rope_theta = getattr(config, "rope_theta", 10000)
170 + rope_scaling = getattr(config, "rope_scaling", None)
171 max_position_embeddings = getattr(config, "max_position_embeddings",
172 8192)
173 self.self_attn = LlamaAttention(
@@ -172,6 +175,7 @@ class LlamaDecoderLayer(nn.Module):
175 num_heads=config.num_attention_heads,
176 num_kv_heads=config.num_key_value_heads,
177 rope_theta=rope_theta,
178 + rope_scaling=rope_scaling,
179 max_position_embeddings=max_position_embeddings,
180 quant_config=quant_config,
181 )