A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
rope_theta and max_position_embeddings from config (#1096)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu> Co-authored-by: wnma3mz <wnma3mz@gmail.com>
Antoni Baum committed
Sep 20, 2023 at 13:35 UTC
3302f0aef39c392321567ac1400101155e365a29
9 files changed
+140
-62
vllm/config.py
+39
-31
@@ -57,7 +57,7 @@ class ModelConfig:
57
load_format: str,
58
dtype: str,
59
seed: int,
60
- revision: Optional[str],
60
+ revision: Optional[str] = None,
61
max_model_len: Optional[int] = None,
62
quantization: Optional[str] = None,
63
) -> None:
@@ -73,19 +73,11 @@ class ModelConfig:
73
74
self.hf_config = get_config(model, trust_remote_code, revision)
75
self.dtype = _get_and_verify_dtype(self.hf_config, dtype)
76
+ self.max_model_len = _get_and_verify_max_len(self.hf_config,
77
+ max_model_len)
78
self._verify_load_format()
79
self._verify_tokenizer_mode()
80
self._verify_quantization()
79
- self.max_model_len = None
80
- if max_model_len is not None:
81
- derived_max_model_len = self.get_max_model_len()
82
- if max_model_len > derived_max_model_len:
83
- logger.warning(
84
- f"User-specified max_model_len ({max_model_len}) is "
85
- f"greater than the derived max_model_len "
86
- f"({derived_max_model_len}). Make sure the value is "
87
- "correct and within the model context size.")
88
- self.max_model_len = max_model_len
81
82
def _verify_load_format(self) -> None:
83
load_format = self.load_format.lower()
@@ -168,26 +160,7 @@ class ModelConfig:
160
return total_num_attention_heads // parallel_config.tensor_parallel_size
161
162
def get_max_model_len(self) -> int:
171
- if self.max_model_len is not None:
172
- return self.max_model_len
173
- max_model_len = float("inf")
174
- possible_keys = [
175
- # OPT
176
- "max_position_embeddings",
177
- # GPT-2
178
- "n_positions",
179
- # MPT
180
- "max_seq_len",
181
- # Others
182
- "max_sequence_length",
183
- "max_seq_length",
184
- "seq_len",
185
- ]
186
- for key in possible_keys:
187
- max_len_key = getattr(self.hf_config, key, None)
188
- if max_len_key is not None:
189
- max_model_len = min(max_model_len, max_len_key)
190
- return max_model_len
163
+ return self.max_model_len
164
165
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
166
total_num_hidden_layers = self.hf_config.num_hidden_layers
@@ -348,3 +321,38 @@ def _get_and_verify_dtype(
321
f"of at least 8.0. Your {gpu_name} GPU has compute capability "
322
f"{compute_capability[0]}.{compute_capability[1]}.")
323
return torch_dtype
324
+
325
+
326
+def _get_and_verify_max_len(
327
+ hf_config: PretrainedConfig,
328
+ max_model_len: Optional[int],
329
+) -> int:
330
+ """Get and verify the model's maximum length."""
331
+ derived_max_model_len = float("inf")
332
+ possible_keys = [
333
+ # OPT
334
+ "max_position_embeddings",
335
+ # GPT-2
336
+ "n_positions",
337
+ # MPT
338
+ "max_seq_len",
339
+ # Others
340
+ "max_sequence_length",
341
+ "max_seq_length",
342
+ "seq_len",
343
+ ]
344
+ for key in possible_keys:
345
+ max_len_key = getattr(hf_config, key, None)
346
+ if max_len_key is not None:
347
+ derived_max_model_len = min(derived_max_model_len, max_len_key)
348
+
349
+ if max_model_len is None:
350
+ max_model_len = derived_max_model_len
351
+ elif max_model_len > derived_max_model_len:
352
+ raise ValueError(
353
+ f"User-specified max_model_len ({max_model_len}) is greater than "
354
+ f"the derived max_model_len ({max_len_key}={derived_max_model_len}"
355
+ " in model's config.json). This may lead to incorrect model "
356
+ "outputs or CUDA errors. Make sure the value is correct and "
357
+ "within the model context size.")
358
+ return max_model_len
vllm/model_executor/models/aquila.py
+11
@@ -105,6 +105,8 @@ class AquilaAttention(nn.Module):
105
hidden_size: int,
106
num_heads: int,
107
num_kv_heads: int,
108
+ rope_theta: float = 10000,
109
+ max_position_embeddings: int = 8192,
110
):
111
super().__init__()
112
self.hidden_size = hidden_size
@@ -119,6 +121,8 @@ class AquilaAttention(nn.Module):
121
self.q_size = self.num_heads * self.head_dim
122
self.kv_size = self.num_kv_heads * self.head_dim
123
self.scaling = self.head_dim**-0.5
124
+ self.rope_theta = rope_theta
125
+ self.max_position_embeddings = max_position_embeddings
126
127
self.qkv_proj = ColumnParallelLinear(
128
hidden_size,
@@ -140,6 +144,8 @@ class AquilaAttention(nn.Module):
144
self.head_dim,
145
self.scaling,
146
rotary_dim=self.head_dim,
147
+ base=self.rope_theta,
148
+ max_position=self.max_position_embeddings,
149
)
150
151
def forward(
@@ -164,10 +170,15 @@ class AquilaDecoderLayer(nn.Module):
170
def __init__(self, config: AquilaConfig):
171
super().__init__()
172
self.hidden_size = config.hidden_size
173
+ rope_theta = getattr(config, "rope_theta", 10000)
174
+ max_position_embeddings = getattr(config, "max_position_embeddings",
175
+ 8192)
176
self.self_attn = AquilaAttention(
177
hidden_size=self.hidden_size,
178
num_heads=config.num_attention_heads,
179
num_kv_heads=config.num_attention_heads,
180
+ rope_theta=rope_theta,
181
+ max_position_embeddings=max_position_embeddings,
182
)
183
self.mlp = AquilaMLP(
184
hidden_size=self.hidden_size,
vllm/model_executor/models/baichuan.py
+16
-4
@@ -111,6 +111,8 @@ class BaiChuanAttention(nn.Module):
111
hidden_size: int,
112
num_heads: int,
113
position_embedding: str,
114
+ rope_theta: float = 10000,
115
+ max_position_embeddings: int = 8192,
116
):
117
super().__init__()
118
self.hidden_size = hidden_size
@@ -122,6 +124,8 @@ class BaiChuanAttention(nn.Module):
124
tensor_model_parallel_world_size)
125
self.head_dim = hidden_size // self.total_num_heads
126
self.postion_embedding = position_embedding
127
+ self.rope_theta = rope_theta
128
+ self.max_position_embeddings = max_position_embeddings
129
130
# pylint: disable=invalid-name
131
self.W_pack = ColumnParallelLinear(
@@ -151,10 +155,13 @@ class BaiChuanAttention(nn.Module):
155
scaling, alibi_slopes)
156
else:
157
self.scaling = self.head_dim**-0.5
154
- self.attn = PagedAttentionWithRoPE(self.num_heads,
155
- self.head_dim,
156
- self.scaling,
157
- rotary_dim=self.head_dim)
158
+ self.attn = PagedAttentionWithRoPE(
159
+ self.num_heads,
160
+ self.head_dim,
161
+ self.scaling,
162
+ rotary_dim=self.head_dim,
163
+ base=self.rope_theta,
164
+ max_position=self.max_position_embeddings)
165
166
def forward(
167
self,
@@ -183,10 +190,15 @@ class BaiChuanDecoderLayer(nn.Module):
190
def __init__(self, config: BaiChuanConfig, position_embedding: str):
191
super().__init__()
192
self.hidden_size = config.hidden_size
193
+ rope_theta = getattr(config, "rope_theta", 10000)
194
+ max_position_embeddings = getattr(config, "max_position_embeddings",
195
+ 8192)
196
self.self_attn = BaiChuanAttention(
197
hidden_size=self.hidden_size,
198
num_heads=config.num_attention_heads,
199
position_embedding=position_embedding,
200
+ rope_theta=rope_theta,
201
+ max_position_embeddings=max_position_embeddings,
202
)
203
self.mlp = BaiChuanMLP(
204
hidden_size=self.hidden_size,
vllm/model_executor/models/falcon.py
+11
-6
@@ -161,12 +161,17 @@ class FalconAttention(nn.Module):
161
"Rotary and alibi are mutually exclusive.")
162
163
if self.use_rotary:
164
- # TODO(zhuohan): Pass in correct `max_position``
165
- self.attn = PagedAttentionWithRoPE(self.num_heads,
166
- self.head_dim,
167
- self.inv_norm_factor,
168
- rotary_dim=self.head_dim,
169
- num_kv_heads=self.num_kv_heads)
164
+ rope_theta = getattr(config, "rope_theta", 10000)
165
+ max_position_embeddings = getattr(config,
166
+ "max_position_embeddings", 8192)
167
+ self.attn = PagedAttentionWithRoPE(
168
+ self.num_heads,
169
+ self.head_dim,
170
+ self.inv_norm_factor,
171
+ base=rope_theta,
172
+ max_position=max_position_embeddings,
173
+ rotary_dim=self.head_dim,
174
+ num_kv_heads=self.num_kv_heads)
175
elif self.use_alibi:
176
tp_rank = get_tensor_model_parallel_rank()
177
head_start = tp_rank * self.num_heads
vllm/model_executor/models/gpt_j.py
+11
-5
@@ -67,11 +67,17 @@ class GPTJAttention(nn.Module):
67
scaling = self.head_size**-0.5
68
assert getattr(config, "rotary", True)
69
assert config.rotary_dim % 2 == 0
70
- self.attn = PagedAttentionWithRoPE(self.num_heads,
71
- self.head_size,
72
- scaling,
73
- config.rotary_dim,
74
- is_neox_style=False)
70
+ rope_theta = getattr(config, "rope_theta", 10000)
71
+ max_position_embeddings = getattr(config, "max_position_embeddings",
72
+ 8192)
73
+ self.attn = PagedAttentionWithRoPE(
74
+ self.num_heads,
75
+ self.head_size,
76
+ scaling,
77
+ config.rotary_dim,
78
+ base=rope_theta,
79
+ max_position=max_position_embeddings,
80
+ is_neox_style=False)
81
self.warmup = False
82
83
def forward(
vllm/model_executor/models/gpt_neox.py
+10
-2
@@ -68,8 +68,16 @@ class GPTNeoXAttention(nn.Module):
68
scaling = self.head_size**-0.5
69
rotary_dim = int(self.head_size * config.rotary_pct)
70
assert rotary_dim % 2 == 0
71
- self.attn = PagedAttentionWithRoPE(self.num_heads, self.head_size,
72
- scaling, rotary_dim)
71
+ rope_theta = getattr(config, "rope_theta", 10000)
72
+ max_position_embeddings = getattr(config, "max_position_embeddings",
73
+ 8192)
74
+ self.attn = PagedAttentionWithRoPE(
75
+ self.num_heads,
76
+ self.head_size,
77
+ scaling,
78
+ rotary_dim,
79
+ base=rope_theta,
80
+ max_position=max_position_embeddings)
81
82
def forward(
83
self,
vllm/model_executor/models/internlm.py
+16
-4
@@ -59,6 +59,8 @@ class InternLMAttention(nn.Module):
59
self,
60
hidden_size: int,
61
num_heads: int,
62
+ rope_theta: float = 10000,
63
+ max_position_embeddings: int = 8192,
64
):
65
super().__init__()
66
self.hidden_size = hidden_size
@@ -70,6 +72,8 @@ class InternLMAttention(nn.Module):
72
tensor_model_parallel_world_size)
73
self.head_dim = hidden_size // self.total_num_heads
74
self.scaling = self.head_dim**-0.5
75
+ self.rope_theta = rope_theta
76
+ self.max_position_embeddings = max_position_embeddings
77
78
self.qkv_proj = ColumnParallelLinear(
79
hidden_size,
@@ -85,10 +89,13 @@ class InternLMAttention(nn.Module):
89
input_is_parallel=True,
90
perform_initialization=False,
91
)
88
- self.attn = PagedAttentionWithRoPE(self.num_heads,
89
- self.head_dim,
90
- self.scaling,
91
- rotary_dim=self.head_dim)
92
+ self.attn = PagedAttentionWithRoPE(
93
+ self.num_heads,
94
+ self.head_dim,
95
+ self.scaling,
96
+ base=self.rope_theta,
97
+ max_position=self.max_position_embeddings,
98
+ rotary_dim=self.head_dim)
99
100
def forward(
101
self,
@@ -112,9 +119,14 @@ class InternLMDecoderLayer(nn.Module):
119
def __init__(self, config: LlamaConfig):
120
super().__init__()
121
self.hidden_size = config.hidden_size
122
+ rope_theta = getattr(config, "rope_theta", 10000)
123
+ max_position_embeddings = getattr(config, "max_position_embeddings",
124
+ 8192)
125
self.self_attn = InternLMAttention(
126
hidden_size=self.hidden_size,
127
num_heads=config.num_attention_heads,
128
+ rope_theta=rope_theta,
129
+ max_position_embeddings=max_position_embeddings,
130
)
131
self.mlp = InternLMMLP(
132
hidden_size=self.hidden_size,
vllm/model_executor/models/llama.py
+13
-6
@@ -92,6 +92,7 @@ class LlamaAttention(nn.Module):
92
num_heads: int,
93
num_kv_heads: int,
94
rope_theta: float = 10000,
95
+ max_position_embeddings: int = 8192,
96
quant_config: Optional[QuantizationConfig] = None,
97
) -> None:
98
super().__init__()
@@ -108,6 +109,7 @@ class LlamaAttention(nn.Module):
109
self.kv_size = self.num_kv_heads * self.head_dim
110
self.scaling = self.head_dim**-0.5
111
self.rope_theta = rope_theta
112
+ self.max_position_embeddings = max_position_embeddings
113
114
self.qkv_proj = ParallelLinear.column(
115
hidden_size,
@@ -126,12 +128,14 @@ class LlamaAttention(nn.Module):
128
perform_initialization=False,
129
quant_config=quant_config,
130
)
129
- self.attn = PagedAttentionWithRoPE(self.num_heads,
130
- self.head_dim,
131
- self.scaling,
132
- base=self.rope_theta,
133
- rotary_dim=self.head_dim,
134
- num_kv_heads=self.num_kv_heads)
131
+ self.attn = PagedAttentionWithRoPE(
132
+ self.num_heads,
133
+ self.head_dim,
134
+ self.scaling,
135
+ base=self.rope_theta,
136
+ max_position=self.max_position_embeddings,
137
+ rotary_dim=self.head_dim,
138
+ num_kv_heads=self.num_kv_heads)
139
140
def forward(
141
self,
@@ -161,11 +165,14 @@ class LlamaDecoderLayer(nn.Module):
165
self.hidden_size = config.hidden_size
166
# Requires transformers > 4.32.0
167
rope_theta = getattr(config, "rope_theta", 10000)
168
+ max_position_embeddings = getattr(config, "max_position_embeddings",
169
+ 8192)
170
self.self_attn = LlamaAttention(
171
hidden_size=self.hidden_size,
172
num_heads=config.num_attention_heads,
173
num_kv_heads=config.num_key_value_heads,
174
rope_theta=rope_theta,
175
+ max_position_embeddings=max_position_embeddings,
176
quant_config=quant_config,
177
)
178
self.mlp = LlamaMLP(
vllm/model_executor/models/qwen.py
+13
-4
@@ -76,8 +76,13 @@ class QWenMLP(nn.Module):
76
77
class QWenAttention(nn.Module):
78
79
- def __init__(self, hidden_size: int, num_heads: int,
80
- max_position_embeddings: int):
79
+ def __init__(
80
+ self,
81
+ hidden_size: int,
82
+ num_heads: int,
83
+ max_position_embeddings: int,
84
+ rope_theta: float = 10000,
85
+ ):
86
super().__init__()
87
self.hidden_size = hidden_size
88
tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
@@ -109,6 +114,7 @@ class QWenAttention(nn.Module):
114
self.head_dim,
115
self.scaling,
116
rotary_dim=self.head_dim,
117
+ base=rope_theta,
118
max_position=max_position_embeddings,
119
)
120
@@ -137,8 +143,11 @@ class QWenBlock(nn.Module):
143
super().__init__()
144
self.ln_1 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
145
140
- self.attn = QWenAttention(config.n_embd, config.num_attention_heads,
141
- config.max_position_embeddings)
146
+ rope_theta = getattr(config, "rope_theta", 10000)
147
+ self.attn = QWenAttention(config.n_embd,
148
+ config.num_attention_heads,
149
+ config.max_position_embeddings,
150
+ rope_theta=rope_theta)
151
152
self.ln_2 = RMSNorm(config.n_embd, eps=config.layer_norm_epsilon)
153