A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix][Model] Refactor OLMo model to support new HF format in transformers 4.40.0 (#4324)
Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Isotr0py committed
Apr 26, 2024 at 00:35 UTC
fbf152d976e655f8734561452a1036e63081ccfd
5 files changed
+150
-162
README.md
+1
-1
@@ -74,7 +74,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
74
- Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
75
- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc.)
76
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
77
-- OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
77
+- OLMo (`allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc.)
78
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
79
- Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
80
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
docs/source/models/supported_models.rst
+1
-1
@@ -101,7 +101,7 @@ Alongside each architecture, we include some popular models that use it.
101
-
102
* - :code:`OLMoForCausalLM`
103
- OLMo
104
- - :code:`allenai/OLMo-1B`, :code:`allenai/OLMo-7B`, etc.
104
+ - :code:`allenai/OLMo-1B-hf`, :code:`allenai/OLMo-7B-hf`, etc.
105
-
106
* - :code:`OPTForCausalLM`
107
- OPT, OPT-IML
requirements-dev.txt
-1
@@ -26,7 +26,6 @@ requests
26
ray
27
peft
28
awscli
29
-ai2-olmo # required for OLMo
29
30
# Benchmarking
31
aiohttp
vllm/model_executor/models/__init__.py
+1
-1
@@ -42,7 +42,7 @@ _MODELS = {
42
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
43
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
45
- "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
45
+ "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
46
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
47
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
48
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
vllm/model_executor/models/olmo.py
+147
-158
@@ -1,53 +1,36 @@
1
# coding=utf-8
2
# Adapted from
3
-# https://github.com/allenai/OLMo/blob/v0.2.4/olmo/model.py and
4
-# https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/modeling_olmo.py
5
-# Copyright 2023 The vLLM team.
6
-# Copyright (c) Microsoft Corporation.
7
-# Licensed under the MIT license.
3
+# https://github.com/huggingface/transformers/blob/v4.40.1/src/transformers/models/olmo/modeling_olmo.py
4
+# Copyright 2024 The vLLM team.
5
+# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
#
9
-# BSD 3-Clause License
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
#
11
-# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
12
-# All rights reserved.
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
#
14
-# Redistribution and use in source and binary forms, with or without
15
-# modification, are permitted provided that the following conditions are met:
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
#
17
-# * Redistributions of source code must retain the above copyright notice, this
18
-# list of conditions and the following disclaimer.
19
-#
20
-# * Redistributions in binary form must reproduce the above copyright notice,
21
-# this list of conditions and the following disclaimer in the documentation
22
-# and/or other materials provided with the distribution.
23
-#
24
-# * Neither the name of the copyright holder nor the names of its
25
-# contributors may be used to endorse or promote products derived from
26
-# this software without specific prior written permission.
27
-#
28
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29
-# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
32
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
34
-# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
35
-# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36
-# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37
-# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
"""Inference-only OLMo model compatible with HuggingFace weights."""
24
from typing import Iterable, List, Optional, Tuple
25
26
import torch
42
-# this model must need this dependency
43
-from hf_olmo import OLMoConfig
27
from torch import nn
28
+from transformers import OlmoConfig
29
30
from vllm.attention import Attention, AttentionMetadata
31
from vllm.distributed import get_tensor_model_parallel_world_size
32
from vllm.model_executor.layers.activation import SiluAndMul
49
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50
- LinearMethodBase,
33
+from vllm.model_executor.layers.linear import (LinearMethodBase,
34
MergedColumnParallelLinear,
35
QKVParallelLinear,
36
RowParallelLinear)
@@ -55,7 +38,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
58
- VocabParallelEmbedding)
41
+ ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43
from vllm.model_executor.sampling_metadata import SamplingMetadata
44
from vllm.sequence import SamplerOutput
@@ -70,55 +53,52 @@ class OlmoAttention(nn.Module):
53
54
def __init__(
55
self,
73
- config: OLMoConfig,
56
+ config: OlmoConfig,
57
linear_method: Optional[LinearMethodBase] = None,
58
):
59
super().__init__()
60
self.config = config
78
- self.hidden_size = config.d_model
79
- assert config.d_model % config.n_heads == 0
61
+ self.hidden_size = config.hidden_size
62
tensor_model_parallel_world_size = (
63
get_tensor_model_parallel_world_size())
82
- self.total_num_heads = self.config.n_heads
64
+ self.total_num_heads = config.num_attention_heads
65
+
66
+ assert self.hidden_size % self.total_num_heads == 0
67
assert self.total_num_heads % tensor_model_parallel_world_size == 0
68
+
69
self.num_heads = (self.total_num_heads //
70
tensor_model_parallel_world_size)
71
self.head_dim = self.hidden_size // self.total_num_heads
72
+ self.max_position_embeddings = config.max_position_embeddings
73
+ self.rope_theta = config.rope_theta
74
+ self.clip_qkv = config.clip_qkv
75
88
- # Layer norms.
89
- self.attn_norm = nn.LayerNorm(config.d_model,
90
- elementwise_affine=False,
91
- bias=False)
76
# Attention input projection. Projects x -> (q, k, v)
93
- self.att_proj = QKVParallelLinear(
94
- config.d_model,
77
+ self.qkv_proj = QKVParallelLinear(
78
+ self.hidden_size,
79
self.head_dim,
80
self.total_num_heads,
97
- bias=config.include_bias,
81
+ bias=config.attention_bias,
82
linear_method=linear_method,
83
)
84
85
# Rotary embeddings.
102
- if self.config.rope:
103
- rope_theta = getattr(config, "rope_theta", 10000)
104
- max_position_embeddings = getattr(config,
105
- "max_position_embeddings", 8192)
106
- self.rotary_emb = get_rope(
107
- self.head_dim,
108
- rotary_dim=self.head_dim,
109
- max_position=max_position_embeddings,
110
- base=rope_theta,
111
- )
86
+ self.rotary_emb = get_rope(
87
+ self.head_dim,
88
+ rotary_dim=self.head_dim,
89
+ max_position=self.max_position_embeddings,
90
+ base=self.rope_theta,
91
+ )
92
self.scaling = self.head_dim**-0.5
93
self.attn = Attention(self.num_heads,
94
self.head_dim,
95
scale=self.scaling)
96
97
# Attention output projection.
118
- self.attn_out = RowParallelLinear(
119
- config.d_model,
120
- config.d_model,
121
- bias=config.include_bias,
98
+ self.o_proj = RowParallelLinear(
99
+ self.hidden_size,
100
+ self.hidden_size,
101
+ bias=config.attention_bias,
102
linear_method=linear_method,
103
)
104
@@ -129,13 +109,13 @@ class OlmoAttention(nn.Module):
109
kv_cache: torch.Tensor,
110
attn_metadata: AttentionMetadata,
111
) -> torch.Tensor:
132
- hidden_states = self.attn_norm(hidden_states)
133
- qkv, _ = self.att_proj(hidden_states)
112
+ qkv, _ = self.qkv_proj(hidden_states)
113
+ if self.clip_qkv is not None:
114
+ qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
115
q, k, v = qkv.chunk(chunks=3, dim=-1)
135
- if self.config.rope:
136
- q, k = self.rotary_emb(positions, q, k)
116
+ q, k = self.rotary_emb(positions, q, k)
117
attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
138
- output, _ = self.attn_out(attn_output)
118
+ output, _ = self.o_proj(attn_output)
119
return output
120
121
@@ -148,37 +128,30 @@ class OlmoMLP(nn.Module):
128
129
def __init__(
130
self,
151
- config: OLMoConfig,
131
+ config: OlmoConfig,
132
linear_method: Optional[LinearMethodBase] = None,
133
):
134
super().__init__()
135
self.config = config
156
- self.hidden_size = (config.mlp_hidden_size if config.mlp_hidden_size
157
- is not None else config.mlp_ratio * config.d_model)
158
-
159
- # Layer norms.
160
- self.ff_norm = nn.LayerNorm(config.d_model,
161
- elementwise_affine=False,
162
- bias=False)
136
+ self.hidden_size = config.hidden_size
137
+ self.intermediate_size = config.intermediate_size
138
139
# Feed-forward input projection.
165
- self.ff_proj = MergedColumnParallelLinear(
166
- config.d_model,
167
- [self.hidden_size // 2] * 2,
168
- bias=config.include_bias,
140
+ self.gate_up_proj = MergedColumnParallelLinear(
141
+ self.hidden_size,
142
+ [self.intermediate_size] * 2,
143
+ bias=False,
144
linear_method=linear_method,
145
)
146
147
# Activation function.
173
- self.act = SiluAndMul()
174
- self.act.output_multiplier = 0.5
175
- assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
148
+ self.act_fn = SiluAndMul()
149
150
# Feed-forward output projection.
178
- self.ff_out = RowParallelLinear(
179
- int(self.act.output_multiplier * self.hidden_size),
180
- config.d_model,
181
- bias=config.include_bias,
151
+ self.down_proj = RowParallelLinear(
152
+ self.intermediate_size,
153
+ self.hidden_size,
154
+ bias=False,
155
linear_method=linear_method,
156
)
157
@@ -186,19 +159,13 @@ class OlmoMLP(nn.Module):
159
self,
160
x: torch.Tensor,
161
) -> torch.Tensor:
189
- # Add feed-forward projection.
190
- # shape: (batch_size, seq_len, d_model)
191
- og_x = x
192
- x = self.ff_norm(x)
193
- x, _ = self.ff_proj(x)
194
- x = self.act(x)
195
- x, _ = self.ff_out(x)
196
- x = og_x + x
197
-
162
+ gate_up, _ = self.gate_up_proj(x)
163
+ x = self.act_fn(gate_up)
164
+ x, _ = self.down_proj(x)
165
return x
166
167
201
-class OlmoBlock(nn.Module):
168
+class OlmoDecoderLayer(nn.Module):
169
"""
170
This is a typical transformer block where the output is
171
computed as ``MLP(LN(x + Attention(LN(x))))``
@@ -206,15 +173,23 @@ class OlmoBlock(nn.Module):
173
"""
174
175
def __init__(self,
209
- config: OLMoConfig,
176
+ config: OlmoConfig,
177
linear_method: Optional[LinearMethodBase] = None):
178
super().__init__()
179
# Attention block.
213
- self.attn = OlmoAttention(config, linear_method)
180
+ self.self_attn = OlmoAttention(config, linear_method)
181
182
# MLP block.
183
self.mlp = OlmoMLP(config, linear_method)
184
185
+ # LayerNorm
186
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
187
+ elementwise_affine=False,
188
+ bias=False)
189
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
190
+ elementwise_affine=False,
191
+ bias=False)
192
+
193
def forward(
194
self,
195
positions: torch.Tensor,
@@ -223,52 +198,37 @@ class OlmoBlock(nn.Module):
198
attn_metadata: AttentionMetadata,
199
) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
200
# Attention block.
226
- og_x = hidden_states
227
- x = self.attn(positions, hidden_states, kv_cache, attn_metadata)
228
- x = x + og_x
201
+ residual = hidden_states
202
+ hidden_states = self.input_layernorm(hidden_states)
203
+ hidden_states = self.self_attn(positions, hidden_states, kv_cache,
204
+ attn_metadata)
205
+ hidden_states = hidden_states + residual
206
207
# MLP block.
231
- hidden_states = self.mlp(x)
208
+ residual = hidden_states
209
+ hidden_states = self.post_attention_layernorm(hidden_states)
210
+ hidden_states = self.mlp(hidden_states)
211
+ hidden_states = residual + hidden_states
212
return hidden_states
213
214
215
class OlmoModel(nn.Module):
216
217
def __init__(self,
238
- config: OLMoConfig,
218
+ config: OlmoConfig,
219
linear_method: Optional[LinearMethodBase] = None):
220
super().__init__()
221
self.config = config
222
243
- self.transformer = nn.ModuleDict(
244
- dict(
245
- wte=VocabParallelEmbedding(
246
- config.embedding_size or config.vocab_size,
247
- config.d_model,
248
- ),
249
- ln_f=nn.LayerNorm(config.d_model,
250
- elementwise_affine=False,
251
- bias=False),
252
- ))
253
-
254
- blocks = [
255
- OlmoBlock(config, linear_method) for i in range(config.n_layers)
256
- ]
257
- if self.config.block_group_size > 1:
258
- raise NotImplementedError("Block group size > 1 not supported yet")
259
- else:
260
- self.transformer.update({"blocks": nn.ModuleList(blocks)})
261
-
262
- if not config.weight_tying:
263
- self.transformer.update({
264
- "ff_out":
265
- ColumnParallelLinear(
266
- config.d_model,
267
- config.embedding_size or config.vocab_size,
268
- bias=config.include_bias,
269
- linear_method=linear_method,
270
- )
271
- })
223
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
224
+ config.hidden_size)
225
+ self.layers = nn.ModuleList([
226
+ OlmoDecoderLayer(config, linear_method)
227
+ for layer_idx in range(config.num_hidden_layers)
228
+ ])
229
+ self.norm = nn.LayerNorm(config.hidden_size,
230
+ elementwise_affine=False,
231
+ bias=False)
232
233
def forward(
234
self,
@@ -282,39 +242,49 @@ class OlmoModel(nn.Module):
242
"""
243
# Get embeddings of input.
244
# shape: (batch_size, seq_len, d_model)
285
- x = self.transformer.wte(input_ids) # type: ignore
245
+ inputs_embeds = self.embed_tokens(input_ids)
246
+
247
+ # embed positions
248
+ hidden_states = inputs_embeds
249
250
# Apply blocks one-by-one.
288
- for block_idx, block in enumerate(self.transformer.blocks):
251
+ for layer_idx, decoder_layer in enumerate(self.layers):
252
# shape: (batch_size, seq_len, d_model)
290
- x = block(
253
+ hidden_states = decoder_layer(
254
positions,
292
- x,
293
- kv_caches[block_idx],
255
+ hidden_states,
256
+ kv_caches[layer_idx],
257
attn_metadata,
258
)
259
260
# Apply final layer norm.
261
# shape: (batch_size, seq_len or 1, d_model)
299
- x = self.transformer.ln_f(x) # type: ignore
300
- return x
262
+ hidden_states = self.norm(hidden_states)
263
+ return hidden_states
264
265
303
-class OLMoForCausalLM(nn.Module):
266
+class OlmoForCausalLM(nn.Module):
267
"""
268
Extremely barebones HF model wrapper.
269
"""
270
271
def __init__(self,
309
- config: OLMoConfig,
272
+ config: OlmoConfig,
273
linear_method: Optional[LinearMethodBase] = None):
274
super().__init__()
275
self.config = config
276
self.linear_method = linear_method
277
self.model = OlmoModel(config, linear_method)
315
- self.lm_head_weight = (self.model.transformer.wte.weight
316
- if config.weight_tying else
317
- self.model.transformer.ff_out.weight)
278
+ if config.tie_word_embeddings:
279
+ self.lm_head_weight = self.model.embed_tokens.weight
280
+ else:
281
+ self.unpadded_vocab_size = config.vocab_size
282
+ self.lm_head = ParallelLMHead(
283
+ self.unpadded_vocab_size,
284
+ config.hidden_size,
285
+ org_num_embeddings=config.vocab_size,
286
+ )
287
+ self.lm_head_weight = self.lm_head.weight
288
self.logits_processor = LogitsProcessor(config.vocab_size)
289
self.sampler = Sampler()
290
@@ -348,20 +318,39 @@ class OLMoForCausalLM(nn.Module):
318
return next_tokens
319
320
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
321
+ stacked_params_mapping = [
322
+ # (param_name, shard_name, shard_id)
323
+ ("qkv_proj", "q_proj", "q"),
324
+ ("qkv_proj", "k_proj", "k"),
325
+ ("qkv_proj", "v_proj", "v"),
326
+ ("gate_up_proj", "gate_proj", 0),
327
+ ("gate_up_proj", "up_proj", 1),
328
+ ]
329
params_dict = dict(self.named_parameters(remove_duplicate=False))
330
for name, loaded_weight in weights:
353
- # attention
354
- if ".att" in name:
355
- name = name.replace(".att", ".attn.att")
356
- # mlp
357
- if ".ff_proj" in name:
358
- name = name.replace(".ff_proj", ".mlp.ff_proj")
359
- # Reverse the weight for the MergeColumnParallelLinear
360
- loaded_weight = torch.concat(loaded_weight.chunk(2)[::-1])
361
- if ".ff_out" in name and "transformer.ff_out" not in name:
362
- name = name.replace(".ff_out", ".mlp.ff_out")
363
- # there is no bias in olmo
364
- param = params_dict[name]
365
- weight_loader = getattr(param, "weight_loader",
366
- default_weight_loader)
367
- weight_loader(param, loaded_weight)
331
+ if "rotary_emb.inv_freq" in name:
332
+ continue
333
+ if ("rotary_emb.cos_cached" in name
334
+ or "rotary_emb.sin_cached" in name):
335
+ # Models trained using ColossalAI may include these tensors in
336
+ # the checkpoint. Skip them.
337
+ continue
338
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
339
+ if weight_name not in name:
340
+ continue
341
+ name = name.replace(weight_name, param_name)
342
+ # Skip loading extra bias for GPTQ models.
343
+ if name.endswith(".bias") and name not in params_dict:
344
+ continue
345
+ param = params_dict[name]
346
+ weight_loader = param.weight_loader
347
+ weight_loader(param, loaded_weight, shard_id)
348
+ break
349
+ else:
350
+ # Skip loading extra bias for GPTQ models.
351
+ if name.endswith(".bias") and name not in params_dict:
352
+ continue
353
+ param = params_dict[name]
354
+ weight_loader = getattr(param, "weight_loader",
355
+ default_weight_loader)
356
+ weight_loader(param, loaded_weight)