@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / fbf152d976
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix][Model] Refactor OLMo model to support new HF format in transformers 4.40.0 (#4324)

Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

Isotr0py committed Apr 26, 2024 at 00:35 UTC fbf152d976e655f8734561452a1036e63081ccfd
5 files changed +150 -162
README.md
+1 -1
@@ -74,7 +74,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
74 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
75 - Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc.)
76 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
77 -- OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
77 +- OLMo (`allenai/OLMo-1B-hf`, `allenai/OLMo-7B-hf`, etc.)
78 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
79 - Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
80 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
docs/source/models/supported_models.rst
+1 -1
@@ -101,7 +101,7 @@ Alongside each architecture, we include some popular models that use it.
101 -
102 * - :code:`OLMoForCausalLM`
103 - OLMo
104 - - :code:`allenai/OLMo-1B`, :code:`allenai/OLMo-7B`, etc.
104 + - :code:`allenai/OLMo-1B-hf`, :code:`allenai/OLMo-7B-hf`, etc.
105 -
106 * - :code:`OPTForCausalLM`
107 - OPT, OPT-IML
requirements-dev.txt
-1
@@ -26,7 +26,6 @@ requests
26 ray
27 peft
28 awscli
29 -ai2-olmo # required for OLMo
29
30 # Benchmarking
31 aiohttp
vllm/model_executor/models/__init__.py
+1 -1
@@ -42,7 +42,7 @@ _MODELS = {
42 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
43 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
44 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
45 - "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
45 + "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
46 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
47 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
48 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
vllm/model_executor/models/olmo.py
+147 -158
@@ -1,53 +1,36 @@
1 # coding=utf-8
2 # Adapted from
3 -# https://github.com/allenai/OLMo/blob/v0.2.4/olmo/model.py and
4 -# https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/modeling_olmo.py
5 -# Copyright 2023 The vLLM team.
6 -# Copyright (c) Microsoft Corporation.
7 -# Licensed under the MIT license.
3 +# https://github.com/huggingface/transformers/blob/v4.40.1/src/transformers/models/olmo/modeling_olmo.py
4 +# Copyright 2024 The vLLM team.
5 +# Copyright 2024 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 #
9 -# BSD 3-Clause License
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 #
11 -# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
12 -# All rights reserved.
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 #
14 -# Redistribution and use in source and binary forms, with or without
15 -# modification, are permitted provided that the following conditions are met:
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 #
17 -# * Redistributions of source code must retain the above copyright notice, this
18 -# list of conditions and the following disclaimer.
19 -#
20 -# * Redistributions in binary form must reproduce the above copyright notice,
21 -# this list of conditions and the following disclaimer in the documentation
22 -# and/or other materials provided with the distribution.
23 -#
24 -# * Neither the name of the copyright holder nor the names of its
25 -# contributors may be used to endorse or promote products derived from
26 -# this software without specific prior written permission.
27 -#
28 -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31 -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
32 -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
34 -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
35 -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36 -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 """Inference-only OLMo model compatible with HuggingFace weights."""
24 from typing import Iterable, List, Optional, Tuple
25
26 import torch
42 -# this model must need this dependency
43 -from hf_olmo import OLMoConfig
27 from torch import nn
28 +from transformers import OlmoConfig
29
30 from vllm.attention import Attention, AttentionMetadata
31 from vllm.distributed import get_tensor_model_parallel_world_size
32 from vllm.model_executor.layers.activation import SiluAndMul
49 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
50 - LinearMethodBase,
33 +from vllm.model_executor.layers.linear import (LinearMethodBase,
34 MergedColumnParallelLinear,
35 QKVParallelLinear,
36 RowParallelLinear)
@@ -55,7 +38,7 @@ from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
58 - VocabParallelEmbedding)
41 + ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 from vllm.model_executor.sampling_metadata import SamplingMetadata
44 from vllm.sequence import SamplerOutput
@@ -70,55 +53,52 @@ class OlmoAttention(nn.Module):
53
54 def __init__(
55 self,
73 - config: OLMoConfig,
56 + config: OlmoConfig,
57 linear_method: Optional[LinearMethodBase] = None,
58 ):
59 super().__init__()
60 self.config = config
78 - self.hidden_size = config.d_model
79 - assert config.d_model % config.n_heads == 0
61 + self.hidden_size = config.hidden_size
62 tensor_model_parallel_world_size = (
63 get_tensor_model_parallel_world_size())
82 - self.total_num_heads = self.config.n_heads
64 + self.total_num_heads = config.num_attention_heads
65 +
66 + assert self.hidden_size % self.total_num_heads == 0
67 assert self.total_num_heads % tensor_model_parallel_world_size == 0
68 +
69 self.num_heads = (self.total_num_heads //
70 tensor_model_parallel_world_size)
71 self.head_dim = self.hidden_size // self.total_num_heads
72 + self.max_position_embeddings = config.max_position_embeddings
73 + self.rope_theta = config.rope_theta
74 + self.clip_qkv = config.clip_qkv
75
88 - # Layer norms.
89 - self.attn_norm = nn.LayerNorm(config.d_model,
90 - elementwise_affine=False,
91 - bias=False)
76 # Attention input projection. Projects x -> (q, k, v)
93 - self.att_proj = QKVParallelLinear(
94 - config.d_model,
77 + self.qkv_proj = QKVParallelLinear(
78 + self.hidden_size,
79 self.head_dim,
80 self.total_num_heads,
97 - bias=config.include_bias,
81 + bias=config.attention_bias,
82 linear_method=linear_method,
83 )
84
85 # Rotary embeddings.
102 - if self.config.rope:
103 - rope_theta = getattr(config, "rope_theta", 10000)
104 - max_position_embeddings = getattr(config,
105 - "max_position_embeddings", 8192)
106 - self.rotary_emb = get_rope(
107 - self.head_dim,
108 - rotary_dim=self.head_dim,
109 - max_position=max_position_embeddings,
110 - base=rope_theta,
111 - )
86 + self.rotary_emb = get_rope(
87 + self.head_dim,
88 + rotary_dim=self.head_dim,
89 + max_position=self.max_position_embeddings,
90 + base=self.rope_theta,
91 + )
92 self.scaling = self.head_dim**-0.5
93 self.attn = Attention(self.num_heads,
94 self.head_dim,
95 scale=self.scaling)
96
97 # Attention output projection.
118 - self.attn_out = RowParallelLinear(
119 - config.d_model,
120 - config.d_model,
121 - bias=config.include_bias,
98 + self.o_proj = RowParallelLinear(
99 + self.hidden_size,
100 + self.hidden_size,
101 + bias=config.attention_bias,
102 linear_method=linear_method,
103 )
104
@@ -129,13 +109,13 @@ class OlmoAttention(nn.Module):
109 kv_cache: torch.Tensor,
110 attn_metadata: AttentionMetadata,
111 ) -> torch.Tensor:
132 - hidden_states = self.attn_norm(hidden_states)
133 - qkv, _ = self.att_proj(hidden_states)
112 + qkv, _ = self.qkv_proj(hidden_states)
113 + if self.clip_qkv is not None:
114 + qkv.clamp_(min=-self.clip_qkv, max=self.clip_qkv)
115 q, k, v = qkv.chunk(chunks=3, dim=-1)
135 - if self.config.rope:
136 - q, k = self.rotary_emb(positions, q, k)
116 + q, k = self.rotary_emb(positions, q, k)
117 attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
138 - output, _ = self.attn_out(attn_output)
118 + output, _ = self.o_proj(attn_output)
119 return output
120
121
@@ -148,37 +128,30 @@ class OlmoMLP(nn.Module):
128
129 def __init__(
130 self,
151 - config: OLMoConfig,
131 + config: OlmoConfig,
132 linear_method: Optional[LinearMethodBase] = None,
133 ):
134 super().__init__()
135 self.config = config
156 - self.hidden_size = (config.mlp_hidden_size if config.mlp_hidden_size
157 - is not None else config.mlp_ratio * config.d_model)
158 -
159 - # Layer norms.
160 - self.ff_norm = nn.LayerNorm(config.d_model,
161 - elementwise_affine=False,
162 - bias=False)
136 + self.hidden_size = config.hidden_size
137 + self.intermediate_size = config.intermediate_size
138
139 # Feed-forward input projection.
165 - self.ff_proj = MergedColumnParallelLinear(
166 - config.d_model,
167 - [self.hidden_size // 2] * 2,
168 - bias=config.include_bias,
140 + self.gate_up_proj = MergedColumnParallelLinear(
141 + self.hidden_size,
142 + [self.intermediate_size] * 2,
143 + bias=False,
144 linear_method=linear_method,
145 )
146
147 # Activation function.
173 - self.act = SiluAndMul()
174 - self.act.output_multiplier = 0.5
175 - assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
148 + self.act_fn = SiluAndMul()
149
150 # Feed-forward output projection.
178 - self.ff_out = RowParallelLinear(
179 - int(self.act.output_multiplier * self.hidden_size),
180 - config.d_model,
181 - bias=config.include_bias,
151 + self.down_proj = RowParallelLinear(
152 + self.intermediate_size,
153 + self.hidden_size,
154 + bias=False,
155 linear_method=linear_method,
156 )
157
@@ -186,19 +159,13 @@ class OlmoMLP(nn.Module):
159 self,
160 x: torch.Tensor,
161 ) -> torch.Tensor:
189 - # Add feed-forward projection.
190 - # shape: (batch_size, seq_len, d_model)
191 - og_x = x
192 - x = self.ff_norm(x)
193 - x, _ = self.ff_proj(x)
194 - x = self.act(x)
195 - x, _ = self.ff_out(x)
196 - x = og_x + x
197 -
162 + gate_up, _ = self.gate_up_proj(x)
163 + x = self.act_fn(gate_up)
164 + x, _ = self.down_proj(x)
165 return x
166
167
201 -class OlmoBlock(nn.Module):
168 +class OlmoDecoderLayer(nn.Module):
169 """
170 This is a typical transformer block where the output is
171 computed as ``MLP(LN(x + Attention(LN(x))))``
@@ -206,15 +173,23 @@ class OlmoBlock(nn.Module):
173 """
174
175 def __init__(self,
209 - config: OLMoConfig,
176 + config: OlmoConfig,
177 linear_method: Optional[LinearMethodBase] = None):
178 super().__init__()
179 # Attention block.
213 - self.attn = OlmoAttention(config, linear_method)
180 + self.self_attn = OlmoAttention(config, linear_method)
181
182 # MLP block.
183 self.mlp = OlmoMLP(config, linear_method)
184
185 + # LayerNorm
186 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
187 + elementwise_affine=False,
188 + bias=False)
189 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
190 + elementwise_affine=False,
191 + bias=False)
192 +
193 def forward(
194 self,
195 positions: torch.Tensor,
@@ -223,52 +198,37 @@ class OlmoBlock(nn.Module):
198 attn_metadata: AttentionMetadata,
199 ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
200 # Attention block.
226 - og_x = hidden_states
227 - x = self.attn(positions, hidden_states, kv_cache, attn_metadata)
228 - x = x + og_x
201 + residual = hidden_states
202 + hidden_states = self.input_layernorm(hidden_states)
203 + hidden_states = self.self_attn(positions, hidden_states, kv_cache,
204 + attn_metadata)
205 + hidden_states = hidden_states + residual
206
207 # MLP block.
231 - hidden_states = self.mlp(x)
208 + residual = hidden_states
209 + hidden_states = self.post_attention_layernorm(hidden_states)
210 + hidden_states = self.mlp(hidden_states)
211 + hidden_states = residual + hidden_states
212 return hidden_states
213
214
215 class OlmoModel(nn.Module):
216
217 def __init__(self,
238 - config: OLMoConfig,
218 + config: OlmoConfig,
219 linear_method: Optional[LinearMethodBase] = None):
220 super().__init__()
221 self.config = config
222
243 - self.transformer = nn.ModuleDict(
244 - dict(
245 - wte=VocabParallelEmbedding(
246 - config.embedding_size or config.vocab_size,
247 - config.d_model,
248 - ),
249 - ln_f=nn.LayerNorm(config.d_model,
250 - elementwise_affine=False,
251 - bias=False),
252 - ))
253 -
254 - blocks = [
255 - OlmoBlock(config, linear_method) for i in range(config.n_layers)
256 - ]
257 - if self.config.block_group_size > 1:
258 - raise NotImplementedError("Block group size > 1 not supported yet")
259 - else:
260 - self.transformer.update({"blocks": nn.ModuleList(blocks)})
261 -
262 - if not config.weight_tying:
263 - self.transformer.update({
264 - "ff_out":
265 - ColumnParallelLinear(
266 - config.d_model,
267 - config.embedding_size or config.vocab_size,
268 - bias=config.include_bias,
269 - linear_method=linear_method,
270 - )
271 - })
223 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
224 + config.hidden_size)
225 + self.layers = nn.ModuleList([
226 + OlmoDecoderLayer(config, linear_method)
227 + for layer_idx in range(config.num_hidden_layers)
228 + ])
229 + self.norm = nn.LayerNorm(config.hidden_size,
230 + elementwise_affine=False,
231 + bias=False)
232
233 def forward(
234 self,
@@ -282,39 +242,49 @@ class OlmoModel(nn.Module):
242 """
243 # Get embeddings of input.
244 # shape: (batch_size, seq_len, d_model)
285 - x = self.transformer.wte(input_ids) # type: ignore
245 + inputs_embeds = self.embed_tokens(input_ids)
246 +
247 + # embed positions
248 + hidden_states = inputs_embeds
249
250 # Apply blocks one-by-one.
288 - for block_idx, block in enumerate(self.transformer.blocks):
251 + for layer_idx, decoder_layer in enumerate(self.layers):
252 # shape: (batch_size, seq_len, d_model)
290 - x = block(
253 + hidden_states = decoder_layer(
254 positions,
292 - x,
293 - kv_caches[block_idx],
255 + hidden_states,
256 + kv_caches[layer_idx],
257 attn_metadata,
258 )
259
260 # Apply final layer norm.
261 # shape: (batch_size, seq_len or 1, d_model)
299 - x = self.transformer.ln_f(x) # type: ignore
300 - return x
262 + hidden_states = self.norm(hidden_states)
263 + return hidden_states
264
265
303 -class OLMoForCausalLM(nn.Module):
266 +class OlmoForCausalLM(nn.Module):
267 """
268 Extremely barebones HF model wrapper.
269 """
270
271 def __init__(self,
309 - config: OLMoConfig,
272 + config: OlmoConfig,
273 linear_method: Optional[LinearMethodBase] = None):
274 super().__init__()
275 self.config = config
276 self.linear_method = linear_method
277 self.model = OlmoModel(config, linear_method)
315 - self.lm_head_weight = (self.model.transformer.wte.weight
316 - if config.weight_tying else
317 - self.model.transformer.ff_out.weight)
278 + if config.tie_word_embeddings:
279 + self.lm_head_weight = self.model.embed_tokens.weight
280 + else:
281 + self.unpadded_vocab_size = config.vocab_size
282 + self.lm_head = ParallelLMHead(
283 + self.unpadded_vocab_size,
284 + config.hidden_size,
285 + org_num_embeddings=config.vocab_size,
286 + )
287 + self.lm_head_weight = self.lm_head.weight
288 self.logits_processor = LogitsProcessor(config.vocab_size)
289 self.sampler = Sampler()
290
@@ -348,20 +318,39 @@ class OLMoForCausalLM(nn.Module):
318 return next_tokens
319
320 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
321 + stacked_params_mapping = [
322 + # (param_name, shard_name, shard_id)
323 + ("qkv_proj", "q_proj", "q"),
324 + ("qkv_proj", "k_proj", "k"),
325 + ("qkv_proj", "v_proj", "v"),
326 + ("gate_up_proj", "gate_proj", 0),
327 + ("gate_up_proj", "up_proj", 1),
328 + ]
329 params_dict = dict(self.named_parameters(remove_duplicate=False))
330 for name, loaded_weight in weights:
353 - # attention
354 - if ".att" in name:
355 - name = name.replace(".att", ".attn.att")
356 - # mlp
357 - if ".ff_proj" in name:
358 - name = name.replace(".ff_proj", ".mlp.ff_proj")
359 - # Reverse the weight for the MergeColumnParallelLinear
360 - loaded_weight = torch.concat(loaded_weight.chunk(2)[::-1])
361 - if ".ff_out" in name and "transformer.ff_out" not in name:
362 - name = name.replace(".ff_out", ".mlp.ff_out")
363 - # there is no bias in olmo
364 - param = params_dict[name]
365 - weight_loader = getattr(param, "weight_loader",
366 - default_weight_loader)
367 - weight_loader(param, loaded_weight)
331 + if "rotary_emb.inv_freq" in name:
332 + continue
333 + if ("rotary_emb.cos_cached" in name
334 + or "rotary_emb.sin_cached" in name):
335 + # Models trained using ColossalAI may include these tensors in
336 + # the checkpoint. Skip them.
337 + continue
338 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
339 + if weight_name not in name:
340 + continue
341 + name = name.replace(weight_name, param_name)
342 + # Skip loading extra bias for GPTQ models.
343 + if name.endswith(".bias") and name not in params_dict:
344 + continue
345 + param = params_dict[name]
346 + weight_loader = param.weight_loader
347 + weight_loader(param, loaded_weight, shard_id)
348 + break
349 + else:
350 + # Skip loading extra bias for GPTQ models.
351 + if name.endswith(".bias") and name not in params_dict:
352 + continue
353 + param = params_dict[name]
354 + weight_loader = getattr(param, "weight_loader",
355 + default_weight_loader)
356 + weight_loader(param, loaded_weight)