A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support OLMo models. (#2832)
Isotr0py committed
Feb 19, 2024 at 13:05 UTC
ab3a5a8259922ce312d01be39d29e27666968039
7 files changed
+471
-5
README.md
+1
@@ -70,6 +70,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
70
- Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
71
- Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
72
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
73
+- OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
74
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
75
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
76
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
docs/source/models/supported_models.rst
+3
@@ -62,6 +62,9 @@ Alongside each architecture, we include some popular models that use it.
62
* - :code:`MPTForCausalLM`
63
- MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
64
- :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
65
+ * - :code:`OLMoForCausalLM`
66
+ - OLMo
67
+ - :code:`allenai/OLMo-1B`, :code:`allenai/OLMo-7B`, etc.
68
* - :code:`OPTForCausalLM`
69
- OPT, OPT-IML
70
- :code:`facebook/opt-66b`, :code:`facebook/opt-iml-max-30b`, etc.
tests/models/test_models.py
+14
-5
@@ -5,11 +5,20 @@ Run `pytest tests/models/test_models.py --forked`.
5
import pytest
6
7
MODELS = [
8
- "facebook/opt-125m", "meta-llama/Llama-2-7b-hf",
9
- "mistralai/Mistral-7B-v0.1", "Deci/DeciLM-7b", "tiiuae/falcon-7b", "gpt2",
10
- "bigcode/tiny_starcoder_py", "EleutherAI/gpt-j-6b",
11
- "EleutherAI/pythia-70m", "bigscience/bloom-560m", "mosaicml/mpt-7b",
12
- "microsoft/phi-2", "stabilityai/stablelm-3b-4e1t"
8
+ "facebook/opt-125m",
9
+ "meta-llama/Llama-2-7b-hf",
10
+ "mistralai/Mistral-7B-v0.1",
11
+ "Deci/DeciLM-7b",
12
+ "tiiuae/falcon-7b",
13
+ "gpt2",
14
+ "bigcode/tiny_starcoder_py",
15
+ "EleutherAI/gpt-j-6b",
16
+ "EleutherAI/pythia-70m",
17
+ "bigscience/bloom-560m",
18
+ "mosaicml/mpt-7b",
19
+ "microsoft/phi-2",
20
+ "stabilityai/stablelm-3b-4e1t",
21
+ "allenai/OLMo-1B",
22
]
23
24
vllm/model_executor/models/__init__.py
+1
@@ -35,6 +35,7 @@ _MODELS = {
35
# transformers's mpt class has lower case
36
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
37
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
38
+ "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
39
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
40
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
41
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
vllm/model_executor/models/olmo.py
new
+378
@@ -0,0 +1,378 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/allenai/OLMo/blob/v0.2.4/olmo/model.py and
4
+# https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/modeling_olmo.py
5
+# Copyright 2023 The vLLM team.
6
+# Copyright (c) Microsoft Corporation.
7
+# Licensed under the MIT license.
8
+#
9
+# BSD 3-Clause License
10
+#
11
+# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
12
+# All rights reserved.
13
+#
14
+# Redistribution and use in source and binary forms, with or without
15
+# modification, are permitted provided that the following conditions are met:
16
+#
17
+# * Redistributions of source code must retain the above copyright notice, this
18
+# list of conditions and the following disclaimer.
19
+#
20
+# * Redistributions in binary form must reproduce the above copyright notice,
21
+# this list of conditions and the following disclaimer in the documentation
22
+# and/or other materials provided with the distribution.
23
+#
24
+# * Neither the name of the copyright holder nor the names of its
25
+# contributors may be used to endorse or promote products derived from
26
+# this software without specific prior written permission.
27
+#
28
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29
+# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
32
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
34
+# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
35
+# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36
+# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38
+"""Inference-only OLMo model compatible with HuggingFace weights."""
39
+from typing import List, Optional, Tuple
40
+
41
+import torch
42
+import torch.nn.functional as F
43
+from torch import nn
44
+
45
+from vllm.model_executor.input_metadata import InputMetadata
46
+from vllm.model_executor.layers.attention import PagedAttention
47
+from vllm.model_executor.layers.linear import (
48
+ ColumnParallelLinear,
49
+ LinearMethodBase,
50
+ QKVParallelLinear,
51
+ RowParallelLinear,
52
+)
53
+from vllm.model_executor.layers.rotary_embedding import get_rope
54
+from vllm.model_executor.layers.sampler import Sampler
55
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
56
+from vllm.model_executor.parallel_utils.parallel_state import (
57
+ get_tensor_model_parallel_world_size, )
58
+from vllm.model_executor.sampling_metadata import SamplingMetadata
59
+from vllm.model_executor.weight_utils import (
60
+ default_weight_loader,
61
+ hf_model_weights_iterator,
62
+)
63
+from vllm.sequence import SamplerOutput
64
+from vllm.transformers_utils.configs.olmo import OLMoConfig
65
+
66
+KVCache = Tuple[torch.Tensor, torch.Tensor]
67
+
68
+
69
+class SwiGLU(nn.Module):
70
+
71
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
72
+ x, gate = x.chunk(2, dim=-1)
73
+ return F.silu(gate) * x
74
+
75
+ @property
76
+ def output_multiplier(self) -> float:
77
+ return 0.5
78
+
79
+
80
+class OlmoAttention(nn.Module):
81
+ """
82
+ This is the attention block where the output is computed as ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
83
+ (plus another skip connection).
84
+ """
85
+
86
+ def __init__(
87
+ self,
88
+ config: OLMoConfig,
89
+ linear_method: Optional[LinearMethodBase] = None,
90
+ ):
91
+ super().__init__()
92
+ self.config = config
93
+ self.hidden_size = config.d_model
94
+ assert config.d_model % config.n_heads == 0
95
+ tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
96
+ )
97
+ self.total_num_heads = self.config.n_heads
98
+ assert self.total_num_heads % tensor_model_parallel_world_size == 0
99
+ self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
100
+ self.head_dim = self.hidden_size // self.total_num_heads
101
+
102
+ # Layer norms.
103
+ self.attn_norm = nn.LayerNorm(config.d_model,
104
+ elementwise_affine=False,
105
+ bias=False)
106
+ # Attention input projection. Projects x -> (q, k, v)
107
+ self.att_proj = QKVParallelLinear(
108
+ config.d_model,
109
+ self.head_dim,
110
+ self.total_num_heads,
111
+ bias=config.include_bias,
112
+ linear_method=linear_method,
113
+ )
114
+
115
+ # Rotary embeddings.
116
+ if self.config.rope:
117
+ rope_theta = getattr(config, "rope_theta", 10000)
118
+ max_position_embeddings = getattr(config,
119
+ "max_position_embeddings", 8192)
120
+ self.rotary_emb = get_rope(
121
+ self.head_dim,
122
+ rotary_dim=self.head_dim,
123
+ max_position=max_position_embeddings,
124
+ base=rope_theta,
125
+ )
126
+ self.scaling = self.head_dim**-0.5
127
+ self.attn = PagedAttention(self.num_heads,
128
+ self.head_dim,
129
+ scale=self.scaling)
130
+
131
+ # Attention output projection.
132
+ self.attn_out = RowParallelLinear(
133
+ config.d_model,
134
+ config.d_model,
135
+ bias=config.include_bias,
136
+ linear_method=linear_method,
137
+ )
138
+
139
+ def forward(
140
+ self,
141
+ positions: torch.Tensor,
142
+ hidden_states: torch.Tensor,
143
+ kv_cache: KVCache,
144
+ input_metadata: InputMetadata,
145
+ ) -> torch.Tensor:
146
+ hidden_states = self.attn_norm(hidden_states)
147
+ qkv, _ = self.att_proj(hidden_states)
148
+ q, k, v = qkv.chunk(chunks=3, dim=-1)
149
+ if self.config.rope:
150
+ q, k = self.rotary_emb(positions, q, k)
151
+ k_cache, v_cache = kv_cache
152
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
153
+ output, _ = self.attn_out(attn_output)
154
+ return output
155
+
156
+
157
+class OlmoMLP(nn.Module):
158
+ """
159
+ This is the MLP block where the output is computed as ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
160
+ (plus another skip connection).
161
+ """
162
+
163
+ def __init__(
164
+ self,
165
+ config: OLMoConfig,
166
+ linear_method: Optional[LinearMethodBase] = None,
167
+ ):
168
+ super().__init__()
169
+ self.config = config
170
+ self.hidden_size = (config.mlp_hidden_size if config.mlp_hidden_size
171
+ is not None else config.mlp_ratio * config.d_model)
172
+
173
+ # Layer norms.
174
+ self.ff_norm = nn.LayerNorm(config.d_model,
175
+ elementwise_affine=False,
176
+ bias=False)
177
+
178
+ # Feed-forward input projection.
179
+ self.ff_proj = ColumnParallelLinear(
180
+ config.d_model,
181
+ self.hidden_size,
182
+ bias=config.include_bias,
183
+ linear_method=linear_method,
184
+ )
185
+
186
+ # Activation function.
187
+ # self.act = SiluAndMul()
188
+ # self.act.output_multiplier = 0.5
189
+ self.act = SwiGLU()
190
+ assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
191
+
192
+ # Feed-forward output projection.
193
+ self.ff_out = RowParallelLinear(
194
+ int(self.act.output_multiplier * self.hidden_size),
195
+ config.d_model,
196
+ bias=config.include_bias,
197
+ linear_method=linear_method,
198
+ )
199
+
200
+ def forward(
201
+ self,
202
+ x: torch.Tensor,
203
+ ) -> torch.Tensor:
204
+ # Add feed-forward projection.
205
+ # shape: (batch_size, seq_len, d_model)
206
+ og_x = x
207
+ x = self.ff_norm(x)
208
+ x, _ = self.ff_proj(x)
209
+ x = self.act(x)
210
+ x, _ = self.ff_out(x)
211
+ x = og_x + x
212
+
213
+ return x
214
+
215
+
216
+class OlmoBlock(nn.Module):
217
+ """
218
+ This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
219
+ (plus another skip connection).
220
+ """
221
+
222
+ def __init__(self,
223
+ config: OLMoConfig,
224
+ linear_method: Optional[LinearMethodBase] = None):
225
+ super().__init__()
226
+ # Attention block.
227
+ self.attn = OlmoAttention(config, linear_method)
228
+
229
+ # MLP block.
230
+ self.mlp = OlmoMLP(config, linear_method)
231
+
232
+ def forward(
233
+ self,
234
+ positions: torch.Tensor,
235
+ hidden_states: torch.Tensor,
236
+ kv_cache: KVCache,
237
+ input_metadata: InputMetadata,
238
+ ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
239
+ # Attention block.
240
+ og_x = hidden_states
241
+ x = self.attn(positions, hidden_states, kv_cache, input_metadata)
242
+ x = x + og_x
243
+
244
+ # MLP block.
245
+ hidden_states = self.mlp(x)
246
+ return hidden_states
247
+
248
+
249
+class OlmoModel(nn.Module):
250
+
251
+ def __init__(self,
252
+ config: OLMoConfig,
253
+ linear_method: Optional[LinearMethodBase] = None):
254
+ super().__init__()
255
+ self.config = config
256
+
257
+ self.transformer = nn.ModuleDict(
258
+ dict(
259
+ wte=VocabParallelEmbedding(
260
+ config.embedding_size or config.vocab_size,
261
+ config.d_model,
262
+ ),
263
+ ln_f=nn.LayerNorm(config.d_model,
264
+ elementwise_affine=False,
265
+ bias=False),
266
+ ))
267
+
268
+ blocks = [
269
+ OlmoBlock(config, linear_method) for i in range(config.n_layers)
270
+ ]
271
+ if self.config.block_group_size > 1:
272
+ raise NotImplementedError("Block group size > 1 not supported yet")
273
+ else:
274
+ self.transformer.update({"blocks": nn.ModuleList(blocks)})
275
+
276
+ if not config.weight_tying:
277
+ self.transformer.update({
278
+ "ff_out":
279
+ ColumnParallelLinear(
280
+ config.d_model,
281
+ config.embedding_size or config.vocab_size,
282
+ bias=config.include_bias,
283
+ linear_method=linear_method,
284
+ )
285
+ })
286
+
287
+ def forward(
288
+ self,
289
+ input_ids: torch.Tensor,
290
+ positions: torch.Tensor,
291
+ kv_caches: List[KVCache],
292
+ input_metadata: InputMetadata,
293
+ ) -> torch.Tensor:
294
+ """
295
+ :param input_ids: A tensor of shape `(batch_size, seq_len)`.
296
+ """
297
+ # Get embeddings of input.
298
+ # shape: (batch_size, seq_len, d_model)
299
+ x = self.transformer.wte(input_ids) # type: ignore
300
+
301
+ # Apply blocks one-by-one.
302
+ for block_idx, block in enumerate(self.transformer.blocks):
303
+ # shape: (batch_size, seq_len, d_model)
304
+ x = block(
305
+ positions,
306
+ x,
307
+ kv_caches[block_idx],
308
+ input_metadata,
309
+ )
310
+
311
+ # Apply final layer norm.
312
+ # shape: (batch_size, seq_len or 1, d_model)
313
+ x = self.transformer.ln_f(x) # type: ignore
314
+ return x
315
+
316
+
317
+class OLMoForCausalLM(nn.Module):
318
+ """
319
+ Extremely barebones HF model wrapper.
320
+ """
321
+
322
+ def __init__(self,
323
+ config: OLMoConfig,
324
+ linear_method: Optional[LinearMethodBase] = None):
325
+ super().__init__()
326
+ self.config = config
327
+ self.linear_method = linear_method
328
+ self.model = OlmoModel(config, linear_method)
329
+ self.lm_head_weight = (self.model.transformer.wte.weight
330
+ if config.weight_tying else
331
+ self.model.transformer.ff_out.weight)
332
+ self.sampler = Sampler(config.vocab_size)
333
+
334
+ def forward(
335
+ self,
336
+ input_ids: torch.Tensor,
337
+ positions: torch.Tensor,
338
+ kv_caches: List[KVCache],
339
+ input_metadata: InputMetadata,
340
+ ) -> torch.Tensor:
341
+ hidden_states = self.model(
342
+ input_ids=input_ids,
343
+ positions=positions,
344
+ kv_caches=kv_caches,
345
+ input_metadata=input_metadata,
346
+ )
347
+ return hidden_states
348
+
349
+ def sample(
350
+ self,
351
+ hidden_states: torch.Tensor,
352
+ sampling_metadata: SamplingMetadata,
353
+ ) -> Optional[SamplerOutput]:
354
+ next_tokens = self.sampler(self.lm_head_weight, hidden_states,
355
+ sampling_metadata)
356
+ return next_tokens
357
+
358
+ def load_weights(
359
+ self,
360
+ model_name_or_path: str,
361
+ cache_dir: Optional[str] = None,
362
+ load_format: str = "auto",
363
+ revision: Optional[str] = None,
364
+ ):
365
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
366
+ for name, loaded_weight in hf_model_weights_iterator(
367
+ model_name_or_path, cache_dir, load_format, revision):
368
+ # attention
369
+ if ".att" in name:
370
+ name = name.replace(".att", ".attn.att")
371
+ # mlp
372
+ if ".ff" in name and "transformer.ff_out" not in name:
373
+ name = name.replace(".ff", ".mlp.ff")
374
+ # there is no bias in olmo
375
+ param = params_dict[name]
376
+ weight_loader = getattr(param, "weight_loader",
377
+ default_weight_loader)
378
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1
from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
2
from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3
from vllm.transformers_utils.configs.mpt import MPTConfig
4
+from vllm.transformers_utils.configs.olmo import OLMoConfig
5
from vllm.transformers_utils.configs.qwen import QWenConfig
6
# RWConfig is for the original tiiuae/falcon-40b(-instruct) and
7
# tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -11,6 +12,7 @@ __all__ = [
12
"BaiChuanConfig",
13
"ChatGLMConfig",
14
"MPTConfig",
15
+ "OLMoConfig",
16
"QWenConfig",
17
"RWConfig",
18
]
vllm/transformers_utils/configs/olmo.py
new
+72
@@ -0,0 +1,72 @@
1
+# coding=utf-8
2
+# adapted from https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/configuration_olmo.py
3
+"""OLMo configuration"""
4
+from transformers import PretrainedConfig
5
+
6
+
7
+class OLMoConfig(PretrainedConfig):
8
+ model_type = 'olmo'
9
+ attribute_map = {
10
+ 'num_attention_heads': 'n_heads',
11
+ 'hidden_size': 'd_model',
12
+ 'num_hidden_layers': 'n_layers',
13
+ }
14
+
15
+ # Note that the defaults for these attributes are equivalent to the base GPT2 model.
16
+ def __init__(
17
+ self,
18
+ d_model=768,
19
+ n_heads=12,
20
+ n_layers=12,
21
+ mlp_ratio=4,
22
+ mlp_hidden_size=None,
23
+ activation_type="swiglu",
24
+ block_type="sequential",
25
+ block_group_size=1,
26
+ alibi=False,
27
+ alibi_bias_max=8.0,
28
+ rope=False,
29
+ rope_full_precision=True,
30
+ multi_query_attention=False,
31
+ attention_layer_norm=False,
32
+ layer_norm_type="default",
33
+ layer_norm_with_affine=True,
34
+ attention_layer_norm_with_affine=True,
35
+ max_sequence_length=1024,
36
+ include_bias=True,
37
+ bias_for_layer_norm=None,
38
+ scale_logits=False,
39
+ vocab_size=50257,
40
+ embedding_size=50304,
41
+ weight_tying=True,
42
+ eos_token_id=50256,
43
+ pad_token_id=50256,
44
+ **kwargs,
45
+ ):
46
+ self.d_model = d_model
47
+ self.n_heads = n_heads
48
+ self.n_layers = n_layers
49
+ self.mlp_ratio = mlp_ratio
50
+ self.mlp_hidden_size = mlp_hidden_size
51
+ self.activation_type = activation_type
52
+ self.block_type = block_type
53
+ self.block_group_size = block_group_size
54
+ self.alibi = alibi
55
+ self.alibi_bias_max = alibi_bias_max
56
+ self.rope = rope
57
+ self.rope_full_precision = rope_full_precision
58
+ self.multi_query_attention = multi_query_attention
59
+ self.attention_layer_norm = attention_layer_norm
60
+ self.layer_norm_type = layer_norm_type
61
+ self.layer_norm_with_affine = layer_norm_with_affine
62
+ self.attention_layer_norm_with_affine = attention_layer_norm_with_affine
63
+ self.max_sequence_length = max_sequence_length
64
+ self.include_bias = include_bias
65
+ self.bias_for_layer_norm = bias_for_layer_norm
66
+ self.scale_logits = scale_logits
67
+ self.vocab_size = vocab_size
68
+ self.embedding_size = embedding_size
69
+ self.weight_tying = weight_tying
70
+ self.eos_token_id = eos_token_id
71
+ self.pad_token_id = pad_token_id
72
+ super().__init__(**kwargs)