@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ab3a5a8259
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Support OLMo models. (#2832)

Isotr0py committed Feb 19, 2024 at 13:05 UTC ab3a5a8259922ce312d01be39d29e27666968039
7 files changed +471 -5
README.md
+1
@@ -70,6 +70,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
70 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
71 - Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
72 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
73 +- OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
74 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
75 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
76 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
docs/source/models/supported_models.rst
+3
@@ -62,6 +62,9 @@ Alongside each architecture, we include some popular models that use it.
62 * - :code:`MPTForCausalLM`
63 - MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter
64 - :code:`mosaicml/mpt-7b`, :code:`mosaicml/mpt-7b-storywriter`, :code:`mosaicml/mpt-30b`, etc.
65 + * - :code:`OLMoForCausalLM`
66 + - OLMo
67 + - :code:`allenai/OLMo-1B`, :code:`allenai/OLMo-7B`, etc.
68 * - :code:`OPTForCausalLM`
69 - OPT, OPT-IML
70 - :code:`facebook/opt-66b`, :code:`facebook/opt-iml-max-30b`, etc.
tests/models/test_models.py
+14 -5
@@ -5,11 +5,20 @@ Run `pytest tests/models/test_models.py --forked`.
5 import pytest
6
7 MODELS = [
8 - "facebook/opt-125m", "meta-llama/Llama-2-7b-hf",
9 - "mistralai/Mistral-7B-v0.1", "Deci/DeciLM-7b", "tiiuae/falcon-7b", "gpt2",
10 - "bigcode/tiny_starcoder_py", "EleutherAI/gpt-j-6b",
11 - "EleutherAI/pythia-70m", "bigscience/bloom-560m", "mosaicml/mpt-7b",
12 - "microsoft/phi-2", "stabilityai/stablelm-3b-4e1t"
8 + "facebook/opt-125m",
9 + "meta-llama/Llama-2-7b-hf",
10 + "mistralai/Mistral-7B-v0.1",
11 + "Deci/DeciLM-7b",
12 + "tiiuae/falcon-7b",
13 + "gpt2",
14 + "bigcode/tiny_starcoder_py",
15 + "EleutherAI/gpt-j-6b",
16 + "EleutherAI/pythia-70m",
17 + "bigscience/bloom-560m",
18 + "mosaicml/mpt-7b",
19 + "microsoft/phi-2",
20 + "stabilityai/stablelm-3b-4e1t",
21 + "allenai/OLMo-1B",
22 ]
23
24
vllm/model_executor/models/__init__.py
+1
@@ -35,6 +35,7 @@ _MODELS = {
35 # transformers's mpt class has lower case
36 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
37 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
38 + "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
39 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
40 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
41 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
vllm/model_executor/models/olmo.py new
+378
@@ -0,0 +1,378 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/allenai/OLMo/blob/v0.2.4/olmo/model.py and
4 +# https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/modeling_olmo.py
5 +# Copyright 2023 The vLLM team.
6 +# Copyright (c) Microsoft Corporation.
7 +# Licensed under the MIT license.
8 +#
9 +# BSD 3-Clause License
10 +#
11 +# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
12 +# All rights reserved.
13 +#
14 +# Redistribution and use in source and binary forms, with or without
15 +# modification, are permitted provided that the following conditions are met:
16 +#
17 +# * Redistributions of source code must retain the above copyright notice, this
18 +# list of conditions and the following disclaimer.
19 +#
20 +# * Redistributions in binary form must reproduce the above copyright notice,
21 +# this list of conditions and the following disclaimer in the documentation
22 +# and/or other materials provided with the distribution.
23 +#
24 +# * Neither the name of the copyright holder nor the names of its
25 +# contributors may be used to endorse or promote products derived from
26 +# this software without specific prior written permission.
27 +#
28 +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
29 +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
30 +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
31 +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
32 +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
33 +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
34 +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
35 +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
36 +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
37 +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 +"""Inference-only OLMo model compatible with HuggingFace weights."""
39 +from typing import List, Optional, Tuple
40 +
41 +import torch
42 +import torch.nn.functional as F
43 +from torch import nn
44 +
45 +from vllm.model_executor.input_metadata import InputMetadata
46 +from vllm.model_executor.layers.attention import PagedAttention
47 +from vllm.model_executor.layers.linear import (
48 + ColumnParallelLinear,
49 + LinearMethodBase,
50 + QKVParallelLinear,
51 + RowParallelLinear,
52 +)
53 +from vllm.model_executor.layers.rotary_embedding import get_rope
54 +from vllm.model_executor.layers.sampler import Sampler
55 +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
56 +from vllm.model_executor.parallel_utils.parallel_state import (
57 + get_tensor_model_parallel_world_size, )
58 +from vllm.model_executor.sampling_metadata import SamplingMetadata
59 +from vllm.model_executor.weight_utils import (
60 + default_weight_loader,
61 + hf_model_weights_iterator,
62 +)
63 +from vllm.sequence import SamplerOutput
64 +from vllm.transformers_utils.configs.olmo import OLMoConfig
65 +
66 +KVCache = Tuple[torch.Tensor, torch.Tensor]
67 +
68 +
69 +class SwiGLU(nn.Module):
70 +
71 + def forward(self, x: torch.Tensor) -> torch.Tensor:
72 + x, gate = x.chunk(2, dim=-1)
73 + return F.silu(gate) * x
74 +
75 + @property
76 + def output_multiplier(self) -> float:
77 + return 0.5
78 +
79 +
80 +class OlmoAttention(nn.Module):
81 + """
82 + This is the attention block where the output is computed as ``Attention(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
83 + (plus another skip connection).
84 + """
85 +
86 + def __init__(
87 + self,
88 + config: OLMoConfig,
89 + linear_method: Optional[LinearMethodBase] = None,
90 + ):
91 + super().__init__()
92 + self.config = config
93 + self.hidden_size = config.d_model
94 + assert config.d_model % config.n_heads == 0
95 + tensor_model_parallel_world_size = get_tensor_model_parallel_world_size(
96 + )
97 + self.total_num_heads = self.config.n_heads
98 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
99 + self.num_heads = self.total_num_heads // tensor_model_parallel_world_size
100 + self.head_dim = self.hidden_size // self.total_num_heads
101 +
102 + # Layer norms.
103 + self.attn_norm = nn.LayerNorm(config.d_model,
104 + elementwise_affine=False,
105 + bias=False)
106 + # Attention input projection. Projects x -> (q, k, v)
107 + self.att_proj = QKVParallelLinear(
108 + config.d_model,
109 + self.head_dim,
110 + self.total_num_heads,
111 + bias=config.include_bias,
112 + linear_method=linear_method,
113 + )
114 +
115 + # Rotary embeddings.
116 + if self.config.rope:
117 + rope_theta = getattr(config, "rope_theta", 10000)
118 + max_position_embeddings = getattr(config,
119 + "max_position_embeddings", 8192)
120 + self.rotary_emb = get_rope(
121 + self.head_dim,
122 + rotary_dim=self.head_dim,
123 + max_position=max_position_embeddings,
124 + base=rope_theta,
125 + )
126 + self.scaling = self.head_dim**-0.5
127 + self.attn = PagedAttention(self.num_heads,
128 + self.head_dim,
129 + scale=self.scaling)
130 +
131 + # Attention output projection.
132 + self.attn_out = RowParallelLinear(
133 + config.d_model,
134 + config.d_model,
135 + bias=config.include_bias,
136 + linear_method=linear_method,
137 + )
138 +
139 + def forward(
140 + self,
141 + positions: torch.Tensor,
142 + hidden_states: torch.Tensor,
143 + kv_cache: KVCache,
144 + input_metadata: InputMetadata,
145 + ) -> torch.Tensor:
146 + hidden_states = self.attn_norm(hidden_states)
147 + qkv, _ = self.att_proj(hidden_states)
148 + q, k, v = qkv.chunk(chunks=3, dim=-1)
149 + if self.config.rope:
150 + q, k = self.rotary_emb(positions, q, k)
151 + k_cache, v_cache = kv_cache
152 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
153 + output, _ = self.attn_out(attn_output)
154 + return output
155 +
156 +
157 +class OlmoMLP(nn.Module):
158 + """
159 + This is the MLP block where the output is computed as ``MLP(LN(x))`` in ``MLP(LN(x + Attention(LN(x))))``
160 + (plus another skip connection).
161 + """
162 +
163 + def __init__(
164 + self,
165 + config: OLMoConfig,
166 + linear_method: Optional[LinearMethodBase] = None,
167 + ):
168 + super().__init__()
169 + self.config = config
170 + self.hidden_size = (config.mlp_hidden_size if config.mlp_hidden_size
171 + is not None else config.mlp_ratio * config.d_model)
172 +
173 + # Layer norms.
174 + self.ff_norm = nn.LayerNorm(config.d_model,
175 + elementwise_affine=False,
176 + bias=False)
177 +
178 + # Feed-forward input projection.
179 + self.ff_proj = ColumnParallelLinear(
180 + config.d_model,
181 + self.hidden_size,
182 + bias=config.include_bias,
183 + linear_method=linear_method,
184 + )
185 +
186 + # Activation function.
187 + # self.act = SiluAndMul()
188 + # self.act.output_multiplier = 0.5
189 + self.act = SwiGLU()
190 + assert (self.act.output_multiplier * self.hidden_size) % 1 == 0
191 +
192 + # Feed-forward output projection.
193 + self.ff_out = RowParallelLinear(
194 + int(self.act.output_multiplier * self.hidden_size),
195 + config.d_model,
196 + bias=config.include_bias,
197 + linear_method=linear_method,
198 + )
199 +
200 + def forward(
201 + self,
202 + x: torch.Tensor,
203 + ) -> torch.Tensor:
204 + # Add feed-forward projection.
205 + # shape: (batch_size, seq_len, d_model)
206 + og_x = x
207 + x = self.ff_norm(x)
208 + x, _ = self.ff_proj(x)
209 + x = self.act(x)
210 + x, _ = self.ff_out(x)
211 + x = og_x + x
212 +
213 + return x
214 +
215 +
216 +class OlmoBlock(nn.Module):
217 + """
218 + This is a typical transformer block where the output is computed as ``MLP(LN(x + Attention(LN(x))))``
219 + (plus another skip connection).
220 + """
221 +
222 + def __init__(self,
223 + config: OLMoConfig,
224 + linear_method: Optional[LinearMethodBase] = None):
225 + super().__init__()
226 + # Attention block.
227 + self.attn = OlmoAttention(config, linear_method)
228 +
229 + # MLP block.
230 + self.mlp = OlmoMLP(config, linear_method)
231 +
232 + def forward(
233 + self,
234 + positions: torch.Tensor,
235 + hidden_states: torch.Tensor,
236 + kv_cache: KVCache,
237 + input_metadata: InputMetadata,
238 + ) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor, torch.Tensor]]]:
239 + # Attention block.
240 + og_x = hidden_states
241 + x = self.attn(positions, hidden_states, kv_cache, input_metadata)
242 + x = x + og_x
243 +
244 + # MLP block.
245 + hidden_states = self.mlp(x)
246 + return hidden_states
247 +
248 +
249 +class OlmoModel(nn.Module):
250 +
251 + def __init__(self,
252 + config: OLMoConfig,
253 + linear_method: Optional[LinearMethodBase] = None):
254 + super().__init__()
255 + self.config = config
256 +
257 + self.transformer = nn.ModuleDict(
258 + dict(
259 + wte=VocabParallelEmbedding(
260 + config.embedding_size or config.vocab_size,
261 + config.d_model,
262 + ),
263 + ln_f=nn.LayerNorm(config.d_model,
264 + elementwise_affine=False,
265 + bias=False),
266 + ))
267 +
268 + blocks = [
269 + OlmoBlock(config, linear_method) for i in range(config.n_layers)
270 + ]
271 + if self.config.block_group_size > 1:
272 + raise NotImplementedError("Block group size > 1 not supported yet")
273 + else:
274 + self.transformer.update({"blocks": nn.ModuleList(blocks)})
275 +
276 + if not config.weight_tying:
277 + self.transformer.update({
278 + "ff_out":
279 + ColumnParallelLinear(
280 + config.d_model,
281 + config.embedding_size or config.vocab_size,
282 + bias=config.include_bias,
283 + linear_method=linear_method,
284 + )
285 + })
286 +
287 + def forward(
288 + self,
289 + input_ids: torch.Tensor,
290 + positions: torch.Tensor,
291 + kv_caches: List[KVCache],
292 + input_metadata: InputMetadata,
293 + ) -> torch.Tensor:
294 + """
295 + :param input_ids: A tensor of shape `(batch_size, seq_len)`.
296 + """
297 + # Get embeddings of input.
298 + # shape: (batch_size, seq_len, d_model)
299 + x = self.transformer.wte(input_ids) # type: ignore
300 +
301 + # Apply blocks one-by-one.
302 + for block_idx, block in enumerate(self.transformer.blocks):
303 + # shape: (batch_size, seq_len, d_model)
304 + x = block(
305 + positions,
306 + x,
307 + kv_caches[block_idx],
308 + input_metadata,
309 + )
310 +
311 + # Apply final layer norm.
312 + # shape: (batch_size, seq_len or 1, d_model)
313 + x = self.transformer.ln_f(x) # type: ignore
314 + return x
315 +
316 +
317 +class OLMoForCausalLM(nn.Module):
318 + """
319 + Extremely barebones HF model wrapper.
320 + """
321 +
322 + def __init__(self,
323 + config: OLMoConfig,
324 + linear_method: Optional[LinearMethodBase] = None):
325 + super().__init__()
326 + self.config = config
327 + self.linear_method = linear_method
328 + self.model = OlmoModel(config, linear_method)
329 + self.lm_head_weight = (self.model.transformer.wte.weight
330 + if config.weight_tying else
331 + self.model.transformer.ff_out.weight)
332 + self.sampler = Sampler(config.vocab_size)
333 +
334 + def forward(
335 + self,
336 + input_ids: torch.Tensor,
337 + positions: torch.Tensor,
338 + kv_caches: List[KVCache],
339 + input_metadata: InputMetadata,
340 + ) -> torch.Tensor:
341 + hidden_states = self.model(
342 + input_ids=input_ids,
343 + positions=positions,
344 + kv_caches=kv_caches,
345 + input_metadata=input_metadata,
346 + )
347 + return hidden_states
348 +
349 + def sample(
350 + self,
351 + hidden_states: torch.Tensor,
352 + sampling_metadata: SamplingMetadata,
353 + ) -> Optional[SamplerOutput]:
354 + next_tokens = self.sampler(self.lm_head_weight, hidden_states,
355 + sampling_metadata)
356 + return next_tokens
357 +
358 + def load_weights(
359 + self,
360 + model_name_or_path: str,
361 + cache_dir: Optional[str] = None,
362 + load_format: str = "auto",
363 + revision: Optional[str] = None,
364 + ):
365 + params_dict = dict(self.named_parameters(remove_duplicate=False))
366 + for name, loaded_weight in hf_model_weights_iterator(
367 + model_name_or_path, cache_dir, load_format, revision):
368 + # attention
369 + if ".att" in name:
370 + name = name.replace(".att", ".attn.att")
371 + # mlp
372 + if ".ff" in name and "transformer.ff_out" not in name:
373 + name = name.replace(".ff", ".mlp.ff")
374 + # there is no bias in olmo
375 + param = params_dict[name]
376 + weight_loader = getattr(param, "weight_loader",
377 + default_weight_loader)
378 + weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/__init__.py
+2
@@ -1,6 +1,7 @@
1 from vllm.transformers_utils.configs.baichuan import BaiChuanConfig
2 from vllm.transformers_utils.configs.chatglm import ChatGLMConfig
3 from vllm.transformers_utils.configs.mpt import MPTConfig
4 +from vllm.transformers_utils.configs.olmo import OLMoConfig
5 from vllm.transformers_utils.configs.qwen import QWenConfig
6 # RWConfig is for the original tiiuae/falcon-40b(-instruct) and
7 # tiiuae/falcon-7b(-instruct) models. Newer Falcon models will use the
@@ -11,6 +12,7 @@ __all__ = [
12 "BaiChuanConfig",
13 "ChatGLMConfig",
14 "MPTConfig",
15 + "OLMoConfig",
16 "QWenConfig",
17 "RWConfig",
18 ]
vllm/transformers_utils/configs/olmo.py new
+72
@@ -0,0 +1,72 @@
1 +# coding=utf-8
2 +# adapted from https://github.com/allenai/OLMo/blob/v0.2.4/hf_olmo/configuration_olmo.py
3 +"""OLMo configuration"""
4 +from transformers import PretrainedConfig
5 +
6 +
7 +class OLMoConfig(PretrainedConfig):
8 + model_type = 'olmo'
9 + attribute_map = {
10 + 'num_attention_heads': 'n_heads',
11 + 'hidden_size': 'd_model',
12 + 'num_hidden_layers': 'n_layers',
13 + }
14 +
15 + # Note that the defaults for these attributes are equivalent to the base GPT2 model.
16 + def __init__(
17 + self,
18 + d_model=768,
19 + n_heads=12,
20 + n_layers=12,
21 + mlp_ratio=4,
22 + mlp_hidden_size=None,
23 + activation_type="swiglu",
24 + block_type="sequential",
25 + block_group_size=1,
26 + alibi=False,
27 + alibi_bias_max=8.0,
28 + rope=False,
29 + rope_full_precision=True,
30 + multi_query_attention=False,
31 + attention_layer_norm=False,
32 + layer_norm_type="default",
33 + layer_norm_with_affine=True,
34 + attention_layer_norm_with_affine=True,
35 + max_sequence_length=1024,
36 + include_bias=True,
37 + bias_for_layer_norm=None,
38 + scale_logits=False,
39 + vocab_size=50257,
40 + embedding_size=50304,
41 + weight_tying=True,
42 + eos_token_id=50256,
43 + pad_token_id=50256,
44 + **kwargs,
45 + ):
46 + self.d_model = d_model
47 + self.n_heads = n_heads
48 + self.n_layers = n_layers
49 + self.mlp_ratio = mlp_ratio
50 + self.mlp_hidden_size = mlp_hidden_size
51 + self.activation_type = activation_type
52 + self.block_type = block_type
53 + self.block_group_size = block_group_size
54 + self.alibi = alibi
55 + self.alibi_bias_max = alibi_bias_max
56 + self.rope = rope
57 + self.rope_full_precision = rope_full_precision
58 + self.multi_query_attention = multi_query_attention
59 + self.attention_layer_norm = attention_layer_norm
60 + self.layer_norm_type = layer_norm_type
61 + self.layer_norm_with_affine = layer_norm_with_affine
62 + self.attention_layer_norm_with_affine = attention_layer_norm_with_affine
63 + self.max_sequence_length = max_sequence_length
64 + self.include_bias = include_bias
65 + self.bias_for_layer_norm = bias_for_layer_norm
66 + self.scale_logits = scale_logits
67 + self.vocab_size = vocab_size
68 + self.embedding_size = embedding_size
69 + self.weight_tying = weight_tying
70 + self.eos_token_id = eos_token_id
71 + self.pad_token_id = pad_token_id
72 + super().__init__(**kwargs)