A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Support Orion model (#2539)
Co-authored-by: zhangdacheng <zhangdacheng@ainirobot.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
张大成 committed
Feb 27, 2024 at 11:17 UTC
48a8f4a7fd18d516ffc0a304219ef722613ea792
4 files changed
+327
README.md
+1
@@ -73,6 +73,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
73
- MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
74
- OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
75
- OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
76
+- Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
77
- Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
78
- Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
79
- Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
docs/source/models/supported_models.rst
+3
@@ -71,6 +71,9 @@ Alongside each architecture, we include some popular models that use it.
71
* - :code:`OPTForCausalLM`
72
- OPT, OPT-IML
73
- :code:`facebook/opt-66b`, :code:`facebook/opt-iml-max-30b`, etc.
74
+ * - :code:`OrionForCausalLM`
75
+ - Orion
76
+ - :code:`OrionStarAI/Orion-14B-Base`, :code:`OrionStarAI/Orion-14B-Chat`, etc.
77
* - :code:`PhiForCausalLM`
78
- Phi
79
- :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -38,6 +38,7 @@ _MODELS = {
38
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
39
"OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
40
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
41
+ "OrionForCausalLM": ("orion", "OrionForCausalLM"),
42
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
43
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
44
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
vllm/model_executor/models/orion.py
new
+322
@@ -0,0 +1,322 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/modeling_orion.py
4
+# Copyright (c) OrionStar Inc.
5
+# LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
6
+"""Inference-only Orion-14B model compatible with HuggingFace weights."""
7
+from typing import Any, Dict, List, Optional, Tuple
8
+
9
+import torch
10
+from torch import nn
11
+from transformers import PretrainedConfig
12
+
13
+from vllm.model_executor.input_metadata import InputMetadata
14
+from vllm.model_executor.layers.activation import SiluAndMul
15
+from vllm.model_executor.layers.attention import PagedAttention
16
+from vllm.model_executor.layers.linear import (LinearMethodBase,
17
+ MergedColumnParallelLinear,
18
+ QKVParallelLinear,
19
+ RowParallelLinear)
20
+from vllm.model_executor.layers.rotary_embedding import get_rope
21
+from vllm.model_executor.layers.sampler import Sampler
22
+from vllm.model_executor.layers.vocab_parallel_embedding import (
23
+ VocabParallelEmbedding, ParallelLMHead)
24
+from vllm.model_executor.parallel_utils.parallel_state import (
25
+ get_tensor_model_parallel_world_size)
26
+from vllm.model_executor.sampling_metadata import SamplingMetadata
27
+from vllm.model_executor.weight_utils import (default_weight_loader,
28
+ hf_model_weights_iterator)
29
+from vllm.sequence import SamplerOutput
30
+
31
+KVCache = Tuple[torch.Tensor, torch.Tensor]
32
+
33
+
34
+class OrionMLP(nn.Module):
35
+
36
+ def __init__(
37
+ self,
38
+ hidden_size: int,
39
+ intermediate_size: int,
40
+ hidden_act: str,
41
+ linear_method: Optional[LinearMethodBase] = None,
42
+ ) -> None:
43
+ super().__init__()
44
+ self.gate_up_proj = MergedColumnParallelLinear(
45
+ hidden_size, [intermediate_size] * 2,
46
+ bias=False,
47
+ linear_method=linear_method)
48
+ self.down_proj = RowParallelLinear(intermediate_size,
49
+ hidden_size,
50
+ bias=False,
51
+ linear_method=linear_method)
52
+ if hidden_act != "silu":
53
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
54
+ "Only silu is supported for now.")
55
+ self.act_fn = SiluAndMul()
56
+
57
+ def forward(self, x):
58
+ gate_up, _ = self.gate_up_proj(x)
59
+ x = self.act_fn(gate_up)
60
+ x, _ = self.down_proj(x)
61
+ return x
62
+
63
+
64
+class OrionAttention(nn.Module):
65
+
66
+ def __init__(
67
+ self,
68
+ hidden_size: int,
69
+ num_heads: int,
70
+ num_kv_heads: int,
71
+ rope_theta: float = 10000,
72
+ rope_scaling: Optional[Dict[str, Any]] = None,
73
+ max_position_embeddings: int = 8192,
74
+ linear_method: Optional[LinearMethodBase] = None,
75
+ ) -> None:
76
+ super().__init__()
77
+ self.hidden_size = hidden_size
78
+ tp_size = get_tensor_model_parallel_world_size()
79
+ self.total_num_heads = num_heads
80
+ assert self.total_num_heads % tp_size == 0
81
+ self.num_heads = self.total_num_heads // tp_size
82
+ self.total_num_kv_heads = num_kv_heads
83
+ if self.total_num_kv_heads >= tp_size:
84
+ # Number of KV heads is greater than TP size, so we partition
85
+ # the KV heads across multiple tensor parallel GPUs.
86
+ assert self.total_num_kv_heads % tp_size == 0
87
+ else:
88
+ # Number of KV heads is less than TP size, so we replicate
89
+ # the KV heads across multiple tensor parallel GPUs.
90
+ assert tp_size % self.total_num_kv_heads == 0
91
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
92
+ self.head_dim = hidden_size // self.total_num_heads
93
+ self.q_size = self.num_heads * self.head_dim
94
+ self.kv_size = self.num_kv_heads * self.head_dim
95
+ self.scaling = self.head_dim**-0.5
96
+ self.rope_theta = rope_theta
97
+ self.max_position_embeddings = max_position_embeddings
98
+
99
+ self.qkv_proj = QKVParallelLinear(
100
+ hidden_size,
101
+ self.head_dim,
102
+ self.total_num_heads,
103
+ self.total_num_kv_heads,
104
+ bias=False,
105
+ linear_method=linear_method,
106
+ )
107
+ self.o_proj = RowParallelLinear(
108
+ self.total_num_heads * self.head_dim,
109
+ hidden_size,
110
+ bias=False,
111
+ linear_method=linear_method,
112
+ )
113
+
114
+ self.rotary_emb = get_rope(
115
+ self.head_dim,
116
+ rotary_dim=self.head_dim,
117
+ max_position=max_position_embeddings,
118
+ base=rope_theta,
119
+ rope_scaling=rope_scaling,
120
+ )
121
+ self.attn = PagedAttention(self.num_heads,
122
+ self.head_dim,
123
+ self.scaling,
124
+ num_kv_heads=self.num_kv_heads)
125
+
126
+ def forward(
127
+ self,
128
+ positions: torch.Tensor,
129
+ hidden_states: torch.Tensor,
130
+ kv_cache: KVCache,
131
+ input_metadata: InputMetadata,
132
+ ) -> torch.Tensor:
133
+ qkv, _ = self.qkv_proj(hidden_states)
134
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
135
+ q, k = self.rotary_emb(positions, q, k)
136
+ k_cache, v_cache = kv_cache
137
+ attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
138
+ output, _ = self.o_proj(attn_output)
139
+ return output
140
+
141
+
142
+class OrionDecoderLayer(nn.Module):
143
+
144
+ def __init__(
145
+ self,
146
+ config: PretrainedConfig,
147
+ linear_method: Optional[LinearMethodBase] = None,
148
+ ) -> None:
149
+ super().__init__()
150
+ self.hidden_size = config.hidden_size
151
+ rope_theta = getattr(config, "rope_theta", 10000)
152
+ rope_scaling = getattr(config, "rope_scaling", None)
153
+ max_position_embeddings = getattr(config, "max_position_embeddings",
154
+ 8192)
155
+ self.self_attn = OrionAttention(
156
+ hidden_size=self.hidden_size,
157
+ num_heads=config.num_attention_heads,
158
+ num_kv_heads=config.num_key_value_heads,
159
+ rope_theta=rope_theta,
160
+ rope_scaling=rope_scaling,
161
+ max_position_embeddings=max_position_embeddings,
162
+ linear_method=linear_method,
163
+ )
164
+ self.mlp = OrionMLP(
165
+ hidden_size=self.hidden_size,
166
+ intermediate_size=config.intermediate_size,
167
+ hidden_act=config.hidden_act,
168
+ linear_method=linear_method,
169
+ )
170
+
171
+ self.input_layernorm = nn.LayerNorm(config.hidden_size,
172
+ eps=config.rms_norm_eps)
173
+ self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
174
+ eps=config.rms_norm_eps)
175
+
176
+ def forward(
177
+ self,
178
+ positions: torch.Tensor,
179
+ hidden_states: torch.Tensor,
180
+ kv_cache: KVCache,
181
+ input_metadata: InputMetadata,
182
+ residual: Optional[torch.Tensor],
183
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
184
+ # Self Attention
185
+ residual = hidden_states
186
+ hidden_states = self.input_layernorm(hidden_states)
187
+ hidden_states = self.self_attn(
188
+ positions=positions,
189
+ hidden_states=hidden_states,
190
+ kv_cache=kv_cache,
191
+ input_metadata=input_metadata,
192
+ )
193
+
194
+ hidden_states = residual + hidden_states
195
+
196
+ # Fully Connected
197
+ residual = hidden_states
198
+ hidden_states = self.post_attention_layernorm(hidden_states)
199
+ hidden_states = self.mlp(hidden_states)
200
+ hidden_states = residual + hidden_states
201
+ return hidden_states, None
202
+
203
+
204
+class OrionModel(nn.Module):
205
+
206
+ def __init__(
207
+ self,
208
+ config: PretrainedConfig,
209
+ linear_method: Optional[LinearMethodBase] = None,
210
+ ) -> None:
211
+ super().__init__()
212
+ self.config = config
213
+ self.padding_idx = config.pad_token_id
214
+ self.vocab_size = config.vocab_size
215
+ self.embed_tokens = VocabParallelEmbedding(
216
+ config.vocab_size,
217
+ config.hidden_size,
218
+ )
219
+ self.layers = nn.ModuleList([
220
+ OrionDecoderLayer(config, linear_method)
221
+ for _ in range(config.num_hidden_layers)
222
+ ])
223
+ self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
224
+
225
+ def forward(
226
+ self,
227
+ input_ids: torch.Tensor,
228
+ positions: torch.Tensor,
229
+ kv_caches: List[KVCache],
230
+ input_metadata: InputMetadata,
231
+ ) -> torch.Tensor:
232
+ hidden_states = self.embed_tokens(input_ids)
233
+ residual = None
234
+ for i in range(len(self.layers)):
235
+ layer = self.layers[i]
236
+ hidden_states, residual = layer(
237
+ positions,
238
+ hidden_states,
239
+ kv_caches[i],
240
+ input_metadata,
241
+ residual,
242
+ )
243
+ hidden_states = self.norm(hidden_states)
244
+ return hidden_states
245
+
246
+
247
+class OrionForCausalLM(nn.Module):
248
+
249
+ def __init__(
250
+ self,
251
+ config: PretrainedConfig,
252
+ linear_method: Optional[LinearMethodBase] = None,
253
+ ) -> None:
254
+ super().__init__()
255
+ self.config = config
256
+ self.linear_method = linear_method
257
+ self.model = OrionModel(config, linear_method)
258
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
259
+ self.sampler = Sampler(config.vocab_size)
260
+
261
+ def forward(
262
+ self,
263
+ input_ids: torch.Tensor,
264
+ positions: torch.Tensor,
265
+ kv_caches: List[KVCache],
266
+ input_metadata: InputMetadata,
267
+ ) -> torch.Tensor:
268
+ hidden_states = self.model(input_ids, positions, kv_caches,
269
+ input_metadata)
270
+ return hidden_states
271
+
272
+ def sample(
273
+ self,
274
+ hidden_states: torch.Tensor,
275
+ sampling_metadata: SamplingMetadata,
276
+ ) -> Optional[SamplerOutput]:
277
+ next_tokens = self.sampler(self.lm_head.weight, hidden_states,
278
+ sampling_metadata)
279
+ return next_tokens
280
+
281
+ def load_weights(self,
282
+ model_name_or_path: str,
283
+ cache_dir: Optional[str] = None,
284
+ load_format: str = "auto",
285
+ revision: Optional[str] = None):
286
+ stacked_params_mapping = [
287
+ # (param_name, shard_name, shard_id)
288
+ ("qkv_proj", "q_proj", "q"),
289
+ ("qkv_proj", "k_proj", "k"),
290
+ ("qkv_proj", "v_proj", "v"),
291
+ ("gate_up_proj", "gate_proj", 0),
292
+ ("gate_up_proj", "up_proj", 1),
293
+ ]
294
+ params_dict = dict(self.named_parameters())
295
+ for name, loaded_weight in hf_model_weights_iterator(
296
+ model_name_or_path, cache_dir, load_format, revision):
297
+ if "rotary_emb.inv_freq" in name:
298
+ continue
299
+ if ("rotary_emb.cos_cached" in name
300
+ or "rotary_emb.sin_cached" in name):
301
+ # Models trained using ColossalAI may include these tensors in
302
+ # the checkpoint. Skip them.
303
+ continue
304
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
305
+ if weight_name not in name:
306
+ continue
307
+ name = name.replace(weight_name, param_name)
308
+ # Skip loading extra bias for GPTQ models.
309
+ if name.endswith(".bias") and name not in params_dict:
310
+ continue
311
+ param = params_dict[name]
312
+ weight_loader = param.weight_loader
313
+ weight_loader(param, loaded_weight, shard_id)
314
+ break
315
+ else:
316
+ # Skip loading extra bias for GPTQ models.
317
+ if name.endswith(".bias") and name not in params_dict:
318
+ continue
319
+ param = params_dict[name]
320
+ weight_loader = getattr(param, "weight_loader",
321
+ default_weight_loader)
322
+ weight_loader(param, loaded_weight)