@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48a8f4a7fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Support Orion model (#2539)

Co-authored-by: zhangdacheng <zhangdacheng@ainirobot.com> Co-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>

张大成 committed Feb 27, 2024 at 11:17 UTC 48a8f4a7fd18d516ffc0a304219ef722613ea792
4 files changed +327
README.md
+1
@@ -73,6 +73,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
73 - MPT (`mosaicml/mpt-7b`, `mosaicml/mpt-30b`, etc.)
74 - OLMo (`allenai/OLMo-1B`, `allenai/OLMo-7B`, etc.)
75 - OPT (`facebook/opt-66b`, `facebook/opt-iml-max-30b`, etc.)
76 +- Orion (`OrionStarAI/Orion-14B-Base`, `OrionStarAI/Orion-14B-Chat`, etc.)
77 - Phi (`microsoft/phi-1_5`, `microsoft/phi-2`, etc.)
78 - Qwen (`Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc.)
79 - Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
docs/source/models/supported_models.rst
+3
@@ -71,6 +71,9 @@ Alongside each architecture, we include some popular models that use it.
71 * - :code:`OPTForCausalLM`
72 - OPT, OPT-IML
73 - :code:`facebook/opt-66b`, :code:`facebook/opt-iml-max-30b`, etc.
74 + * - :code:`OrionForCausalLM`
75 + - Orion
76 + - :code:`OrionStarAI/Orion-14B-Base`, :code:`OrionStarAI/Orion-14B-Chat`, etc.
77 * - :code:`PhiForCausalLM`
78 - Phi
79 - :code:`microsoft/phi-1_5`, :code:`microsoft/phi-2`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -38,6 +38,7 @@ _MODELS = {
38 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
39 "OLMoForCausalLM": ("olmo", "OLMoForCausalLM"),
40 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
41 + "OrionForCausalLM": ("orion", "OrionForCausalLM"),
42 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
43 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
44 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
vllm/model_executor/models/orion.py new
+322
@@ -0,0 +1,322 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/modeling_orion.py
4 +# Copyright (c) OrionStar Inc.
5 +# LICENSE: https://huggingface.co/OrionStarAI/Orion-14B-Base/blob/main/LICENSE
6 +"""Inference-only Orion-14B model compatible with HuggingFace weights."""
7 +from typing import Any, Dict, List, Optional, Tuple
8 +
9 +import torch
10 +from torch import nn
11 +from transformers import PretrainedConfig
12 +
13 +from vllm.model_executor.input_metadata import InputMetadata
14 +from vllm.model_executor.layers.activation import SiluAndMul
15 +from vllm.model_executor.layers.attention import PagedAttention
16 +from vllm.model_executor.layers.linear import (LinearMethodBase,
17 + MergedColumnParallelLinear,
18 + QKVParallelLinear,
19 + RowParallelLinear)
20 +from vllm.model_executor.layers.rotary_embedding import get_rope
21 +from vllm.model_executor.layers.sampler import Sampler
22 +from vllm.model_executor.layers.vocab_parallel_embedding import (
23 + VocabParallelEmbedding, ParallelLMHead)
24 +from vllm.model_executor.parallel_utils.parallel_state import (
25 + get_tensor_model_parallel_world_size)
26 +from vllm.model_executor.sampling_metadata import SamplingMetadata
27 +from vllm.model_executor.weight_utils import (default_weight_loader,
28 + hf_model_weights_iterator)
29 +from vllm.sequence import SamplerOutput
30 +
31 +KVCache = Tuple[torch.Tensor, torch.Tensor]
32 +
33 +
34 +class OrionMLP(nn.Module):
35 +
36 + def __init__(
37 + self,
38 + hidden_size: int,
39 + intermediate_size: int,
40 + hidden_act: str,
41 + linear_method: Optional[LinearMethodBase] = None,
42 + ) -> None:
43 + super().__init__()
44 + self.gate_up_proj = MergedColumnParallelLinear(
45 + hidden_size, [intermediate_size] * 2,
46 + bias=False,
47 + linear_method=linear_method)
48 + self.down_proj = RowParallelLinear(intermediate_size,
49 + hidden_size,
50 + bias=False,
51 + linear_method=linear_method)
52 + if hidden_act != "silu":
53 + raise ValueError(f"Unsupported activation: {hidden_act}. "
54 + "Only silu is supported for now.")
55 + self.act_fn = SiluAndMul()
56 +
57 + def forward(self, x):
58 + gate_up, _ = self.gate_up_proj(x)
59 + x = self.act_fn(gate_up)
60 + x, _ = self.down_proj(x)
61 + return x
62 +
63 +
64 +class OrionAttention(nn.Module):
65 +
66 + def __init__(
67 + self,
68 + hidden_size: int,
69 + num_heads: int,
70 + num_kv_heads: int,
71 + rope_theta: float = 10000,
72 + rope_scaling: Optional[Dict[str, Any]] = None,
73 + max_position_embeddings: int = 8192,
74 + linear_method: Optional[LinearMethodBase] = None,
75 + ) -> None:
76 + super().__init__()
77 + self.hidden_size = hidden_size
78 + tp_size = get_tensor_model_parallel_world_size()
79 + self.total_num_heads = num_heads
80 + assert self.total_num_heads % tp_size == 0
81 + self.num_heads = self.total_num_heads // tp_size
82 + self.total_num_kv_heads = num_kv_heads
83 + if self.total_num_kv_heads >= tp_size:
84 + # Number of KV heads is greater than TP size, so we partition
85 + # the KV heads across multiple tensor parallel GPUs.
86 + assert self.total_num_kv_heads % tp_size == 0
87 + else:
88 + # Number of KV heads is less than TP size, so we replicate
89 + # the KV heads across multiple tensor parallel GPUs.
90 + assert tp_size % self.total_num_kv_heads == 0
91 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
92 + self.head_dim = hidden_size // self.total_num_heads
93 + self.q_size = self.num_heads * self.head_dim
94 + self.kv_size = self.num_kv_heads * self.head_dim
95 + self.scaling = self.head_dim**-0.5
96 + self.rope_theta = rope_theta
97 + self.max_position_embeddings = max_position_embeddings
98 +
99 + self.qkv_proj = QKVParallelLinear(
100 + hidden_size,
101 + self.head_dim,
102 + self.total_num_heads,
103 + self.total_num_kv_heads,
104 + bias=False,
105 + linear_method=linear_method,
106 + )
107 + self.o_proj = RowParallelLinear(
108 + self.total_num_heads * self.head_dim,
109 + hidden_size,
110 + bias=False,
111 + linear_method=linear_method,
112 + )
113 +
114 + self.rotary_emb = get_rope(
115 + self.head_dim,
116 + rotary_dim=self.head_dim,
117 + max_position=max_position_embeddings,
118 + base=rope_theta,
119 + rope_scaling=rope_scaling,
120 + )
121 + self.attn = PagedAttention(self.num_heads,
122 + self.head_dim,
123 + self.scaling,
124 + num_kv_heads=self.num_kv_heads)
125 +
126 + def forward(
127 + self,
128 + positions: torch.Tensor,
129 + hidden_states: torch.Tensor,
130 + kv_cache: KVCache,
131 + input_metadata: InputMetadata,
132 + ) -> torch.Tensor:
133 + qkv, _ = self.qkv_proj(hidden_states)
134 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
135 + q, k = self.rotary_emb(positions, q, k)
136 + k_cache, v_cache = kv_cache
137 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
138 + output, _ = self.o_proj(attn_output)
139 + return output
140 +
141 +
142 +class OrionDecoderLayer(nn.Module):
143 +
144 + def __init__(
145 + self,
146 + config: PretrainedConfig,
147 + linear_method: Optional[LinearMethodBase] = None,
148 + ) -> None:
149 + super().__init__()
150 + self.hidden_size = config.hidden_size
151 + rope_theta = getattr(config, "rope_theta", 10000)
152 + rope_scaling = getattr(config, "rope_scaling", None)
153 + max_position_embeddings = getattr(config, "max_position_embeddings",
154 + 8192)
155 + self.self_attn = OrionAttention(
156 + hidden_size=self.hidden_size,
157 + num_heads=config.num_attention_heads,
158 + num_kv_heads=config.num_key_value_heads,
159 + rope_theta=rope_theta,
160 + rope_scaling=rope_scaling,
161 + max_position_embeddings=max_position_embeddings,
162 + linear_method=linear_method,
163 + )
164 + self.mlp = OrionMLP(
165 + hidden_size=self.hidden_size,
166 + intermediate_size=config.intermediate_size,
167 + hidden_act=config.hidden_act,
168 + linear_method=linear_method,
169 + )
170 +
171 + self.input_layernorm = nn.LayerNorm(config.hidden_size,
172 + eps=config.rms_norm_eps)
173 + self.post_attention_layernorm = nn.LayerNorm(config.hidden_size,
174 + eps=config.rms_norm_eps)
175 +
176 + def forward(
177 + self,
178 + positions: torch.Tensor,
179 + hidden_states: torch.Tensor,
180 + kv_cache: KVCache,
181 + input_metadata: InputMetadata,
182 + residual: Optional[torch.Tensor],
183 + ) -> Tuple[torch.Tensor, torch.Tensor]:
184 + # Self Attention
185 + residual = hidden_states
186 + hidden_states = self.input_layernorm(hidden_states)
187 + hidden_states = self.self_attn(
188 + positions=positions,
189 + hidden_states=hidden_states,
190 + kv_cache=kv_cache,
191 + input_metadata=input_metadata,
192 + )
193 +
194 + hidden_states = residual + hidden_states
195 +
196 + # Fully Connected
197 + residual = hidden_states
198 + hidden_states = self.post_attention_layernorm(hidden_states)
199 + hidden_states = self.mlp(hidden_states)
200 + hidden_states = residual + hidden_states
201 + return hidden_states, None
202 +
203 +
204 +class OrionModel(nn.Module):
205 +
206 + def __init__(
207 + self,
208 + config: PretrainedConfig,
209 + linear_method: Optional[LinearMethodBase] = None,
210 + ) -> None:
211 + super().__init__()
212 + self.config = config
213 + self.padding_idx = config.pad_token_id
214 + self.vocab_size = config.vocab_size
215 + self.embed_tokens = VocabParallelEmbedding(
216 + config.vocab_size,
217 + config.hidden_size,
218 + )
219 + self.layers = nn.ModuleList([
220 + OrionDecoderLayer(config, linear_method)
221 + for _ in range(config.num_hidden_layers)
222 + ])
223 + self.norm = nn.LayerNorm(config.hidden_size, eps=config.rms_norm_eps)
224 +
225 + def forward(
226 + self,
227 + input_ids: torch.Tensor,
228 + positions: torch.Tensor,
229 + kv_caches: List[KVCache],
230 + input_metadata: InputMetadata,
231 + ) -> torch.Tensor:
232 + hidden_states = self.embed_tokens(input_ids)
233 + residual = None
234 + for i in range(len(self.layers)):
235 + layer = self.layers[i]
236 + hidden_states, residual = layer(
237 + positions,
238 + hidden_states,
239 + kv_caches[i],
240 + input_metadata,
241 + residual,
242 + )
243 + hidden_states = self.norm(hidden_states)
244 + return hidden_states
245 +
246 +
247 +class OrionForCausalLM(nn.Module):
248 +
249 + def __init__(
250 + self,
251 + config: PretrainedConfig,
252 + linear_method: Optional[LinearMethodBase] = None,
253 + ) -> None:
254 + super().__init__()
255 + self.config = config
256 + self.linear_method = linear_method
257 + self.model = OrionModel(config, linear_method)
258 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
259 + self.sampler = Sampler(config.vocab_size)
260 +
261 + def forward(
262 + self,
263 + input_ids: torch.Tensor,
264 + positions: torch.Tensor,
265 + kv_caches: List[KVCache],
266 + input_metadata: InputMetadata,
267 + ) -> torch.Tensor:
268 + hidden_states = self.model(input_ids, positions, kv_caches,
269 + input_metadata)
270 + return hidden_states
271 +
272 + def sample(
273 + self,
274 + hidden_states: torch.Tensor,
275 + sampling_metadata: SamplingMetadata,
276 + ) -> Optional[SamplerOutput]:
277 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
278 + sampling_metadata)
279 + return next_tokens
280 +
281 + def load_weights(self,
282 + model_name_or_path: str,
283 + cache_dir: Optional[str] = None,
284 + load_format: str = "auto",
285 + revision: Optional[str] = None):
286 + stacked_params_mapping = [
287 + # (param_name, shard_name, shard_id)
288 + ("qkv_proj", "q_proj", "q"),
289 + ("qkv_proj", "k_proj", "k"),
290 + ("qkv_proj", "v_proj", "v"),
291 + ("gate_up_proj", "gate_proj", 0),
292 + ("gate_up_proj", "up_proj", 1),
293 + ]
294 + params_dict = dict(self.named_parameters())
295 + for name, loaded_weight in hf_model_weights_iterator(
296 + model_name_or_path, cache_dir, load_format, revision):
297 + if "rotary_emb.inv_freq" in name:
298 + continue
299 + if ("rotary_emb.cos_cached" in name
300 + or "rotary_emb.sin_cached" in name):
301 + # Models trained using ColossalAI may include these tensors in
302 + # the checkpoint. Skip them.
303 + continue
304 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
305 + if weight_name not in name:
306 + continue
307 + name = name.replace(weight_name, param_name)
308 + # Skip loading extra bias for GPTQ models.
309 + if name.endswith(".bias") and name not in params_dict:
310 + continue
311 + param = params_dict[name]
312 + weight_loader = param.weight_loader
313 + weight_loader(param, loaded_weight, shard_id)
314 + break
315 + else:
316 + # Skip loading extra bias for GPTQ models.
317 + if name.endswith(".bias") and name not in params_dict:
318 + continue
319 + param = params_dict[name]
320 + weight_loader = getattr(param, "weight_loader",
321 + default_weight_loader)
322 + weight_loader(param, loaded_weight)