A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Model] Add support for xverse (#3610)
Co-authored-by: willhe <hexin@xverse.cn> Co-authored-by: root <root@localhost.localdomain>
hxer7963 committed
Mar 28, 2024 at 09:12 UTC
098e1776ba72805bbf8f68a44b10f499413e5189
3 files changed
+374
README.md
+1
@@ -91,6 +91,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
91
- Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
92
- StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
93
- Starcoder2(`bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc.)
94
+- Xverse (`xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc.)
95
- Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
96
97
Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
vllm/model_executor/models/__init__.py
+1
@@ -51,6 +51,7 @@ _MODELS = {
51
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
52
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
53
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
54
+ "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
55
}
56
57
# Models not supported by ROCm.
vllm/model_executor/models/xverse.py
new
+372
@@ -0,0 +1,372 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py
4
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5
+#
6
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7
+# and OPT implementations in this library. It has been modified from its
8
+# original forms to accommodate minor architectural differences compared
9
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10
+#
11
+# Licensed under the Apache License, Version 2.0 (the "License");
12
+# you may not use this file except in compliance with the License.
13
+# You may obtain a copy of the License at
14
+#
15
+# http://www.apache.org/licenses/LICENSE-2.0
16
+#
17
+# Unless required by applicable law or agreed to in writing, software
18
+# distributed under the License is distributed on an "AS IS" BASIS,
19
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20
+# See the License for the specific language governing permissions and
21
+# limitations under the License.
22
+"""Inference-only Xverse model compatible with HuggingFace weights."""
23
+from typing import Any, Dict, List, Optional, Tuple
24
+
25
+import torch
26
+from torch import nn
27
+from transformers import PretrainedConfig
28
+
29
+from vllm.attention import Attention, AttentionMetadata
30
+from vllm.config import LoRAConfig
31
+from vllm.model_executor.layers.activation import SiluAndMul
32
+from vllm.model_executor.layers.layernorm import RMSNorm
33
+from vllm.model_executor.layers.linear import (LinearMethodBase,
34
+ MergedColumnParallelLinear,
35
+ QKVParallelLinear,
36
+ RowParallelLinear)
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
+from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.vocab_parallel_embedding import (
41
+ ParallelLMHead, VocabParallelEmbedding)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
+from vllm.sequence import SamplerOutput
48
+
49
+
50
+class XverseMLP(nn.Module):
51
+
52
+ def __init__(
53
+ self,
54
+ hidden_size: int,
55
+ intermediate_size: int,
56
+ hidden_act: str,
57
+ linear_method: Optional[LinearMethodBase] = None,
58
+ ) -> None:
59
+ super().__init__()
60
+ self.gate_up_proj = MergedColumnParallelLinear(
61
+ hidden_size, [intermediate_size] * 2,
62
+ bias=False,
63
+ linear_method=linear_method)
64
+ self.down_proj = RowParallelLinear(intermediate_size,
65
+ hidden_size,
66
+ bias=False,
67
+ linear_method=linear_method)
68
+ if hidden_act != "silu":
69
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
70
+ "Only silu is supported for now.")
71
+ self.act_fn = SiluAndMul()
72
+
73
+ def forward(self, x):
74
+ gate, _ = self.gate_up_proj(x)
75
+ x = self.act_fn(gate)
76
+ x, _ = self.down_proj(x)
77
+ return x
78
+
79
+
80
+class XverseAttention(nn.Module):
81
+
82
+ def __init__(
83
+ self,
84
+ hidden_size: int,
85
+ num_heads: int,
86
+ num_kv_heads: int,
87
+ rope_theta: float = 10000,
88
+ rope_scaling: Optional[Dict[str, Any]] = None,
89
+ max_position_embeddings: int = 8192,
90
+ linear_method: Optional[LinearMethodBase] = None,
91
+ bias: bool = False,
92
+ sliding_window: Optional[int] = None,
93
+ ) -> None:
94
+ super().__init__()
95
+ self.hidden_size = hidden_size
96
+ tp_size = get_tensor_model_parallel_world_size()
97
+ self.total_num_heads = num_heads
98
+ assert self.total_num_heads % tp_size == 0
99
+ self.num_heads = self.total_num_heads // tp_size
100
+ self.total_num_kv_heads = num_kv_heads
101
+ # partition the KV heads across multiple tensor parallel GPUs.
102
+ assert self.total_num_kv_heads % tp_size == 0
103
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
104
+ self.head_dim = hidden_size // self.total_num_heads
105
+ self.q_size = self.num_heads * self.head_dim
106
+ self.kv_size = self.num_kv_heads * self.head_dim
107
+ self.scaling = self.head_dim**-0.5
108
+ self.rope_theta = rope_theta
109
+ self.max_position_embeddings = max_position_embeddings
110
+
111
+ self.qkv_proj = QKVParallelLinear(
112
+ hidden_size,
113
+ self.head_dim,
114
+ self.total_num_heads,
115
+ self.total_num_kv_heads,
116
+ bias=bias,
117
+ linear_method=linear_method,
118
+ )
119
+ self.o_proj = RowParallelLinear(
120
+ self.total_num_heads * self.head_dim,
121
+ hidden_size,
122
+ bias=bias,
123
+ linear_method=linear_method,
124
+ )
125
+
126
+ self.rotary_emb = get_rope(
127
+ self.head_dim,
128
+ rotary_dim=self.head_dim,
129
+ max_position=max_position_embeddings,
130
+ base=rope_theta,
131
+ rope_scaling=rope_scaling,
132
+ )
133
+ self.attn = Attention(self.num_heads,
134
+ self.head_dim,
135
+ self.scaling,
136
+ num_kv_heads=self.num_kv_heads,
137
+ sliding_window=sliding_window)
138
+
139
+ def forward(
140
+ self,
141
+ positions: torch.Tensor,
142
+ hidden_states: torch.Tensor,
143
+ kv_cache: torch.Tensor,
144
+ attn_metadata: AttentionMetadata,
145
+ ) -> torch.Tensor:
146
+ qkv, _ = self.qkv_proj(hidden_states)
147
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
148
+ q, k = self.rotary_emb(positions, q, k)
149
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
150
+ output, _ = self.o_proj(attn_output)
151
+ return output
152
+
153
+
154
+class XverseDecoderLayer(nn.Module):
155
+
156
+ def __init__(
157
+ self,
158
+ config: PretrainedConfig,
159
+ linear_method: Optional[LinearMethodBase] = None,
160
+ ) -> None:
161
+ super().__init__()
162
+ self.hidden_size = config.hidden_size
163
+ rope_theta = getattr(config, "rope_theta", 10000)
164
+ rope_scaling = getattr(config, "rope_scaling", None)
165
+ max_position_embeddings = getattr(config, "max_position_embeddings",
166
+ 8192)
167
+ sliding_window = getattr(config, "sliding_window", None)
168
+ self.self_attn = XverseAttention(
169
+ hidden_size=self.hidden_size,
170
+ num_heads=config.num_attention_heads,
171
+ num_kv_heads=getattr(config, "num_key_value_heads",
172
+ config.num_attention_heads),
173
+ rope_theta=rope_theta,
174
+ rope_scaling=rope_scaling,
175
+ max_position_embeddings=max_position_embeddings,
176
+ linear_method=linear_method,
177
+ bias=getattr(config, "bias", False),
178
+ sliding_window=sliding_window,
179
+ )
180
+ self.mlp = XverseMLP(
181
+ hidden_size=self.hidden_size,
182
+ intermediate_size=config.intermediate_size,
183
+ hidden_act=config.hidden_act,
184
+ linear_method=linear_method,
185
+ )
186
+ self.input_layernorm = RMSNorm(config.hidden_size,
187
+ eps=config.rms_norm_eps)
188
+ self.post_attention_layernorm = RMSNorm(config.hidden_size,
189
+ eps=config.rms_norm_eps)
190
+
191
+ def forward(
192
+ self,
193
+ positions: torch.Tensor,
194
+ hidden_states: torch.Tensor,
195
+ kv_cache: torch.Tensor,
196
+ attn_metadata: AttentionMetadata,
197
+ residual: Optional[torch.Tensor],
198
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
199
+ # Self Attention
200
+ if residual is None:
201
+ residual = hidden_states
202
+ hidden_states = self.input_layernorm(hidden_states)
203
+ else:
204
+ hidden_states, residual = self.input_layernorm(
205
+ hidden_states, residual)
206
+ hidden_states = self.self_attn(
207
+ positions=positions,
208
+ hidden_states=hidden_states,
209
+ kv_cache=kv_cache,
210
+ attn_metadata=attn_metadata,
211
+ )
212
+
213
+ # Fully Connected
214
+ hidden_states, residual = self.post_attention_layernorm(
215
+ hidden_states, residual)
216
+ hidden_states = self.mlp(hidden_states)
217
+ return hidden_states, residual
218
+
219
+
220
+class XverseModel(nn.Module):
221
+
222
+ def __init__(
223
+ self,
224
+ config: PretrainedConfig,
225
+ linear_method: Optional[LinearMethodBase] = None,
226
+ lora_config: Optional[LoRAConfig] = None,
227
+ ) -> None:
228
+ super().__init__()
229
+ self.config = config
230
+ self.padding_idx = config.pad_token_id
231
+ lora_vocab = (lora_config.lora_extra_vocab_size *
232
+ (lora_config.max_loras or 1)) if lora_config else 0
233
+ self.vocab_size = config.vocab_size + lora_vocab
234
+ self.org_vocab_size = config.vocab_size
235
+ self.embed_tokens = VocabParallelEmbedding(
236
+ self.vocab_size,
237
+ config.hidden_size,
238
+ org_num_embeddings=config.vocab_size,
239
+ )
240
+ self.layers = nn.ModuleList([
241
+ XverseDecoderLayer(config, linear_method)
242
+ for _ in range(config.num_hidden_layers)
243
+ ])
244
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245
+
246
+ def forward(
247
+ self,
248
+ input_ids: torch.Tensor,
249
+ positions: torch.Tensor,
250
+ kv_caches: List[torch.Tensor],
251
+ attn_metadata: AttentionMetadata,
252
+ ) -> torch.Tensor:
253
+ hidden_states = self.embed_tokens(input_ids)
254
+ residual = None
255
+ for i in range(len(self.layers)):
256
+ layer = self.layers[i]
257
+ hidden_states, residual = layer(
258
+ positions,
259
+ hidden_states,
260
+ kv_caches[i],
261
+ attn_metadata,
262
+ residual,
263
+ )
264
+ hidden_states, _ = self.norm(hidden_states, residual)
265
+ return hidden_states
266
+
267
+
268
+class XverseForCausalLM(nn.Module):
269
+ packed_modules_mapping = {
270
+ "qkv_proj": [
271
+ "q_proj",
272
+ "k_proj",
273
+ "v_proj",
274
+ ],
275
+ "gate_up_proj": [
276
+ "gate_proj",
277
+ "up_proj",
278
+ ],
279
+ }
280
+
281
+ # LoRA specific attributes
282
+ supported_lora_modules = [
283
+ "qkv_proj",
284
+ "o_proj",
285
+ "gate_up_proj",
286
+ "down_proj",
287
+ "embed_tokens",
288
+ "lm_head",
289
+ ]
290
+ embedding_modules = {
291
+ "embed_tokens": "input_embeddings",
292
+ "lm_head": "output_embeddings",
293
+ }
294
+ embedding_padding_modules = ["lm_head"]
295
+
296
+ def __init__(
297
+ self,
298
+ config: PretrainedConfig,
299
+ linear_method: Optional[LinearMethodBase] = None,
300
+ lora_config=None,
301
+ ) -> None:
302
+ super().__init__()
303
+ self.config = config
304
+ self.linear_method = linear_method
305
+ self.model = XverseModel(config, linear_method)
306
+ self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
307
+ self.logits_processor = LogitsProcessor(config.vocab_size)
308
+ self.sampler = Sampler()
309
+
310
+ def forward(
311
+ self,
312
+ input_ids: torch.Tensor,
313
+ positions: torch.Tensor,
314
+ kv_caches: List[torch.Tensor],
315
+ attn_metadata: AttentionMetadata,
316
+ ) -> torch.Tensor:
317
+ hidden_states = self.model(input_ids, positions, kv_caches,
318
+ attn_metadata)
319
+ return hidden_states
320
+
321
+ def compute_logits(self, hidden_states: torch.Tensor,
322
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
323
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
324
+ sampling_metadata)
325
+ return logits
326
+
327
+ def sample(
328
+ self,
329
+ logits: torch.Tensor,
330
+ sampling_metadata: SamplingMetadata,
331
+ ) -> Optional[SamplerOutput]:
332
+ next_tokens = self.sampler(logits, sampling_metadata)
333
+ return next_tokens
334
+
335
+ def load_weights(self,
336
+ model_name_or_path: str,
337
+ cache_dir: Optional[str] = None,
338
+ load_format: str = "auto",
339
+ revision: Optional[str] = None):
340
+ stacked_params_mapping = [
341
+ ("qkv_proj", "q_proj", "q"),
342
+ ("qkv_proj", "k_proj", "k"),
343
+ ("qkv_proj", "v_proj", "v"),
344
+ ("gate_up_proj", "gate_proj", 0),
345
+ ("gate_up_proj", "up_proj", 1),
346
+ ]
347
+ params_dict = dict(self.named_parameters())
348
+ for name, loaded_weight in hf_model_weights_iterator(
349
+ model_name_or_path, cache_dir, load_format, revision):
350
+ if ("rotary_emb.inv_freq" in name
351
+ or "rotary_emb.cos_cached" in name
352
+ or "rotary_emb.sin_cached" in name):
353
+ continue
354
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
355
+ if weight_name not in name:
356
+ continue
357
+ name = name.replace(weight_name, param_name)
358
+ # Skip loading extra bias for GPTQ models.
359
+ if name.endswith(".bias") and name not in params_dict:
360
+ continue
361
+ param = params_dict[name]
362
+ weight_loader = param.weight_loader
363
+ weight_loader(param, loaded_weight, shard_id)
364
+ break
365
+ else:
366
+ # Skip loading extra bias for GPTQ models.
367
+ if name.endswith(".bias") and name not in params_dict:
368
+ continue
369
+ param = params_dict[name]
370
+ weight_loader = getattr(param, "weight_loader",
371
+ default_weight_loader)
372
+ weight_loader(param, loaded_weight)