@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 098e1776ba
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Model] Add support for xverse (#3610)

Co-authored-by: willhe <hexin@xverse.cn> Co-authored-by: root <root@localhost.localdomain>

hxer7963 committed Mar 28, 2024 at 09:12 UTC 098e1776ba72805bbf8f68a44b10f499413e5189
3 files changed +374
README.md
+1
@@ -91,6 +91,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
91 - Qwen2 (`Qwen/Qwen2-7B-beta`, `Qwen/Qwen-7B-Chat-beta`, etc.)
92 - StableLM(`stabilityai/stablelm-3b-4e1t`, `stabilityai/stablelm-base-alpha-7b-v2`, etc.)
93 - Starcoder2(`bigcode/starcoder2-3b`, `bigcode/starcoder2-7b`, `bigcode/starcoder2-15b`, etc.)
94 +- Xverse (`xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc.)
95 - Yi (`01-ai/Yi-6B`, `01-ai/Yi-34B`, etc.)
96
97 Install vLLM with pip or [from source](https://vllm.readthedocs.io/en/latest/getting_started/installation.html#build-from-source):
vllm/model_executor/models/__init__.py
+1
@@ -51,6 +51,7 @@ _MODELS = {
51 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
52 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
53 "Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
54 + "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
55 }
56
57 # Models not supported by ROCm.
vllm/model_executor/models/xverse.py new
+372
@@ -0,0 +1,372 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://huggingface.co/xverse/XVERSE-7B/blob/main/modeling_xverse.py
4 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
5 +#
6 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
7 +# and OPT implementations in this library. It has been modified from its
8 +# original forms to accommodate minor architectural differences compared
9 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
10 +#
11 +# Licensed under the Apache License, Version 2.0 (the "License");
12 +# you may not use this file except in compliance with the License.
13 +# You may obtain a copy of the License at
14 +#
15 +# http://www.apache.org/licenses/LICENSE-2.0
16 +#
17 +# Unless required by applicable law or agreed to in writing, software
18 +# distributed under the License is distributed on an "AS IS" BASIS,
19 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
20 +# See the License for the specific language governing permissions and
21 +# limitations under the License.
22 +"""Inference-only Xverse model compatible with HuggingFace weights."""
23 +from typing import Any, Dict, List, Optional, Tuple
24 +
25 +import torch
26 +from torch import nn
27 +from transformers import PretrainedConfig
28 +
29 +from vllm.attention import Attention, AttentionMetadata
30 +from vllm.config import LoRAConfig
31 +from vllm.model_executor.layers.activation import SiluAndMul
32 +from vllm.model_executor.layers.layernorm import RMSNorm
33 +from vllm.model_executor.layers.linear import (LinearMethodBase,
34 + MergedColumnParallelLinear,
35 + QKVParallelLinear,
36 + RowParallelLinear)
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 +from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.vocab_parallel_embedding import (
41 + ParallelLMHead, VocabParallelEmbedding)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 +from vllm.sequence import SamplerOutput
48 +
49 +
50 +class XverseMLP(nn.Module):
51 +
52 + def __init__(
53 + self,
54 + hidden_size: int,
55 + intermediate_size: int,
56 + hidden_act: str,
57 + linear_method: Optional[LinearMethodBase] = None,
58 + ) -> None:
59 + super().__init__()
60 + self.gate_up_proj = MergedColumnParallelLinear(
61 + hidden_size, [intermediate_size] * 2,
62 + bias=False,
63 + linear_method=linear_method)
64 + self.down_proj = RowParallelLinear(intermediate_size,
65 + hidden_size,
66 + bias=False,
67 + linear_method=linear_method)
68 + if hidden_act != "silu":
69 + raise ValueError(f"Unsupported activation: {hidden_act}. "
70 + "Only silu is supported for now.")
71 + self.act_fn = SiluAndMul()
72 +
73 + def forward(self, x):
74 + gate, _ = self.gate_up_proj(x)
75 + x = self.act_fn(gate)
76 + x, _ = self.down_proj(x)
77 + return x
78 +
79 +
80 +class XverseAttention(nn.Module):
81 +
82 + def __init__(
83 + self,
84 + hidden_size: int,
85 + num_heads: int,
86 + num_kv_heads: int,
87 + rope_theta: float = 10000,
88 + rope_scaling: Optional[Dict[str, Any]] = None,
89 + max_position_embeddings: int = 8192,
90 + linear_method: Optional[LinearMethodBase] = None,
91 + bias: bool = False,
92 + sliding_window: Optional[int] = None,
93 + ) -> None:
94 + super().__init__()
95 + self.hidden_size = hidden_size
96 + tp_size = get_tensor_model_parallel_world_size()
97 + self.total_num_heads = num_heads
98 + assert self.total_num_heads % tp_size == 0
99 + self.num_heads = self.total_num_heads // tp_size
100 + self.total_num_kv_heads = num_kv_heads
101 + # partition the KV heads across multiple tensor parallel GPUs.
102 + assert self.total_num_kv_heads % tp_size == 0
103 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
104 + self.head_dim = hidden_size // self.total_num_heads
105 + self.q_size = self.num_heads * self.head_dim
106 + self.kv_size = self.num_kv_heads * self.head_dim
107 + self.scaling = self.head_dim**-0.5
108 + self.rope_theta = rope_theta
109 + self.max_position_embeddings = max_position_embeddings
110 +
111 + self.qkv_proj = QKVParallelLinear(
112 + hidden_size,
113 + self.head_dim,
114 + self.total_num_heads,
115 + self.total_num_kv_heads,
116 + bias=bias,
117 + linear_method=linear_method,
118 + )
119 + self.o_proj = RowParallelLinear(
120 + self.total_num_heads * self.head_dim,
121 + hidden_size,
122 + bias=bias,
123 + linear_method=linear_method,
124 + )
125 +
126 + self.rotary_emb = get_rope(
127 + self.head_dim,
128 + rotary_dim=self.head_dim,
129 + max_position=max_position_embeddings,
130 + base=rope_theta,
131 + rope_scaling=rope_scaling,
132 + )
133 + self.attn = Attention(self.num_heads,
134 + self.head_dim,
135 + self.scaling,
136 + num_kv_heads=self.num_kv_heads,
137 + sliding_window=sliding_window)
138 +
139 + def forward(
140 + self,
141 + positions: torch.Tensor,
142 + hidden_states: torch.Tensor,
143 + kv_cache: torch.Tensor,
144 + attn_metadata: AttentionMetadata,
145 + ) -> torch.Tensor:
146 + qkv, _ = self.qkv_proj(hidden_states)
147 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
148 + q, k = self.rotary_emb(positions, q, k)
149 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
150 + output, _ = self.o_proj(attn_output)
151 + return output
152 +
153 +
154 +class XverseDecoderLayer(nn.Module):
155 +
156 + def __init__(
157 + self,
158 + config: PretrainedConfig,
159 + linear_method: Optional[LinearMethodBase] = None,
160 + ) -> None:
161 + super().__init__()
162 + self.hidden_size = config.hidden_size
163 + rope_theta = getattr(config, "rope_theta", 10000)
164 + rope_scaling = getattr(config, "rope_scaling", None)
165 + max_position_embeddings = getattr(config, "max_position_embeddings",
166 + 8192)
167 + sliding_window = getattr(config, "sliding_window", None)
168 + self.self_attn = XverseAttention(
169 + hidden_size=self.hidden_size,
170 + num_heads=config.num_attention_heads,
171 + num_kv_heads=getattr(config, "num_key_value_heads",
172 + config.num_attention_heads),
173 + rope_theta=rope_theta,
174 + rope_scaling=rope_scaling,
175 + max_position_embeddings=max_position_embeddings,
176 + linear_method=linear_method,
177 + bias=getattr(config, "bias", False),
178 + sliding_window=sliding_window,
179 + )
180 + self.mlp = XverseMLP(
181 + hidden_size=self.hidden_size,
182 + intermediate_size=config.intermediate_size,
183 + hidden_act=config.hidden_act,
184 + linear_method=linear_method,
185 + )
186 + self.input_layernorm = RMSNorm(config.hidden_size,
187 + eps=config.rms_norm_eps)
188 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
189 + eps=config.rms_norm_eps)
190 +
191 + def forward(
192 + self,
193 + positions: torch.Tensor,
194 + hidden_states: torch.Tensor,
195 + kv_cache: torch.Tensor,
196 + attn_metadata: AttentionMetadata,
197 + residual: Optional[torch.Tensor],
198 + ) -> Tuple[torch.Tensor, torch.Tensor]:
199 + # Self Attention
200 + if residual is None:
201 + residual = hidden_states
202 + hidden_states = self.input_layernorm(hidden_states)
203 + else:
204 + hidden_states, residual = self.input_layernorm(
205 + hidden_states, residual)
206 + hidden_states = self.self_attn(
207 + positions=positions,
208 + hidden_states=hidden_states,
209 + kv_cache=kv_cache,
210 + attn_metadata=attn_metadata,
211 + )
212 +
213 + # Fully Connected
214 + hidden_states, residual = self.post_attention_layernorm(
215 + hidden_states, residual)
216 + hidden_states = self.mlp(hidden_states)
217 + return hidden_states, residual
218 +
219 +
220 +class XverseModel(nn.Module):
221 +
222 + def __init__(
223 + self,
224 + config: PretrainedConfig,
225 + linear_method: Optional[LinearMethodBase] = None,
226 + lora_config: Optional[LoRAConfig] = None,
227 + ) -> None:
228 + super().__init__()
229 + self.config = config
230 + self.padding_idx = config.pad_token_id
231 + lora_vocab = (lora_config.lora_extra_vocab_size *
232 + (lora_config.max_loras or 1)) if lora_config else 0
233 + self.vocab_size = config.vocab_size + lora_vocab
234 + self.org_vocab_size = config.vocab_size
235 + self.embed_tokens = VocabParallelEmbedding(
236 + self.vocab_size,
237 + config.hidden_size,
238 + org_num_embeddings=config.vocab_size,
239 + )
240 + self.layers = nn.ModuleList([
241 + XverseDecoderLayer(config, linear_method)
242 + for _ in range(config.num_hidden_layers)
243 + ])
244 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
245 +
246 + def forward(
247 + self,
248 + input_ids: torch.Tensor,
249 + positions: torch.Tensor,
250 + kv_caches: List[torch.Tensor],
251 + attn_metadata: AttentionMetadata,
252 + ) -> torch.Tensor:
253 + hidden_states = self.embed_tokens(input_ids)
254 + residual = None
255 + for i in range(len(self.layers)):
256 + layer = self.layers[i]
257 + hidden_states, residual = layer(
258 + positions,
259 + hidden_states,
260 + kv_caches[i],
261 + attn_metadata,
262 + residual,
263 + )
264 + hidden_states, _ = self.norm(hidden_states, residual)
265 + return hidden_states
266 +
267 +
268 +class XverseForCausalLM(nn.Module):
269 + packed_modules_mapping = {
270 + "qkv_proj": [
271 + "q_proj",
272 + "k_proj",
273 + "v_proj",
274 + ],
275 + "gate_up_proj": [
276 + "gate_proj",
277 + "up_proj",
278 + ],
279 + }
280 +
281 + # LoRA specific attributes
282 + supported_lora_modules = [
283 + "qkv_proj",
284 + "o_proj",
285 + "gate_up_proj",
286 + "down_proj",
287 + "embed_tokens",
288 + "lm_head",
289 + ]
290 + embedding_modules = {
291 + "embed_tokens": "input_embeddings",
292 + "lm_head": "output_embeddings",
293 + }
294 + embedding_padding_modules = ["lm_head"]
295 +
296 + def __init__(
297 + self,
298 + config: PretrainedConfig,
299 + linear_method: Optional[LinearMethodBase] = None,
300 + lora_config=None,
301 + ) -> None:
302 + super().__init__()
303 + self.config = config
304 + self.linear_method = linear_method
305 + self.model = XverseModel(config, linear_method)
306 + self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
307 + self.logits_processor = LogitsProcessor(config.vocab_size)
308 + self.sampler = Sampler()
309 +
310 + def forward(
311 + self,
312 + input_ids: torch.Tensor,
313 + positions: torch.Tensor,
314 + kv_caches: List[torch.Tensor],
315 + attn_metadata: AttentionMetadata,
316 + ) -> torch.Tensor:
317 + hidden_states = self.model(input_ids, positions, kv_caches,
318 + attn_metadata)
319 + return hidden_states
320 +
321 + def compute_logits(self, hidden_states: torch.Tensor,
322 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
323 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
324 + sampling_metadata)
325 + return logits
326 +
327 + def sample(
328 + self,
329 + logits: torch.Tensor,
330 + sampling_metadata: SamplingMetadata,
331 + ) -> Optional[SamplerOutput]:
332 + next_tokens = self.sampler(logits, sampling_metadata)
333 + return next_tokens
334 +
335 + def load_weights(self,
336 + model_name_or_path: str,
337 + cache_dir: Optional[str] = None,
338 + load_format: str = "auto",
339 + revision: Optional[str] = None):
340 + stacked_params_mapping = [
341 + ("qkv_proj", "q_proj", "q"),
342 + ("qkv_proj", "k_proj", "k"),
343 + ("qkv_proj", "v_proj", "v"),
344 + ("gate_up_proj", "gate_proj", 0),
345 + ("gate_up_proj", "up_proj", 1),
346 + ]
347 + params_dict = dict(self.named_parameters())
348 + for name, loaded_weight in hf_model_weights_iterator(
349 + model_name_or_path, cache_dir, load_format, revision):
350 + if ("rotary_emb.inv_freq" in name
351 + or "rotary_emb.cos_cached" in name
352 + or "rotary_emb.sin_cached" in name):
353 + continue
354 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
355 + if weight_name not in name:
356 + continue
357 + name = name.replace(weight_name, param_name)
358 + # Skip loading extra bias for GPTQ models.
359 + if name.endswith(".bias") and name not in params_dict:
360 + continue
361 + param = params_dict[name]
362 + weight_loader = param.weight_loader
363 + weight_loader(param, loaded_weight, shard_id)
364 + break
365 + else:
366 + # Skip loading extra bias for GPTQ models.
367 + if name.endswith(".bias") and name not in params_dict:
368 + continue
369 + param = params_dict[name]
370 + weight_loader = getattr(param, "weight_loader",
371 + default_weight_loader)
372 + weight_loader(param, loaded_weight)