@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cd9e60c76c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add Internlm2 (#2666)

Fengzhe Zhou committed Feb 2, 2024 at 01:27 UTC cd9e60c76c776c42431b7ae523fcfe7835546d74
4 files changed +330
README.md
+1
@@ -73,6 +73,7 @@ vLLM seamlessly supports many Hugging Face models, including the following archi
73 - GPT-J (`EleutherAI/gpt-j-6b`, `nomic-ai/gpt4all-j`, etc.)
74 - GPT-NeoX (`EleutherAI/gpt-neox-20b`, `databricks/dolly-v2-12b`, `stabilityai/stablelm-tuned-alpha-7b`, etc.)
75 - InternLM (`internlm/internlm-7b`, `internlm/internlm-chat-7b`, etc.)
76 +- InternLM2 (`internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.)
77 - LLaMA & LLaMA-2 (`meta-llama/Llama-2-70b-hf`, `lmsys/vicuna-13b-v1.3`, `young-geng/koala`, `openlm-research/open_llama_13b`, etc.)
78 - Mistral (`mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc.)
79 - Mixtral (`mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, etc.)
docs/source/models/supported_models.rst
+3
@@ -47,6 +47,9 @@ Alongside each architecture, we include some popular models that use it.
47 * - :code:`InternLMForCausalLM`
48 - InternLM
49 - :code:`internlm/internlm-7b`, :code:`internlm/internlm-chat-7b`, etc.
50 + * - :code:`InternLM2ForCausalLM`
51 + - InternLM2
52 + - :code:`internlm/internlm2-7b`, :code:`internlm/internlm2-chat-7b`, etc.
53 * - :code:`LlamaForCausalLM`
54 - LLaMA, LLaMA-2, Vicuna, Alpaca, Koala, Guanaco
55 - :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`young-geng/koala`, etc.
vllm/model_executor/models/__init__.py
+1
@@ -25,6 +25,7 @@ _MODELS = {
25 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27 "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
28 + "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30 # For decapoda-research/llama-*
31 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
vllm/model_executor/models/internlm2.py new
+325
@@ -0,0 +1,325 @@
1 +# -*- coding: utf-8 -*-
2 +from typing import Any, Dict, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import PretrainedConfig
7 +
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.activation import SiluAndMul
10 +from vllm.model_executor.layers.attention import PagedAttention
11 +from vllm.model_executor.layers.layernorm import RMSNorm
12 +from vllm.model_executor.layers.linear import (LinearMethodBase,
13 + MergedColumnParallelLinear,
14 + QKVParallelLinear,
15 + RowParallelLinear)
16 +from vllm.model_executor.layers.rotary_embedding import get_rope
17 +from vllm.model_executor.layers.sampler import Sampler
18 +from vllm.model_executor.layers.vocab_parallel_embedding import (
19 + VocabParallelEmbedding, ParallelLMHead)
20 +from vllm.model_executor.parallel_utils.parallel_state import (
21 + get_tensor_model_parallel_world_size)
22 +from vllm.model_executor.sampling_metadata import SamplingMetadata
23 +from vllm.model_executor.weight_utils import (default_weight_loader,
24 + hf_model_weights_iterator)
25 +from vllm.sequence import SamplerOutput
26 +
27 +KVCache = Tuple[torch.Tensor, torch.Tensor]
28 +
29 +
30 +class InternLM2MLP(nn.Module):
31 +
32 + def __init__(
33 + self,
34 + hidden_size: int,
35 + intermediate_size: int,
36 + hidden_act: str,
37 + linear_method: Optional[LinearMethodBase] = None,
38 + ) -> None:
39 + super().__init__()
40 + self.gate_up_proj = MergedColumnParallelLinear(
41 + hidden_size, [intermediate_size] * 2,
42 + bias=False,
43 + linear_method=linear_method)
44 + self.w2 = RowParallelLinear(intermediate_size,
45 + hidden_size,
46 + bias=False,
47 + linear_method=linear_method)
48 + if hidden_act != "silu":
49 + raise ValueError(f"Unsupported activation: {hidden_act}. "
50 + "Only silu is supported for now.")
51 + self.act_fn = SiluAndMul()
52 +
53 + def forward(self, x):
54 + gate_up, _ = self.gate_up_proj(x)
55 + x = self.act_fn(gate_up)
56 + x, _ = self.w2(x)
57 + return x
58 +
59 +
60 +class InternLM2Attention(nn.Module):
61 +
62 + def __init__(
63 + self,
64 + hidden_size: int,
65 + num_heads: int,
66 + num_kv_heads: int,
67 + rope_theta: float = 10000,
68 + rope_scaling: Optional[Dict[str, Any]] = None,
69 + max_position_embeddings: int = 8192,
70 + linear_method: Optional[LinearMethodBase] = None,
71 + ) -> None:
72 + super().__init__()
73 + self.hidden_size = hidden_size
74 + tp_size = get_tensor_model_parallel_world_size()
75 + self.total_num_heads = num_heads
76 + assert self.total_num_heads % tp_size == 0
77 + self.num_heads = self.total_num_heads // tp_size
78 + self.total_num_kv_heads = num_kv_heads
79 + if self.total_num_kv_heads >= tp_size:
80 + # Number of KV heads is greater than TP size, so we partition
81 + # the KV heads across multiple tensor parallel GPUs.
82 + assert self.total_num_kv_heads % tp_size == 0
83 + else:
84 + # Number of KV heads is less than TP size, so we replicate
85 + # the KV heads across multiple tensor parallel GPUs.
86 + assert tp_size % self.total_num_kv_heads == 0
87 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
88 + self.head_dim = hidden_size // self.total_num_heads
89 + self.q_size = self.num_heads * self.head_dim
90 + self.kv_size = self.num_kv_heads * self.head_dim
91 + self.scaling = self.head_dim**-0.5
92 + self.rope_theta = rope_theta
93 + self.max_position_embeddings = max_position_embeddings
94 +
95 + self.wqkv = QKVParallelLinear(
96 + hidden_size,
97 + self.head_dim,
98 + self.total_num_heads,
99 + self.total_num_kv_heads,
100 + bias=False,
101 + linear_method=linear_method,
102 + )
103 + self.wo = RowParallelLinear(
104 + self.total_num_heads * self.head_dim,
105 + hidden_size,
106 + bias=False,
107 + linear_method=linear_method,
108 + )
109 +
110 + self.rotary_emb = get_rope(
111 + self.head_dim,
112 + rotary_dim=self.head_dim,
113 + max_position=max_position_embeddings,
114 + base=rope_theta,
115 + rope_scaling=rope_scaling,
116 + )
117 + self.attn = PagedAttention(self.num_heads,
118 + self.head_dim,
119 + self.scaling,
120 + num_kv_heads=self.num_kv_heads)
121 +
122 + def forward(
123 + self,
124 + positions: torch.Tensor,
125 + hidden_states: torch.Tensor,
126 + kv_cache: KVCache,
127 + input_metadata: InputMetadata,
128 + ) -> torch.Tensor:
129 + qkv, _ = self.wqkv(hidden_states)
130 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
131 + q, k = self.rotary_emb(positions, q, k)
132 + k_cache, v_cache = kv_cache
133 + attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
134 + output, _ = self.wo(attn_output)
135 + return output
136 +
137 +
138 +class InternLMDecoderLayer(nn.Module):
139 +
140 + def __init__(
141 + self,
142 + config: PretrainedConfig,
143 + linear_method: Optional[LinearMethodBase] = None,
144 + ) -> None:
145 + super().__init__()
146 + self.hidden_size = config.hidden_size
147 + rope_theta = getattr(config, "rope_theta", 10000)
148 + rope_scaling = getattr(config, "rope_scaling", None)
149 + max_position_embeddings = getattr(config, "max_position_embeddings",
150 + 8192)
151 + self.attention = InternLM2Attention(
152 + hidden_size=self.hidden_size,
153 + num_heads=config.num_attention_heads,
154 + num_kv_heads=config.num_key_value_heads,
155 + rope_theta=rope_theta,
156 + rope_scaling=rope_scaling,
157 + max_position_embeddings=max_position_embeddings,
158 + linear_method=linear_method,
159 + )
160 + self.feed_forward = InternLM2MLP(
161 + hidden_size=self.hidden_size,
162 + intermediate_size=config.intermediate_size,
163 + hidden_act=config.hidden_act,
164 + linear_method=linear_method,
165 + )
166 + self.attention_norm = RMSNorm(config.hidden_size,
167 + eps=config.rms_norm_eps)
168 + self.ffn_norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
169 +
170 + def forward(
171 + self,
172 + positions: torch.Tensor,
173 + hidden_states: torch.Tensor,
174 + kv_cache: KVCache,
175 + input_metadata: InputMetadata,
176 + residual: Optional[torch.Tensor],
177 + ) -> Tuple[torch.Tensor, torch.Tensor]:
178 + # Self Attention
179 + if residual is None:
180 + residual = hidden_states
181 + hidden_states = self.attention_norm(hidden_states)
182 + else:
183 + hidden_states, residual = self.attention_norm(
184 + hidden_states, residual)
185 + hidden_states = self.attention(
186 + positions=positions,
187 + hidden_states=hidden_states,
188 + kv_cache=kv_cache,
189 + input_metadata=input_metadata,
190 + )
191 +
192 + # Fully Connected
193 + hidden_states, residual = self.ffn_norm(hidden_states, residual)
194 + hidden_states = self.feed_forward(hidden_states)
195 + return hidden_states, residual
196 +
197 +
198 +class InternLM2Model(nn.Module):
199 +
200 + def __init__(
201 + self,
202 + config: PretrainedConfig,
203 + linear_method: Optional[LinearMethodBase] = None,
204 + ) -> None:
205 + super().__init__()
206 + self.config = config
207 + self.padding_idx = config.pad_token_id
208 + self.vocab_size = config.vocab_size
209 + self.tok_embeddings = VocabParallelEmbedding(
210 + config.vocab_size,
211 + config.hidden_size,
212 + )
213 + self.layers = nn.ModuleList([
214 + InternLMDecoderLayer(config, linear_method)
215 + for _ in range(config.num_hidden_layers)
216 + ])
217 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
218 +
219 + def forward(
220 + self,
221 + input_ids: torch.Tensor,
222 + positions: torch.Tensor,
223 + kv_caches: List[KVCache],
224 + input_metadata: InputMetadata,
225 + ) -> torch.Tensor:
226 + hidden_states = self.tok_embeddings(input_ids)
227 + residual = None
228 + for i in range(len(self.layers)):
229 + layer = self.layers[i]
230 + hidden_states, residual = layer(
231 + positions,
232 + hidden_states,
233 + kv_caches[i],
234 + input_metadata,
235 + residual,
236 + )
237 + hidden_states, _ = self.norm(hidden_states, residual)
238 + return hidden_states
239 +
240 +
241 +class InternLM2ForCausalLM(nn.Module):
242 +
243 + def __init__(
244 + self,
245 + config: PretrainedConfig,
246 + linear_method: Optional[LinearMethodBase] = None,
247 + ) -> None:
248 + super().__init__()
249 + self.config = config
250 + self.linear_method = linear_method
251 + self.model = InternLM2Model(config, linear_method)
252 + self.output = ParallelLMHead(config.vocab_size, config.hidden_size)
253 + self.sampler = Sampler(config.vocab_size)
254 +
255 + def forward(
256 + self,
257 + input_ids: torch.Tensor,
258 + positions: torch.Tensor,
259 + kv_caches: List[KVCache],
260 + input_metadata: InputMetadata,
261 + ) -> torch.Tensor:
262 + hidden_states = self.model(input_ids, positions, kv_caches,
263 + input_metadata)
264 + return hidden_states
265 +
266 + def sample(
267 + self,
268 + hidden_states: torch.Tensor,
269 + sampling_metadata: SamplingMetadata,
270 + ) -> Optional[SamplerOutput]:
271 + next_tokens = self.sampler(self.output.weight, hidden_states,
272 + sampling_metadata)
273 + return next_tokens
274 +
275 + def load_weights(self,
276 + model_name_or_path: str,
277 + cache_dir: Optional[str] = None,
278 + load_format: str = "auto",
279 + revision: Optional[str] = None):
280 + stacked_params_mapping = [
281 + # (param_name, shard_name, shard_id)
282 + ("gate_up_proj", "w1", 0),
283 + ("gate_up_proj", "w3", 1),
284 + ]
285 + params_dict = dict(self.named_parameters())
286 + for name, loaded_weight in hf_model_weights_iterator(
287 + model_name_or_path, cache_dir, load_format, revision):
288 + if "rotary_emb.inv_freq" in name:
289 + continue
290 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
291 + if weight_name not in name:
292 + continue
293 + name = name.replace(weight_name, param_name)
294 + # Skip loading extra bias for GPTQ models.
295 + if name.endswith(".bias") and name not in params_dict:
296 + continue
297 + param = params_dict[name]
298 + weight_loader = param.weight_loader
299 + weight_loader(param, loaded_weight, shard_id)
300 + break
301 + else:
302 + # Skip loading extra bias for GPTQ models.
303 + if name.endswith(".bias") and name not in params_dict:
304 + continue
305 + param = params_dict[name]
306 + if "wqkv" in name:
307 + config = self.config
308 + kv_groups = config.num_attention_heads // config.num_key_value_heads
309 + head_dim = config.hidden_size // config.num_attention_heads
310 + loaded_weight = loaded_weight.view(-1, 2 + kv_groups,
311 + head_dim,
312 + loaded_weight.shape[-1])
313 + wq, wk, wv = torch.split(loaded_weight, [kv_groups, 1, 1],
314 + dim=1)
315 + wq = wq.reshape(-1, wq.shape[-1])
316 + wk = wk.reshape(-1, wk.shape[-1])
317 + wv = wv.reshape(-1, wv.shape[-1])
318 + weight_loader = param.weight_loader
319 + weight_loader(param, wq, 'q')
320 + weight_loader(param, wk, 'k')
321 + weight_loader(param, wv, 'v')
322 + else:
323 + weight_loader = getattr(param, "weight_loader",
324 + default_weight_loader)
325 + weight_loader(param, loaded_weight)