@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 735ecfff61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

add internlm model (#528)

Jia Guoqing committed Aug 9, 2023 at 07:35 UTC 735ecfff61f62b78bd1455c65e8e27dc36febce1
3 files changed +314 -4
vllm/model_executor/model_loader.py
+1
@@ -19,6 +19,7 @@ _MODEL_REGISTRY = {
19 "GPTBigCodeForCausalLM": GPTBigCodeForCausalLM,
20 "GPTJForCausalLM": GPTJForCausalLM,
21 "GPTNeoXForCausalLM": GPTNeoXForCausalLM,
22 + "InternLMForCausalLM": InternLMForCausalLM,
23 "LlamaForCausalLM": LlamaForCausalLM,
24 "LLaMAForCausalLM": LlamaForCausalLM, # For decapoda-research/llama-*
25 "MPTForCausalLM": MPTForCausalLM,
vllm/model_executor/models/__init__.py
+14 -4
@@ -6,14 +6,24 @@ from vllm.model_executor.models.gpt2 import GPT2LMHeadModel
6 from vllm.model_executor.models.gpt_bigcode import GPTBigCodeForCausalLM
7 from vllm.model_executor.models.gpt_j import GPTJForCausalLM
8 from vllm.model_executor.models.gpt_neox import GPTNeoXForCausalLM
9 +from vllm.model_executor.models.internlm import InternLMForCausalLM
10 from vllm.model_executor.models.llama import LlamaForCausalLM
11 from vllm.model_executor.models.mpt import MPTForCausalLM
12 from vllm.model_executor.models.opt import OPTForCausalLM
13 from vllm.model_executor.models.qwen import QWenLMHeadModel
14
15 __all__ = [
15 - "BaiChuanForCausalLM", "BaichuanForCausalLM", "BloomForCausalLM",
16 - "FalconForCausalLM", "GPT2LMHeadModel", "GPTBigCodeForCausalLM",
17 - "GPTJForCausalLM", "GPTNeoXForCausalLM", "LlamaForCausalLM",
18 - "MPTForCausalLM", "OPTForCausalLM", "QWenLMHeadModel"
16 + "BaiChuanForCausalLM",
17 + "BaichuanForCausalLM",
18 + "BloomForCausalLM",
19 + "FalconForCausalLM",
20 + "GPT2LMHeadModel",
21 + "GPTBigCodeForCausalLM",
22 + "GPTJForCausalLM",
23 + "GPTNeoXForCausalLM",
24 + "InternLMForCausalLM",
25 + "LlamaForCausalLM",
26 + "MPTForCausalLM",
27 + "OPTForCausalLM",
28 + "QWenLMHeadModel",
29 ]
vllm/model_executor/models/internlm.py new
+299
@@ -0,0 +1,299 @@
1 +# -*- coding: utf-8 -*-
2 +from typing import Dict, List, Optional, Tuple
3 +
4 +import torch
5 +from torch import nn
6 +from transformers import LlamaConfig
7 +
8 +from vllm.model_executor.input_metadata import InputMetadata
9 +from vllm.model_executor.layers.activation import SiluAndMul
10 +from vllm.model_executor.layers.layernorm import RMSNorm
11 +from vllm.model_executor.layers.attention import PagedAttentionWithRoPE
12 +from vllm.model_executor.layers.sampler import Sampler
13 +from vllm.model_executor.weight_utils import (hf_model_weights_iterator,
14 + load_tensor_parallel_weights)
15 +from vllm.model_executor.parallel_utils.parallel_state import (
16 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
17 +from vllm.model_executor.parallel_utils.tensor_parallel import (
18 + VocabParallelEmbedding, ColumnParallelLinear, RowParallelLinear)
19 +from vllm.sequence import SequenceOutputs
20 +
21 +KVCache = Tuple[torch.Tensor, torch.Tensor]
22 +
23 +
24 +class InternLMMLP(nn.Module):
25 +
26 + def __init__(
27 + self,
28 + hidden_size: int,
29 + intermediate_size: int,
30 + hidden_act: str,
31 + ):
32 + super().__init__()
33 + self.gate_up_proj = ColumnParallelLinear(hidden_size,
34 + 2 * intermediate_size,
35 + bias=True,
36 + gather_output=False,
37 + perform_initialization=False)
38 + self.down_proj = RowParallelLinear(intermediate_size,
39 + hidden_size,
40 + bias=True,
41 + input_is_parallel=True,
42 + perform_initialization=False)
43 + if hidden_act != "silu":
44 + raise ValueError(f"Unsupported activation: {hidden_act}. "
45 + "Only silu is supported for now.")
46 + self.act_fn = SiluAndMul()
47 +
48 + def forward(self, x):
49 + gate_up, _ = self.gate_up_proj(x)
50 + x = self.act_fn(gate_up)
51 + x, _ = self.down_proj(x)
52 + return x
53 +
54 +
55 +class InternLMAttention(nn.Module):
56 +
57 + def __init__(
58 + self,
59 + hidden_size: int,
60 + num_heads: int,
61 + ):
62 + super().__init__()
63 + self.hidden_size = hidden_size
64 + tensor_model_parallel_world_size = (
65 + get_tensor_model_parallel_world_size())
66 + self.total_num_heads = num_heads
67 + assert self.total_num_heads % tensor_model_parallel_world_size == 0
68 + self.num_heads = (self.total_num_heads //
69 + tensor_model_parallel_world_size)
70 + self.head_dim = hidden_size // self.total_num_heads
71 + self.scaling = self.head_dim**-0.5
72 +
73 + self.qkv_proj = ColumnParallelLinear(
74 + hidden_size,
75 + 3 * self.total_num_heads * self.head_dim,
76 + bias=True,
77 + gather_output=False,
78 + perform_initialization=False,
79 + )
80 + self.o_proj = RowParallelLinear(
81 + self.total_num_heads * self.head_dim,
82 + hidden_size,
83 + bias=True,
84 + input_is_parallel=True,
85 + perform_initialization=False,
86 + )
87 + self.attn = PagedAttentionWithRoPE(self.num_heads,
88 + self.head_dim,
89 + self.scaling,
90 + rotary_dim=self.head_dim)
91 +
92 + def forward(
93 + self,
94 + positions: torch.Tensor,
95 + hidden_states: torch.Tensor,
96 + kv_cache: KVCache,
97 + input_metadata: InputMetadata,
98 + cache_event: Optional[torch.cuda.Event],
99 + ) -> torch.Tensor:
100 + qkv, _ = self.qkv_proj(hidden_states)
101 + q, k, v = qkv.chunk(chunks=3, dim=-1)
102 + k_cache, v_cache = kv_cache
103 + attn_output = self.attn(positions, q, k, v, k_cache, v_cache,
104 + input_metadata, cache_event)
105 + output, _ = self.o_proj(attn_output)
106 + return output
107 +
108 +
109 +class InternLMDecoderLayer(nn.Module):
110 +
111 + def __init__(self, config: LlamaConfig):
112 + super().__init__()
113 + self.hidden_size = config.hidden_size
114 + self.self_attn = InternLMAttention(
115 + hidden_size=self.hidden_size,
116 + num_heads=config.num_attention_heads,
117 + )
118 + self.mlp = InternLMMLP(
119 + hidden_size=self.hidden_size,
120 + intermediate_size=config.intermediate_size,
121 + hidden_act=config.hidden_act,
122 + )
123 + self.input_layernorm = RMSNorm(config.hidden_size,
124 + eps=config.rms_norm_eps)
125 + self.post_attention_layernorm = RMSNorm(config.hidden_size,
126 + eps=config.rms_norm_eps)
127 +
128 + def forward(
129 + self,
130 + positions: torch.Tensor,
131 + hidden_states: torch.Tensor,
132 + kv_cache: KVCache,
133 + input_metadata: InputMetadata,
134 + cache_event: Optional[torch.cuda.Event],
135 + ) -> torch.Tensor:
136 + # Self Attention
137 + residual = hidden_states
138 + hidden_states = self.input_layernorm(hidden_states)
139 + hidden_states = self.self_attn(
140 + positions=positions,
141 + hidden_states=hidden_states,
142 + kv_cache=kv_cache,
143 + input_metadata=input_metadata,
144 + cache_event=cache_event,
145 + )
146 + hidden_states = residual + hidden_states
147 +
148 + # Fully Connected
149 + residual = hidden_states
150 + hidden_states = self.post_attention_layernorm(hidden_states)
151 + hidden_states = self.mlp(hidden_states)
152 + hidden_states = residual + hidden_states
153 + return hidden_states
154 +
155 +
156 +class InternLMModel(nn.Module):
157 +
158 + def __init__(self, config: LlamaConfig):
159 + super().__init__()
160 + self.config = config
161 + self.padding_idx = config.pad_token_id
162 + self.vocab_size = config.vocab_size
163 +
164 + vocab_size = ((config.vocab_size + 63) // 64) * 64
165 + self.embed_tokens = VocabParallelEmbedding(
166 + vocab_size, config.hidden_size, perform_initialization=False)
167 + self.layers = nn.ModuleList([
168 + InternLMDecoderLayer(config)
169 + for _ in range(config.num_hidden_layers)
170 + ])
171 + self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
172 +
173 + def forward(
174 + self,
175 + input_ids: torch.Tensor,
176 + positions: torch.Tensor,
177 + kv_caches: List[KVCache],
178 + input_metadata: InputMetadata,
179 + cache_events: Optional[List[torch.cuda.Event]],
180 + ) -> torch.Tensor:
181 + hidden_states = self.embed_tokens(input_ids)
182 + for i in range(len(self.layers)):
183 + if cache_events is None:
184 + cache_event = None
185 + else:
186 + cache_event = cache_events[i]
187 + layer = self.layers[i]
188 + hidden_states = layer(
189 + positions,
190 + hidden_states,
191 + kv_caches[i],
192 + input_metadata,
193 + cache_event,
194 + )
195 + hidden_states = self.norm(hidden_states)
196 + return hidden_states
197 +
198 +
199 +class InternLMForCausalLM(nn.Module):
200 +
201 + def __init__(self, config):
202 + super().__init__()
203 + self.config = config
204 + self.model = InternLMModel(config)
205 + vocab_size = ((config.vocab_size + 63) // 64) * 64
206 + self.lm_head = ColumnParallelLinear(config.hidden_size,
207 + vocab_size,
208 + bias=False,
209 + gather_output=False,
210 + perform_initialization=False)
211 + self.sampler = Sampler(config.vocab_size)
212 +
213 + def forward(
214 + self,
215 + input_ids: torch.Tensor,
216 + positions: torch.Tensor,
217 + kv_caches: List[KVCache],
218 + input_metadata: InputMetadata,
219 + cache_events: Optional[List[torch.cuda.Event]],
220 + ) -> Dict[int, SequenceOutputs]:
221 + hidden_states = self.model(input_ids, positions, kv_caches,
222 + input_metadata, cache_events)
223 + next_tokens = self.sampler(self.lm_head.weight, hidden_states,
224 + input_metadata)
225 + return next_tokens
226 +
227 + _column_parallel_weights = [
228 + "embed_tokens.weight", "lm_head.weight", "qkv_proj.weight",
229 + "gate_proj.weight", "up_proj.weight"
230 + ]
231 + _row_parallel_weights = ["o_proj.weight", "down_proj.weight"]
232 +
233 + def load_weights(self,
234 + model_name_or_path: str,
235 + cache_dir: Optional[str] = None,
236 + use_np_cache: bool = False):
237 + tensor_model_parallel_world_size = (
238 + get_tensor_model_parallel_world_size())
239 + tensor_model_parallel_rank = get_tensor_model_parallel_rank()
240 + state_dict = self.state_dict()
241 +
242 + for name, loaded_weight in hf_model_weights_iterator(
243 + model_name_or_path, cache_dir, use_np_cache):
244 + if "rotary_emb.inv_freq" in name:
245 + continue
246 +
247 + if "embed_tokens" in name or "lm_head" in name:
248 + param = state_dict[name]
249 + # Consider padding in the vocab size.
250 + padded_vocab_size = (param.shape[0] *
251 + tensor_model_parallel_world_size)
252 + num_extra_rows = padded_vocab_size - self.config.vocab_size
253 + extra_rows = torch.empty(num_extra_rows,
254 + loaded_weight.shape[1])
255 + extra_rows = extra_rows.to(loaded_weight)
256 + loaded_weight = torch.cat([loaded_weight, extra_rows], dim=0)
257 +
258 + is_attention_weight = False
259 + for stride_id, att_weight_name in enumerate(
260 + ["q_proj", "k_proj", "v_proj"]):
261 + if att_weight_name not in name:
262 + continue
263 + param = state_dict[name.replace(att_weight_name, "qkv_proj")]
264 + shard_size = param.shape[0] // 3
265 + loaded_weight = loaded_weight[
266 + shard_size * tensor_model_parallel_rank:shard_size *
267 + (tensor_model_parallel_rank + 1)]
268 + param_slice = param.data[shard_size * stride_id:shard_size *
269 + (stride_id + 1)]
270 + assert param_slice.shape == loaded_weight.shape
271 + param_slice.copy_(loaded_weight)
272 + is_attention_weight = True
273 + break
274 + if is_attention_weight:
275 + continue
276 +
277 + is_gate_up_weight = False
278 + for stride_id, weight_name in enumerate(["gate_proj", "up_proj"]):
279 + if weight_name not in name:
280 + continue
281 + param = state_dict[name.replace(weight_name, "gate_up_proj")]
282 + shard_size = param.shape[0] // 2
283 + loaded_weight = loaded_weight[
284 + shard_size * tensor_model_parallel_rank:shard_size *
285 + (tensor_model_parallel_rank + 1)]
286 + param_slice = param.data[shard_size * stride_id:shard_size *
287 + (stride_id + 1)]
288 + assert param_slice.shape == loaded_weight.shape
289 + param_slice.copy_(loaded_weight)
290 + is_gate_up_weight = True
291 + break
292 + if is_gate_up_weight:
293 + continue
294 +
295 + param = state_dict[name]
296 + load_tensor_parallel_weights(param, loaded_weight, name,
297 + self._column_parallel_weights,
298 + self._row_parallel_weights,
299 + tensor_model_parallel_rank)