@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7eacffd951
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Migrate InternLMForCausalLM to LlamaForCausalLM (#2860)

Co-authored-by: Roy <jasonailu87@gmail.com>

Philipp Moritz committed Feb 13, 2024 at 17:12 UTC 7eacffd9512c29bfcce0963b5a19da0cd66cc22f
3 files changed +5 -302
vllm/model_executor/models/__init__.py
+1 -1
@@ -24,7 +24,7 @@ _MODELS = {
24 "GPTBigCodeForCausalLM": ("gpt_bigcode", "GPTBigCodeForCausalLM"),
25 "GPTJForCausalLM": ("gpt_j", "GPTJForCausalLM"),
26 "GPTNeoXForCausalLM": ("gpt_neox", "GPTNeoXForCausalLM"),
27 - "InternLMForCausalLM": ("internlm", "InternLMForCausalLM"),
27 + "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
28 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
29 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
30 # For decapoda-research/llama-*
vllm/model_executor/models/internlm.py deleted
-299
@@ -1,299 +0,0 @@
1 -# -*- coding: utf-8 -*-
2 -from typing import Any, Dict, List, Optional, Tuple
3 -
4 -import torch
5 -from torch import nn
6 -from transformers import LlamaConfig
7 -
8 -from vllm.model_executor.input_metadata import InputMetadata
9 -from vllm.model_executor.layers.activation import SiluAndMul
10 -from vllm.model_executor.layers.attention import PagedAttention
11 -from vllm.model_executor.layers.layernorm import RMSNorm
12 -from vllm.model_executor.layers.linear import (LinearMethodBase,
13 - MergedColumnParallelLinear,
14 - QKVParallelLinear,
15 - RowParallelLinear)
16 -from vllm.model_executor.layers.rotary_embedding import get_rope
17 -from vllm.model_executor.layers.sampler import Sampler
18 -from vllm.model_executor.layers.vocab_parallel_embedding import (
19 - VocabParallelEmbedding, ParallelLMHead)
20 -from vllm.model_executor.parallel_utils.parallel_state import (
21 - get_tensor_model_parallel_world_size)
22 -from vllm.model_executor.sampling_metadata import SamplingMetadata
23 -from vllm.model_executor.weight_utils import (default_weight_loader,
24 - hf_model_weights_iterator)
25 -from vllm.sequence import SamplerOutput
26 -
27 -KVCache = Tuple[torch.Tensor, torch.Tensor]
28 -
29 -
30 -class InternLMMLP(nn.Module):
31 -
32 - def __init__(
33 - self,
34 - hidden_size: int,
35 - intermediate_size: int,
36 - hidden_act: str,
37 - linear_method: Optional[LinearMethodBase] = None,
38 - ):
39 - super().__init__()
40 - self.gate_up_proj = MergedColumnParallelLinear(
41 - hidden_size, [intermediate_size] * 2,
42 - bias=False,
43 - linear_method=linear_method)
44 - self.down_proj = RowParallelLinear(intermediate_size,
45 - hidden_size,
46 - bias=False,
47 - linear_method=linear_method)
48 - if hidden_act != "silu":
49 - raise ValueError(f"Unsupported activation: {hidden_act}. "
50 - "Only silu is supported for now.")
51 - self.act_fn = SiluAndMul()
52 -
53 - def forward(self, x):
54 - gate_up, _ = self.gate_up_proj(x)
55 - x = self.act_fn(gate_up)
56 - x, _ = self.down_proj(x)
57 - return x
58 -
59 -
60 -class InternLMAttention(nn.Module):
61 -
62 - def __init__(
63 - self,
64 - hidden_size: int,
65 - num_heads: int,
66 - bias: bool,
67 - rope_theta: float = 10000,
68 - max_position_embeddings: int = 8192,
69 - linear_method: Optional[LinearMethodBase] = None,
70 - rope_scaling: Optional[Dict[str, Any]] = None,
71 - ):
72 - super().__init__()
73 - self.hidden_size = hidden_size
74 - tensor_model_parallel_world_size = (
75 - get_tensor_model_parallel_world_size())
76 - self.total_num_heads = num_heads
77 - assert self.total_num_heads % tensor_model_parallel_world_size == 0
78 - self.num_heads = (self.total_num_heads //
79 - tensor_model_parallel_world_size)
80 - self.head_dim = hidden_size // self.total_num_heads
81 - self.scaling = self.head_dim**-0.5
82 - self.rope_theta = rope_theta
83 - self.max_position_embeddings = max_position_embeddings
84 -
85 - self.qkv_proj = QKVParallelLinear(
86 - hidden_size,
87 - self.head_dim,
88 - self.total_num_heads,
89 - bias=bias,
90 - linear_method=linear_method,
91 - )
92 - self.o_proj = RowParallelLinear(
93 - self.total_num_heads * self.head_dim,
94 - hidden_size,
95 - bias=bias,
96 - linear_method=linear_method,
97 - )
98 - self.rotary_emb = get_rope(
99 - self.head_dim,
100 - rotary_dim=self.head_dim,
101 - max_position=self.max_position_embeddings,
102 - base=self.rope_theta,
103 - rope_scaling=rope_scaling,
104 - )
105 - self.attn = PagedAttention(self.num_heads, self.head_dim, self.scaling)
106 -
107 - def forward(
108 - self,
109 - positions: torch.Tensor,
110 - hidden_states: torch.Tensor,
111 - kv_cache: KVCache,
112 - input_metadata: InputMetadata,
113 - ) -> torch.Tensor:
114 - qkv, _ = self.qkv_proj(hidden_states)
115 - q, k, v = qkv.chunk(chunks=3, dim=-1)
116 - q, k = self.rotary_emb(positions, q, k)
117 - k_cache, v_cache = kv_cache
118 - attn_output = self.attn(q, k, v, k_cache, v_cache, input_metadata)
119 - output, _ = self.o_proj(attn_output)
120 - return output
121 -
122 -
123 -class InternLMDecoderLayer(nn.Module):
124 -
125 - def __init__(
126 - self,
127 - config: LlamaConfig,
128 - linear_method: Optional[LinearMethodBase] = None,
129 - ):
130 - super().__init__()
131 - self.hidden_size = config.hidden_size
132 - rope_theta = getattr(config, "rope_theta", 10000)
133 - max_position_embeddings = getattr(config, "max_position_embeddings",
134 - 8192)
135 - self.self_attn = InternLMAttention(
136 - hidden_size=self.hidden_size,
137 - num_heads=config.num_attention_heads,
138 - bias=config.bias,
139 - rope_theta=rope_theta,
140 - max_position_embeddings=max_position_embeddings,
141 - linear_method=linear_method,
142 - rope_scaling=getattr(config, "rope_scaling", None),
143 - )
144 - self.mlp = InternLMMLP(
145 - hidden_size=self.hidden_size,
146 - intermediate_size=config.intermediate_size,
147 - hidden_act=config.hidden_act,
148 - linear_method=linear_method,
149 - )
150 - self.input_layernorm = RMSNorm(config.hidden_size,
151 - eps=config.rms_norm_eps)
152 - self.post_attention_layernorm = RMSNorm(config.hidden_size,
153 - eps=config.rms_norm_eps)
154 -
155 - def forward(
156 - self,
157 - positions: torch.Tensor,
158 - hidden_states: torch.Tensor,
159 - kv_cache: KVCache,
160 - input_metadata: InputMetadata,
161 - residual: Optional[torch.Tensor],
162 - ) -> Tuple[torch.Tensor, torch.Tensor]:
163 - # Self Attention
164 - if residual is None:
165 - residual = hidden_states
166 - hidden_states = self.input_layernorm(hidden_states)
167 - else:
168 - hidden_states, residual = self.input_layernorm(
169 - hidden_states, residual)
170 - hidden_states = self.self_attn(
171 - positions=positions,
172 - hidden_states=hidden_states,
173 - kv_cache=kv_cache,
174 - input_metadata=input_metadata,
175 - )
176 -
177 - # Fully Connected
178 - hidden_states, residual = self.post_attention_layernorm(
179 - hidden_states, residual)
180 - hidden_states = self.mlp(hidden_states)
181 - return hidden_states, residual
182 -
183 -
184 -class InternLMModel(nn.Module):
185 -
186 - def __init__(
187 - self,
188 - config: LlamaConfig,
189 - linear_method: Optional[LinearMethodBase] = None,
190 - ):
191 - super().__init__()
192 - self.config = config
193 - self.padding_idx = config.pad_token_id
194 - self.vocab_size = config.vocab_size
195 -
196 - vocab_size = ((config.vocab_size + 63) // 64) * 64
197 - self.embed_tokens = VocabParallelEmbedding(
198 - vocab_size,
199 - config.hidden_size,
200 - )
201 - self.layers = nn.ModuleList([
202 - InternLMDecoderLayer(config, linear_method)
203 - for _ in range(config.num_hidden_layers)
204 - ])
205 - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
206 -
207 - def forward(
208 - self,
209 - input_ids: torch.Tensor,
210 - positions: torch.Tensor,
211 - kv_caches: List[KVCache],
212 - input_metadata: InputMetadata,
213 - ) -> torch.Tensor:
214 - hidden_states = self.embed_tokens(input_ids)
215 - residual = None
216 - for i in range(len(self.layers)):
217 - layer = self.layers[i]
218 - hidden_states, residual = layer(
219 - positions,
220 - hidden_states,
221 - kv_caches[i],
222 - input_metadata,
223 - residual,
224 - )
225 - hidden_states, _ = self.norm(hidden_states, residual)
226 - return hidden_states
227 -
228 -
229 -class InternLMForCausalLM(nn.Module):
230 -
231 - def __init__(
232 - self,
233 - config,
234 - linear_method: Optional[LinearMethodBase] = None,
235 - ):
236 - super().__init__()
237 - self.config = config
238 - self.linear_method = linear_method
239 - self.model = InternLMModel(config, linear_method)
240 - self.lm_head = ParallelLMHead(config.vocab_size, config.hidden_size)
241 - self.sampler = Sampler(config.vocab_size)
242 -
243 - def forward(
244 - self,
245 - input_ids: torch.Tensor,
246 - positions: torch.Tensor,
247 - kv_caches: List[KVCache],
248 - input_metadata: InputMetadata,
249 - ) -> torch.Tensor:
250 - hidden_states = self.model(input_ids, positions, kv_caches,
251 - input_metadata)
252 - return hidden_states
253 -
254 - def sample(
255 - self,
256 - hidden_states: torch.Tensor,
257 - sampling_metadata: SamplingMetadata,
258 - ) -> Optional[SamplerOutput]:
259 - next_tokens = self.sampler(self.lm_head.weight, hidden_states,
260 - sampling_metadata)
261 - return next_tokens
262 -
263 - def load_weights(self,
264 - model_name_or_path: str,
265 - cache_dir: Optional[str] = None,
266 - load_format: str = "auto",
267 - revision: Optional[str] = None):
268 - stacked_params_mapping = [
269 - # (param_name, shard_name, shard_id)
270 - ("qkv_proj", "q_proj", "q"),
271 - ("qkv_proj", "k_proj", "k"),
272 - ("qkv_proj", "v_proj", "v"),
273 - ("gate_up_proj", "gate_proj", 0),
274 - ("gate_up_proj", "up_proj", 1),
275 - ]
276 - params_dict = dict(self.named_parameters())
277 - for name, loaded_weight in hf_model_weights_iterator(
278 - model_name_or_path, cache_dir, load_format, revision):
279 - if "rotary_emb.inv_freq" in name:
280 - continue
281 - for (param_name, weight_name, shard_id) in stacked_params_mapping:
282 - if weight_name not in name:
283 - continue
284 - name = name.replace(weight_name, param_name)
285 - # Skip loading extra bias for GPTQ models.
286 - if name.endswith(".bias") and name not in params_dict:
287 - continue
288 - param = params_dict[name]
289 - weight_loader = param.weight_loader
290 - weight_loader(param, loaded_weight, shard_id)
291 - break
292 - else:
293 - # Skip loading extra bias for GPTQ models.
294 - if name.endswith(".bias") and name not in params_dict:
295 - continue
296 - param = params_dict[name]
297 - weight_loader = getattr(param, "weight_loader",
298 - default_weight_loader)
299 - weight_loader(param, loaded_weight)
vllm/model_executor/models/llama.py
+4 -2
@@ -91,6 +91,7 @@ class LlamaAttention(nn.Module):
91 rope_scaling: Optional[Dict[str, Any]] = None,
92 max_position_embeddings: int = 8192,
93 linear_method: Optional[LinearMethodBase] = None,
94 + bias: bool = False,
95 ) -> None:
96 super().__init__()
97 self.hidden_size = hidden_size
@@ -120,13 +121,13 @@ class LlamaAttention(nn.Module):
121 self.head_dim,
122 self.total_num_heads,
123 self.total_num_kv_heads,
123 - bias=False,
124 + bias=bias,
125 linear_method=linear_method,
126 )
127 self.o_proj = RowParallelLinear(
128 self.total_num_heads * self.head_dim,
129 hidden_size,
129 - bias=False,
130 + bias=bias,
131 linear_method=linear_method,
132 )
133
@@ -179,6 +180,7 @@ class LlamaDecoderLayer(nn.Module):
180 rope_scaling=rope_scaling,
181 max_position_embeddings=max_position_embeddings,
182 linear_method=linear_method,
183 + bias=getattr(config, "bias", False),
184 )
185 self.mlp = LlamaMLP(
186 hidden_size=self.hidden_size,