A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Add support for Cohere's Command-R model (#3433)
Co-authored-by: José Maria Pombal <jose.pombal@unbabel.com> Co-authored-by: youkaichao <youkaichao@gmail.com>
zeppombal committed
Mar 27, 2024 at 21:19 UTC
1182607e182f139e24a298ec3a3a5863f7ba0ace
2 files changed
+338
vllm/model_executor/models/__init__.py
+1
@@ -17,6 +17,7 @@ _MODELS = {
17
"BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18
"ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19
"ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20
+ "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
21
"DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
22
"DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23
"DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
vllm/model_executor/models/commandr.py
new
+337
@@ -0,0 +1,337 @@
1
+# coding=utf-8
2
+# Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved.
3
+#
4
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5
+# and OPT implementations in this library. It has been modified from its
6
+# original forms to accommodate minor architectural differences compared
7
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8
+#
9
+# Licensed under the Apache License, Version 2.0 (the "License");
10
+# you may not use this file except in compliance with the License.
11
+# You may obtain a copy of the License at
12
+#
13
+# http://www.apache.org/licenses/LICENSE-2.0
14
+#
15
+# Unless required by applicable law or agreed to in writing, software
16
+# distributed under the License is distributed on an "AS IS" BASIS,
17
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18
+# See the License for the specific language governing permissions and
19
+# limitations under the License.
20
+
21
+# This file is based on the LLama model definition file in transformers
22
+"""PyTorch Cohere model."""
23
+from typing import List, Optional, Tuple
24
+
25
+import torch
26
+import torch.utils.checkpoint
27
+from torch import nn
28
+from transformers import CohereConfig
29
+from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
30
+
31
+from vllm.attention import Attention, AttentionMetadata
32
+from vllm.model_executor.layers.activation import SiluAndMul
33
+from vllm.model_executor.layers.linear import (LinearMethodBase,
34
+ MergedColumnParallelLinear,
35
+ QKVParallelLinear,
36
+ RowParallelLinear)
37
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
38
+from vllm.model_executor.layers.rotary_embedding import get_rope
39
+from vllm.model_executor.layers.sampler import Sampler
40
+from vllm.model_executor.layers.vocab_parallel_embedding import (
41
+ VocabParallelEmbedding)
42
+from vllm.model_executor.parallel_utils.parallel_state import (
43
+ get_tensor_model_parallel_world_size)
44
+from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.weight_utils import (default_weight_loader,
46
+ hf_model_weights_iterator)
47
+from vllm.sequence import SamplerOutput
48
+
49
+KVCache = Tuple[torch.Tensor, torch.Tensor]
50
+
51
+
52
+class LayerNorm(nn.Module):
53
+
54
+ def __init__(self, hidden_size, eps=1e-5, bias=False):
55
+ super().__init__()
56
+ self.weight = nn.Parameter(torch.ones(hidden_size))
57
+ self.bias = nn.Parameter(torch.zeros(hidden_size)) if bias else None
58
+ self.variance_epsilon = eps
59
+
60
+ def forward(self, hidden_states, residuals=None):
61
+ input_dtype = hidden_states.dtype
62
+ hidden_states = hidden_states.to(torch.float32)
63
+ mean = hidden_states.mean(-1, keepdim=True)
64
+ variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True)
65
+ hidden_states = (hidden_states -
66
+ mean) * torch.rsqrt(variance + self.variance_epsilon)
67
+ hidden_states = self.weight.to(torch.float32) * hidden_states
68
+ if self.bias is not None:
69
+ hidden_states = hidden_states + self.bias.to(torch.float32)
70
+ return hidden_states.to(input_dtype), residuals
71
+
72
+
73
+ALL_LAYERNORM_LAYERS.append(LayerNorm)
74
+
75
+
76
+# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
77
+class CohereMLP(nn.Module):
78
+
79
+ def __init__(
80
+ self,
81
+ config,
82
+ linear_method: Optional[LinearMethodBase] = None,
83
+ ):
84
+ super().__init__()
85
+ self.config = config
86
+ self.hidden_size = config.hidden_size
87
+ self.intermediate_size = config.intermediate_size
88
+ self.gate_up_proj = MergedColumnParallelLinear(
89
+ self.hidden_size,
90
+ [self.intermediate_size] * 2,
91
+ bias=False,
92
+ linear_method=linear_method,
93
+ )
94
+ self.down_proj = RowParallelLinear(
95
+ self.intermediate_size,
96
+ self.hidden_size,
97
+ bias=False,
98
+ linear_method=linear_method,
99
+ )
100
+ self.act_fn = SiluAndMul()
101
+
102
+ def forward(self, x):
103
+ gate_up, _ = self.gate_up_proj(x)
104
+ x = self.act_fn(gate_up)
105
+ x, _ = self.down_proj(x)
106
+ return x
107
+
108
+
109
+class CohereAttention(nn.Module):
110
+
111
+ def __init__(
112
+ self,
113
+ config: CohereConfig,
114
+ linear_method: Optional[LinearMethodBase] = None,
115
+ ):
116
+ super().__init__()
117
+ tp_size = get_tensor_model_parallel_world_size()
118
+ self.config = config
119
+ self.attention_dropout = config.attention_dropout
120
+ self.hidden_size = config.hidden_size
121
+ self.total_num_heads = config.num_attention_heads
122
+ self.num_heads = self.total_num_heads // tp_size
123
+ self.head_dim = self.hidden_size // self.total_num_heads
124
+ self.total_num_kv_heads = config.num_key_value_heads
125
+ if self.total_num_kv_heads >= tp_size:
126
+ # Number of KV heads is greater than TP size, so we partition
127
+ # the KV heads across multiple tensor parallel GPUs.
128
+ assert self.total_num_kv_heads % tp_size == 0
129
+ else:
130
+ # Number of KV heads is less than TP size, so we replicate
131
+ # the KV heads across multiple tensor parallel GPUs.
132
+ assert tp_size % self.total_num_kv_heads == 0
133
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
134
+ self.q_size = self.num_heads * self.head_dim
135
+ self.kv_size = self.num_kv_heads * self.head_dim
136
+ self.scaling = self.head_dim**-0.5
137
+ self.max_position_embeddings = config.max_position_embeddings
138
+ self.rope_theta = config.rope_theta
139
+ self.rope_scaling = getattr(config, "rope_scaling", None)
140
+ self.is_causal = True
141
+ self.qkv_proj = QKVParallelLinear(
142
+ self.hidden_size,
143
+ self.head_dim,
144
+ self.total_num_heads,
145
+ self.total_num_kv_heads,
146
+ bias=False,
147
+ linear_method=linear_method,
148
+ )
149
+ self.o_proj = RowParallelLinear(
150
+ self.total_num_heads * self.head_dim,
151
+ self.hidden_size,
152
+ bias=False,
153
+ linear_method=linear_method,
154
+ )
155
+ self.rotary_emb = get_rope(
156
+ self.head_dim,
157
+ rotary_dim=self.head_dim,
158
+ max_position=self.max_position_embeddings,
159
+ base=self.rope_theta,
160
+ rope_scaling=self.rope_scaling,
161
+ is_neox_style=False,
162
+ )
163
+ self.attn = Attention(
164
+ self.num_heads,
165
+ self.head_dim,
166
+ self.scaling,
167
+ num_kv_heads=self.num_kv_heads,
168
+ )
169
+
170
+ def forward(
171
+ self,
172
+ positions: torch.Tensor,
173
+ hidden_states: torch.Tensor,
174
+ kv_cache: KVCache,
175
+ attn_metadata: AttentionMetadata,
176
+ ) -> torch.Tensor:
177
+ qkv, _ = self.qkv_proj(hidden_states)
178
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
179
+ q, k = self.rotary_emb(positions, q, k)
180
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
181
+ output, _ = self.o_proj(attn_output)
182
+ return output
183
+
184
+
185
+class CohereDecoderLayer(nn.Module):
186
+
187
+ def __init__(self,
188
+ config: CohereConfig,
189
+ linear_method: Optional[LinearMethodBase] = None):
190
+ super().__init__()
191
+ self.hidden_size = config.hidden_size
192
+
193
+ self.self_attn = CohereAttention(config, linear_method=linear_method)
194
+
195
+ self.mlp = CohereMLP(config, linear_method=linear_method)
196
+ self.input_layernorm = LayerNorm(config.hidden_size,
197
+ eps=config.layer_norm_eps)
198
+
199
+ def forward(
200
+ self,
201
+ positions: torch.Tensor,
202
+ hidden_states: torch.Tensor,
203
+ kv_cache: KVCache,
204
+ attn_metadata: AttentionMetadata,
205
+ residual: Optional[torch.Tensor],
206
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
207
+ # Self Attention
208
+ residual = hidden_states
209
+ hidden_states, residual = self.input_layernorm(hidden_states, residual)
210
+ hidden_states_attention = self.self_attn(
211
+ positions=positions,
212
+ hidden_states=hidden_states,
213
+ kv_cache=kv_cache,
214
+ attn_metadata=attn_metadata,
215
+ )
216
+ hidden_states_mlp = self.mlp(hidden_states)
217
+ # Add everything together
218
+ hidden_states = residual + hidden_states_attention + hidden_states_mlp
219
+
220
+ return hidden_states, residual
221
+
222
+
223
+class CohereModel(nn.Module):
224
+
225
+ def __init__(
226
+ self,
227
+ config: CohereConfig,
228
+ linear_method: Optional[LinearMethodBase] = None,
229
+ ):
230
+ super().__init__()
231
+ self.config = config
232
+ self.vocab_size = config.vocab_size
233
+ self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
234
+ config.hidden_size)
235
+ self.layers = nn.ModuleList([
236
+ CohereDecoderLayer(config, linear_method=linear_method)
237
+ for _ in range(config.num_hidden_layers)
238
+ ])
239
+ self.norm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
240
+
241
+ def forward(
242
+ self,
243
+ input_ids: torch.Tensor,
244
+ positions: torch.Tensor,
245
+ kv_caches: List[KVCache],
246
+ attn_metadata: AttentionMetadata,
247
+ ) -> torch.Tensor:
248
+ hidden_states = self.embed_tokens(input_ids)
249
+ residual = None
250
+ for i in range(len(self.layers)):
251
+ layer = self.layers[i]
252
+ hidden_states, residual = layer(
253
+ positions,
254
+ hidden_states,
255
+ kv_caches[i],
256
+ attn_metadata,
257
+ residual,
258
+ )
259
+ hidden_states, _ = self.norm(hidden_states, residual)
260
+ return hidden_states
261
+
262
+
263
+class CohereForCausalLM(nn.Module):
264
+
265
+ def __init__(
266
+ self,
267
+ config: CohereConfig,
268
+ linear_method: Optional[LinearMethodBase] = None,
269
+ ) -> None:
270
+ super().__init__()
271
+ self.config = config
272
+ self.unpadded_vocab_size = config.vocab_size
273
+ self.linear_method = linear_method
274
+ self.logits_processor = LogitsProcessor(config.vocab_size,
275
+ scale=config.logit_scale)
276
+ self.model = CohereModel(config, linear_method)
277
+ self.sampler = Sampler()
278
+
279
+ @torch.no_grad()
280
+ def forward(
281
+ self,
282
+ input_ids: torch.Tensor,
283
+ positions: torch.Tensor,
284
+ kv_caches: List[KVCache],
285
+ attn_metadata: AttentionMetadata,
286
+ ) -> torch.Tensor:
287
+ hidden_states = self.model(input_ids, positions, kv_caches,
288
+ attn_metadata)
289
+ return hidden_states
290
+
291
+ def compute_logits(self, hidden_states: torch.Tensor,
292
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
293
+ logits = self.logits_processor(self.model.embed_tokens.weight,
294
+ hidden_states, sampling_metadata)
295
+ return logits
296
+
297
+ def sample(
298
+ self,
299
+ logits: torch.Tensor,
300
+ sampling_metadata: SamplingMetadata,
301
+ ) -> Optional[SamplerOutput]:
302
+ next_tokens = self.sampler(logits, sampling_metadata)
303
+ return next_tokens
304
+
305
+ def load_weights(
306
+ self,
307
+ model_name_or_path: str,
308
+ cache_dir: Optional[str] = None,
309
+ load_format: str = "auto",
310
+ revision: Optional[str] = None,
311
+ ):
312
+ stacked_params_mapping = [
313
+ # (param_name, shard_name, shard_id)
314
+ ("qkv_proj", "q_proj", "q"),
315
+ ("qkv_proj", "k_proj", "k"),
316
+ ("qkv_proj", "v_proj", "v"),
317
+ ("gate_up_proj", "gate_proj", 0),
318
+ ("gate_up_proj", "up_proj", 1),
319
+ ]
320
+ params_dict = dict(self.named_parameters())
321
+ loaded_params = set()
322
+ for name, loaded_weight in hf_model_weights_iterator(
323
+ model_name_or_path, cache_dir, load_format, revision):
324
+ for param_name, shard_name, shard_id in stacked_params_mapping:
325
+ if shard_name not in name:
326
+ continue
327
+ name = name.replace(shard_name, param_name)
328
+ param = params_dict[name]
329
+ weight_loader = param.weight_loader
330
+ weight_loader(param, loaded_weight, shard_id)
331
+ break
332
+ else:
333
+ param = params_dict[name]
334
+ weight_loader = getattr(param, "weight_loader",
335
+ default_weight_loader)
336
+ weight_loader(param, loaded_weight)
337
+ loaded_params.add(name)