@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1182607e18
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Add support for Cohere's Command-R model (#3433)

Co-authored-by: José Maria Pombal <jose.pombal@unbabel.com> Co-authored-by: youkaichao <youkaichao@gmail.com>

zeppombal committed Mar 27, 2024 at 21:19 UTC 1182607e182f139e24a298ec3a3a5863f7ba0ace
2 files changed +338
vllm/model_executor/models/__init__.py
+1
@@ -17,6 +17,7 @@ _MODELS = {
17 "BloomForCausalLM": ("bloom", "BloomForCausalLM"),
18 "ChatGLMModel": ("chatglm", "ChatGLMForCausalLM"),
19 "ChatGLMForConditionalGeneration": ("chatglm", "ChatGLMForCausalLM"),
20 + "CohereForCausalLM": ("commandr", "CohereForCausalLM"),
21 "DbrxForCausalLM": ("dbrx", "DbrxForCausalLM"),
22 "DeciLMForCausalLM": ("decilm", "DeciLMForCausalLM"),
23 "DeepseekForCausalLM": ("deepseek", "DeepseekForCausalLM"),
vllm/model_executor/models/commandr.py new
+337
@@ -0,0 +1,337 @@
1 +# coding=utf-8
2 +# Copyright 2024 Cohere and the HuggingFace Inc. team. All rights reserved.
3 +#
4 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
5 +# and OPT implementations in this library. It has been modified from its
6 +# original forms to accommodate minor architectural differences compared
7 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
8 +#
9 +# Licensed under the Apache License, Version 2.0 (the "License");
10 +# you may not use this file except in compliance with the License.
11 +# You may obtain a copy of the License at
12 +#
13 +# http://www.apache.org/licenses/LICENSE-2.0
14 +#
15 +# Unless required by applicable law or agreed to in writing, software
16 +# distributed under the License is distributed on an "AS IS" BASIS,
17 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 +# See the License for the specific language governing permissions and
19 +# limitations under the License.
20 +
21 +# This file is based on the LLama model definition file in transformers
22 +"""PyTorch Cohere model."""
23 +from typing import List, Optional, Tuple
24 +
25 +import torch
26 +import torch.utils.checkpoint
27 +from torch import nn
28 +from transformers import CohereConfig
29 +from transformers.pytorch_utils import ALL_LAYERNORM_LAYERS
30 +
31 +from vllm.attention import Attention, AttentionMetadata
32 +from vllm.model_executor.layers.activation import SiluAndMul
33 +from vllm.model_executor.layers.linear import (LinearMethodBase,
34 + MergedColumnParallelLinear,
35 + QKVParallelLinear,
36 + RowParallelLinear)
37 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
38 +from vllm.model_executor.layers.rotary_embedding import get_rope
39 +from vllm.model_executor.layers.sampler import Sampler
40 +from vllm.model_executor.layers.vocab_parallel_embedding import (
41 + VocabParallelEmbedding)
42 +from vllm.model_executor.parallel_utils.parallel_state import (
43 + get_tensor_model_parallel_world_size)
44 +from vllm.model_executor.sampling_metadata import SamplingMetadata
45 +from vllm.model_executor.weight_utils import (default_weight_loader,
46 + hf_model_weights_iterator)
47 +from vllm.sequence import SamplerOutput
48 +
49 +KVCache = Tuple[torch.Tensor, torch.Tensor]
50 +
51 +
52 +class LayerNorm(nn.Module):
53 +
54 + def __init__(self, hidden_size, eps=1e-5, bias=False):
55 + super().__init__()
56 + self.weight = nn.Parameter(torch.ones(hidden_size))
57 + self.bias = nn.Parameter(torch.zeros(hidden_size)) if bias else None
58 + self.variance_epsilon = eps
59 +
60 + def forward(self, hidden_states, residuals=None):
61 + input_dtype = hidden_states.dtype
62 + hidden_states = hidden_states.to(torch.float32)
63 + mean = hidden_states.mean(-1, keepdim=True)
64 + variance = (hidden_states - mean).pow(2).mean(-1, keepdim=True)
65 + hidden_states = (hidden_states -
66 + mean) * torch.rsqrt(variance + self.variance_epsilon)
67 + hidden_states = self.weight.to(torch.float32) * hidden_states
68 + if self.bias is not None:
69 + hidden_states = hidden_states + self.bias.to(torch.float32)
70 + return hidden_states.to(input_dtype), residuals
71 +
72 +
73 +ALL_LAYERNORM_LAYERS.append(LayerNorm)
74 +
75 +
76 +# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
77 +class CohereMLP(nn.Module):
78 +
79 + def __init__(
80 + self,
81 + config,
82 + linear_method: Optional[LinearMethodBase] = None,
83 + ):
84 + super().__init__()
85 + self.config = config
86 + self.hidden_size = config.hidden_size
87 + self.intermediate_size = config.intermediate_size
88 + self.gate_up_proj = MergedColumnParallelLinear(
89 + self.hidden_size,
90 + [self.intermediate_size] * 2,
91 + bias=False,
92 + linear_method=linear_method,
93 + )
94 + self.down_proj = RowParallelLinear(
95 + self.intermediate_size,
96 + self.hidden_size,
97 + bias=False,
98 + linear_method=linear_method,
99 + )
100 + self.act_fn = SiluAndMul()
101 +
102 + def forward(self, x):
103 + gate_up, _ = self.gate_up_proj(x)
104 + x = self.act_fn(gate_up)
105 + x, _ = self.down_proj(x)
106 + return x
107 +
108 +
109 +class CohereAttention(nn.Module):
110 +
111 + def __init__(
112 + self,
113 + config: CohereConfig,
114 + linear_method: Optional[LinearMethodBase] = None,
115 + ):
116 + super().__init__()
117 + tp_size = get_tensor_model_parallel_world_size()
118 + self.config = config
119 + self.attention_dropout = config.attention_dropout
120 + self.hidden_size = config.hidden_size
121 + self.total_num_heads = config.num_attention_heads
122 + self.num_heads = self.total_num_heads // tp_size
123 + self.head_dim = self.hidden_size // self.total_num_heads
124 + self.total_num_kv_heads = config.num_key_value_heads
125 + if self.total_num_kv_heads >= tp_size:
126 + # Number of KV heads is greater than TP size, so we partition
127 + # the KV heads across multiple tensor parallel GPUs.
128 + assert self.total_num_kv_heads % tp_size == 0
129 + else:
130 + # Number of KV heads is less than TP size, so we replicate
131 + # the KV heads across multiple tensor parallel GPUs.
132 + assert tp_size % self.total_num_kv_heads == 0
133 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
134 + self.q_size = self.num_heads * self.head_dim
135 + self.kv_size = self.num_kv_heads * self.head_dim
136 + self.scaling = self.head_dim**-0.5
137 + self.max_position_embeddings = config.max_position_embeddings
138 + self.rope_theta = config.rope_theta
139 + self.rope_scaling = getattr(config, "rope_scaling", None)
140 + self.is_causal = True
141 + self.qkv_proj = QKVParallelLinear(
142 + self.hidden_size,
143 + self.head_dim,
144 + self.total_num_heads,
145 + self.total_num_kv_heads,
146 + bias=False,
147 + linear_method=linear_method,
148 + )
149 + self.o_proj = RowParallelLinear(
150 + self.total_num_heads * self.head_dim,
151 + self.hidden_size,
152 + bias=False,
153 + linear_method=linear_method,
154 + )
155 + self.rotary_emb = get_rope(
156 + self.head_dim,
157 + rotary_dim=self.head_dim,
158 + max_position=self.max_position_embeddings,
159 + base=self.rope_theta,
160 + rope_scaling=self.rope_scaling,
161 + is_neox_style=False,
162 + )
163 + self.attn = Attention(
164 + self.num_heads,
165 + self.head_dim,
166 + self.scaling,
167 + num_kv_heads=self.num_kv_heads,
168 + )
169 +
170 + def forward(
171 + self,
172 + positions: torch.Tensor,
173 + hidden_states: torch.Tensor,
174 + kv_cache: KVCache,
175 + attn_metadata: AttentionMetadata,
176 + ) -> torch.Tensor:
177 + qkv, _ = self.qkv_proj(hidden_states)
178 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
179 + q, k = self.rotary_emb(positions, q, k)
180 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
181 + output, _ = self.o_proj(attn_output)
182 + return output
183 +
184 +
185 +class CohereDecoderLayer(nn.Module):
186 +
187 + def __init__(self,
188 + config: CohereConfig,
189 + linear_method: Optional[LinearMethodBase] = None):
190 + super().__init__()
191 + self.hidden_size = config.hidden_size
192 +
193 + self.self_attn = CohereAttention(config, linear_method=linear_method)
194 +
195 + self.mlp = CohereMLP(config, linear_method=linear_method)
196 + self.input_layernorm = LayerNorm(config.hidden_size,
197 + eps=config.layer_norm_eps)
198 +
199 + def forward(
200 + self,
201 + positions: torch.Tensor,
202 + hidden_states: torch.Tensor,
203 + kv_cache: KVCache,
204 + attn_metadata: AttentionMetadata,
205 + residual: Optional[torch.Tensor],
206 + ) -> Tuple[torch.Tensor, torch.Tensor]:
207 + # Self Attention
208 + residual = hidden_states
209 + hidden_states, residual = self.input_layernorm(hidden_states, residual)
210 + hidden_states_attention = self.self_attn(
211 + positions=positions,
212 + hidden_states=hidden_states,
213 + kv_cache=kv_cache,
214 + attn_metadata=attn_metadata,
215 + )
216 + hidden_states_mlp = self.mlp(hidden_states)
217 + # Add everything together
218 + hidden_states = residual + hidden_states_attention + hidden_states_mlp
219 +
220 + return hidden_states, residual
221 +
222 +
223 +class CohereModel(nn.Module):
224 +
225 + def __init__(
226 + self,
227 + config: CohereConfig,
228 + linear_method: Optional[LinearMethodBase] = None,
229 + ):
230 + super().__init__()
231 + self.config = config
232 + self.vocab_size = config.vocab_size
233 + self.embed_tokens = VocabParallelEmbedding(config.vocab_size,
234 + config.hidden_size)
235 + self.layers = nn.ModuleList([
236 + CohereDecoderLayer(config, linear_method=linear_method)
237 + for _ in range(config.num_hidden_layers)
238 + ])
239 + self.norm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
240 +
241 + def forward(
242 + self,
243 + input_ids: torch.Tensor,
244 + positions: torch.Tensor,
245 + kv_caches: List[KVCache],
246 + attn_metadata: AttentionMetadata,
247 + ) -> torch.Tensor:
248 + hidden_states = self.embed_tokens(input_ids)
249 + residual = None
250 + for i in range(len(self.layers)):
251 + layer = self.layers[i]
252 + hidden_states, residual = layer(
253 + positions,
254 + hidden_states,
255 + kv_caches[i],
256 + attn_metadata,
257 + residual,
258 + )
259 + hidden_states, _ = self.norm(hidden_states, residual)
260 + return hidden_states
261 +
262 +
263 +class CohereForCausalLM(nn.Module):
264 +
265 + def __init__(
266 + self,
267 + config: CohereConfig,
268 + linear_method: Optional[LinearMethodBase] = None,
269 + ) -> None:
270 + super().__init__()
271 + self.config = config
272 + self.unpadded_vocab_size = config.vocab_size
273 + self.linear_method = linear_method
274 + self.logits_processor = LogitsProcessor(config.vocab_size,
275 + scale=config.logit_scale)
276 + self.model = CohereModel(config, linear_method)
277 + self.sampler = Sampler()
278 +
279 + @torch.no_grad()
280 + def forward(
281 + self,
282 + input_ids: torch.Tensor,
283 + positions: torch.Tensor,
284 + kv_caches: List[KVCache],
285 + attn_metadata: AttentionMetadata,
286 + ) -> torch.Tensor:
287 + hidden_states = self.model(input_ids, positions, kv_caches,
288 + attn_metadata)
289 + return hidden_states
290 +
291 + def compute_logits(self, hidden_states: torch.Tensor,
292 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
293 + logits = self.logits_processor(self.model.embed_tokens.weight,
294 + hidden_states, sampling_metadata)
295 + return logits
296 +
297 + def sample(
298 + self,
299 + logits: torch.Tensor,
300 + sampling_metadata: SamplingMetadata,
301 + ) -> Optional[SamplerOutput]:
302 + next_tokens = self.sampler(logits, sampling_metadata)
303 + return next_tokens
304 +
305 + def load_weights(
306 + self,
307 + model_name_or_path: str,
308 + cache_dir: Optional[str] = None,
309 + load_format: str = "auto",
310 + revision: Optional[str] = None,
311 + ):
312 + stacked_params_mapping = [
313 + # (param_name, shard_name, shard_id)
314 + ("qkv_proj", "q_proj", "q"),
315 + ("qkv_proj", "k_proj", "k"),
316 + ("qkv_proj", "v_proj", "v"),
317 + ("gate_up_proj", "gate_proj", 0),
318 + ("gate_up_proj", "up_proj", 1),
319 + ]
320 + params_dict = dict(self.named_parameters())
321 + loaded_params = set()
322 + for name, loaded_weight in hf_model_weights_iterator(
323 + model_name_or_path, cache_dir, load_format, revision):
324 + for param_name, shard_name, shard_id in stacked_params_mapping:
325 + if shard_name not in name:
326 + continue
327 + name = name.replace(shard_name, param_name)
328 + param = params_dict[name]
329 + weight_loader = param.weight_loader
330 + weight_loader(param, loaded_weight, shard_id)
331 + break
332 + else:
333 + param = params_dict[name]
334 + weight_loader = getattr(param, "weight_loader",
335 + default_weight_loader)
336 + weight_loader(param, loaded_weight)
337 + loaded_params.add(name)