A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Model] Cohere CommandR+ (#3829)
Saurabh Dash committed
Apr 5, 2024 at 02:01 UTC
9117f892f0e4d3b0f07bf0b9b409321bc743dabc
1 file changed
+40
-8
vllm/model_executor/models/commandr.py
+40
-8
@@ -25,6 +25,7 @@ from typing import List, Optional, Tuple
25
import torch
26
import torch.utils.checkpoint
27
from torch import nn
28
+from torch.nn.parameter import Parameter
29
from transformers import CohereConfig
30
31
from vllm.attention import Attention, AttentionMetadata
@@ -39,8 +40,9 @@ from vllm.model_executor.layers.sampler import Sampler
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
VocabParallelEmbedding)
42
from vllm.model_executor.parallel_utils.parallel_state import (
42
- get_tensor_model_parallel_world_size)
43
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
45
+from vllm.model_executor.utils import set_weight_attrs
46
from vllm.model_executor.weight_utils import (default_weight_loader,
47
hf_model_weights_iterator)
48
from vllm.sequence import SamplerOutput
@@ -48,11 +50,11 @@ from vllm.sequence import SamplerOutput
50
51
class LayerNorm(nn.Module):
52
51
- def __init__(self, hidden_size, eps=1e-5, bias=False):
53
+ def __init__(self, param_shape=None, eps=1e-5):
54
super().__init__()
53
- self.weight = nn.Parameter(torch.ones(hidden_size))
54
- self.bias = nn.Parameter(torch.zeros(hidden_size)) if bias else None
55
+ self.weight = nn.Parameter(torch.ones(param_shape))
56
self.variance_epsilon = eps
57
+ set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
58
59
def forward(self, hidden_states, residuals=None):
60
input_dtype = hidden_states.dtype
@@ -62,10 +64,20 @@ class LayerNorm(nn.Module):
64
hidden_states = (hidden_states -
65
mean) * torch.rsqrt(variance + self.variance_epsilon)
66
hidden_states = self.weight.to(torch.float32) * hidden_states
65
- if self.bias is not None:
66
- hidden_states = hidden_states + self.bias.to(torch.float32)
67
return hidden_states.to(input_dtype), residuals
68
69
+ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
70
+ tp_rank = get_tensor_model_parallel_rank()
71
+ shard_dim = 0 if param.dim() != 1 else None
72
+ param_data = param.data
73
+ if shard_dim is not None:
74
+ shard_size = param_data.shape[shard_dim]
75
+ start_idx = tp_rank * shard_size
76
+ loaded_weight = loaded_weight.narrow(shard_dim, start_idx,
77
+ shard_size)
78
+ assert param_data.shape == loaded_weight.shape
79
+ param_data.copy_(loaded_weight)
80
+
81
82
# Copied from transformers.models.llama.modeling_llama.LlamaMLP Llama->Cohere
83
class CohereMLP(nn.Module):
@@ -131,6 +143,7 @@ class CohereAttention(nn.Module):
143
self.max_position_embeddings = config.max_position_embeddings
144
self.rope_theta = config.rope_theta
145
self.rope_scaling = getattr(config, "rope_scaling", None)
146
+ self.use_qk_norm = getattr(config, "use_qk_norm", False)
147
self.qkv_proj = QKVParallelLinear(
148
self.hidden_size,
149
self.head_dim,
@@ -159,6 +172,22 @@ class CohereAttention(nn.Module):
172
self.scaling,
173
num_kv_heads=self.num_kv_heads,
174
)
175
+ if self.use_qk_norm:
176
+ self.q_norm = LayerNorm(param_shape=(self.num_heads,
177
+ self.head_dim),
178
+ eps=config.layer_norm_eps)
179
+ self.k_norm = LayerNorm(param_shape=(self.num_kv_heads,
180
+ self.head_dim),
181
+ eps=config.layer_norm_eps)
182
+
183
+ def _apply_qk_norm(self, q, k):
184
+ q = q.view(*q.shape[:-1], -1, self.head_dim)
185
+ k = k.view(*k.shape[:-1], -1, self.head_dim)
186
+ q, _ = self.q_norm(q)
187
+ k, _ = self.k_norm(k)
188
+ q = q.view(*q.shape[:-2], -1)
189
+ k = k.view(*k.shape[:-2], -1)
190
+ return q, k
191
192
def forward(
193
self,
@@ -169,6 +198,8 @@ class CohereAttention(nn.Module):
198
) -> torch.Tensor:
199
qkv, _ = self.qkv_proj(hidden_states)
200
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
201
+ if self.use_qk_norm:
202
+ q, k = self._apply_qk_norm(q, k)
203
q, k = self.rotary_emb(positions, q, k)
204
attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
205
output, _ = self.o_proj(attn_output)
@@ -186,7 +217,7 @@ class CohereDecoderLayer(nn.Module):
217
self.self_attn = CohereAttention(config, linear_method=linear_method)
218
219
self.mlp = CohereMLP(config, linear_method=linear_method)
189
- self.input_layernorm = LayerNorm(config.hidden_size,
220
+ self.input_layernorm = LayerNorm(param_shape=(config.hidden_size),
221
eps=config.layer_norm_eps)
222
223
def forward(
@@ -229,7 +260,8 @@ class CohereModel(nn.Module):
260
CohereDecoderLayer(config, linear_method=linear_method)
261
for _ in range(config.num_hidden_layers)
262
])
232
- self.norm = LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
263
+ self.norm = LayerNorm(param_shape=(config.hidden_size),
264
+ eps=config.layer_norm_eps)
265
266
def forward(
267
self,