A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model] factoring out MambaMixer out of Jamba (#8993)
Signed-off-by: mzusman <mor.zusmann@gmail.com>
Mor Zusman committed
Nov 4, 2024 at 20:00 UTC
603a661ae8ccadd8401284f7db8563164b232651
3 files changed
+245
-374
vllm/model_executor/layers/mamba/mamba_mixer.py
new
+217
@@ -0,0 +1,217 @@
1
+import torch
2
+from torch import nn
3
+from torch.nn.parameter import Parameter
4
+
5
+from vllm.attention.backends.abstract import AttentionMetadata
6
+from vllm.distributed.parallel_state import (
7
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
8
+from vllm.model_executor.custom_op import CustomOp
9
+from vllm.model_executor.layers.layernorm import RMSNorm
10
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
11
+ MergedColumnParallelLinear,
12
+ RowParallelLinear)
13
+from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
14
+ causal_conv1d_fn, causal_conv1d_update)
15
+from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
16
+ selective_scan_fn, selective_state_update)
17
+from vllm.model_executor.models.mamba_cache import MambaCacheParams
18
+from vllm.model_executor.utils import set_weight_attrs
19
+
20
+
21
+# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
22
+@CustomOp.register("mamba_mixer")
23
+class MambaMixer(CustomOp):
24
+ """
25
+ Compute ∆, A, B, C, and D the state space parameters and compute
26
+ the `contextualized_states`. A, D are input independent
27
+ (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
28
+ for why A isn't selective) ∆, B, C are input-dependent
29
+ (this is a key difference between Mamba and the linear time
30
+ invariant S4, and is why Mamba is called
31
+ **selective** state spaces)
32
+ """
33
+
34
+ def __init__(self,
35
+ hidden_size: int,
36
+ ssm_state_size: int,
37
+ conv_kernel_size: int,
38
+ intermediate_size: int,
39
+ time_step_rank: int,
40
+ use_conv_bias: bool,
41
+ use_bias: bool,
42
+ use_rms_norm: bool,
43
+ rms_norm_eps: float = 1e-5,
44
+ activation="silu"):
45
+ super().__init__()
46
+ self.time_step_rank = time_step_rank
47
+ self.ssm_state_size = ssm_state_size
48
+ self.use_rms_norm = use_rms_norm
49
+ self.activation = activation
50
+
51
+ self.conv1d = ColumnParallelLinear(
52
+ input_size=conv_kernel_size,
53
+ output_size=intermediate_size,
54
+ bias=use_conv_bias,
55
+ )
56
+ # unsqueeze to fit conv1d weights shape into the linear weights shape.
57
+ # Can't do this in `weight_loader` since it already exists in
58
+ # `ColumnParallelLinear` and `set_weight_attrs`
59
+ # doesn't allow to override it
60
+ self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
61
+
62
+ self.in_proj = MergedColumnParallelLinear(hidden_size,
63
+ [intermediate_size] * 2,
64
+ bias=use_bias)
65
+ # selective projection used to make dt, B and C input dependent
66
+ self.x_proj = RowParallelLinear(
67
+ intermediate_size,
68
+ time_step_rank + ssm_state_size * 2,
69
+ bias=False,
70
+ )
71
+ # time step projection (discretization) -
72
+ # In the forward we need to apply dt_proj without the bias,
73
+ # as the bias is added in the selective scan kernel.
74
+ self.dt_proj = ColumnParallelLinear(time_step_rank,
75
+ intermediate_size,
76
+ bias=True,
77
+ skip_bias_add=True)
78
+
79
+ def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
80
+ tp_rank = get_tensor_model_parallel_rank()
81
+ tp_size = get_tensor_model_parallel_world_size()
82
+ param.data.copy_(
83
+ loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
84
+ dim=0)[tp_rank])
85
+
86
+ def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
87
+ weight_loader(param, -torch.exp(loaded_weight.float()))
88
+
89
+ tp_size = get_tensor_model_parallel_world_size()
90
+ self.A = nn.Parameter(
91
+ torch.empty(
92
+ intermediate_size // tp_size,
93
+ ssm_state_size,
94
+ dtype=torch.float32,
95
+ ))
96
+ self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
97
+
98
+ set_weight_attrs(self.D, {"weight_loader": weight_loader})
99
+ set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
100
+
101
+ self.out_proj = RowParallelLinear(
102
+ intermediate_size,
103
+ hidden_size,
104
+ bias=use_bias,
105
+ input_is_parallel=True,
106
+ )
107
+
108
+ self.dt_layernorm = RMSNorm(time_step_rank,
109
+ eps=rms_norm_eps) if use_rms_norm else None
110
+
111
+ self.b_layernorm = RMSNorm(ssm_state_size,
112
+ eps=rms_norm_eps) if use_rms_norm else None
113
+
114
+ self.c_layernorm = RMSNorm(ssm_state_size,
115
+ eps=rms_norm_eps) if use_rms_norm else None
116
+
117
+ def forward_native(self, hidden_states: torch.Tensor,
118
+ attn_metadata: AttentionMetadata,
119
+ conv_state: torch.Tensor, ssm_state: torch.Tensor):
120
+ pass
121
+
122
+ def forward_cuda(self, hidden_states: torch.Tensor,
123
+ attn_metadata: AttentionMetadata,
124
+ mamba_cache_params: MambaCacheParams):
125
+
126
+ # 1. Gated MLP's linear projection
127
+ projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
128
+ hidden_states, gate = projected_states.chunk(2, dim=-2)
129
+
130
+ # 2. Convolution sequence transformation
131
+ conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
132
+ self.conv1d.weight.size(2))
133
+
134
+ if attn_metadata.query_start_loc is not None \
135
+ and attn_metadata.context_lens_tensor is not None:
136
+ # |---------- N-1 iteration --------|
137
+ # |---------------- N iteration ---------------------|
138
+ # |- tokenA -|......................|-- newTokens ---|
139
+ # |---------- context_len ----------|
140
+ # |-------------------- seq_len ---------------------|
141
+ # |-- query_len ---|
142
+ hidden_states = causal_conv1d_fn(
143
+ hidden_states,
144
+ conv_weights,
145
+ self.conv1d.bias,
146
+ activation=self.activation,
147
+ conv_states=mamba_cache_params.conv_state,
148
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
149
+ cache_indices=mamba_cache_params.state_indices_tensor,
150
+ query_start_loc=attn_metadata.query_start_loc)
151
+ else:
152
+ hidden_states = causal_conv1d_update(
153
+ hidden_states.transpose(0, 1),
154
+ mamba_cache_params.conv_state,
155
+ conv_weights,
156
+ self.conv1d.bias,
157
+ self.activation,
158
+ conv_state_indices=mamba_cache_params.state_indices_tensor)
159
+ hidden_states = hidden_states.transpose(0, 1)
160
+
161
+ # 3. State Space Model sequence transformation
162
+ # 3.a. input varying initialization of time_step, B and C
163
+ ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
164
+
165
+ time_step, B, C = torch.split(
166
+ ssm_parameters,
167
+ [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
168
+ dim=-1,
169
+ )
170
+ if self.use_rms_norm:
171
+ assert self.dt_layernorm is not None
172
+ assert self.b_layernorm is not None
173
+ assert self.c_layernorm is not None
174
+ time_step = self.dt_layernorm(time_step.contiguous())
175
+ B = self.b_layernorm(B.contiguous())
176
+ C = self.c_layernorm(C.contiguous())
177
+
178
+ discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
179
+ # 3.c perform the recurrence y ← SSM(A, B, C)(x)
180
+ time_proj_bias = (self.dt_proj.bias.float() if hasattr(
181
+ self.dt_proj, "bias") else None)
182
+
183
+ if attn_metadata.query_start_loc is not None \
184
+ and attn_metadata.context_lens_tensor is not None:
185
+ scan_outputs = selective_scan_fn(
186
+ hidden_states,
187
+ mamba_cache_params.ssm_state,
188
+ discrete_time_step,
189
+ self.A,
190
+ B.transpose(-2, -1),
191
+ C.transpose(-2, -1),
192
+ self.D.float(),
193
+ gate,
194
+ time_proj_bias,
195
+ delta_softplus=True,
196
+ cache_indices=mamba_cache_params.state_indices_tensor,
197
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
198
+ query_start_loc=attn_metadata.query_start_loc)
199
+ else:
200
+ scan_outputs = selective_state_update(
201
+ mamba_cache_params.ssm_state,
202
+ hidden_states.transpose(0, 1),
203
+ discrete_time_step.transpose(0, 1),
204
+ self.A,
205
+ B,
206
+ C,
207
+ self.D,
208
+ gate.transpose(0, 1),
209
+ time_proj_bias,
210
+ dt_softplus=True,
211
+ state_batch_indices=mamba_cache_params.state_indices_tensor)
212
+ scan_outputs = scan_outputs.transpose(0, 1)
213
+
214
+ # 4. Final linear projection
215
+ contextualized_states = self.out_proj(scan_outputs.transpose(-2,
216
+ -1))[0]
217
+ return contextualized_states
vllm/model_executor/models/jamba.py
+14
-185
@@ -12,26 +12,19 @@ from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12
from vllm.distributed import get_tensor_model_parallel_world_size
13
from vllm.model_executor.layers.fused_moe import FusedMoE
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
- MergedColumnParallelLinear,
17
- QKVParallelLinear,
15
+from vllm.model_executor.layers.linear import (QKVParallelLinear,
16
ReplicatedLinear,
17
RowParallelLinear)
18
from vllm.model_executor.layers.logits_processor import LogitsProcessor
21
-from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
22
- causal_conv1d_fn, causal_conv1d_update)
23
-from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
24
- selective_scan_fn, selective_state_update)
19
+from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
20
from vllm.model_executor.layers.quantization import QuantizationConfig
21
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
22
from vllm.model_executor.layers.vocab_parallel_embedding import (
23
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
29
-from vllm.model_executor.model_loader.weight_utils import (
30
- composed_weight_loader, default_weight_loader, sharded_weight_loader)
24
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25
from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
26
MambaCacheParams)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
-from vllm.model_executor.utils import set_weight_attrs
28
from vllm.sequence import IntermediateTensors
29
from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
30
_get_graph_batch_size)
@@ -41,179 +34,6 @@ from .interfaces import HasInnerState, SupportsLoRA
34
KVCache = Tuple[torch.Tensor, torch.Tensor]
35
36
44
-# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
45
-class JambaMambaMixer(nn.Module):
46
- """
47
- Compute ∆, A, B, C, and D the state space parameters and compute
48
- the `contextualized_states`. A, D are input independent
49
- (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
50
- for why A isn't selective) ∆, B, C are input-dependent
51
- (this is a key difference between Mamba and the linear time
52
- invariant S4, and is why Mamba is called
53
- **selective** state spaces)
54
- """
55
-
56
- def __init__(self, config: JambaConfig):
57
- super().__init__()
58
- self.config = config
59
- self.hidden_size = config.hidden_size
60
- self.ssm_state_size = config.mamba_d_state
61
- self.conv_kernel_size = config.mamba_d_conv
62
- self.intermediate_size = config.mamba_expand * config.hidden_size
63
- self.time_step_rank = config.mamba_dt_rank
64
- self.use_conv_bias = config.mamba_conv_bias
65
- self.use_bias = config.mamba_proj_bias
66
- self.conv1d = ColumnParallelLinear(
67
- input_size=self.conv_kernel_size,
68
- output_size=self.intermediate_size,
69
- bias=self.use_conv_bias,
70
- )
71
- # unsqueeze to fit conv1d weights shape into the linear weights shape.
72
- # Can't do this in `weight_loader` since it already exists in
73
- # `ColumnParallelLinear` and `set_weight_attrs`
74
- # doesn't allow to override it
75
- self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
76
-
77
- self.in_proj = MergedColumnParallelLinear(self.hidden_size,
78
- [self.intermediate_size] * 2,
79
- bias=self.use_bias)
80
- # selective projection used to make dt, B and C input dependent
81
- self.x_proj = RowParallelLinear(
82
- self.intermediate_size,
83
- self.time_step_rank + self.ssm_state_size * 2,
84
- bias=False,
85
- )
86
- # time step projection (discretization) -
87
- # In the forward we need to apply dt_proj without the bias,
88
- # as the bias is added in the selective scan kernel.
89
- self.dt_proj = ColumnParallelLinear(self.time_step_rank,
90
- self.intermediate_size,
91
- bias=True,
92
- skip_bias_add=True)
93
-
94
- tp_size = get_tensor_model_parallel_world_size()
95
- self.A = nn.Parameter(
96
- torch.empty(
97
- self.intermediate_size // tp_size,
98
- self.ssm_state_size,
99
- dtype=torch.float32,
100
- ))
101
- self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
102
-
103
- set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
104
- a_weight_loader = composed_weight_loader(
105
- sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
106
- set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
107
-
108
- self.out_proj = RowParallelLinear(
109
- self.intermediate_size,
110
- self.hidden_size,
111
- bias=self.use_bias,
112
- input_is_parallel=True,
113
- )
114
- self.activation = config.hidden_act
115
-
116
- self.dt_layernorm = RMSNorm(self.time_step_rank,
117
- eps=config.rms_norm_eps)
118
- self.b_layernorm = RMSNorm(self.ssm_state_size,
119
- eps=config.rms_norm_eps)
120
- self.c_layernorm = RMSNorm(self.ssm_state_size,
121
- eps=config.rms_norm_eps)
122
-
123
- def forward(self, hidden_states: torch.Tensor,
124
- attn_metadata: AttentionMetadata,
125
- mamba_cache_params: MambaCacheParams):
126
-
127
- # 1. Gated MLP's linear projection
128
- projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
129
- hidden_states, gate = projected_states.chunk(2, dim=-2)
130
-
131
- # 2. Convolution sequence transformation
132
- conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
133
- self.conv1d.weight.size(2))
134
-
135
- if attn_metadata.query_start_loc is not None \
136
- and attn_metadata.context_lens_tensor is not None:
137
- # |---------- N-1 iteration --------|
138
- # |---------------- N iteration ---------------------|
139
- # |- tokenA -|......................|-- newTokens ---|
140
- # |---------- context_len ----------|
141
- # |-------------------- seq_len ---------------------|
142
- # |-- query_len ---|
143
- hidden_states = causal_conv1d_fn(
144
- hidden_states,
145
- conv_weights,
146
- self.conv1d.bias,
147
- activation=self.activation,
148
- conv_states=mamba_cache_params.conv_state,
149
- has_initial_state=attn_metadata.context_lens_tensor > 0,
150
- cache_indices=mamba_cache_params.state_indices_tensor,
151
- query_start_loc=attn_metadata.query_start_loc)
152
- else:
153
- hidden_states = causal_conv1d_update(
154
- hidden_states.transpose(0, 1),
155
- mamba_cache_params.conv_state,
156
- conv_weights,
157
- self.conv1d.bias,
158
- self.activation,
159
- conv_state_indices=mamba_cache_params.state_indices_tensor)
160
- hidden_states = hidden_states.transpose(0, 1)
161
-
162
- # 3. State Space Model sequence transformation
163
- # 3.a. input varying initialization of time_step, B and C
164
- ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
165
-
166
- time_step, B, C = torch.split(
167
- ssm_parameters,
168
- [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
169
- dim=-1,
170
- )
171
- time_step = self.dt_layernorm(time_step.contiguous())
172
- B = self.b_layernorm(B.contiguous())
173
- C = self.c_layernorm(C.contiguous())
174
-
175
- discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176
- # 3.c perform the recurrence y ← SSM(A, B, C)(x)
177
- time_proj_bias = (self.dt_proj.bias.float() if hasattr(
178
- self.dt_proj, "bias") else None)
179
-
180
- if attn_metadata.query_start_loc is not None \
181
- and attn_metadata.context_lens_tensor is not None:
182
- scan_outputs = selective_scan_fn(
183
- hidden_states,
184
- mamba_cache_params.ssm_state,
185
- discrete_time_step,
186
- self.A,
187
- B.transpose(-2, -1),
188
- C.transpose(-2, -1),
189
- self.D.float(),
190
- gate,
191
- time_proj_bias,
192
- delta_softplus=True,
193
- cache_indices=mamba_cache_params.state_indices_tensor,
194
- has_initial_state=attn_metadata.context_lens_tensor > 0,
195
- query_start_loc=attn_metadata.query_start_loc)
196
- else:
197
- scan_outputs = selective_state_update(
198
- mamba_cache_params.ssm_state,
199
- hidden_states.transpose(0, 1),
200
- discrete_time_step.transpose(0, 1),
201
- self.A,
202
- B,
203
- C,
204
- self.D,
205
- gate.transpose(0, 1),
206
- time_proj_bias,
207
- dt_softplus=True,
208
- state_batch_indices=mamba_cache_params.state_indices_tensor)
209
- scan_outputs = scan_outputs.transpose(0, 1)
210
-
211
- # 4. Final linear projection
212
- contextualized_states = self.out_proj(scan_outputs.transpose(-2,
213
- -1))[0]
214
- return contextualized_states
215
-
216
-
37
class JambaMoE(nn.Module):
38
39
def __init__(self,
@@ -284,9 +104,18 @@ class JambaMambaDecoderLayer(nn.Module):
104
cache_config: Optional[CacheConfig] = None,
105
quant_config: Optional[QuantizationConfig] = None) -> None:
106
super().__init__()
287
- self.layer_idx = layer_idx
107
self.config = config
289
- self.mamba = JambaMambaMixer(config)
108
+ self.mamba = MambaMixer(hidden_size= config.hidden_size,
109
+ ssm_state_size = config.mamba_d_state,
110
+ conv_kernel_size = config.mamba_d_conv,
111
+ intermediate_size = config.mamba_expand *\
112
+ config.hidden_size,
113
+ time_step_rank = config.mamba_dt_rank,
114
+ use_conv_bias = config.mamba_conv_bias,
115
+ use_bias = config.mamba_proj_bias,
116
+ use_rms_norm=True,
117
+ rms_norm_eps=config.rms_norm_eps,
118
+ activation=config.hidden_act)
119
120
num_experts = config.layers_num_experts[layer_idx]
121
ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
vllm/model_executor/models/mamba.py
+14
-189
@@ -10,27 +10,19 @@ from vllm.attention.backends.abstract import AttentionMetadata
10
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
from vllm.model_executor.layers.layernorm import RMSNorm
13
-from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
- MergedColumnParallelLinear,
15
- RowParallelLinear)
13
from vllm.model_executor.layers.logits_processor import LogitsProcessor
17
-from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
18
- causal_conv1d_fn, causal_conv1d_update)
19
-from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
20
- selective_scan_fn, selective_state_update)
14
+from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
15
from vllm.model_executor.layers.quantization.base_config import (
16
QuantizationConfig)
17
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18
from vllm.model_executor.layers.vocab_parallel_embedding import (
19
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
26
-from vllm.model_executor.model_loader.weight_utils import (
27
- composed_weight_loader, default_weight_loader, sharded_weight_loader)
20
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
from vllm.model_executor.models.interfaces import (HasInnerState,
22
IsAttentionFree)
23
from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
24
MambaCacheParams)
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
33
-from vllm.model_executor.utils import set_weight_attrs
26
from vllm.sequence import IntermediateTensors
27
from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
28
_get_graph_batch_size)
@@ -38,194 +30,27 @@ from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
30
KVCache = Tuple[torch.Tensor, torch.Tensor]
31
32
41
-# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
42
-class MambaMixer(nn.Module):
43
- """
44
- Compute ∆, A, B, C, and D the state space parameters and compute
45
- the `contextualized_states`. A, D are input independent
46
- (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
47
- for why A isn't selective) ∆, B, C are input-dependent
48
- (this is a key difference between Mamba and the linear time
49
- invariant S4, and is why Mamba is called
50
- **selective** state spaces)
51
- """
52
-
53
- def __init__(self, config: MambaConfig, layer_idx):
54
- super().__init__()
55
- self.config = config
56
- self.layer_idx = layer_idx
57
- self.hidden_size = config.hidden_size
58
- self.ssm_state_size = config.state_size
59
- self.conv_kernel_size = config.conv_kernel
60
- self.intermediate_size = config.intermediate_size
61
- self.time_step_rank = int(config.time_step_rank)
62
- self.is_falcon_mamba = config.model_type == "falcon_mamba"
63
- self.conv1d = ColumnParallelLinear(
64
- input_size=self.conv_kernel_size,
65
- output_size=self.intermediate_size,
66
- bias=config.use_conv_bias,
67
- )
68
- # unsqueeze to fit conv1d weights shape into the linear weights shape.
69
- # Can't do this in `weight_loader` since it already exists in
70
- # `ColumnParallelLinear` and `set_weight_attrs`
71
- # doesn't allow to override it
72
- self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
73
-
74
- self.in_proj = MergedColumnParallelLinear(self.hidden_size,
75
- [self.intermediate_size] * 2,
76
- bias=config.use_bias)
77
- # selective projection used to make dt, B and C input dependent
78
- self.x_proj = RowParallelLinear(
79
- self.intermediate_size,
80
- self.time_step_rank + self.ssm_state_size * 2,
81
- bias=False,
82
- )
83
- # time step projection (discretization) -
84
- # In the forward we need to apply dt_proj without the bias,
85
- # as the bias is added in the selective scan kernel.
86
- self.dt_proj = ColumnParallelLinear(self.time_step_rank,
87
- self.intermediate_size,
88
- bias=True,
89
- skip_bias_add=True)
90
-
91
- tp_size = get_tensor_model_parallel_world_size()
92
- self.A = nn.Parameter(
93
- torch.empty(
94
- self.intermediate_size // tp_size,
95
- self.ssm_state_size,
96
- dtype=torch.float32,
97
- ))
98
- self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
99
-
100
- set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
101
- a_weight_loader = composed_weight_loader(
102
- sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
103
- set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
104
-
105
- self.out_proj = RowParallelLinear(
106
- self.intermediate_size,
107
- self.hidden_size,
108
- bias=config.use_bias,
109
- input_is_parallel=True,
110
- )
111
- self.activation = config.hidden_act
112
- if self.is_falcon_mamba:
113
- self.dt_layernorm = RMSNorm(self.time_step_rank,
114
- eps=config.mixer_rms_eps)
115
- self.b_layernorm = RMSNorm(self.ssm_state_size,
116
- eps=config.mixer_rms_eps)
117
- self.c_layernorm = RMSNorm(self.ssm_state_size,
118
- eps=config.mixer_rms_eps)
119
-
120
- def forward(self, hidden_states: torch.Tensor,
121
- attn_metadata: AttentionMetadata,
122
- mamba_cache_params: MambaCacheParams):
123
-
124
- # 1. Gated MLP's linear projection
125
- projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
126
- hidden_states, gate = projected_states.chunk(2, dim=-2)
127
-
128
- # 2. Convolution sequence transformation
129
- conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
130
- self.conv1d.weight.size(2))
131
-
132
- if attn_metadata.query_start_loc is not None \
133
- and attn_metadata.context_lens_tensor is not None:
134
- # |---------- N-1 iteration --------|
135
- # |---------------- N iteration ---------------------|
136
- # |- tokenA -|......................|-- newTokens ---|
137
- # |---------- context_len ----------|
138
- # |-------------------- seq_len ---------------------|
139
- # |-- query_len ---|
140
- hidden_states = causal_conv1d_fn(
141
- hidden_states,
142
- conv_weights,
143
- self.conv1d.bias,
144
- activation=self.activation,
145
- conv_states=mamba_cache_params.conv_state,
146
- has_initial_state=attn_metadata.context_lens_tensor > 0,
147
- cache_indices=mamba_cache_params.state_indices_tensor,
148
- query_start_loc=attn_metadata.query_start_loc)
149
- else:
150
- hidden_states = causal_conv1d_update(
151
- hidden_states.transpose(0, 1),
152
- mamba_cache_params.conv_state,
153
- conv_weights,
154
- self.conv1d.bias,
155
- self.activation,
156
- conv_state_indices=mamba_cache_params.state_indices_tensor)
157
- hidden_states = hidden_states.transpose(0, 1)
158
-
159
- # 3. State Space Model sequence transformation
160
- # 3.a. input varying initialization of time_step, B and C
161
- ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
162
-
163
- time_step, B, C = torch.split(
164
- ssm_parameters,
165
- [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166
- dim=-1,
167
- )
168
- # Note that Jamba and FalconMamba normalizes B, C, and time_step here
169
- # but Mamba doesn't.
170
- if self.is_falcon_mamba:
171
- time_step = self.dt_layernorm(time_step.contiguous())
172
- B = self.b_layernorm(B.contiguous())
173
- C = self.c_layernorm(C.contiguous())
174
-
175
- discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176
- # 3.c perform the recurrence y ← SSM(A, B, C)(x)
177
- time_proj_bias = (self.dt_proj.bias.float() if hasattr(
178
- self.dt_proj, "bias") else None)
179
-
180
- if attn_metadata.query_start_loc is not None \
181
- and attn_metadata.context_lens_tensor is not None:
182
- scan_outputs = selective_scan_fn(
183
- hidden_states,
184
- mamba_cache_params.ssm_state,
185
- discrete_time_step,
186
- self.A,
187
- B.transpose(-2, -1),
188
- C.transpose(-2, -1),
189
- self.D.float(),
190
- gate,
191
- time_proj_bias,
192
- delta_softplus=True,
193
- cache_indices=mamba_cache_params.state_indices_tensor,
194
- has_initial_state=attn_metadata.context_lens_tensor > 0,
195
- query_start_loc=attn_metadata.query_start_loc)
196
- else:
197
- scan_outputs = selective_state_update(
198
- mamba_cache_params.ssm_state,
199
- hidden_states.transpose(0, 1),
200
- discrete_time_step.transpose(0, 1),
201
- self.A,
202
- B,
203
- C,
204
- self.D,
205
- gate.transpose(0, 1),
206
- time_proj_bias,
207
- dt_softplus=True,
208
- state_batch_indices=mamba_cache_params.state_indices_tensor)
209
- scan_outputs = scan_outputs.transpose(0, 1)
210
-
211
- # 4. Final linear projection
212
- contextualized_states = self.out_proj(scan_outputs.transpose(-2,
213
- -1))[0]
214
- return contextualized_states
215
-
216
-
33
class MambaDecoderLayer(nn.Module):
34
35
def __init__(self,
36
config: MambaConfig,
221
- layer_idx: int,
37
cache_config: Optional[CacheConfig] = None,
38
quant_config: Optional[QuantizationConfig] = None) -> None:
39
super().__init__()
225
- self.layer_idx = layer_idx
40
self.config = config
41
self.is_falcon_mamba = config.model_type == "falcon_mamba"
228
- self.mixer = MambaMixer(config, layer_idx)
42
+ mixer_rms_rps = config.mixer_rms_rps if self.is_falcon_mamba else None
43
+ self.mamba = MambaMixer(hidden_size=config.hidden_size,
44
+ ssm_state_size=config.state_size,
45
+ conv_kernel_size=config.conv_kernel,
46
+ intermediate_size=config.intermediate_size,
47
+ time_step_rank=config.time_step_rank,
48
+ use_conv_bias=config.use_conv_bias,
49
+ use_bias=config.use_bias,
50
+ use_rms_norm=self.is_falcon_mamba,
51
+ rms_norm_eps=mixer_rms_rps,
52
+ activation=config.hidden_act)
53
+
54
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
55
56
def forward(