@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 603a661ae8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model] factoring out MambaMixer out of Jamba (#8993)

Signed-off-by: mzusman <mor.zusmann@gmail.com>

Mor Zusman committed Nov 4, 2024 at 20:00 UTC 603a661ae8ccadd8401284f7db8563164b232651
3 files changed +245 -374
vllm/model_executor/layers/mamba/mamba_mixer.py new
+217
@@ -0,0 +1,217 @@
1 +import torch
2 +from torch import nn
3 +from torch.nn.parameter import Parameter
4 +
5 +from vllm.attention.backends.abstract import AttentionMetadata
6 +from vllm.distributed.parallel_state import (
7 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
8 +from vllm.model_executor.custom_op import CustomOp
9 +from vllm.model_executor.layers.layernorm import RMSNorm
10 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
11 + MergedColumnParallelLinear,
12 + RowParallelLinear)
13 +from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
14 + causal_conv1d_fn, causal_conv1d_update)
15 +from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
16 + selective_scan_fn, selective_state_update)
17 +from vllm.model_executor.models.mamba_cache import MambaCacheParams
18 +from vllm.model_executor.utils import set_weight_attrs
19 +
20 +
21 +# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
22 +@CustomOp.register("mamba_mixer")
23 +class MambaMixer(CustomOp):
24 + """
25 + Compute ∆, A, B, C, and D the state space parameters and compute
26 + the `contextualized_states`. A, D are input independent
27 + (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
28 + for why A isn't selective) ∆, B, C are input-dependent
29 + (this is a key difference between Mamba and the linear time
30 + invariant S4, and is why Mamba is called
31 + **selective** state spaces)
32 + """
33 +
34 + def __init__(self,
35 + hidden_size: int,
36 + ssm_state_size: int,
37 + conv_kernel_size: int,
38 + intermediate_size: int,
39 + time_step_rank: int,
40 + use_conv_bias: bool,
41 + use_bias: bool,
42 + use_rms_norm: bool,
43 + rms_norm_eps: float = 1e-5,
44 + activation="silu"):
45 + super().__init__()
46 + self.time_step_rank = time_step_rank
47 + self.ssm_state_size = ssm_state_size
48 + self.use_rms_norm = use_rms_norm
49 + self.activation = activation
50 +
51 + self.conv1d = ColumnParallelLinear(
52 + input_size=conv_kernel_size,
53 + output_size=intermediate_size,
54 + bias=use_conv_bias,
55 + )
56 + # unsqueeze to fit conv1d weights shape into the linear weights shape.
57 + # Can't do this in `weight_loader` since it already exists in
58 + # `ColumnParallelLinear` and `set_weight_attrs`
59 + # doesn't allow to override it
60 + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
61 +
62 + self.in_proj = MergedColumnParallelLinear(hidden_size,
63 + [intermediate_size] * 2,
64 + bias=use_bias)
65 + # selective projection used to make dt, B and C input dependent
66 + self.x_proj = RowParallelLinear(
67 + intermediate_size,
68 + time_step_rank + ssm_state_size * 2,
69 + bias=False,
70 + )
71 + # time step projection (discretization) -
72 + # In the forward we need to apply dt_proj without the bias,
73 + # as the bias is added in the selective scan kernel.
74 + self.dt_proj = ColumnParallelLinear(time_step_rank,
75 + intermediate_size,
76 + bias=True,
77 + skip_bias_add=True)
78 +
79 + def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
80 + tp_rank = get_tensor_model_parallel_rank()
81 + tp_size = get_tensor_model_parallel_world_size()
82 + param.data.copy_(
83 + loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
84 + dim=0)[tp_rank])
85 +
86 + def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
87 + weight_loader(param, -torch.exp(loaded_weight.float()))
88 +
89 + tp_size = get_tensor_model_parallel_world_size()
90 + self.A = nn.Parameter(
91 + torch.empty(
92 + intermediate_size // tp_size,
93 + ssm_state_size,
94 + dtype=torch.float32,
95 + ))
96 + self.D = nn.Parameter(torch.ones(intermediate_size // tp_size))
97 +
98 + set_weight_attrs(self.D, {"weight_loader": weight_loader})
99 + set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
100 +
101 + self.out_proj = RowParallelLinear(
102 + intermediate_size,
103 + hidden_size,
104 + bias=use_bias,
105 + input_is_parallel=True,
106 + )
107 +
108 + self.dt_layernorm = RMSNorm(time_step_rank,
109 + eps=rms_norm_eps) if use_rms_norm else None
110 +
111 + self.b_layernorm = RMSNorm(ssm_state_size,
112 + eps=rms_norm_eps) if use_rms_norm else None
113 +
114 + self.c_layernorm = RMSNorm(ssm_state_size,
115 + eps=rms_norm_eps) if use_rms_norm else None
116 +
117 + def forward_native(self, hidden_states: torch.Tensor,
118 + attn_metadata: AttentionMetadata,
119 + conv_state: torch.Tensor, ssm_state: torch.Tensor):
120 + pass
121 +
122 + def forward_cuda(self, hidden_states: torch.Tensor,
123 + attn_metadata: AttentionMetadata,
124 + mamba_cache_params: MambaCacheParams):
125 +
126 + # 1. Gated MLP's linear projection
127 + projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
128 + hidden_states, gate = projected_states.chunk(2, dim=-2)
129 +
130 + # 2. Convolution sequence transformation
131 + conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
132 + self.conv1d.weight.size(2))
133 +
134 + if attn_metadata.query_start_loc is not None \
135 + and attn_metadata.context_lens_tensor is not None:
136 + # |---------- N-1 iteration --------|
137 + # |---------------- N iteration ---------------------|
138 + # |- tokenA -|......................|-- newTokens ---|
139 + # |---------- context_len ----------|
140 + # |-------------------- seq_len ---------------------|
141 + # |-- query_len ---|
142 + hidden_states = causal_conv1d_fn(
143 + hidden_states,
144 + conv_weights,
145 + self.conv1d.bias,
146 + activation=self.activation,
147 + conv_states=mamba_cache_params.conv_state,
148 + has_initial_state=attn_metadata.context_lens_tensor > 0,
149 + cache_indices=mamba_cache_params.state_indices_tensor,
150 + query_start_loc=attn_metadata.query_start_loc)
151 + else:
152 + hidden_states = causal_conv1d_update(
153 + hidden_states.transpose(0, 1),
154 + mamba_cache_params.conv_state,
155 + conv_weights,
156 + self.conv1d.bias,
157 + self.activation,
158 + conv_state_indices=mamba_cache_params.state_indices_tensor)
159 + hidden_states = hidden_states.transpose(0, 1)
160 +
161 + # 3. State Space Model sequence transformation
162 + # 3.a. input varying initialization of time_step, B and C
163 + ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
164 +
165 + time_step, B, C = torch.split(
166 + ssm_parameters,
167 + [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
168 + dim=-1,
169 + )
170 + if self.use_rms_norm:
171 + assert self.dt_layernorm is not None
172 + assert self.b_layernorm is not None
173 + assert self.c_layernorm is not None
174 + time_step = self.dt_layernorm(time_step.contiguous())
175 + B = self.b_layernorm(B.contiguous())
176 + C = self.c_layernorm(C.contiguous())
177 +
178 + discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
179 + # 3.c perform the recurrence y ← SSM(A, B, C)(x)
180 + time_proj_bias = (self.dt_proj.bias.float() if hasattr(
181 + self.dt_proj, "bias") else None)
182 +
183 + if attn_metadata.query_start_loc is not None \
184 + and attn_metadata.context_lens_tensor is not None:
185 + scan_outputs = selective_scan_fn(
186 + hidden_states,
187 + mamba_cache_params.ssm_state,
188 + discrete_time_step,
189 + self.A,
190 + B.transpose(-2, -1),
191 + C.transpose(-2, -1),
192 + self.D.float(),
193 + gate,
194 + time_proj_bias,
195 + delta_softplus=True,
196 + cache_indices=mamba_cache_params.state_indices_tensor,
197 + has_initial_state=attn_metadata.context_lens_tensor > 0,
198 + query_start_loc=attn_metadata.query_start_loc)
199 + else:
200 + scan_outputs = selective_state_update(
201 + mamba_cache_params.ssm_state,
202 + hidden_states.transpose(0, 1),
203 + discrete_time_step.transpose(0, 1),
204 + self.A,
205 + B,
206 + C,
207 + self.D,
208 + gate.transpose(0, 1),
209 + time_proj_bias,
210 + dt_softplus=True,
211 + state_batch_indices=mamba_cache_params.state_indices_tensor)
212 + scan_outputs = scan_outputs.transpose(0, 1)
213 +
214 + # 4. Final linear projection
215 + contextualized_states = self.out_proj(scan_outputs.transpose(-2,
216 + -1))[0]
217 + return contextualized_states
vllm/model_executor/models/jamba.py
+14 -185
@@ -12,26 +12,19 @@ from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12 from vllm.distributed import get_tensor_model_parallel_world_size
13 from vllm.model_executor.layers.fused_moe import FusedMoE
14 from vllm.model_executor.layers.layernorm import RMSNorm
15 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 - MergedColumnParallelLinear,
17 - QKVParallelLinear,
15 +from vllm.model_executor.layers.linear import (QKVParallelLinear,
16 ReplicatedLinear,
17 RowParallelLinear)
18 from vllm.model_executor.layers.logits_processor import LogitsProcessor
21 -from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
22 - causal_conv1d_fn, causal_conv1d_update)
23 -from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
24 - selective_scan_fn, selective_state_update)
19 +from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
20 from vllm.model_executor.layers.quantization import QuantizationConfig
21 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
22 from vllm.model_executor.layers.vocab_parallel_embedding import (
23 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
29 -from vllm.model_executor.model_loader.weight_utils import (
30 - composed_weight_loader, default_weight_loader, sharded_weight_loader)
24 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
25 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
26 MambaCacheParams)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 -from vllm.model_executor.utils import set_weight_attrs
28 from vllm.sequence import IntermediateTensors
29 from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
30 _get_graph_batch_size)
@@ -41,179 +34,6 @@ from .interfaces import HasInnerState, SupportsLoRA
34 KVCache = Tuple[torch.Tensor, torch.Tensor]
35
36
44 -# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
45 -class JambaMambaMixer(nn.Module):
46 - """
47 - Compute ∆, A, B, C, and D the state space parameters and compute
48 - the `contextualized_states`. A, D are input independent
49 - (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
50 - for why A isn't selective) ∆, B, C are input-dependent
51 - (this is a key difference between Mamba and the linear time
52 - invariant S4, and is why Mamba is called
53 - **selective** state spaces)
54 - """
55 -
56 - def __init__(self, config: JambaConfig):
57 - super().__init__()
58 - self.config = config
59 - self.hidden_size = config.hidden_size
60 - self.ssm_state_size = config.mamba_d_state
61 - self.conv_kernel_size = config.mamba_d_conv
62 - self.intermediate_size = config.mamba_expand * config.hidden_size
63 - self.time_step_rank = config.mamba_dt_rank
64 - self.use_conv_bias = config.mamba_conv_bias
65 - self.use_bias = config.mamba_proj_bias
66 - self.conv1d = ColumnParallelLinear(
67 - input_size=self.conv_kernel_size,
68 - output_size=self.intermediate_size,
69 - bias=self.use_conv_bias,
70 - )
71 - # unsqueeze to fit conv1d weights shape into the linear weights shape.
72 - # Can't do this in `weight_loader` since it already exists in
73 - # `ColumnParallelLinear` and `set_weight_attrs`
74 - # doesn't allow to override it
75 - self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
76 -
77 - self.in_proj = MergedColumnParallelLinear(self.hidden_size,
78 - [self.intermediate_size] * 2,
79 - bias=self.use_bias)
80 - # selective projection used to make dt, B and C input dependent
81 - self.x_proj = RowParallelLinear(
82 - self.intermediate_size,
83 - self.time_step_rank + self.ssm_state_size * 2,
84 - bias=False,
85 - )
86 - # time step projection (discretization) -
87 - # In the forward we need to apply dt_proj without the bias,
88 - # as the bias is added in the selective scan kernel.
89 - self.dt_proj = ColumnParallelLinear(self.time_step_rank,
90 - self.intermediate_size,
91 - bias=True,
92 - skip_bias_add=True)
93 -
94 - tp_size = get_tensor_model_parallel_world_size()
95 - self.A = nn.Parameter(
96 - torch.empty(
97 - self.intermediate_size // tp_size,
98 - self.ssm_state_size,
99 - dtype=torch.float32,
100 - ))
101 - self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
102 -
103 - set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
104 - a_weight_loader = composed_weight_loader(
105 - sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
106 - set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
107 -
108 - self.out_proj = RowParallelLinear(
109 - self.intermediate_size,
110 - self.hidden_size,
111 - bias=self.use_bias,
112 - input_is_parallel=True,
113 - )
114 - self.activation = config.hidden_act
115 -
116 - self.dt_layernorm = RMSNorm(self.time_step_rank,
117 - eps=config.rms_norm_eps)
118 - self.b_layernorm = RMSNorm(self.ssm_state_size,
119 - eps=config.rms_norm_eps)
120 - self.c_layernorm = RMSNorm(self.ssm_state_size,
121 - eps=config.rms_norm_eps)
122 -
123 - def forward(self, hidden_states: torch.Tensor,
124 - attn_metadata: AttentionMetadata,
125 - mamba_cache_params: MambaCacheParams):
126 -
127 - # 1. Gated MLP's linear projection
128 - projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
129 - hidden_states, gate = projected_states.chunk(2, dim=-2)
130 -
131 - # 2. Convolution sequence transformation
132 - conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
133 - self.conv1d.weight.size(2))
134 -
135 - if attn_metadata.query_start_loc is not None \
136 - and attn_metadata.context_lens_tensor is not None:
137 - # |---------- N-1 iteration --------|
138 - # |---------------- N iteration ---------------------|
139 - # |- tokenA -|......................|-- newTokens ---|
140 - # |---------- context_len ----------|
141 - # |-------------------- seq_len ---------------------|
142 - # |-- query_len ---|
143 - hidden_states = causal_conv1d_fn(
144 - hidden_states,
145 - conv_weights,
146 - self.conv1d.bias,
147 - activation=self.activation,
148 - conv_states=mamba_cache_params.conv_state,
149 - has_initial_state=attn_metadata.context_lens_tensor > 0,
150 - cache_indices=mamba_cache_params.state_indices_tensor,
151 - query_start_loc=attn_metadata.query_start_loc)
152 - else:
153 - hidden_states = causal_conv1d_update(
154 - hidden_states.transpose(0, 1),
155 - mamba_cache_params.conv_state,
156 - conv_weights,
157 - self.conv1d.bias,
158 - self.activation,
159 - conv_state_indices=mamba_cache_params.state_indices_tensor)
160 - hidden_states = hidden_states.transpose(0, 1)
161 -
162 - # 3. State Space Model sequence transformation
163 - # 3.a. input varying initialization of time_step, B and C
164 - ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
165 -
166 - time_step, B, C = torch.split(
167 - ssm_parameters,
168 - [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
169 - dim=-1,
170 - )
171 - time_step = self.dt_layernorm(time_step.contiguous())
172 - B = self.b_layernorm(B.contiguous())
173 - C = self.c_layernorm(C.contiguous())
174 -
175 - discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176 - # 3.c perform the recurrence y ← SSM(A, B, C)(x)
177 - time_proj_bias = (self.dt_proj.bias.float() if hasattr(
178 - self.dt_proj, "bias") else None)
179 -
180 - if attn_metadata.query_start_loc is not None \
181 - and attn_metadata.context_lens_tensor is not None:
182 - scan_outputs = selective_scan_fn(
183 - hidden_states,
184 - mamba_cache_params.ssm_state,
185 - discrete_time_step,
186 - self.A,
187 - B.transpose(-2, -1),
188 - C.transpose(-2, -1),
189 - self.D.float(),
190 - gate,
191 - time_proj_bias,
192 - delta_softplus=True,
193 - cache_indices=mamba_cache_params.state_indices_tensor,
194 - has_initial_state=attn_metadata.context_lens_tensor > 0,
195 - query_start_loc=attn_metadata.query_start_loc)
196 - else:
197 - scan_outputs = selective_state_update(
198 - mamba_cache_params.ssm_state,
199 - hidden_states.transpose(0, 1),
200 - discrete_time_step.transpose(0, 1),
201 - self.A,
202 - B,
203 - C,
204 - self.D,
205 - gate.transpose(0, 1),
206 - time_proj_bias,
207 - dt_softplus=True,
208 - state_batch_indices=mamba_cache_params.state_indices_tensor)
209 - scan_outputs = scan_outputs.transpose(0, 1)
210 -
211 - # 4. Final linear projection
212 - contextualized_states = self.out_proj(scan_outputs.transpose(-2,
213 - -1))[0]
214 - return contextualized_states
215 -
216 -
37 class JambaMoE(nn.Module):
38
39 def __init__(self,
@@ -284,9 +104,18 @@ class JambaMambaDecoderLayer(nn.Module):
104 cache_config: Optional[CacheConfig] = None,
105 quant_config: Optional[QuantizationConfig] = None) -> None:
106 super().__init__()
287 - self.layer_idx = layer_idx
107 self.config = config
289 - self.mamba = JambaMambaMixer(config)
108 + self.mamba = MambaMixer(hidden_size= config.hidden_size,
109 + ssm_state_size = config.mamba_d_state,
110 + conv_kernel_size = config.mamba_d_conv,
111 + intermediate_size = config.mamba_expand *\
112 + config.hidden_size,
113 + time_step_rank = config.mamba_dt_rank,
114 + use_conv_bias = config.mamba_conv_bias,
115 + use_bias = config.mamba_proj_bias,
116 + use_rms_norm=True,
117 + rms_norm_eps=config.rms_norm_eps,
118 + activation=config.hidden_act)
119
120 num_experts = config.layers_num_experts[layer_idx]
121 ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
vllm/model_executor/models/mamba.py
+14 -189
@@ -10,27 +10,19 @@ from vllm.attention.backends.abstract import AttentionMetadata
10 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 from vllm.model_executor.layers.layernorm import RMSNorm
13 -from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14 - MergedColumnParallelLinear,
15 - RowParallelLinear)
13 from vllm.model_executor.layers.logits_processor import LogitsProcessor
17 -from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
18 - causal_conv1d_fn, causal_conv1d_update)
19 -from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
20 - selective_scan_fn, selective_state_update)
14 +from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
15 from vllm.model_executor.layers.quantization.base_config import (
16 QuantizationConfig)
17 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
18 from vllm.model_executor.layers.vocab_parallel_embedding import (
19 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
26 -from vllm.model_executor.model_loader.weight_utils import (
27 - composed_weight_loader, default_weight_loader, sharded_weight_loader)
20 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.model_executor.models.interfaces import (HasInnerState,
22 IsAttentionFree)
23 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
24 MambaCacheParams)
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
33 -from vllm.model_executor.utils import set_weight_attrs
26 from vllm.sequence import IntermediateTensors
27 from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
28 _get_graph_batch_size)
@@ -38,194 +30,27 @@ from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
30 KVCache = Tuple[torch.Tensor, torch.Tensor]
31
32
41 -# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
42 -class MambaMixer(nn.Module):
43 - """
44 - Compute ∆, A, B, C, and D the state space parameters and compute
45 - the `contextualized_states`. A, D are input independent
46 - (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
47 - for why A isn't selective) ∆, B, C are input-dependent
48 - (this is a key difference between Mamba and the linear time
49 - invariant S4, and is why Mamba is called
50 - **selective** state spaces)
51 - """
52 -
53 - def __init__(self, config: MambaConfig, layer_idx):
54 - super().__init__()
55 - self.config = config
56 - self.layer_idx = layer_idx
57 - self.hidden_size = config.hidden_size
58 - self.ssm_state_size = config.state_size
59 - self.conv_kernel_size = config.conv_kernel
60 - self.intermediate_size = config.intermediate_size
61 - self.time_step_rank = int(config.time_step_rank)
62 - self.is_falcon_mamba = config.model_type == "falcon_mamba"
63 - self.conv1d = ColumnParallelLinear(
64 - input_size=self.conv_kernel_size,
65 - output_size=self.intermediate_size,
66 - bias=config.use_conv_bias,
67 - )
68 - # unsqueeze to fit conv1d weights shape into the linear weights shape.
69 - # Can't do this in `weight_loader` since it already exists in
70 - # `ColumnParallelLinear` and `set_weight_attrs`
71 - # doesn't allow to override it
72 - self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
73 -
74 - self.in_proj = MergedColumnParallelLinear(self.hidden_size,
75 - [self.intermediate_size] * 2,
76 - bias=config.use_bias)
77 - # selective projection used to make dt, B and C input dependent
78 - self.x_proj = RowParallelLinear(
79 - self.intermediate_size,
80 - self.time_step_rank + self.ssm_state_size * 2,
81 - bias=False,
82 - )
83 - # time step projection (discretization) -
84 - # In the forward we need to apply dt_proj without the bias,
85 - # as the bias is added in the selective scan kernel.
86 - self.dt_proj = ColumnParallelLinear(self.time_step_rank,
87 - self.intermediate_size,
88 - bias=True,
89 - skip_bias_add=True)
90 -
91 - tp_size = get_tensor_model_parallel_world_size()
92 - self.A = nn.Parameter(
93 - torch.empty(
94 - self.intermediate_size // tp_size,
95 - self.ssm_state_size,
96 - dtype=torch.float32,
97 - ))
98 - self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
99 -
100 - set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
101 - a_weight_loader = composed_weight_loader(
102 - sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
103 - set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
104 -
105 - self.out_proj = RowParallelLinear(
106 - self.intermediate_size,
107 - self.hidden_size,
108 - bias=config.use_bias,
109 - input_is_parallel=True,
110 - )
111 - self.activation = config.hidden_act
112 - if self.is_falcon_mamba:
113 - self.dt_layernorm = RMSNorm(self.time_step_rank,
114 - eps=config.mixer_rms_eps)
115 - self.b_layernorm = RMSNorm(self.ssm_state_size,
116 - eps=config.mixer_rms_eps)
117 - self.c_layernorm = RMSNorm(self.ssm_state_size,
118 - eps=config.mixer_rms_eps)
119 -
120 - def forward(self, hidden_states: torch.Tensor,
121 - attn_metadata: AttentionMetadata,
122 - mamba_cache_params: MambaCacheParams):
123 -
124 - # 1. Gated MLP's linear projection
125 - projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
126 - hidden_states, gate = projected_states.chunk(2, dim=-2)
127 -
128 - # 2. Convolution sequence transformation
129 - conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
130 - self.conv1d.weight.size(2))
131 -
132 - if attn_metadata.query_start_loc is not None \
133 - and attn_metadata.context_lens_tensor is not None:
134 - # |---------- N-1 iteration --------|
135 - # |---------------- N iteration ---------------------|
136 - # |- tokenA -|......................|-- newTokens ---|
137 - # |---------- context_len ----------|
138 - # |-------------------- seq_len ---------------------|
139 - # |-- query_len ---|
140 - hidden_states = causal_conv1d_fn(
141 - hidden_states,
142 - conv_weights,
143 - self.conv1d.bias,
144 - activation=self.activation,
145 - conv_states=mamba_cache_params.conv_state,
146 - has_initial_state=attn_metadata.context_lens_tensor > 0,
147 - cache_indices=mamba_cache_params.state_indices_tensor,
148 - query_start_loc=attn_metadata.query_start_loc)
149 - else:
150 - hidden_states = causal_conv1d_update(
151 - hidden_states.transpose(0, 1),
152 - mamba_cache_params.conv_state,
153 - conv_weights,
154 - self.conv1d.bias,
155 - self.activation,
156 - conv_state_indices=mamba_cache_params.state_indices_tensor)
157 - hidden_states = hidden_states.transpose(0, 1)
158 -
159 - # 3. State Space Model sequence transformation
160 - # 3.a. input varying initialization of time_step, B and C
161 - ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
162 -
163 - time_step, B, C = torch.split(
164 - ssm_parameters,
165 - [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166 - dim=-1,
167 - )
168 - # Note that Jamba and FalconMamba normalizes B, C, and time_step here
169 - # but Mamba doesn't.
170 - if self.is_falcon_mamba:
171 - time_step = self.dt_layernorm(time_step.contiguous())
172 - B = self.b_layernorm(B.contiguous())
173 - C = self.c_layernorm(C.contiguous())
174 -
175 - discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176 - # 3.c perform the recurrence y ← SSM(A, B, C)(x)
177 - time_proj_bias = (self.dt_proj.bias.float() if hasattr(
178 - self.dt_proj, "bias") else None)
179 -
180 - if attn_metadata.query_start_loc is not None \
181 - and attn_metadata.context_lens_tensor is not None:
182 - scan_outputs = selective_scan_fn(
183 - hidden_states,
184 - mamba_cache_params.ssm_state,
185 - discrete_time_step,
186 - self.A,
187 - B.transpose(-2, -1),
188 - C.transpose(-2, -1),
189 - self.D.float(),
190 - gate,
191 - time_proj_bias,
192 - delta_softplus=True,
193 - cache_indices=mamba_cache_params.state_indices_tensor,
194 - has_initial_state=attn_metadata.context_lens_tensor > 0,
195 - query_start_loc=attn_metadata.query_start_loc)
196 - else:
197 - scan_outputs = selective_state_update(
198 - mamba_cache_params.ssm_state,
199 - hidden_states.transpose(0, 1),
200 - discrete_time_step.transpose(0, 1),
201 - self.A,
202 - B,
203 - C,
204 - self.D,
205 - gate.transpose(0, 1),
206 - time_proj_bias,
207 - dt_softplus=True,
208 - state_batch_indices=mamba_cache_params.state_indices_tensor)
209 - scan_outputs = scan_outputs.transpose(0, 1)
210 -
211 - # 4. Final linear projection
212 - contextualized_states = self.out_proj(scan_outputs.transpose(-2,
213 - -1))[0]
214 - return contextualized_states
215 -
216 -
33 class MambaDecoderLayer(nn.Module):
34
35 def __init__(self,
36 config: MambaConfig,
221 - layer_idx: int,
37 cache_config: Optional[CacheConfig] = None,
38 quant_config: Optional[QuantizationConfig] = None) -> None:
39 super().__init__()
225 - self.layer_idx = layer_idx
40 self.config = config
41 self.is_falcon_mamba = config.model_type == "falcon_mamba"
228 - self.mixer = MambaMixer(config, layer_idx)
42 + mixer_rms_rps = config.mixer_rms_rps if self.is_falcon_mamba else None
43 + self.mamba = MambaMixer(hidden_size=config.hidden_size,
44 + ssm_state_size=config.state_size,
45 + conv_kernel_size=config.conv_kernel,
46 + intermediate_size=config.intermediate_size,
47 + time_step_rank=config.time_step_rank,
48 + use_conv_bias=config.use_conv_bias,
49 + use_bias=config.use_bias,
50 + use_rms_norm=self.is_falcon_mamba,
51 + rms_norm_eps=mixer_rms_rps,
52 + activation=config.hidden_act)
53 +
54 self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
55
56 def forward(