A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] FalconMamba Support (#9325)
Dhia Eddine Rhaiem committed
Oct 21, 2024 at 20:50 UTC
f6b97293aa7d52e52e9c5144cc98330733a8cf0d
5 files changed
+35
-12
docs/source/models/supported_models.rst
+5
@@ -87,6 +87,11 @@ Text Generation
87
- :code:`tiiuae/falcon-7b`, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
88
-
89
- ✅︎
90
+ * - :code:`FalconMambaForCausalLM`
91
+ - FalconMamba
92
+ - :code:`tiiuae/falcon-mamba-7b`, :code:`tiiuae/falcon-mamba-7b-instruct`, etc.
93
+ - ✅︎
94
+ -
95
* - :code:`GemmaForCausalLM`
96
- Gemma
97
- :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
tests/models/decoder_only/language/test_mamba.py
+1
-1
@@ -10,7 +10,7 @@ from vllm.worker.model_runner import _get_graph_batch_size
10
11
from ...utils import check_outputs_equal
12
13
-MODELS = ["state-spaces/mamba-130m-hf"]
13
+MODELS = ["state-spaces/mamba-130m-hf", "tiiuae/falcon-mamba-tiny-dev"]
14
15
16
# Use lower-level interfaces to create this greedy generator, as mamba will
vllm/model_executor/layers/layernorm.py
-1
@@ -27,7 +27,6 @@ class RMSNorm(CustomOp):
27
self.variance_epsilon = eps
28
self.variance_size_override = (None if var_hidden_size == hidden_size
29
else var_hidden_size)
30
-
30
self.weight = nn.Parameter(torch.ones(hidden_size))
31
32
def forward_native(
vllm/model_executor/models/mamba.py
+28
-10
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.quantization.base_config import (
22
QuantizationConfig)
23
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24
from vllm.model_executor.layers.vocab_parallel_embedding import (
25
- VocabParallelEmbedding)
25
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
26
from vllm.model_executor.model_loader.weight_utils import (
27
composed_weight_loader, default_weight_loader, sharded_weight_loader)
28
from vllm.model_executor.models.interfaces import (HasInnerState,
@@ -59,7 +59,7 @@ class MambaMixer(nn.Module):
59
self.conv_kernel_size = config.conv_kernel
60
self.intermediate_size = config.intermediate_size
61
self.time_step_rank = int(config.time_step_rank)
62
-
62
+ self.is_falcon_mamba = config.model_type == "falcon_mamba"
63
self.conv1d = ColumnParallelLinear(
64
input_size=self.conv_kernel_size,
65
output_size=self.intermediate_size,
@@ -109,6 +109,13 @@ class MambaMixer(nn.Module):
109
input_is_parallel=True,
110
)
111
self.activation = config.hidden_act
112
+ if self.is_falcon_mamba:
113
+ self.dt_layernorm = RMSNorm(self.time_step_rank,
114
+ eps=config.mixer_rms_eps)
115
+ self.b_layernorm = RMSNorm(self.ssm_state_size,
116
+ eps=config.mixer_rms_eps)
117
+ self.c_layernorm = RMSNorm(self.ssm_state_size,
118
+ eps=config.mixer_rms_eps)
119
120
def forward(self, hidden_states: torch.Tensor,
121
attn_metadata: AttentionMetadata,
@@ -158,8 +165,12 @@ class MambaMixer(nn.Module):
165
[self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166
dim=-1,
167
)
161
-
162
- # Note that Jamba normalizes B, C, and time_step here but Mamba doesn't.
168
+ # Note that Jamba and FalconMamba normalizes B, C, and time_step here
169
+ # but Mamba doesn't.
170
+ if self.is_falcon_mamba:
171
+ time_step = self.dt_layernorm(time_step.contiguous())
172
+ B = self.b_layernorm(B.contiguous())
173
+ C = self.c_layernorm(C.contiguous())
174
175
discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176
# 3.c perform the recurrence y ← SSM(A, B, C)(x)
@@ -213,11 +224,9 @@ class MambaDecoderLayer(nn.Module):
224
super().__init__()
225
self.layer_idx = layer_idx
226
self.config = config
227
+ self.is_falcon_mamba = config.model_type == "falcon_mamba"
228
self.mixer = MambaMixer(config, layer_idx)
217
-
229
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
219
- self.pre_ff_layernorm = RMSNorm(config.hidden_size,
220
- eps=config.layer_norm_epsilon)
230
231
def forward(
232
self,
@@ -319,8 +328,18 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
328
self.unpadded_vocab_size = config.vocab_size
329
if lora_config:
330
self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
322
-
323
- self.lm_head = self.backbone.embeddings
331
+ if config.tie_word_embeddings:
332
+ self.lm_head = self.backbone.embeddings
333
+ else:
334
+ self.lm_head = ParallelLMHead(
335
+ self.unpadded_vocab_size,
336
+ config.hidden_size,
337
+ org_num_embeddings=config.vocab_size,
338
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
339
+ # We need bigger padding if using lora for kernel
340
+ # compatibility
341
+ if not lora_config else lora_config.lora_vocab_padding_size,
342
+ )
343
344
# Used to track and store by the Mamba cache between steps.
345
self.mamba_cache: Optional[MambaCacheManager] = None
@@ -398,7 +417,6 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
417
for name, loaded_weight in weights:
418
if "A_log" in name:
419
name = name.replace("A_log", "A")
401
-
420
# Skip loading extra bias for GPTQ models.
421
if name.endswith(".bias") and name not in params_dict:
422
continue
vllm/model_executor/models/registry.py
+1
@@ -53,6 +53,7 @@ _TEXT_GENERATION_MODELS = {
53
# For decapoda-research/llama-*
54
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
55
"MambaForCausalLM": ("mamba", "MambaForCausalLM"),
56
+ "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
57
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
58
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
59
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),