@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f6b97293aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] FalconMamba Support (#9325)

Dhia Eddine Rhaiem committed Oct 21, 2024 at 20:50 UTC f6b97293aa7d52e52e9c5144cc98330733a8cf0d
5 files changed +35 -12
docs/source/models/supported_models.rst
+5
@@ -87,6 +87,11 @@ Text Generation
87 - :code:`tiiuae/falcon-7b`, :code:`tiiuae/falcon-40b`, :code:`tiiuae/falcon-rw-7b`, etc.
88 -
89 - ✅︎
90 + * - :code:`FalconMambaForCausalLM`
91 + - FalconMamba
92 + - :code:`tiiuae/falcon-mamba-7b`, :code:`tiiuae/falcon-mamba-7b-instruct`, etc.
93 + - ✅︎
94 + -
95 * - :code:`GemmaForCausalLM`
96 - Gemma
97 - :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
tests/models/decoder_only/language/test_mamba.py
+1 -1
@@ -10,7 +10,7 @@ from vllm.worker.model_runner import _get_graph_batch_size
10
11 from ...utils import check_outputs_equal
12
13 -MODELS = ["state-spaces/mamba-130m-hf"]
13 +MODELS = ["state-spaces/mamba-130m-hf", "tiiuae/falcon-mamba-tiny-dev"]
14
15
16 # Use lower-level interfaces to create this greedy generator, as mamba will
vllm/model_executor/layers/layernorm.py
-1
@@ -27,7 +27,6 @@ class RMSNorm(CustomOp):
27 self.variance_epsilon = eps
28 self.variance_size_override = (None if var_hidden_size == hidden_size
29 else var_hidden_size)
30 -
30 self.weight = nn.Parameter(torch.ones(hidden_size))
31
32 def forward_native(
vllm/model_executor/models/mamba.py
+28 -10
@@ -22,7 +22,7 @@ from vllm.model_executor.layers.quantization.base_config import (
22 QuantizationConfig)
23 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
24 from vllm.model_executor.layers.vocab_parallel_embedding import (
25 - VocabParallelEmbedding)
25 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
26 from vllm.model_executor.model_loader.weight_utils import (
27 composed_weight_loader, default_weight_loader, sharded_weight_loader)
28 from vllm.model_executor.models.interfaces import (HasInnerState,
@@ -59,7 +59,7 @@ class MambaMixer(nn.Module):
59 self.conv_kernel_size = config.conv_kernel
60 self.intermediate_size = config.intermediate_size
61 self.time_step_rank = int(config.time_step_rank)
62 -
62 + self.is_falcon_mamba = config.model_type == "falcon_mamba"
63 self.conv1d = ColumnParallelLinear(
64 input_size=self.conv_kernel_size,
65 output_size=self.intermediate_size,
@@ -109,6 +109,13 @@ class MambaMixer(nn.Module):
109 input_is_parallel=True,
110 )
111 self.activation = config.hidden_act
112 + if self.is_falcon_mamba:
113 + self.dt_layernorm = RMSNorm(self.time_step_rank,
114 + eps=config.mixer_rms_eps)
115 + self.b_layernorm = RMSNorm(self.ssm_state_size,
116 + eps=config.mixer_rms_eps)
117 + self.c_layernorm = RMSNorm(self.ssm_state_size,
118 + eps=config.mixer_rms_eps)
119
120 def forward(self, hidden_states: torch.Tensor,
121 attn_metadata: AttentionMetadata,
@@ -158,8 +165,12 @@ class MambaMixer(nn.Module):
165 [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166 dim=-1,
167 )
161 -
162 - # Note that Jamba normalizes B, C, and time_step here but Mamba doesn't.
168 + # Note that Jamba and FalconMamba normalizes B, C, and time_step here
169 + # but Mamba doesn't.
170 + if self.is_falcon_mamba:
171 + time_step = self.dt_layernorm(time_step.contiguous())
172 + B = self.b_layernorm(B.contiguous())
173 + C = self.c_layernorm(C.contiguous())
174
175 discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
176 # 3.c perform the recurrence y ← SSM(A, B, C)(x)
@@ -213,11 +224,9 @@ class MambaDecoderLayer(nn.Module):
224 super().__init__()
225 self.layer_idx = layer_idx
226 self.config = config
227 + self.is_falcon_mamba = config.model_type == "falcon_mamba"
228 self.mixer = MambaMixer(config, layer_idx)
217 -
229 self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
219 - self.pre_ff_layernorm = RMSNorm(config.hidden_size,
220 - eps=config.layer_norm_epsilon)
230
231 def forward(
232 self,
@@ -319,8 +328,18 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
328 self.unpadded_vocab_size = config.vocab_size
329 if lora_config:
330 self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
322 -
323 - self.lm_head = self.backbone.embeddings
331 + if config.tie_word_embeddings:
332 + self.lm_head = self.backbone.embeddings
333 + else:
334 + self.lm_head = ParallelLMHead(
335 + self.unpadded_vocab_size,
336 + config.hidden_size,
337 + org_num_embeddings=config.vocab_size,
338 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
339 + # We need bigger padding if using lora for kernel
340 + # compatibility
341 + if not lora_config else lora_config.lora_vocab_padding_size,
342 + )
343
344 # Used to track and store by the Mamba cache between steps.
345 self.mamba_cache: Optional[MambaCacheManager] = None
@@ -398,7 +417,6 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
417 for name, loaded_weight in weights:
418 if "A_log" in name:
419 name = name.replace("A_log", "A")
401 -
420 # Skip loading extra bias for GPTQ models.
421 if name.endswith(".bias") and name not in params_dict:
422 continue
vllm/model_executor/models/registry.py
+1
@@ -53,6 +53,7 @@ _TEXT_GENERATION_MODELS = {
53 # For decapoda-research/llama-*
54 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
55 "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
56 + "FalconMambaForCausalLM": ("mamba", "MambaForCausalLM"),
57 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
58 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
59 "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),