@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 890aa93d27
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Model] Add support for falcon-11B (#5069)

Isotr0py committed May 28, 2024 at 07:41 UTC 890aa93d275a2b75313629614ab9ed278a13f6d7
1 file changed +40 -15
vllm/model_executor/models/falcon.py
+40 -15
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.quantization.base_config import (
41 from vllm.model_executor.layers.rotary_embedding import get_rope
42 from vllm.model_executor.layers.sampler import Sampler
43 from vllm.model_executor.layers.vocab_parallel_embedding import (
44 - VocabParallelEmbedding)
44 + ParallelLMHead, VocabParallelEmbedding)
45 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 from vllm.model_executor.sampling_metadata import SamplingMetadata
47 from vllm.sequence import SamplerOutput
@@ -246,18 +246,26 @@ class FalconDecoderLayer(nn.Module):
246 self.mlp = FalconMLP(config, quant_config)
247 self.config = config
248
249 - if config.new_decoder_architecture:
250 - # The layer norm before self-attention
251 - self.ln_attn = LayerNorm(hidden_size,
252 - eps=config.layer_norm_epsilon)
253 - # The layer norm before the MLP
254 - self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
255 - else:
249 + if (config.num_ln_in_parallel_attn is None
250 + and config.new_decoder_architecture):
251 + config.num_ln_in_parallel_attn = 2
252 +
253 + if not config.parallel_attn:
254 + self.post_attention_layernorm = LayerNorm(
255 + hidden_size, eps=config.layer_norm_epsilon)
256 self.input_layernorm = LayerNorm(hidden_size,
257 eps=config.layer_norm_epsilon)
258 - if not config.parallel_attn:
259 - self.post_attention_layernorm = LayerNorm(
260 - hidden_size, eps=config.layer_norm_epsilon)
258 + else:
259 + if config.num_ln_in_parallel_attn == 2:
260 + # The layer norm before self-attention
261 + self.ln_attn = LayerNorm(hidden_size,
262 + eps=config.layer_norm_epsilon)
263 + # The layer norm before the MLP
264 + self.ln_mlp = LayerNorm(hidden_size,
265 + eps=config.layer_norm_epsilon)
266 + else:
267 + self.input_layernorm = LayerNorm(hidden_size,
268 + eps=config.layer_norm_epsilon)
269
270 self.reduce_row_parallel_results = not (config.new_decoder_architecture
271 or config.parallel_attn)
@@ -271,7 +279,7 @@ class FalconDecoderLayer(nn.Module):
279 ) -> torch.Tensor:
280 residual = hidden_states
281
274 - if self.config.new_decoder_architecture:
282 + if self.config.num_ln_in_parallel_attn == 2:
283 attention_layernorm_out = self.ln_attn(hidden_states)
284 mlp_layernorm_out = self.ln_mlp(hidden_states)
285 else:
@@ -294,6 +302,10 @@ class FalconDecoderLayer(nn.Module):
302 residual += attention_output
303 mlp_layernorm_out = self.post_attention_layernorm(residual)
304
305 + if (self.config.new_decoder_architecture and self.config.parallel_attn
306 + and self.config.num_ln_in_parallel_attn == 1):
307 + mlp_layernorm_out = attention_layernorm_out
308 +
309 # MLP.
310 mlp_output, mlp_bias = self.mlp(mlp_layernorm_out)
311 if self.reduce_row_parallel_results and mlp_bias is not None:
@@ -375,7 +387,20 @@ class FalconForCausalLM(nn.Module):
387 self.config = config
388 self.quant_config = quant_config
389 self.transformer = FalconModel(config, cache_config, quant_config)
378 - self.lm_head_weight = self.transformer.word_embeddings.weight
390 + # only Falcon-11B doesn't share lm_head weight with word embeddings
391 + # and previous Falcon model doesn't have tie_word_embeddings config
392 + # so we set tie_word_embeddings to True by default
393 + self.tie_word_embeddings = (config.tie_word_embeddings
394 + if config.tie_word_embeddings is not None
395 + else True)
396 + if self.tie_word_embeddings:
397 + self.lm_head_weight = self.transformer.word_embeddings.weight
398 + else:
399 + self.lm_head = ParallelLMHead(
400 + config.vocab_size,
401 + config.hidden_size,
402 + )
403 + self.lm_head_weight = self.lm_head.weight
404 self.logits_processor = LogitsProcessor(config.vocab_size)
405 self.sampler = Sampler()
406
@@ -419,8 +444,8 @@ class FalconForCausalLM(nn.Module):
444 num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
445 params_dict = dict(self.named_parameters(remove_duplicate=False))
446 for name, loaded_weight in weights:
422 - if name == "lm_head.weight":
423 - # Falcon uses tied embeddings.
447 + if name == "lm_head.weight" and self.tie_word_embeddings:
448 + # Falcon uses tied embeddings except Falcon-11b.
449 continue
450 # Skip loading extra bias for GPTQ models.
451 if name.endswith(".bias") and name not in params_dict: