A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Model] Add support for falcon-11B (#5069)
Isotr0py committed
May 28, 2024 at 07:41 UTC
890aa93d275a2b75313629614ab9ed278a13f6d7
1 file changed
+40
-15
vllm/model_executor/models/falcon.py
+40
-15
@@ -41,7 +41,7 @@ from vllm.model_executor.layers.quantization.base_config import (
41
from vllm.model_executor.layers.rotary_embedding import get_rope
42
from vllm.model_executor.layers.sampler import Sampler
43
from vllm.model_executor.layers.vocab_parallel_embedding import (
44
- VocabParallelEmbedding)
44
+ ParallelLMHead, VocabParallelEmbedding)
45
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
from vllm.model_executor.sampling_metadata import SamplingMetadata
47
from vllm.sequence import SamplerOutput
@@ -246,18 +246,26 @@ class FalconDecoderLayer(nn.Module):
246
self.mlp = FalconMLP(config, quant_config)
247
self.config = config
248
249
- if config.new_decoder_architecture:
250
- # The layer norm before self-attention
251
- self.ln_attn = LayerNorm(hidden_size,
252
- eps=config.layer_norm_epsilon)
253
- # The layer norm before the MLP
254
- self.ln_mlp = LayerNorm(hidden_size, eps=config.layer_norm_epsilon)
255
- else:
249
+ if (config.num_ln_in_parallel_attn is None
250
+ and config.new_decoder_architecture):
251
+ config.num_ln_in_parallel_attn = 2
252
+
253
+ if not config.parallel_attn:
254
+ self.post_attention_layernorm = LayerNorm(
255
+ hidden_size, eps=config.layer_norm_epsilon)
256
self.input_layernorm = LayerNorm(hidden_size,
257
eps=config.layer_norm_epsilon)
258
- if not config.parallel_attn:
259
- self.post_attention_layernorm = LayerNorm(
260
- hidden_size, eps=config.layer_norm_epsilon)
258
+ else:
259
+ if config.num_ln_in_parallel_attn == 2:
260
+ # The layer norm before self-attention
261
+ self.ln_attn = LayerNorm(hidden_size,
262
+ eps=config.layer_norm_epsilon)
263
+ # The layer norm before the MLP
264
+ self.ln_mlp = LayerNorm(hidden_size,
265
+ eps=config.layer_norm_epsilon)
266
+ else:
267
+ self.input_layernorm = LayerNorm(hidden_size,
268
+ eps=config.layer_norm_epsilon)
269
270
self.reduce_row_parallel_results = not (config.new_decoder_architecture
271
or config.parallel_attn)
@@ -271,7 +279,7 @@ class FalconDecoderLayer(nn.Module):
279
) -> torch.Tensor:
280
residual = hidden_states
281
274
- if self.config.new_decoder_architecture:
282
+ if self.config.num_ln_in_parallel_attn == 2:
283
attention_layernorm_out = self.ln_attn(hidden_states)
284
mlp_layernorm_out = self.ln_mlp(hidden_states)
285
else:
@@ -294,6 +302,10 @@ class FalconDecoderLayer(nn.Module):
302
residual += attention_output
303
mlp_layernorm_out = self.post_attention_layernorm(residual)
304
305
+ if (self.config.new_decoder_architecture and self.config.parallel_attn
306
+ and self.config.num_ln_in_parallel_attn == 1):
307
+ mlp_layernorm_out = attention_layernorm_out
308
+
309
# MLP.
310
mlp_output, mlp_bias = self.mlp(mlp_layernorm_out)
311
if self.reduce_row_parallel_results and mlp_bias is not None:
@@ -375,7 +387,20 @@ class FalconForCausalLM(nn.Module):
387
self.config = config
388
self.quant_config = quant_config
389
self.transformer = FalconModel(config, cache_config, quant_config)
378
- self.lm_head_weight = self.transformer.word_embeddings.weight
390
+ # only Falcon-11B doesn't share lm_head weight with word embeddings
391
+ # and previous Falcon model doesn't have tie_word_embeddings config
392
+ # so we set tie_word_embeddings to True by default
393
+ self.tie_word_embeddings = (config.tie_word_embeddings
394
+ if config.tie_word_embeddings is not None
395
+ else True)
396
+ if self.tie_word_embeddings:
397
+ self.lm_head_weight = self.transformer.word_embeddings.weight
398
+ else:
399
+ self.lm_head = ParallelLMHead(
400
+ config.vocab_size,
401
+ config.hidden_size,
402
+ )
403
+ self.lm_head_weight = self.lm_head.weight
404
self.logits_processor = LogitsProcessor(config.vocab_size)
405
self.sampler = Sampler()
406
@@ -419,8 +444,8 @@ class FalconForCausalLM(nn.Module):
444
num_query_heads_per_kv_head = total_num_heads // total_num_kv_heads
445
params_dict = dict(self.named_parameters(remove_duplicate=False))
446
for name, loaded_weight in weights:
422
- if name == "lm_head.weight":
423
- # Falcon uses tied embeddings.
447
+ if name == "lm_head.weight" and self.tie_word_embeddings:
448
+ # Falcon uses tied embeddings except Falcon-11b.
449
continue
450
# Skip loading extra bias for GPTQ models.
451
if name.endswith(".bias") and name not in params_dict: