A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Model] Add support for IBM Granite Code models (#4636)
Yikang Shen committed
May 12, 2024 at 00:27 UTC
6eaccb7353cfe84d77981da726f6d82a8aefd2be
1 file changed
+6
-2
vllm/model_executor/models/llama.py
+6
-2
@@ -58,15 +58,16 @@ class LlamaMLP(nn.Module):
58
intermediate_size: int,
59
hidden_act: str,
60
quant_config: Optional[QKVParallelLinear] = None,
61
+ bias: bool = False,
62
) -> None:
63
super().__init__()
64
self.gate_up_proj = MergedColumnParallelLinear(
65
hidden_size, [intermediate_size] * 2,
65
- bias=False,
66
+ bias=bias,
67
quant_config=quant_config)
68
self.down_proj = RowParallelLinear(intermediate_size,
69
hidden_size,
69
- bias=False,
70
+ bias=bias,
71
quant_config=quant_config)
72
if hidden_act != "silu":
73
raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -209,6 +210,7 @@ class LlamaDecoderLayer(nn.Module):
210
intermediate_size=config.intermediate_size,
211
hidden_act=config.hidden_act,
212
quant_config=quant_config,
213
+ bias=getattr(config, "mlp_bias", False),
214
)
215
self.input_layernorm = RMSNorm(config.hidden_size,
216
eps=config.rms_norm_eps)
@@ -348,6 +350,8 @@ class LlamaForCausalLM(nn.Module):
350
# compatibility
351
if not lora_config else lora_config.lora_vocab_padding_size,
352
)
353
+ if config.tie_word_embeddings:
354
+ self.lm_head.weight = self.model.embed_tokens.weight
355
356
logit_scale = getattr(config, "logit_scale", 1.0)
357
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,