@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 6eaccb7353
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Model] Add support for IBM Granite Code models (#4636)

Yikang Shen committed May 12, 2024 at 00:27 UTC 6eaccb7353cfe84d77981da726f6d82a8aefd2be
1 file changed +6 -2
vllm/model_executor/models/llama.py
+6 -2
@@ -58,15 +58,16 @@ class LlamaMLP(nn.Module):
58 intermediate_size: int,
59 hidden_act: str,
60 quant_config: Optional[QKVParallelLinear] = None,
61 + bias: bool = False,
62 ) -> None:
63 super().__init__()
64 self.gate_up_proj = MergedColumnParallelLinear(
65 hidden_size, [intermediate_size] * 2,
65 - bias=False,
66 + bias=bias,
67 quant_config=quant_config)
68 self.down_proj = RowParallelLinear(intermediate_size,
69 hidden_size,
69 - bias=False,
70 + bias=bias,
71 quant_config=quant_config)
72 if hidden_act != "silu":
73 raise ValueError(f"Unsupported activation: {hidden_act}. "
@@ -209,6 +210,7 @@ class LlamaDecoderLayer(nn.Module):
210 intermediate_size=config.intermediate_size,
211 hidden_act=config.hidden_act,
212 quant_config=quant_config,
213 + bias=getattr(config, "mlp_bias", False),
214 )
215 self.input_layernorm = RMSNorm(config.hidden_size,
216 eps=config.rms_norm_eps)
@@ -348,6 +350,8 @@ class LlamaForCausalLM(nn.Module):
350 # compatibility
351 if not lora_config else lora_config.lora_vocab_padding_size,
352 )
353 + if config.tie_word_embeddings:
354 + self.lm_head.weight = self.model.embed_tokens.weight
355
356 logit_scale = getattr(config, "logit_scale", 1.0)
357 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,