@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7193774b1f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Misc] Support quantization of MllamaForCausalLM (#8822)

Michael Goin committed Sep 25, 2024 at 17:46 UTC 7193774b1ff8603ad5bf4598e5efba0d9a39b436
1 file changed +9 -2
vllm/model_executor/models/mllama.py
+9 -2
@@ -624,6 +624,7 @@ class MllamaTextCrossAttention(nn.Module):
624 self,
625 config: Optional[config_mllama.MllamaTextConfig] = None,
626 layer_idx: Optional[int] = None,
627 + quant_config: Optional[QuantizationConfig] = None,
628 ):
629 super().__init__()
630 self.config = config
@@ -648,12 +649,14 @@ class MllamaTextCrossAttention(nn.Module):
649 self.num_heads,
650 self.num_key_value_heads,
651 bias=False,
652 + quant_config=quant_config,
653 )
654 self.o_proj = RowParallelLinear(
655 self.num_heads * self.head_dim,
656 self.hidden_size,
657 bias=False,
658 input_is_parallel=True,
659 + quant_config=quant_config,
660 )
661 # vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
662 # use huggingface's instead
@@ -708,13 +711,15 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
711 """Cross-attention transformer block with tanh-gated attention
712 and feedforward."""
713
711 - def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int) \
714 + def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int,
715 + quant_config: Optional[QuantizationConfig]) \
716 -> None:
717 super().__init__()
718 self.layer_idx = layer_idx
719 self.cross_attn = MllamaTextCrossAttention(
720 config=config,
721 layer_idx=layer_idx,
722 + quant_config=quant_config,
723 )
724
725 self.input_layernorm = RMSNorm(config.hidden_size,
@@ -725,6 +730,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
730 hidden_size=config.hidden_size,
731 intermediate_size=config.intermediate_size,
732 hidden_act=config.hidden_act,
733 + quant_config=quant_config,
734 )
735 self.post_attention_layernorm = RMSNorm(config.hidden_size,
736 eps=config.rms_norm_eps)
@@ -780,7 +786,8 @@ class MllamaTextModel(nn.Module):
786 for layer_idx in range(config.num_hidden_layers):
787 if layer_idx in self.cross_attention_layers:
788 layers.append(
783 - MllamaCrossAttentionDecoderLayer(config, layer_idx))
789 + MllamaCrossAttentionDecoderLayer(
790 + config, layer_idx, quant_config=quant_config))
791 else:
792 # TODO: force LlamaDecoderLayer to config.attention_bias=False
793 layers.append(