A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Misc] Support quantization of MllamaForCausalLM (#8822)
Michael Goin committed
Sep 25, 2024 at 17:46 UTC
7193774b1ff8603ad5bf4598e5efba0d9a39b436
1 file changed
+9
-2
vllm/model_executor/models/mllama.py
+9
-2
@@ -624,6 +624,7 @@ class MllamaTextCrossAttention(nn.Module):
624
self,
625
config: Optional[config_mllama.MllamaTextConfig] = None,
626
layer_idx: Optional[int] = None,
627
+ quant_config: Optional[QuantizationConfig] = None,
628
):
629
super().__init__()
630
self.config = config
@@ -648,12 +649,14 @@ class MllamaTextCrossAttention(nn.Module):
649
self.num_heads,
650
self.num_key_value_heads,
651
bias=False,
652
+ quant_config=quant_config,
653
)
654
self.o_proj = RowParallelLinear(
655
self.num_heads * self.head_dim,
656
self.hidden_size,
657
bias=False,
658
input_is_parallel=True,
659
+ quant_config=quant_config,
660
)
661
# vllm.model_executor.layers.layernorm.RMSNorm has precision issue,
662
# use huggingface's instead
@@ -708,13 +711,15 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
711
"""Cross-attention transformer block with tanh-gated attention
712
and feedforward."""
713
711
- def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int) \
714
+ def __init__(self, config: config_mllama.MllamaTextConfig, layer_idx: int,
715
+ quant_config: Optional[QuantizationConfig]) \
716
-> None:
717
super().__init__()
718
self.layer_idx = layer_idx
719
self.cross_attn = MllamaTextCrossAttention(
720
config=config,
721
layer_idx=layer_idx,
722
+ quant_config=quant_config,
723
)
724
725
self.input_layernorm = RMSNorm(config.hidden_size,
@@ -725,6 +730,7 @@ class MllamaCrossAttentionDecoderLayer(torch.nn.Module):
730
hidden_size=config.hidden_size,
731
intermediate_size=config.intermediate_size,
732
hidden_act=config.hidden_act,
733
+ quant_config=quant_config,
734
)
735
self.post_attention_layernorm = RMSNorm(config.hidden_size,
736
eps=config.rms_norm_eps)
@@ -780,7 +786,8 @@ class MllamaTextModel(nn.Module):
786
for layer_idx in range(config.num_hidden_layers):
787
if layer_idx in self.cross_attention_layers:
788
layers.append(
783
- MllamaCrossAttentionDecoderLayer(config, layer_idx))
789
+ MllamaCrossAttentionDecoderLayer(
790
+ config, layer_idx, quant_config=quant_config))
791
else:
792
# TODO: force LlamaDecoderLayer to config.attention_bias=False
793
layers.append(