@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a53046b16f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Model] Support quantization of PixtralHFTransformer for PixtralHF (#9921)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 5, 2024 at 13:42 UTC a53046b16fd11436eb2b15421079b7c5b353f955
2 files changed +90 -40
vllm/model_executor/layers/activation.py
+30
@@ -299,3 +299,33 @@ def get_act_fn(
299 return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
300 params_dtype)
301 return act_fn
302 +
303 +
304 +_ACTIVATION_AND_MUL_REGISTRY = LazyDict({
305 + "gelu": lambda: GeluAndMul(),
306 + "silu": lambda: SiluAndMul(),
307 +})
308 +
309 +
310 +def get_act_and_mul_fn(
311 + act_fn_name: str,
312 + quant_config: Optional[QuantizationConfig] = None,
313 + intermediate_size: Optional[int] = None,
314 + input_is_parallel: bool = True,
315 + params_dtype: Optional[torch.dtype] = None,
316 +) -> nn.Module:
317 + """Get an activation-and-mul (i.e. SiluAndMul) function by name."""
318 + act_fn_name = act_fn_name.lower()
319 + if act_fn_name not in _ACTIVATION_AND_MUL_REGISTRY:
320 + raise ValueError(
321 + f"Activation function {act_fn_name!r} is not supported.")
322 +
323 + act_fn = _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
324 + if (quant_config is not None
325 + and act_fn_name in quant_config.get_scaled_act_names()):
326 + if intermediate_size is None:
327 + raise ValueError("intermediate_size must be specified for scaled "
328 + "activation functions.")
329 + return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
330 + params_dtype)
331 + return act_fn
vllm/model_executor/models/pixtral.py
+60 -40
@@ -19,8 +19,11 @@ from vllm.attention import AttentionMetadata
19 from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
20 from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21 InputContext, token_inputs)
22 -from vllm.model_executor.layers.activation import get_act_fn
22 +from vllm.model_executor.layers.activation import get_act_and_mul_fn
23 from vllm.model_executor.layers.layernorm import RMSNorm
24 +from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
25 + QKVParallelLinear,
26 + RowParallelLinear)
27 from vllm.model_executor.layers.quantization import QuantizationConfig
28 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -798,20 +801,24 @@ class PixtralHFMLP(nn.Module):
801 super().__init__()
802
803 assert config.intermediate_size is not None
801 - # TODO: Use quant_config and prefix after optimizing this
802 - self.gate_proj = nn.Linear(config.hidden_size,
803 - config.intermediate_size,
804 - bias=False)
805 - self.up_proj = nn.Linear(config.hidden_size,
806 - config.intermediate_size,
807 - bias=False)
808 - self.down_proj = nn.Linear(config.intermediate_size,
809 - config.hidden_size,
810 - bias=False)
811 - self.act = get_act_fn(config.hidden_act)
804 + self.gate_up_proj = MergedColumnParallelLinear(
805 + input_size=config.hidden_size,
806 + output_sizes=[config.intermediate_size] * 2,
807 + bias=False,
808 + quant_config=quant_config,
809 + prefix=f"{prefix}.gate_up_proj")
810 + self.down_proj = RowParallelLinear(input_size=config.intermediate_size,
811 + output_size=config.hidden_size,
812 + bias=False,
813 + quant_config=quant_config,
814 + prefix=f"{prefix}.down_proj")
815 + self.act_and_mul = get_act_and_mul_fn(config.hidden_act)
816
817 def forward(self, x: torch.Tensor) -> torch.Tensor:
814 - return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
818 + gate_up, _ = self.gate_up_proj(x)
819 + x = self.act_and_mul(gate_up)
820 + x, _ = self.down_proj(x)
821 + return x
822
823
824 class PixtralHFAttention(nn.Module):
@@ -830,21 +837,21 @@ class PixtralHFAttention(nn.Module):
837 self.n_heads = config.num_attention_heads
838 self.head_dim = config.hidden_size // config.num_attention_heads
839
833 - self.scale = self.head_dim**-0.5
834 -
835 - # TODO: Use quant_config and prefix after optimizing this
836 - self.q_proj = nn.Linear(config.hidden_size,
837 - config.hidden_size,
838 - bias=False)
839 - self.k_proj = nn.Linear(config.hidden_size,
840 - config.hidden_size,
841 - bias=False)
842 - self.v_proj = nn.Linear(config.hidden_size,
843 - config.hidden_size,
844 - bias=False)
845 - self.o_proj = nn.Linear(config.hidden_size,
846 - config.hidden_size,
847 - bias=False)
840 + self.qkv_proj = QKVParallelLinear(
841 + hidden_size=config.hidden_size,
842 + head_size=self.head_dim,
843 + total_num_heads=self.n_heads,
844 + bias=False,
845 + quant_config=quant_config,
846 + prefix=f"{prefix}.qkv_proj",
847 + )
848 + self.o_proj = RowParallelLinear(
849 + input_size=config.hidden_size,
850 + output_size=config.hidden_size,
851 + bias=False,
852 + quant_config=quant_config,
853 + prefix=f"{prefix}.o_proj",
854 + )
855
856 def forward(
857 self,
@@ -854,13 +861,13 @@ class PixtralHFAttention(nn.Module):
861 ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
862 batch, patches, _ = hidden_states.size()
863
857 - q = self.q_proj(hidden_states)
858 - k = self.k_proj(hidden_states)
859 - v = self.v_proj(hidden_states)
864 + qkv_states, _ = self.qkv_proj(hidden_states)
865 + q, k, v = qkv_states.chunk(3, dim=-1)
866
867 # Transpose q and k to apply HF's Rotary Position Embedding
868 q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
869 k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
870 + v = v.view(batch, patches, self.n_heads, self.head_dim)
871 cos, sin = position_embeddings
872 q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
873
@@ -868,22 +875,21 @@ class PixtralHFAttention(nn.Module):
875 # Transpose q and k back for attention
876 q = q.transpose(1, 2).contiguous()
877 k = k.transpose(1, 2).contiguous()
871 - v = v.reshape(batch, patches, self.n_heads, self.head_dim)
878
879 out = xops.memory_efficient_attention(q,
880 k,
881 v,
882 attn_bias=attention_mask)
883 else:
878 - v = v.reshape(batch, patches, self.n_heads,
879 - self.head_dim).transpose(1, 2)
884 + v = v.transpose(1, 2)
885 out = nn.functional.scaled_dot_product_attention(
886 q, k, v, attn_mask=attention_mask)
887 out = out.transpose(1, 2)
888
884 - out = out.reshape(batch, patches, self.n_heads * self.head_dim)
889 + out = out.view(batch, patches, self.n_heads * self.head_dim)
890 + attn_output, _ = self.o_proj(out)
891
886 - return self.o_proj(out)
892 + return attn_output, None
893
894
895 class PixtralHFTransformerBlock(nn.Module):
@@ -912,9 +918,9 @@ class PixtralHFTransformerBlock(nn.Module):
918 attention_mask: torch.Tensor,
919 position_embeddings: torch.Tensor,
920 ) -> torch.Tensor:
915 - r = self.attention.forward(self.attention_norm(hidden_states),
916 - attention_mask=attention_mask,
917 - position_embeddings=position_embeddings)
921 + r, _ = self.attention.forward(self.attention_norm(hidden_states),
922 + attention_mask=attention_mask,
923 + position_embeddings=position_embeddings)
924 h = hidden_states + r
925 r = self.feed_forward.forward(self.ffn_norm(h))
926 out = h + r
@@ -1053,10 +1059,24 @@ class PixtralHFVisionModel(nn.Module):
1059 # (TODO) Add prefix argument for filtering out weights to be loaded
1060 # ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
1061 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1056 - stacked_params_mapping = []
1062 + stacked_params_mapping = [
1063 + # (param_name, shard_name, shard_id)
1064 + (".qkv_proj", ".q_proj", "q"),
1065 + (".qkv_proj", ".k_proj", "k"),
1066 + (".qkv_proj", ".v_proj", "v"),
1067 + (".gate_up_proj", ".gate_proj", 0),
1068 + (".gate_up_proj", ".up_proj", 1),
1069 + ]
1070 params_dict = dict(self.named_parameters())
1071 + layer_count = len(self.transformer.layers)
1072
1073 for name, loaded_weight in weights:
1074 + # omit layers when num_hidden_layers_override is set
1075 + if name.startswith("transformer.layers"):
1076 + layer_idx = int(name.split(".")[2])
1077 + if layer_idx >= layer_count:
1078 + continue
1079 +
1080 for (param_name, weight_name, shard_id) in stacked_params_mapping:
1081 if weight_name not in name:
1082 continue