A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Model] Support quantization of PixtralHFTransformer for PixtralHF (#9921)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 5, 2024 at 13:42 UTC
a53046b16fd11436eb2b15421079b7c5b353f955
2 files changed
+90
-40
vllm/model_executor/layers/activation.py
+30
@@ -299,3 +299,33 @@ def get_act_fn(
299
return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
300
params_dtype)
301
return act_fn
302
+
303
+
304
+_ACTIVATION_AND_MUL_REGISTRY = LazyDict({
305
+ "gelu": lambda: GeluAndMul(),
306
+ "silu": lambda: SiluAndMul(),
307
+})
308
+
309
+
310
+def get_act_and_mul_fn(
311
+ act_fn_name: str,
312
+ quant_config: Optional[QuantizationConfig] = None,
313
+ intermediate_size: Optional[int] = None,
314
+ input_is_parallel: bool = True,
315
+ params_dtype: Optional[torch.dtype] = None,
316
+) -> nn.Module:
317
+ """Get an activation-and-mul (i.e. SiluAndMul) function by name."""
318
+ act_fn_name = act_fn_name.lower()
319
+ if act_fn_name not in _ACTIVATION_AND_MUL_REGISTRY:
320
+ raise ValueError(
321
+ f"Activation function {act_fn_name!r} is not supported.")
322
+
323
+ act_fn = _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
324
+ if (quant_config is not None
325
+ and act_fn_name in quant_config.get_scaled_act_names()):
326
+ if intermediate_size is None:
327
+ raise ValueError("intermediate_size must be specified for scaled "
328
+ "activation functions.")
329
+ return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
330
+ params_dtype)
331
+ return act_fn
vllm/model_executor/models/pixtral.py
+60
-40
@@ -19,8 +19,11 @@ from vllm.attention import AttentionMetadata
19
from vllm.config import CacheConfig, ModelConfig, MultiModalConfig
20
from vllm.inputs import (INPUT_REGISTRY, DecoderOnlyInputs, DummyData,
21
InputContext, token_inputs)
22
-from vllm.model_executor.layers.activation import get_act_fn
22
+from vllm.model_executor.layers.activation import get_act_and_mul_fn
23
from vllm.model_executor.layers.layernorm import RMSNorm
24
+from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
25
+ QKVParallelLinear,
26
+ RowParallelLinear)
27
from vllm.model_executor.layers.quantization import QuantizationConfig
28
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
29
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -798,20 +801,24 @@ class PixtralHFMLP(nn.Module):
801
super().__init__()
802
803
assert config.intermediate_size is not None
801
- # TODO: Use quant_config and prefix after optimizing this
802
- self.gate_proj = nn.Linear(config.hidden_size,
803
- config.intermediate_size,
804
- bias=False)
805
- self.up_proj = nn.Linear(config.hidden_size,
806
- config.intermediate_size,
807
- bias=False)
808
- self.down_proj = nn.Linear(config.intermediate_size,
809
- config.hidden_size,
810
- bias=False)
811
- self.act = get_act_fn(config.hidden_act)
804
+ self.gate_up_proj = MergedColumnParallelLinear(
805
+ input_size=config.hidden_size,
806
+ output_sizes=[config.intermediate_size] * 2,
807
+ bias=False,
808
+ quant_config=quant_config,
809
+ prefix=f"{prefix}.gate_up_proj")
810
+ self.down_proj = RowParallelLinear(input_size=config.intermediate_size,
811
+ output_size=config.hidden_size,
812
+ bias=False,
813
+ quant_config=quant_config,
814
+ prefix=f"{prefix}.down_proj")
815
+ self.act_and_mul = get_act_and_mul_fn(config.hidden_act)
816
817
def forward(self, x: torch.Tensor) -> torch.Tensor:
814
- return self.down_proj(self.act(self.gate_proj(x)) * self.up_proj(x))
818
+ gate_up, _ = self.gate_up_proj(x)
819
+ x = self.act_and_mul(gate_up)
820
+ x, _ = self.down_proj(x)
821
+ return x
822
823
824
class PixtralHFAttention(nn.Module):
@@ -830,21 +837,21 @@ class PixtralHFAttention(nn.Module):
837
self.n_heads = config.num_attention_heads
838
self.head_dim = config.hidden_size // config.num_attention_heads
839
833
- self.scale = self.head_dim**-0.5
834
-
835
- # TODO: Use quant_config and prefix after optimizing this
836
- self.q_proj = nn.Linear(config.hidden_size,
837
- config.hidden_size,
838
- bias=False)
839
- self.k_proj = nn.Linear(config.hidden_size,
840
- config.hidden_size,
841
- bias=False)
842
- self.v_proj = nn.Linear(config.hidden_size,
843
- config.hidden_size,
844
- bias=False)
845
- self.o_proj = nn.Linear(config.hidden_size,
846
- config.hidden_size,
847
- bias=False)
840
+ self.qkv_proj = QKVParallelLinear(
841
+ hidden_size=config.hidden_size,
842
+ head_size=self.head_dim,
843
+ total_num_heads=self.n_heads,
844
+ bias=False,
845
+ quant_config=quant_config,
846
+ prefix=f"{prefix}.qkv_proj",
847
+ )
848
+ self.o_proj = RowParallelLinear(
849
+ input_size=config.hidden_size,
850
+ output_size=config.hidden_size,
851
+ bias=False,
852
+ quant_config=quant_config,
853
+ prefix=f"{prefix}.o_proj",
854
+ )
855
856
def forward(
857
self,
@@ -854,13 +861,13 @@ class PixtralHFAttention(nn.Module):
861
) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
862
batch, patches, _ = hidden_states.size()
863
857
- q = self.q_proj(hidden_states)
858
- k = self.k_proj(hidden_states)
859
- v = self.v_proj(hidden_states)
864
+ qkv_states, _ = self.qkv_proj(hidden_states)
865
+ q, k, v = qkv_states.chunk(3, dim=-1)
866
867
# Transpose q and k to apply HF's Rotary Position Embedding
868
q = q.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
869
k = k.view(batch, patches, self.n_heads, self.head_dim).transpose(1, 2)
870
+ v = v.view(batch, patches, self.n_heads, self.head_dim)
871
cos, sin = position_embeddings
872
q, k = apply_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=0)
873
@@ -868,22 +875,21 @@ class PixtralHFAttention(nn.Module):
875
# Transpose q and k back for attention
876
q = q.transpose(1, 2).contiguous()
877
k = k.transpose(1, 2).contiguous()
871
- v = v.reshape(batch, patches, self.n_heads, self.head_dim)
878
879
out = xops.memory_efficient_attention(q,
880
k,
881
v,
882
attn_bias=attention_mask)
883
else:
878
- v = v.reshape(batch, patches, self.n_heads,
879
- self.head_dim).transpose(1, 2)
884
+ v = v.transpose(1, 2)
885
out = nn.functional.scaled_dot_product_attention(
886
q, k, v, attn_mask=attention_mask)
887
out = out.transpose(1, 2)
888
884
- out = out.reshape(batch, patches, self.n_heads * self.head_dim)
889
+ out = out.view(batch, patches, self.n_heads * self.head_dim)
890
+ attn_output, _ = self.o_proj(out)
891
886
- return self.o_proj(out)
892
+ return attn_output, None
893
894
895
class PixtralHFTransformerBlock(nn.Module):
@@ -912,9 +918,9 @@ class PixtralHFTransformerBlock(nn.Module):
918
attention_mask: torch.Tensor,
919
position_embeddings: torch.Tensor,
920
) -> torch.Tensor:
915
- r = self.attention.forward(self.attention_norm(hidden_states),
916
- attention_mask=attention_mask,
917
- position_embeddings=position_embeddings)
921
+ r, _ = self.attention.forward(self.attention_norm(hidden_states),
922
+ attention_mask=attention_mask,
923
+ position_embeddings=position_embeddings)
924
h = hidden_states + r
925
r = self.feed_forward.forward(self.ffn_norm(h))
926
out = h + r
@@ -1053,10 +1059,24 @@ class PixtralHFVisionModel(nn.Module):
1059
# (TODO) Add prefix argument for filtering out weights to be loaded
1060
# ref: https://github.com/vllm-project/vllm/pull/7186#discussion_r1734163986
1061
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1056
- stacked_params_mapping = []
1062
+ stacked_params_mapping = [
1063
+ # (param_name, shard_name, shard_id)
1064
+ (".qkv_proj", ".q_proj", "q"),
1065
+ (".qkv_proj", ".k_proj", "k"),
1066
+ (".qkv_proj", ".v_proj", "v"),
1067
+ (".gate_up_proj", ".gate_proj", 0),
1068
+ (".gate_up_proj", ".up_proj", 1),
1069
+ ]
1070
params_dict = dict(self.named_parameters())
1071
+ layer_count = len(self.transformer.layers)
1072
1073
for name, loaded_weight in weights:
1074
+ # omit layers when num_hidden_layers_override is set
1075
+ if name.startswith("transformer.layers"):
1076
+ layer_idx = int(name.split(".")[2])
1077
+ if layer_idx >= layer_count:
1078
+ continue
1079
+
1080
for (param_name, weight_name, shard_id) in stacked_params_mapping:
1081
if weight_name not in name:
1082
continue