@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 8d17774f92
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Add AWQ support for all models (#1714)

Woosuk Kwon committed Nov 18, 2023 at 17:56 UTC 8d17774f924da6a3b730289f351205f3b17095c6
13 files changed +90 -17
vllm/model_executor/layers/activation.py
+47 -5
@@ -1,8 +1,11 @@
1 """Custom activation functions."""
2 +from typing import Optional
3 +
4 import torch
5 import torch.nn as nn
6
7 from vllm import activation_ops
8 +from vllm.model_executor.layers.quantization import QuantizationConfig
9
10
11 class SiluAndMul(nn.Module):
@@ -39,6 +42,27 @@ class FastGELU(nn.Module):
42 return out
43
44
45 +class ScaledActivation(nn.Module):
46 + """An activation function with post-scale parameters.
47 +
48 + This is used for some quantization methods like AWQ.
49 + """
50 +
51 + def __init__(
52 + self,
53 + act_module: nn.Module,
54 + hidden_size: int,
55 + params_dtype: torch.dtype,
56 + ):
57 + super().__init__()
58 + self.act = act_module
59 + self.scales = nn.Parameter(
60 + torch.empty(hidden_size, dtype=params_dtype, device="cuda"))
61 +
62 + def forward(self, x: torch.Tensor):
63 + return self.act(x) / self.scales
64 +
65 +
66 _ACTIVATION_REGISTRY = {
67 "gelu": nn.GELU(),
68 "gelu_fast": FastGELU(),
@@ -48,9 +72,27 @@ _ACTIVATION_REGISTRY = {
72 }
73
74
51 -def get_act_fn(act_fn: str) -> nn.Module:
75 +def get_act_fn(
76 + act_fn_name: str,
77 + quant_config: Optional[QuantizationConfig] = None,
78 + intermediate_size: Optional[int] = None,
79 +) -> nn.Module:
80 """Get an activation function by name."""
53 - act_fn = act_fn.lower()
54 - if act_fn in _ACTIVATION_REGISTRY:
55 - return _ACTIVATION_REGISTRY[act_fn]
56 - raise ValueError(f"Activation function {act_fn!r} is not supported.")
81 + act_fn_name = act_fn_name.lower()
82 + if act_fn_name not in _ACTIVATION_REGISTRY:
83 + raise ValueError(
84 + f"Activation function {act_fn_name!r} is not supported.")
85 +
86 + act_fn = _ACTIVATION_REGISTRY[act_fn_name]
87 + if quant_config is not None:
88 + if act_fn_name in quant_config.get_scaled_act_names():
89 + if intermediate_size is None:
90 + raise ValueError(
91 + "intermediate_size must be specified for scaled "
92 + "activation functions.")
93 + return ScaledActivation(
94 + act_fn,
95 + intermediate_size,
96 + params_dtype=torch.get_default_dtype(),
97 + )
98 + return act_fn
vllm/model_executor/layers/quantization/awq.py
+3
@@ -63,6 +63,9 @@ class AWQConfig(QuantizationConfig):
63 def get_linear_method(self) -> "AWQLinearMethod":
64 return AWQLinearMethod(self)
65
66 + def get_scaled_act_names(self) -> List[str]:
67 + return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
68 +
69
70 class AWQLinearMethod(LinearMethodBase):
71 """Linear method for AWQ.
vllm/model_executor/layers/quantization/base_config.py
+8
@@ -54,3 +54,11 @@ class QuantizationConfig(ABC):
54 def get_linear_method(self) -> LinearMethodBase:
55 """Get the linear method to use for the quantized linear layer."""
56 raise NotImplementedError
57 +
58 + @abstractmethod
59 + def get_scaled_act_names(self) -> List[str]:
60 + """Returns the activation function names that should be post-scaled.
61 +
62 + For now, this is only used by AWQ.
63 + """
64 + raise NotImplementedError
vllm/model_executor/layers/quantization/squeezellm.py
+3
@@ -52,6 +52,9 @@ class SqueezeLLMConfig(QuantizationConfig):
52 def get_linear_method(self) -> "SqueezeLLMLinearMethod":
53 return SqueezeLLMLinearMethod(self)
54
55 + def get_scaled_act_names(self) -> List[str]:
56 + return []
57 +
58
59 class SqueezeLLMLinearMethod(LinearMethodBase):
60 """Linear method for SqueezeLLM.
vllm/model_executor/models/bloom.py
+3 -2
@@ -145,7 +145,8 @@ class BloomMLP(nn.Module):
145 4 * hidden_size,
146 linear_method=linear_method,
147 )
148 - self.act = get_act_fn("gelu")
148 + quant_config = getattr(linear_method, "quant_config", None)
149 + self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
150 self.dense_4h_to_h = RowParallelLinear(
151 4 * hidden_size,
152 hidden_size,
@@ -154,7 +155,7 @@ class BloomMLP(nn.Module):
155
156 def forward(self, x: torch.Tensor) -> torch.Tensor:
157 x, _ = self.dense_h_to_4h(x)
157 - x = self.act(x)
158 + x = self.gelu_impl(x)
159 x, _ = self.dense_4h_to_h(x)
160 return x
161
vllm/model_executor/models/falcon.py
+4 -1
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27 from transformers import FalconConfig as HF_FalconConfig
28
29 from vllm.model_executor.input_metadata import InputMetadata
30 +from vllm.model_executor.layers.activation import get_act_fn
31 from vllm.model_executor.layers.attention import (PagedAttention,
32 PagedAttentionWithALiBi,
33 PagedAttentionWithRoPE)
@@ -131,6 +132,7 @@ class FalconAttention(nn.Module):
132 self.hidden_size,
133 bias=config.bias,
134 skip_bias_add=True,
135 + linear_method=linear_method,
136 reduce_results=self.reduce_row_parallel_results)
137
138 self.use_rotary = config.rotary
@@ -206,7 +208,8 @@ class FalconMLP(nn.Module):
208 bias=config.bias,
209 skip_bias_add=True,
210 linear_method=linear_method)
209 - self.act = nn.GELU()
211 + quant_config = getattr(linear_method, "quant_config", None)
212 + self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
213 self.reduce_row_parallel_results = not (config.new_decoder_architecture
214 or config.parallel_attn)
215 self.dense_4h_to_h = RowParallelLinear(
vllm/model_executor/models/gpt2.py
+3 -1
@@ -118,7 +118,9 @@ class GPT2MLP(nn.Module):
118 bias=True,
119 linear_method=linear_method,
120 )
121 - self.act = get_act_fn(config.activation_function)
121 + quant_config = getattr(linear_method, "quant_config", None)
122 + self.act = get_act_fn(config.activation_function, quant_config,
123 + intermediate_size)
124
125 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
126 hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_bigcode.py
+3 -1
@@ -137,7 +137,9 @@ class GPTBigMLP(nn.Module):
137 bias=True,
138 linear_method=linear_method,
139 )
140 - self.act = get_act_fn(config.activation_function)
140 + quant_config = getattr(linear_method, "quant_config", None)
141 + self.act = get_act_fn(config.activation_function, quant_config,
142 + intermediate_size)
143
144 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
145 hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_j.py
+3 -1
@@ -128,7 +128,9 @@ class GPTJMLP(nn.Module):
128 hidden_size,
129 linear_method=linear_method,
130 )
131 - self.act = get_act_fn(config.activation_function)
131 + quant_config = getattr(linear_method, "quant_config", None)
132 + self.act = get_act_fn(config.activation_function, quant_config,
133 + intermediate_size)
134
135 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136 hidden_states, _ = self.fc_in(hidden_states)
vllm/model_executor/models/gpt_neox.py
+3 -1
@@ -124,7 +124,9 @@ class GPTNeoXMLP(nn.Module):
124 config.hidden_size,
125 linear_method=linear_method,
126 )
127 - self.act = get_act_fn(config.hidden_act)
127 + quant_config = getattr(linear_method, "quant_config", None)
128 + self.act = get_act_fn(config.hidden_act, quant_config,
129 + config.intermediate_size)
130
131 def forward(self, hidden_states):
132 hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/mpt.py
+2 -1
@@ -130,7 +130,8 @@ class MPTMLP(nn.Module):
130 bias=not config.no_bias,
131 linear_method=linear_method,
132 )
133 - self.act = get_act_fn("gelu")
133 + quant_config = getattr(linear_method, "quant_config", None)
134 + self.act = get_act_fn("gelu", quant_config, intermediate_size)
135 self.down_proj = RowParallelLinear(
136 intermediate_size,
137 hidden_size,
vllm/model_executor/models/opt.py
+5 -3
@@ -129,7 +129,9 @@ class OPTDecoderLayer(nn.Module):
129 linear_method=linear_method,
130 )
131 self.do_layer_norm_before = config.do_layer_norm_before
132 - self.activation_fn = get_act_fn(config.activation_function)
132 + quant_config = getattr(linear_method, "quant_config", None)
133 + self.activation_fn = get_act_fn(config.activation_function,
134 + quant_config, config.ffn_dim)
135
136 self.self_attn_layer_norm = nn.LayerNorm(
137 self.embed_dim,
@@ -251,7 +253,7 @@ class OPTDecoder(nn.Module):
253 inputs_embeds = self.embed_tokens(input_ids)
254 pos_embeds = self.embed_positions(positions)
255 if self.project_in is not None:
254 - inputs_embeds = self.project_in(inputs_embeds)
256 + inputs_embeds, _ = self.project_in(inputs_embeds)
257 hidden_states = inputs_embeds + pos_embeds
258
259 for i in range(len(self.layers)):
@@ -266,7 +268,7 @@ class OPTDecoder(nn.Module):
268 if self.final_layer_norm is not None:
269 hidden_states = self.final_layer_norm(hidden_states)
270 if self.project_out is not None:
269 - hidden_states = self.project_out(hidden_states)
271 + hidden_states, _ = self.project_out(hidden_states)
272 return hidden_states
273
274
vllm/model_executor/models/phi_1_5.py
+3 -1
@@ -168,7 +168,9 @@ class PhiMLP(nn.Module):
168 config.hidden_size,
169 linear_method=linear_method,
170 )
171 - self.act = get_act_fn(config.activation_function)
171 + quant_config = getattr(linear_method, "quant_config", None)
172 + self.act = get_act_fn(config.activation_function, quant_config,
173 + n_inner)
174
175 def forward(self, hidden_states):
176 hidden_states, _ = self.fc1(hidden_states)