A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Add AWQ support for all models (#1714)
Woosuk Kwon committed
Nov 18, 2023 at 17:56 UTC
8d17774f924da6a3b730289f351205f3b17095c6
13 files changed
+90
-17
vllm/model_executor/layers/activation.py
+47
-5
@@ -1,8 +1,11 @@
1
"""Custom activation functions."""
2
+from typing import Optional
3
+
4
import torch
5
import torch.nn as nn
6
7
from vllm import activation_ops
8
+from vllm.model_executor.layers.quantization import QuantizationConfig
9
10
11
class SiluAndMul(nn.Module):
@@ -39,6 +42,27 @@ class FastGELU(nn.Module):
42
return out
43
44
45
+class ScaledActivation(nn.Module):
46
+ """An activation function with post-scale parameters.
47
+
48
+ This is used for some quantization methods like AWQ.
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ act_module: nn.Module,
54
+ hidden_size: int,
55
+ params_dtype: torch.dtype,
56
+ ):
57
+ super().__init__()
58
+ self.act = act_module
59
+ self.scales = nn.Parameter(
60
+ torch.empty(hidden_size, dtype=params_dtype, device="cuda"))
61
+
62
+ def forward(self, x: torch.Tensor):
63
+ return self.act(x) / self.scales
64
+
65
+
66
_ACTIVATION_REGISTRY = {
67
"gelu": nn.GELU(),
68
"gelu_fast": FastGELU(),
@@ -48,9 +72,27 @@ _ACTIVATION_REGISTRY = {
72
}
73
74
51
-def get_act_fn(act_fn: str) -> nn.Module:
75
+def get_act_fn(
76
+ act_fn_name: str,
77
+ quant_config: Optional[QuantizationConfig] = None,
78
+ intermediate_size: Optional[int] = None,
79
+) -> nn.Module:
80
"""Get an activation function by name."""
53
- act_fn = act_fn.lower()
54
- if act_fn in _ACTIVATION_REGISTRY:
55
- return _ACTIVATION_REGISTRY[act_fn]
56
- raise ValueError(f"Activation function {act_fn!r} is not supported.")
81
+ act_fn_name = act_fn_name.lower()
82
+ if act_fn_name not in _ACTIVATION_REGISTRY:
83
+ raise ValueError(
84
+ f"Activation function {act_fn_name!r} is not supported.")
85
+
86
+ act_fn = _ACTIVATION_REGISTRY[act_fn_name]
87
+ if quant_config is not None:
88
+ if act_fn_name in quant_config.get_scaled_act_names():
89
+ if intermediate_size is None:
90
+ raise ValueError(
91
+ "intermediate_size must be specified for scaled "
92
+ "activation functions.")
93
+ return ScaledActivation(
94
+ act_fn,
95
+ intermediate_size,
96
+ params_dtype=torch.get_default_dtype(),
97
+ )
98
+ return act_fn
vllm/model_executor/layers/quantization/awq.py
+3
@@ -63,6 +63,9 @@ class AWQConfig(QuantizationConfig):
63
def get_linear_method(self) -> "AWQLinearMethod":
64
return AWQLinearMethod(self)
65
66
+ def get_scaled_act_names(self) -> List[str]:
67
+ return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
68
+
69
70
class AWQLinearMethod(LinearMethodBase):
71
"""Linear method for AWQ.
vllm/model_executor/layers/quantization/base_config.py
+8
@@ -54,3 +54,11 @@ class QuantizationConfig(ABC):
54
def get_linear_method(self) -> LinearMethodBase:
55
"""Get the linear method to use for the quantized linear layer."""
56
raise NotImplementedError
57
+
58
+ @abstractmethod
59
+ def get_scaled_act_names(self) -> List[str]:
60
+ """Returns the activation function names that should be post-scaled.
61
+
62
+ For now, this is only used by AWQ.
63
+ """
64
+ raise NotImplementedError
vllm/model_executor/layers/quantization/squeezellm.py
+3
@@ -52,6 +52,9 @@ class SqueezeLLMConfig(QuantizationConfig):
52
def get_linear_method(self) -> "SqueezeLLMLinearMethod":
53
return SqueezeLLMLinearMethod(self)
54
55
+ def get_scaled_act_names(self) -> List[str]:
56
+ return []
57
+
58
59
class SqueezeLLMLinearMethod(LinearMethodBase):
60
"""Linear method for SqueezeLLM.
vllm/model_executor/models/bloom.py
+3
-2
@@ -145,7 +145,8 @@ class BloomMLP(nn.Module):
145
4 * hidden_size,
146
linear_method=linear_method,
147
)
148
- self.act = get_act_fn("gelu")
148
+ quant_config = getattr(linear_method, "quant_config", None)
149
+ self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
150
self.dense_4h_to_h = RowParallelLinear(
151
4 * hidden_size,
152
hidden_size,
@@ -154,7 +155,7 @@ class BloomMLP(nn.Module):
155
156
def forward(self, x: torch.Tensor) -> torch.Tensor:
157
x, _ = self.dense_h_to_4h(x)
157
- x = self.act(x)
158
+ x = self.gelu_impl(x)
159
x, _ = self.dense_4h_to_h(x)
160
return x
161
vllm/model_executor/models/falcon.py
+4
-1
@@ -27,6 +27,7 @@ from torch.nn import LayerNorm
27
from transformers import FalconConfig as HF_FalconConfig
28
29
from vllm.model_executor.input_metadata import InputMetadata
30
+from vllm.model_executor.layers.activation import get_act_fn
31
from vllm.model_executor.layers.attention import (PagedAttention,
32
PagedAttentionWithALiBi,
33
PagedAttentionWithRoPE)
@@ -131,6 +132,7 @@ class FalconAttention(nn.Module):
132
self.hidden_size,
133
bias=config.bias,
134
skip_bias_add=True,
135
+ linear_method=linear_method,
136
reduce_results=self.reduce_row_parallel_results)
137
138
self.use_rotary = config.rotary
@@ -206,7 +208,8 @@ class FalconMLP(nn.Module):
208
bias=config.bias,
209
skip_bias_add=True,
210
linear_method=linear_method)
209
- self.act = nn.GELU()
211
+ quant_config = getattr(linear_method, "quant_config", None)
212
+ self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
213
self.reduce_row_parallel_results = not (config.new_decoder_architecture
214
or config.parallel_attn)
215
self.dense_4h_to_h = RowParallelLinear(
vllm/model_executor/models/gpt2.py
+3
-1
@@ -118,7 +118,9 @@ class GPT2MLP(nn.Module):
118
bias=True,
119
linear_method=linear_method,
120
)
121
- self.act = get_act_fn(config.activation_function)
121
+ quant_config = getattr(linear_method, "quant_config", None)
122
+ self.act = get_act_fn(config.activation_function, quant_config,
123
+ intermediate_size)
124
125
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
126
hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_bigcode.py
+3
-1
@@ -137,7 +137,9 @@ class GPTBigMLP(nn.Module):
137
bias=True,
138
linear_method=linear_method,
139
)
140
- self.act = get_act_fn(config.activation_function)
140
+ quant_config = getattr(linear_method, "quant_config", None)
141
+ self.act = get_act_fn(config.activation_function, quant_config,
142
+ intermediate_size)
143
144
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
145
hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_j.py
+3
-1
@@ -128,7 +128,9 @@ class GPTJMLP(nn.Module):
128
hidden_size,
129
linear_method=linear_method,
130
)
131
- self.act = get_act_fn(config.activation_function)
131
+ quant_config = getattr(linear_method, "quant_config", None)
132
+ self.act = get_act_fn(config.activation_function, quant_config,
133
+ intermediate_size)
134
135
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136
hidden_states, _ = self.fc_in(hidden_states)
vllm/model_executor/models/gpt_neox.py
+3
-1
@@ -124,7 +124,9 @@ class GPTNeoXMLP(nn.Module):
124
config.hidden_size,
125
linear_method=linear_method,
126
)
127
- self.act = get_act_fn(config.hidden_act)
127
+ quant_config = getattr(linear_method, "quant_config", None)
128
+ self.act = get_act_fn(config.hidden_act, quant_config,
129
+ config.intermediate_size)
130
131
def forward(self, hidden_states):
132
hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/mpt.py
+2
-1
@@ -130,7 +130,8 @@ class MPTMLP(nn.Module):
130
bias=not config.no_bias,
131
linear_method=linear_method,
132
)
133
- self.act = get_act_fn("gelu")
133
+ quant_config = getattr(linear_method, "quant_config", None)
134
+ self.act = get_act_fn("gelu", quant_config, intermediate_size)
135
self.down_proj = RowParallelLinear(
136
intermediate_size,
137
hidden_size,
vllm/model_executor/models/opt.py
+5
-3
@@ -129,7 +129,9 @@ class OPTDecoderLayer(nn.Module):
129
linear_method=linear_method,
130
)
131
self.do_layer_norm_before = config.do_layer_norm_before
132
- self.activation_fn = get_act_fn(config.activation_function)
132
+ quant_config = getattr(linear_method, "quant_config", None)
133
+ self.activation_fn = get_act_fn(config.activation_function,
134
+ quant_config, config.ffn_dim)
135
136
self.self_attn_layer_norm = nn.LayerNorm(
137
self.embed_dim,
@@ -251,7 +253,7 @@ class OPTDecoder(nn.Module):
253
inputs_embeds = self.embed_tokens(input_ids)
254
pos_embeds = self.embed_positions(positions)
255
if self.project_in is not None:
254
- inputs_embeds = self.project_in(inputs_embeds)
256
+ inputs_embeds, _ = self.project_in(inputs_embeds)
257
hidden_states = inputs_embeds + pos_embeds
258
259
for i in range(len(self.layers)):
@@ -266,7 +268,7 @@ class OPTDecoder(nn.Module):
268
if self.final_layer_norm is not None:
269
hidden_states = self.final_layer_norm(hidden_states)
270
if self.project_out is not None:
269
- hidden_states = self.project_out(hidden_states)
271
+ hidden_states, _ = self.project_out(hidden_states)
272
return hidden_states
273
274
vllm/model_executor/models/phi_1_5.py
+3
-1
@@ -168,7 +168,9 @@ class PhiMLP(nn.Module):
168
config.hidden_size,
169
linear_method=linear_method,
170
)
171
- self.act = get_act_fn(config.activation_function)
171
+ quant_config = getattr(linear_method, "quant_config", None)
172
+ self.act = get_act_fn(config.activation_function, quant_config,
173
+ n_inner)
174
175
def forward(self, hidden_states):
176
hidden_states, _ = self.fc1(hidden_states)