A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
Remove ScaledActivation for AWQ (#10057)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 6, 2024 at 09:27 UTC
399c7986088ed66184e69ac6ae2b28003b642711
34 files changed
+19
-124
vllm/model_executor/layers/activation.py
+4
-33
@@ -9,7 +9,6 @@ import torch.nn.functional as F
9
from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10
get_tensor_model_parallel_world_size)
11
from vllm.model_executor.custom_op import CustomOp
12
-from vllm.model_executor.layers.quantization import QuantizationConfig
12
from vllm.model_executor.utils import set_weight_attrs
13
from vllm.utils import LazyDict
14
@@ -277,28 +276,14 @@ _ACTIVATION_REGISTRY = LazyDict({
276
})
277
278
280
-def get_act_fn(
281
- act_fn_name: str,
282
- quant_config: Optional[QuantizationConfig] = None,
283
- intermediate_size: Optional[int] = None,
284
- input_is_parallel: bool = True,
285
- params_dtype: Optional[torch.dtype] = None,
286
-) -> nn.Module:
279
+def get_act_fn(act_fn_name: str) -> nn.Module:
280
"""Get an activation function by name."""
281
act_fn_name = act_fn_name.lower()
282
if act_fn_name not in _ACTIVATION_REGISTRY:
283
raise ValueError(
284
f"Activation function {act_fn_name!r} is not supported.")
285
293
- act_fn = _ACTIVATION_REGISTRY[act_fn_name]
294
- if (quant_config is not None
295
- and act_fn_name in quant_config.get_scaled_act_names()):
296
- if intermediate_size is None:
297
- raise ValueError("intermediate_size must be specified for scaled "
298
- "activation functions.")
299
- return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
300
- params_dtype)
301
- return act_fn
286
+ return _ACTIVATION_REGISTRY[act_fn_name]
287
288
289
_ACTIVATION_AND_MUL_REGISTRY = LazyDict({
@@ -307,25 +292,11 @@ _ACTIVATION_AND_MUL_REGISTRY = LazyDict({
292
})
293
294
310
-def get_act_and_mul_fn(
311
- act_fn_name: str,
312
- quant_config: Optional[QuantizationConfig] = None,
313
- intermediate_size: Optional[int] = None,
314
- input_is_parallel: bool = True,
315
- params_dtype: Optional[torch.dtype] = None,
316
-) -> nn.Module:
295
+def get_act_and_mul_fn(act_fn_name: str) -> nn.Module:
296
"""Get an activation-and-mul (i.e. SiluAndMul) function by name."""
297
act_fn_name = act_fn_name.lower()
298
if act_fn_name not in _ACTIVATION_AND_MUL_REGISTRY:
299
raise ValueError(
300
f"Activation function {act_fn_name!r} is not supported.")
301
323
- act_fn = _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
324
- if (quant_config is not None
325
- and act_fn_name in quant_config.get_scaled_act_names()):
326
- if intermediate_size is None:
327
- raise ValueError("intermediate_size must be specified for scaled "
328
- "activation functions.")
329
- return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
330
- params_dtype)
331
- return act_fn
302
+ return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
vllm/model_executor/layers/quantization/aqlm.py
-3
@@ -213,9 +213,6 @@ class AQLMConfig(QuantizationConfig):
213
return AQLMLinearMethod(self)
214
return None
215
216
- def get_scaled_act_names(self) -> List[str]:
217
- return []
218
-
216
217
class AQLMLinearMethod(LinearMethodBase):
218
"""Linear method for AQLM.
vllm/model_executor/layers/quantization/awq.py
-3
@@ -77,9 +77,6 @@ class AWQConfig(QuantizationConfig):
77
return AWQLinearMethod(self)
78
return None
79
80
- def get_scaled_act_names(self) -> List[str]:
81
- return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
82
-
80
81
def is_layer_skipped_awq(prefix: str, modules_to_not_convert: List[str]):
82
return any(module_name in prefix for module_name in modules_to_not_convert)
vllm/model_executor/layers/quantization/awq_marlin.py
-3
@@ -127,9 +127,6 @@ class AWQMarlinConfig(QuantizationConfig):
127
return AWQMoEMethod(self)
128
return None
129
130
- def get_scaled_act_names(self) -> List[str]:
131
- return []
132
-
130
@classmethod
131
def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
132
# Extract data from quant config.
vllm/model_executor/layers/quantization/base_config.py
-8
@@ -133,11 +133,3 @@ class QuantizationConfig(ABC):
133
method.
134
"""
135
raise NotImplementedError
136
-
137
- @abstractmethod
138
- def get_scaled_act_names(self) -> List[str]:
139
- """Returns the activation function names that should be post-scaled.
140
-
141
- For now, this is only used by AWQ.
142
- """
143
- raise NotImplementedError
vllm/model_executor/layers/quantization/bitsandbytes.py
-3
@@ -114,9 +114,6 @@ class BitsAndBytesConfig(QuantizationConfig):
114
return BitsAndBytesLinearMethod(self)
115
return None
116
117
- def get_scaled_act_names(self) -> List[str]:
118
- return []
119
-
117
118
def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
119
# Split the prefix into its dot-separated components
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-3
@@ -45,9 +45,6 @@ class CompressedTensorsConfig(QuantizationConfig):
45
def get_linear_method(self) -> "CompressedTensorsLinearMethod":
46
return CompressedTensorsLinearMethod(self)
47
48
- def get_scaled_act_names(self) -> List[str]:
49
- return []
50
-
48
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
49
return [torch.float16, torch.bfloat16]
50
vllm/model_executor/layers/quantization/deepspeedfp.py
-3
@@ -50,9 +50,6 @@ class DeepSpeedFPConfig(QuantizationConfig):
50
def get_linear_method(self) -> "DeepSpeedFPLinearMethod":
51
return DeepSpeedFPLinearMethod(self)
52
53
- def get_scaled_act_names(self) -> List[str]:
54
- return []
55
-
53
@classmethod
54
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
55
return [torch.half, torch.bfloat16]
vllm/model_executor/layers/quantization/experts_int8.py
-3
@@ -45,9 +45,6 @@ class ExpertsInt8Config(QuantizationConfig):
45
return ExpertsInt8MoEMethod(self)
46
return None
47
48
- def get_scaled_act_names(self) -> List[str]:
49
- return []
50
-
48
49
class ExpertsInt8MoEMethod(FusedMoEMethodBase):
50
vllm/model_executor/layers/quantization/fbgemm_fp8.py
-3
@@ -64,9 +64,6 @@ class FBGEMMFp8Config(QuantizationConfig):
64
return FBGEMMFp8LinearMethod(self)
65
return None
66
67
- def get_scaled_act_names(self) -> List[str]:
68
- return []
69
-
67
68
class FBGEMMFp8LinearMethod(LinearMethodBase):
69
vllm/model_executor/layers/quantization/fp8.py
-3
@@ -92,9 +92,6 @@ class Fp8Config(QuantizationConfig):
92
return Fp8KVCacheMethod(self)
93
return None
94
95
- def get_scaled_act_names(self) -> List[str]:
96
- return []
97
-
95
96
class Fp8LinearMethod(LinearMethodBase):
97
"""Linear method for FP8.
vllm/model_executor/layers/quantization/gguf.py
-3
@@ -48,9 +48,6 @@ class GGUFConfig(QuantizationConfig):
48
return GGUFEmbeddingMethod(self)
49
return None
50
51
- def get_scaled_act_names(self) -> List[str]:
52
- return []
53
-
51
52
def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
53
qweight_type: int) -> torch.Tensor:
vllm/model_executor/layers/quantization/gptq.py
-3
@@ -80,9 +80,6 @@ class GPTQConfig(QuantizationConfig):
80
return GPTQLinearMethod(self)
81
return None
82
83
- def get_scaled_act_names(self) -> List[str]:
84
- return []
85
-
83
84
class ExllamaState(Enum):
85
vllm/model_executor/layers/quantization/gptq_marlin.py
-3
@@ -125,9 +125,6 @@ class GPTQMarlinConfig(QuantizationConfig):
125
return GPTQMarlinMoEMethod(self)
126
return None
127
128
- def get_scaled_act_names(self) -> List[str]:
129
- return []
130
-
128
@classmethod
129
def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
130
# Extract data from quant config.
vllm/model_executor/layers/quantization/gptq_marlin_24.py
-3
@@ -127,9 +127,6 @@ class GPTQMarlin24Config(QuantizationConfig):
127
return GPTQMarlin24LinearMethod(self)
128
return None
129
130
- def get_scaled_act_names(self) -> List[str]:
131
- return []
132
-
130
131
class GPTQMarlin24LinearMethod(LinearMethodBase):
132
"""Linear method for Marlin24.
vllm/model_executor/layers/quantization/ipex_quant.py
-6
@@ -93,12 +93,6 @@ class IPEXConfig(QuantizationConfig):
93
return self.quant_method(self)
94
return None
95
96
- def get_scaled_act_names(self) -> List[str]:
97
- if self.method == "awq":
98
- return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
99
- else:
100
- return []
101
-
96
97
class IPEXAWQLinearMethod(AWQLinearMethod):
98
"""AWQ linear method using IPEX for the CPU backend.
vllm/model_executor/layers/quantization/marlin.py
-3
@@ -110,9 +110,6 @@ class MarlinConfig(QuantizationConfig):
110
return MarlinLinearMethod(self)
111
return None
112
113
- def get_scaled_act_names(self) -> List[str]:
114
- return []
115
-
113
114
class MarlinLinearMethod(LinearMethodBase):
115
"""Linear method for Marlin.
vllm/model_executor/layers/quantization/modelopt.py
-3
@@ -68,9 +68,6 @@ class ModelOptFp8Config(QuantizationConfig):
68
return ModelOptFp8KVCacheMethod(self)
69
return None
70
71
- def get_scaled_act_names(self) -> List[str]:
72
- return []
73
-
71
72
class ModelOptFp8KVCacheMethod(BaseKVCacheMethod):
73
"""
vllm/model_executor/layers/quantization/neuron_quant.py
-3
@@ -57,9 +57,6 @@ class NeuronQuantConfig(QuantizationConfig):
57
"Neuron Quantization is only supported through"
58
" transformers_neuronx.")
59
60
- def get_scaled_act_names(self) -> List[str]:
61
- return []
62
-
60
def get_quantization_config(self):
61
from transformers_neuronx.config import QuantizationConfig
62
return QuantizationConfig(quant_dtype=self.quant_dtype,
vllm/model_executor/layers/quantization/qqq.py
-3
@@ -112,9 +112,6 @@ class QQQConfig(QuantizationConfig):
112
return QQQLinearMethod(self)
113
return None
114
115
- def get_scaled_act_names(self) -> List[str]:
116
- return []
117
-
115
116
class QQQLinearMethod(LinearMethodBase):
117
"""Linear method for QQQ.
vllm/model_executor/layers/quantization/tpu_int8.py
-3
@@ -50,9 +50,6 @@ class Int8TpuConfig(QuantizationConfig):
50
return TPUInt8LinearMethod(self)
51
return None
52
53
- def get_scaled_act_names(self) -> List[str]:
54
- return []
55
-
53
54
class TPUInt8LinearMethod(LinearMethodBase):
55
"""Int8 Linear method for TPU Quant. """
vllm/model_executor/models/bart.py
+3
-5
@@ -393,8 +393,7 @@ class BartEncoderLayer(nn.Module):
393
cache_config=cache_config,
394
quant_config=quant_config)
395
self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
396
- self.activation_fn = get_act_fn(config.activation_function,
397
- quant_config)
396
+ self.activation_fn = get_act_fn(config.activation_function)
397
398
ffn_hidden_size = self.embed_dim
399
ffn_intermediate_size = config.encoder_ffn_dim
@@ -405,7 +404,7 @@ class BartEncoderLayer(nn.Module):
404
bias=ffn_has_bias,
405
quant_config=quant_config,
406
)
408
- self.act = get_act_fn("gelu", quant_config, ffn_intermediate_size)
407
+ self.act = get_act_fn("gelu")
408
self.fc2 = RowParallelLinear(
409
ffn_intermediate_size,
410
ffn_hidden_size,
@@ -473,8 +472,7 @@ class BartDecoderLayer(nn.Module):
472
config=config,
473
cache_config=cache_config,
474
quant_config=quant_config)
476
- self.activation_fn = get_act_fn(config.activation_function,
477
- quant_config)
475
+ self.activation_fn = get_act_fn(config.activation_function)
476
477
self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
478
'''
vllm/model_executor/models/bloom.py
+1
-1
@@ -146,7 +146,7 @@ class BloomMLP(nn.Module):
146
4 * hidden_size,
147
quant_config=quant_config,
148
)
149
- self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
149
+ self.gelu_impl = get_act_fn("gelu")
150
self.dense_4h_to_h = RowParallelLinear(
151
4 * hidden_size,
152
hidden_size,
vllm/model_executor/models/falcon.py
+1
-1
@@ -212,7 +212,7 @@ class FalconMLP(nn.Module):
212
bias=config.bias,
213
skip_bias_add=True,
214
quant_config=quant_config)
215
- self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
215
+ self.act = get_act_fn("gelu")
216
self.reduce_row_parallel_results = not (config.new_decoder_architecture
217
or config.parallel_attn)
218
self.dense_4h_to_h = RowParallelLinear(
vllm/model_executor/models/gpt2.py
+1
-2
@@ -123,8 +123,7 @@ class GPT2MLP(nn.Module):
123
quant_config=quant_config,
124
prefix=f"{prefix}.c_proj",
125
)
126
- self.act = get_act_fn(config.activation_function, quant_config,
127
- intermediate_size)
126
+ self.act = get_act_fn(config.activation_function)
127
128
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129
hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_bigcode.py
+1
-2
@@ -135,8 +135,7 @@ class GPTBigMLP(nn.Module):
135
bias=True,
136
quant_config=quant_config,
137
)
138
- self.act = get_act_fn(config.activation_function, quant_config,
139
- intermediate_size)
138
+ self.act = get_act_fn(config.activation_function)
139
140
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
141
hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_j.py
+1
-2
@@ -130,8 +130,7 @@ class GPTJMLP(nn.Module):
130
hidden_size,
131
quant_config=quant_config,
132
)
133
- self.act = get_act_fn(config.activation_function, quant_config,
134
- intermediate_size)
133
+ self.act = get_act_fn(config.activation_function)
134
135
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136
hidden_states, _ = self.fc_in(hidden_states)
vllm/model_executor/models/gpt_neox.py
+1
-2
@@ -128,8 +128,7 @@ class GPTNeoXMLP(nn.Module):
128
config.hidden_size,
129
quant_config=quant_config,
130
)
131
- self.act = get_act_fn(config.hidden_act, quant_config,
132
- config.intermediate_size)
131
+ self.act = get_act_fn(config.hidden_act)
132
133
def forward(self, hidden_states):
134
hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/mpt.py
+1
-1
@@ -153,7 +153,7 @@ class MPTMLP(nn.Module):
153
bias=not config.no_bias,
154
quant_config=quant_config,
155
)
156
- self.act = get_act_fn("gelu", quant_config, intermediate_size)
156
+ self.act = get_act_fn("gelu")
157
self.down_proj = RowParallelLinear(
158
intermediate_size,
159
hidden_size,
vllm/model_executor/models/opt.py
+1
-2
@@ -147,8 +147,7 @@ class OPTDecoderLayer(nn.Module):
147
quant_config=quant_config,
148
prefix=f"{prefix}.fc1",
149
)
150
- self.activation_fn = get_act_fn(config.activation_function,
151
- quant_config, config.ffn_dim)
150
+ self.activation_fn = get_act_fn(config.activation_function)
151
self.fc2 = RowParallelLinear(
152
config.ffn_dim,
153
self.embed_dim,
vllm/model_executor/models/persimmon.py
+1
-1
@@ -60,7 +60,7 @@ class PersimmonMLP(nn.Module):
60
self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
61
config.hidden_size,
62
quant_config=quant_config)
63
- self.act = get_act_fn(config.hidden_act, quant_config)
63
+ self.act = get_act_fn(config.hidden_act)
64
65
def forward(self, hidden_states) -> torch.Tensor:
66
hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/phi.py
+1
-1
@@ -152,7 +152,7 @@ class PhiMLP(nn.Module):
152
config.hidden_size,
153
quant_config=quant_config,
154
)
155
- self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
155
+ self.act = get_act_fn(config.hidden_act)
156
157
def forward(self, hidden_states):
158
hidden_states, _ = self.fc1(hidden_states)
vllm/model_executor/models/qwen.py
+1
-1
@@ -203,7 +203,7 @@ class QwenVMLP(nn.Module):
203
intermediate_size,
204
bias=True,
205
quant_config=quant_config)
206
- self.act_fn = get_act_fn("gelu", quant_config, intermediate_size)
206
+ self.act_fn = get_act_fn("gelu")
207
self.c_proj = RowParallelLinear(
208
intermediate_size,
209
hidden_size,
vllm/model_executor/models/starcoder2.py
+1
-2
@@ -139,8 +139,7 @@ class Starcoder2MLP(nn.Module):
139
bias=config.use_bias,
140
quant_config=quant_config,
141
)
142
- self.act = get_act_fn(config.hidden_act, quant_config,
143
- config.intermediate_size)
142
+ self.act = get_act_fn(config.hidden_act)
143
144
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
145
hidden_states, _ = self.c_fc(hidden_states)