@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 399c798608
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

Remove ScaledActivation for AWQ (#10057)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 6, 2024 at 09:27 UTC 399c7986088ed66184e69ac6ae2b28003b642711
34 files changed +19 -124
vllm/model_executor/layers/activation.py
+4 -33
@@ -9,7 +9,6 @@ import torch.nn.functional as F
9 from vllm.distributed import (divide, get_tensor_model_parallel_rank,
10 get_tensor_model_parallel_world_size)
11 from vllm.model_executor.custom_op import CustomOp
12 -from vllm.model_executor.layers.quantization import QuantizationConfig
12 from vllm.model_executor.utils import set_weight_attrs
13 from vllm.utils import LazyDict
14
@@ -277,28 +276,14 @@ _ACTIVATION_REGISTRY = LazyDict({
276 })
277
278
280 -def get_act_fn(
281 - act_fn_name: str,
282 - quant_config: Optional[QuantizationConfig] = None,
283 - intermediate_size: Optional[int] = None,
284 - input_is_parallel: bool = True,
285 - params_dtype: Optional[torch.dtype] = None,
286 -) -> nn.Module:
279 +def get_act_fn(act_fn_name: str) -> nn.Module:
280 """Get an activation function by name."""
281 act_fn_name = act_fn_name.lower()
282 if act_fn_name not in _ACTIVATION_REGISTRY:
283 raise ValueError(
284 f"Activation function {act_fn_name!r} is not supported.")
285
293 - act_fn = _ACTIVATION_REGISTRY[act_fn_name]
294 - if (quant_config is not None
295 - and act_fn_name in quant_config.get_scaled_act_names()):
296 - if intermediate_size is None:
297 - raise ValueError("intermediate_size must be specified for scaled "
298 - "activation functions.")
299 - return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
300 - params_dtype)
301 - return act_fn
286 + return _ACTIVATION_REGISTRY[act_fn_name]
287
288
289 _ACTIVATION_AND_MUL_REGISTRY = LazyDict({
@@ -307,25 +292,11 @@ _ACTIVATION_AND_MUL_REGISTRY = LazyDict({
292 })
293
294
310 -def get_act_and_mul_fn(
311 - act_fn_name: str,
312 - quant_config: Optional[QuantizationConfig] = None,
313 - intermediate_size: Optional[int] = None,
314 - input_is_parallel: bool = True,
315 - params_dtype: Optional[torch.dtype] = None,
316 -) -> nn.Module:
295 +def get_act_and_mul_fn(act_fn_name: str) -> nn.Module:
296 """Get an activation-and-mul (i.e. SiluAndMul) function by name."""
297 act_fn_name = act_fn_name.lower()
298 if act_fn_name not in _ACTIVATION_AND_MUL_REGISTRY:
299 raise ValueError(
300 f"Activation function {act_fn_name!r} is not supported.")
301
323 - act_fn = _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
324 - if (quant_config is not None
325 - and act_fn_name in quant_config.get_scaled_act_names()):
326 - if intermediate_size is None:
327 - raise ValueError("intermediate_size must be specified for scaled "
328 - "activation functions.")
329 - return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
330 - params_dtype)
331 - return act_fn
302 + return _ACTIVATION_AND_MUL_REGISTRY[act_fn_name]
vllm/model_executor/layers/quantization/aqlm.py
-3
@@ -213,9 +213,6 @@ class AQLMConfig(QuantizationConfig):
213 return AQLMLinearMethod(self)
214 return None
215
216 - def get_scaled_act_names(self) -> List[str]:
217 - return []
218 -
216
217 class AQLMLinearMethod(LinearMethodBase):
218 """Linear method for AQLM.
vllm/model_executor/layers/quantization/awq.py
-3
@@ -77,9 +77,6 @@ class AWQConfig(QuantizationConfig):
77 return AWQLinearMethod(self)
78 return None
79
80 - def get_scaled_act_names(self) -> List[str]:
81 - return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
82 -
80
81 def is_layer_skipped_awq(prefix: str, modules_to_not_convert: List[str]):
82 return any(module_name in prefix for module_name in modules_to_not_convert)
vllm/model_executor/layers/quantization/awq_marlin.py
-3
@@ -127,9 +127,6 @@ class AWQMarlinConfig(QuantizationConfig):
127 return AWQMoEMethod(self)
128 return None
129
130 - def get_scaled_act_names(self) -> List[str]:
131 - return []
132 -
130 @classmethod
131 def is_awq_marlin_compatible(cls, quant_config: Dict[str, Any]):
132 # Extract data from quant config.
vllm/model_executor/layers/quantization/base_config.py
-8
@@ -133,11 +133,3 @@ class QuantizationConfig(ABC):
133 method.
134 """
135 raise NotImplementedError
136 -
137 - @abstractmethod
138 - def get_scaled_act_names(self) -> List[str]:
139 - """Returns the activation function names that should be post-scaled.
140 -
141 - For now, this is only used by AWQ.
142 - """
143 - raise NotImplementedError
vllm/model_executor/layers/quantization/bitsandbytes.py
-3
@@ -114,9 +114,6 @@ class BitsAndBytesConfig(QuantizationConfig):
114 return BitsAndBytesLinearMethod(self)
115 return None
116
117 - def get_scaled_act_names(self) -> List[str]:
118 - return []
119 -
117
118 def is_layer_skipped_bnb(prefix: str, llm_int8_skip_modules: List[str]):
119 # Split the prefix into its dot-separated components
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
-3
@@ -45,9 +45,6 @@ class CompressedTensorsConfig(QuantizationConfig):
45 def get_linear_method(self) -> "CompressedTensorsLinearMethod":
46 return CompressedTensorsLinearMethod(self)
47
48 - def get_scaled_act_names(self) -> List[str]:
49 - return []
50 -
48 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
49 return [torch.float16, torch.bfloat16]
50
vllm/model_executor/layers/quantization/deepspeedfp.py
-3
@@ -50,9 +50,6 @@ class DeepSpeedFPConfig(QuantizationConfig):
50 def get_linear_method(self) -> "DeepSpeedFPLinearMethod":
51 return DeepSpeedFPLinearMethod(self)
52
53 - def get_scaled_act_names(self) -> List[str]:
54 - return []
55 -
53 @classmethod
54 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
55 return [torch.half, torch.bfloat16]
vllm/model_executor/layers/quantization/experts_int8.py
-3
@@ -45,9 +45,6 @@ class ExpertsInt8Config(QuantizationConfig):
45 return ExpertsInt8MoEMethod(self)
46 return None
47
48 - def get_scaled_act_names(self) -> List[str]:
49 - return []
50 -
48
49 class ExpertsInt8MoEMethod(FusedMoEMethodBase):
50
vllm/model_executor/layers/quantization/fbgemm_fp8.py
-3
@@ -64,9 +64,6 @@ class FBGEMMFp8Config(QuantizationConfig):
64 return FBGEMMFp8LinearMethod(self)
65 return None
66
67 - def get_scaled_act_names(self) -> List[str]:
68 - return []
69 -
67
68 class FBGEMMFp8LinearMethod(LinearMethodBase):
69
vllm/model_executor/layers/quantization/fp8.py
-3
@@ -92,9 +92,6 @@ class Fp8Config(QuantizationConfig):
92 return Fp8KVCacheMethod(self)
93 return None
94
95 - def get_scaled_act_names(self) -> List[str]:
96 - return []
97 -
95
96 class Fp8LinearMethod(LinearMethodBase):
97 """Linear method for FP8.
vllm/model_executor/layers/quantization/gguf.py
-3
@@ -48,9 +48,6 @@ class GGUFConfig(QuantizationConfig):
48 return GGUFEmbeddingMethod(self)
49 return None
50
51 - def get_scaled_act_names(self) -> List[str]:
52 - return []
53 -
51
52 def _fuse_mul_mat(x: torch.Tensor, qweight: torch.Tensor,
53 qweight_type: int) -> torch.Tensor:
vllm/model_executor/layers/quantization/gptq.py
-3
@@ -80,9 +80,6 @@ class GPTQConfig(QuantizationConfig):
80 return GPTQLinearMethod(self)
81 return None
82
83 - def get_scaled_act_names(self) -> List[str]:
84 - return []
85 -
83
84 class ExllamaState(Enum):
85
vllm/model_executor/layers/quantization/gptq_marlin.py
-3
@@ -125,9 +125,6 @@ class GPTQMarlinConfig(QuantizationConfig):
125 return GPTQMarlinMoEMethod(self)
126 return None
127
128 - def get_scaled_act_names(self) -> List[str]:
129 - return []
130 -
128 @classmethod
129 def is_gptq_marlin_compatible(cls, quant_config: Dict[str, Any]):
130 # Extract data from quant config.
vllm/model_executor/layers/quantization/gptq_marlin_24.py
-3
@@ -127,9 +127,6 @@ class GPTQMarlin24Config(QuantizationConfig):
127 return GPTQMarlin24LinearMethod(self)
128 return None
129
130 - def get_scaled_act_names(self) -> List[str]:
131 - return []
132 -
130
131 class GPTQMarlin24LinearMethod(LinearMethodBase):
132 """Linear method for Marlin24.
vllm/model_executor/layers/quantization/ipex_quant.py
-6
@@ -93,12 +93,6 @@ class IPEXConfig(QuantizationConfig):
93 return self.quant_method(self)
94 return None
95
96 - def get_scaled_act_names(self) -> List[str]:
97 - if self.method == "awq":
98 - return ["gelu", "gelu_fast", "gelu_new", "gelu_pytorch_tanh"]
99 - else:
100 - return []
101 -
96
97 class IPEXAWQLinearMethod(AWQLinearMethod):
98 """AWQ linear method using IPEX for the CPU backend.
vllm/model_executor/layers/quantization/marlin.py
-3
@@ -110,9 +110,6 @@ class MarlinConfig(QuantizationConfig):
110 return MarlinLinearMethod(self)
111 return None
112
113 - def get_scaled_act_names(self) -> List[str]:
114 - return []
115 -
113
114 class MarlinLinearMethod(LinearMethodBase):
115 """Linear method for Marlin.
vllm/model_executor/layers/quantization/modelopt.py
-3
@@ -68,9 +68,6 @@ class ModelOptFp8Config(QuantizationConfig):
68 return ModelOptFp8KVCacheMethod(self)
69 return None
70
71 - def get_scaled_act_names(self) -> List[str]:
72 - return []
73 -
71
72 class ModelOptFp8KVCacheMethod(BaseKVCacheMethod):
73 """
vllm/model_executor/layers/quantization/neuron_quant.py
-3
@@ -57,9 +57,6 @@ class NeuronQuantConfig(QuantizationConfig):
57 "Neuron Quantization is only supported through"
58 " transformers_neuronx.")
59
60 - def get_scaled_act_names(self) -> List[str]:
61 - return []
62 -
60 def get_quantization_config(self):
61 from transformers_neuronx.config import QuantizationConfig
62 return QuantizationConfig(quant_dtype=self.quant_dtype,
vllm/model_executor/layers/quantization/qqq.py
-3
@@ -112,9 +112,6 @@ class QQQConfig(QuantizationConfig):
112 return QQQLinearMethod(self)
113 return None
114
115 - def get_scaled_act_names(self) -> List[str]:
116 - return []
117 -
115
116 class QQQLinearMethod(LinearMethodBase):
117 """Linear method for QQQ.
vllm/model_executor/layers/quantization/tpu_int8.py
-3
@@ -50,9 +50,6 @@ class Int8TpuConfig(QuantizationConfig):
50 return TPUInt8LinearMethod(self)
51 return None
52
53 - def get_scaled_act_names(self) -> List[str]:
54 - return []
55 -
53
54 class TPUInt8LinearMethod(LinearMethodBase):
55 """Int8 Linear method for TPU Quant. """
vllm/model_executor/models/bart.py
+3 -5
@@ -393,8 +393,7 @@ class BartEncoderLayer(nn.Module):
393 cache_config=cache_config,
394 quant_config=quant_config)
395 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
396 - self.activation_fn = get_act_fn(config.activation_function,
397 - quant_config)
396 + self.activation_fn = get_act_fn(config.activation_function)
397
398 ffn_hidden_size = self.embed_dim
399 ffn_intermediate_size = config.encoder_ffn_dim
@@ -405,7 +404,7 @@ class BartEncoderLayer(nn.Module):
404 bias=ffn_has_bias,
405 quant_config=quant_config,
406 )
408 - self.act = get_act_fn("gelu", quant_config, ffn_intermediate_size)
407 + self.act = get_act_fn("gelu")
408 self.fc2 = RowParallelLinear(
409 ffn_intermediate_size,
410 ffn_hidden_size,
@@ -473,8 +472,7 @@ class BartDecoderLayer(nn.Module):
472 config=config,
473 cache_config=cache_config,
474 quant_config=quant_config)
476 - self.activation_fn = get_act_fn(config.activation_function,
477 - quant_config)
475 + self.activation_fn = get_act_fn(config.activation_function)
476
477 self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim)
478 '''
vllm/model_executor/models/bloom.py
+1 -1
@@ -146,7 +146,7 @@ class BloomMLP(nn.Module):
146 4 * hidden_size,
147 quant_config=quant_config,
148 )
149 - self.gelu_impl = get_act_fn("gelu", quant_config, 4 * hidden_size)
149 + self.gelu_impl = get_act_fn("gelu")
150 self.dense_4h_to_h = RowParallelLinear(
151 4 * hidden_size,
152 hidden_size,
vllm/model_executor/models/falcon.py
+1 -1
@@ -212,7 +212,7 @@ class FalconMLP(nn.Module):
212 bias=config.bias,
213 skip_bias_add=True,
214 quant_config=quant_config)
215 - self.act = get_act_fn("gelu", quant_config, 4 * hidden_size)
215 + self.act = get_act_fn("gelu")
216 self.reduce_row_parallel_results = not (config.new_decoder_architecture
217 or config.parallel_attn)
218 self.dense_4h_to_h = RowParallelLinear(
vllm/model_executor/models/gpt2.py
+1 -2
@@ -123,8 +123,7 @@ class GPT2MLP(nn.Module):
123 quant_config=quant_config,
124 prefix=f"{prefix}.c_proj",
125 )
126 - self.act = get_act_fn(config.activation_function, quant_config,
127 - intermediate_size)
126 + self.act = get_act_fn(config.activation_function)
127
128 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
129 hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_bigcode.py
+1 -2
@@ -135,8 +135,7 @@ class GPTBigMLP(nn.Module):
135 bias=True,
136 quant_config=quant_config,
137 )
138 - self.act = get_act_fn(config.activation_function, quant_config,
139 - intermediate_size)
138 + self.act = get_act_fn(config.activation_function)
139
140 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
141 hidden_states, _ = self.c_fc(hidden_states)
vllm/model_executor/models/gpt_j.py
+1 -2
@@ -130,8 +130,7 @@ class GPTJMLP(nn.Module):
130 hidden_size,
131 quant_config=quant_config,
132 )
133 - self.act = get_act_fn(config.activation_function, quant_config,
134 - intermediate_size)
133 + self.act = get_act_fn(config.activation_function)
134
135 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
136 hidden_states, _ = self.fc_in(hidden_states)
vllm/model_executor/models/gpt_neox.py
+1 -2
@@ -128,8 +128,7 @@ class GPTNeoXMLP(nn.Module):
128 config.hidden_size,
129 quant_config=quant_config,
130 )
131 - self.act = get_act_fn(config.hidden_act, quant_config,
132 - config.intermediate_size)
131 + self.act = get_act_fn(config.hidden_act)
132
133 def forward(self, hidden_states):
134 hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/mpt.py
+1 -1
@@ -153,7 +153,7 @@ class MPTMLP(nn.Module):
153 bias=not config.no_bias,
154 quant_config=quant_config,
155 )
156 - self.act = get_act_fn("gelu", quant_config, intermediate_size)
156 + self.act = get_act_fn("gelu")
157 self.down_proj = RowParallelLinear(
158 intermediate_size,
159 hidden_size,
vllm/model_executor/models/opt.py
+1 -2
@@ -147,8 +147,7 @@ class OPTDecoderLayer(nn.Module):
147 quant_config=quant_config,
148 prefix=f"{prefix}.fc1",
149 )
150 - self.activation_fn = get_act_fn(config.activation_function,
151 - quant_config, config.ffn_dim)
150 + self.activation_fn = get_act_fn(config.activation_function)
151 self.fc2 = RowParallelLinear(
152 config.ffn_dim,
153 self.embed_dim,
vllm/model_executor/models/persimmon.py
+1 -1
@@ -60,7 +60,7 @@ class PersimmonMLP(nn.Module):
60 self.dense_4h_to_h = RowParallelLinear(config.intermediate_size,
61 config.hidden_size,
62 quant_config=quant_config)
63 - self.act = get_act_fn(config.hidden_act, quant_config)
63 + self.act = get_act_fn(config.hidden_act)
64
65 def forward(self, hidden_states) -> torch.Tensor:
66 hidden_states, _ = self.dense_h_to_4h(hidden_states)
vllm/model_executor/models/phi.py
+1 -1
@@ -152,7 +152,7 @@ class PhiMLP(nn.Module):
152 config.hidden_size,
153 quant_config=quant_config,
154 )
155 - self.act = get_act_fn(config.hidden_act, quant_config, n_inner)
155 + self.act = get_act_fn(config.hidden_act)
156
157 def forward(self, hidden_states):
158 hidden_states, _ = self.fc1(hidden_states)
vllm/model_executor/models/qwen.py
+1 -1
@@ -203,7 +203,7 @@ class QwenVMLP(nn.Module):
203 intermediate_size,
204 bias=True,
205 quant_config=quant_config)
206 - self.act_fn = get_act_fn("gelu", quant_config, intermediate_size)
206 + self.act_fn = get_act_fn("gelu")
207 self.c_proj = RowParallelLinear(
208 intermediate_size,
209 hidden_size,
vllm/model_executor/models/starcoder2.py
+1 -2
@@ -139,8 +139,7 @@ class Starcoder2MLP(nn.Module):
139 bias=config.use_bias,
140 quant_config=quant_config,
141 )
142 - self.act = get_act_fn(config.hidden_act, quant_config,
143 - config.intermediate_size)
142 + self.act = get_act_fn(config.hidden_act)
143
144 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
145 hidden_states, _ = self.c_fc(hidden_states)