A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
[BugFix] Fix TP support for AWQ (#1731)
Woosuk Kwon committed
Nov 20, 2023 at 21:42 UTC
cf35d8f3d7210c7cdf282e96d1bce10613db5279
2 files changed
+38
-14
vllm/model_executor/layers/activation.py
+35
-11
@@ -6,6 +6,10 @@ import torch.nn as nn
6
7
from vllm import activation_ops
8
from vllm.model_executor.layers.quantization import QuantizationConfig
9
+from vllm.model_executor.parallel_utils.parallel_state import (
10
+ get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11
+from vllm.model_executor.parallel_utils.utils import divide
12
+from vllm.model_executor.utils import set_weight_attrs
13
14
15
class SiluAndMul(nn.Module):
@@ -51,17 +55,38 @@ class ScaledActivation(nn.Module):
55
def __init__(
56
self,
57
act_module: nn.Module,
54
- hidden_size: int,
55
- params_dtype: torch.dtype,
58
+ intermediate_size: int,
59
+ input_is_parallel: bool = True,
60
+ params_dtype: Optional[torch.dtype] = None,
61
):
62
super().__init__()
63
self.act = act_module
64
+ if input_is_parallel:
65
+ tp_size = get_tensor_model_parallel_world_size()
66
+ intermediate_size_per_partition = divide(intermediate_size,
67
+ tp_size)
68
+ else:
69
+ intermediate_size_per_partition = intermediate_size
70
+ if params_dtype is None:
71
+ params_dtype = torch.get_default_dtype()
72
self.scales = nn.Parameter(
60
- torch.empty(hidden_size, dtype=params_dtype, device="cuda"))
73
+ torch.empty(intermediate_size_per_partition,
74
+ dtype=params_dtype,
75
+ device="cuda"))
76
+ set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})
77
62
- def forward(self, x: torch.Tensor):
78
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
79
return self.act(x) / self.scales
80
81
+ def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
82
+ tp_rank = get_tensor_model_parallel_rank()
83
+ param_data = param.data
84
+ shard_size = param_data.shape[0]
85
+ start_idx = tp_rank * shard_size
86
+ loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
87
+ assert param_data.shape == loaded_weight.shape
88
+ param_data.copy_(loaded_weight)
89
+
90
91
_ACTIVATION_REGISTRY = {
92
"gelu": nn.GELU(),
@@ -76,6 +101,8 @@ def get_act_fn(
101
act_fn_name: str,
102
quant_config: Optional[QuantizationConfig] = None,
103
intermediate_size: Optional[int] = None,
104
+ input_is_parallel: bool = True,
105
+ params_dtype: Optional[torch.dtype] = None,
106
) -> nn.Module:
107
"""Get an activation function by name."""
108
act_fn_name = act_fn_name.lower()
@@ -84,14 +111,11 @@ def get_act_fn(
111
f"Activation function {act_fn_name!r} is not supported.")
112
113
act_fn = _ACTIVATION_REGISTRY[act_fn_name]
87
- if quant_config is not None and act_fn_name in quant_config.get_scaled_act_names(
88
- ):
114
+ if (quant_config is not None
115
+ and act_fn_name in quant_config.get_scaled_act_names()):
116
if intermediate_size is None:
117
raise ValueError("intermediate_size must be specified for scaled "
118
"activation functions.")
92
- return ScaledActivation(
93
- act_fn,
94
- intermediate_size,
95
- params_dtype=torch.get_default_dtype(),
96
- )
119
+ return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
120
+ params_dtype)
121
return act_fn
vllm/model_executor/models/opt.py
+3
-3
@@ -129,9 +129,6 @@ class OPTDecoderLayer(nn.Module):
129
linear_method=linear_method,
130
)
131
self.do_layer_norm_before = config.do_layer_norm_before
132
- quant_config = getattr(linear_method, "quant_config", None)
133
- self.activation_fn = get_act_fn(config.activation_function,
134
- quant_config, config.ffn_dim)
132
133
self.self_attn_layer_norm = nn.LayerNorm(
134
self.embed_dim,
@@ -142,6 +139,9 @@ class OPTDecoderLayer(nn.Module):
139
bias=config.enable_bias,
140
linear_method=linear_method,
141
)
142
+ quant_config = getattr(linear_method, "quant_config", None)
143
+ self.activation_fn = get_act_fn(config.activation_function,
144
+ quant_config, config.ffn_dim)
145
self.fc2 = RowParallelLinear(
146
config.ffn_dim,
147
self.embed_dim,