@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cf35d8f3d7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

[BugFix] Fix TP support for AWQ (#1731)

Woosuk Kwon committed Nov 20, 2023 at 21:42 UTC cf35d8f3d7210c7cdf282e96d1bce10613db5279
2 files changed +38 -14
vllm/model_executor/layers/activation.py
+35 -11
@@ -6,6 +6,10 @@ import torch.nn as nn
6
7 from vllm import activation_ops
8 from vllm.model_executor.layers.quantization import QuantizationConfig
9 +from vllm.model_executor.parallel_utils.parallel_state import (
10 + get_tensor_model_parallel_rank, get_tensor_model_parallel_world_size)
11 +from vllm.model_executor.parallel_utils.utils import divide
12 +from vllm.model_executor.utils import set_weight_attrs
13
14
15 class SiluAndMul(nn.Module):
@@ -51,17 +55,38 @@ class ScaledActivation(nn.Module):
55 def __init__(
56 self,
57 act_module: nn.Module,
54 - hidden_size: int,
55 - params_dtype: torch.dtype,
58 + intermediate_size: int,
59 + input_is_parallel: bool = True,
60 + params_dtype: Optional[torch.dtype] = None,
61 ):
62 super().__init__()
63 self.act = act_module
64 + if input_is_parallel:
65 + tp_size = get_tensor_model_parallel_world_size()
66 + intermediate_size_per_partition = divide(intermediate_size,
67 + tp_size)
68 + else:
69 + intermediate_size_per_partition = intermediate_size
70 + if params_dtype is None:
71 + params_dtype = torch.get_default_dtype()
72 self.scales = nn.Parameter(
60 - torch.empty(hidden_size, dtype=params_dtype, device="cuda"))
73 + torch.empty(intermediate_size_per_partition,
74 + dtype=params_dtype,
75 + device="cuda"))
76 + set_weight_attrs(self.scales, {"weight_loader": self.weight_loader})
77
62 - def forward(self, x: torch.Tensor):
78 + def forward(self, x: torch.Tensor) -> torch.Tensor:
79 return self.act(x) / self.scales
80
81 + def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
82 + tp_rank = get_tensor_model_parallel_rank()
83 + param_data = param.data
84 + shard_size = param_data.shape[0]
85 + start_idx = tp_rank * shard_size
86 + loaded_weight = loaded_weight.narrow(0, start_idx, shard_size)
87 + assert param_data.shape == loaded_weight.shape
88 + param_data.copy_(loaded_weight)
89 +
90
91 _ACTIVATION_REGISTRY = {
92 "gelu": nn.GELU(),
@@ -76,6 +101,8 @@ def get_act_fn(
101 act_fn_name: str,
102 quant_config: Optional[QuantizationConfig] = None,
103 intermediate_size: Optional[int] = None,
104 + input_is_parallel: bool = True,
105 + params_dtype: Optional[torch.dtype] = None,
106 ) -> nn.Module:
107 """Get an activation function by name."""
108 act_fn_name = act_fn_name.lower()
@@ -84,14 +111,11 @@ def get_act_fn(
111 f"Activation function {act_fn_name!r} is not supported.")
112
113 act_fn = _ACTIVATION_REGISTRY[act_fn_name]
87 - if quant_config is not None and act_fn_name in quant_config.get_scaled_act_names(
88 - ):
114 + if (quant_config is not None
115 + and act_fn_name in quant_config.get_scaled_act_names()):
116 if intermediate_size is None:
117 raise ValueError("intermediate_size must be specified for scaled "
118 "activation functions.")
92 - return ScaledActivation(
93 - act_fn,
94 - intermediate_size,
95 - params_dtype=torch.get_default_dtype(),
96 - )
119 + return ScaledActivation(act_fn, intermediate_size, input_is_parallel,
120 + params_dtype)
121 return act_fn
vllm/model_executor/models/opt.py
+3 -3
@@ -129,9 +129,6 @@ class OPTDecoderLayer(nn.Module):
129 linear_method=linear_method,
130 )
131 self.do_layer_norm_before = config.do_layer_norm_before
132 - quant_config = getattr(linear_method, "quant_config", None)
133 - self.activation_fn = get_act_fn(config.activation_function,
134 - quant_config, config.ffn_dim)
132
133 self.self_attn_layer_norm = nn.LayerNorm(
134 self.embed_dim,
@@ -142,6 +139,9 @@ class OPTDecoderLayer(nn.Module):
139 bias=config.enable_bias,
140 linear_method=linear_method,
141 )
142 + quant_config = getattr(linear_method, "quant_config", None)
143 + self.activation_fn = get_act_fn(config.activation_function,
144 + quant_config, config.ffn_dim)
145 self.fc2 = RowParallelLinear(
146 config.ffn_dim,
147 self.embed_dim,