A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
[Model] Add starcoder2 awq support (#3569)
少年 committed
Mar 25, 2024 at 12:07 UTC
b0dfa91dd77e954be71e1e21ef8a4041a0e98cea
1 file changed
+3
-2
vllm/model_executor/models/starcoder2.py
+3
-2
@@ -141,8 +141,9 @@ class Starcoder2MLP(nn.Module):
141
bias=config.use_bias,
142
linear_method=linear_method,
143
)
144
- self.act = get_act_fn(config.hidden_act,
145
- intermediate_size=config.intermediate_size)
144
+ quant_config = getattr(linear_method, "quant_config", None)
145
+ self.act = get_act_fn(config.hidden_act, quant_config,
146
+ config.intermediate_size)
147
148
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
149
hidden_states, _ = self.c_fc(hidden_states)