@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b0dfa91dd7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

[Model] Add starcoder2 awq support (#3569)

少年 committed Mar 25, 2024 at 12:07 UTC b0dfa91dd77e954be71e1e21ef8a4041a0e98cea
1 file changed +3 -2
vllm/model_executor/models/starcoder2.py
+3 -2
@@ -141,8 +141,9 @@ class Starcoder2MLP(nn.Module):
141 bias=config.use_bias,
142 linear_method=linear_method,
143 )
144 - self.act = get_act_fn(config.hidden_act,
145 - intermediate_size=config.intermediate_size)
144 + quant_config = getattr(linear_method, "quant_config", None)
145 + self.act = get_act_fn(config.hidden_act, quant_config,
146 + config.intermediate_size)
147
148 def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
149 hidden_states, _ = self.c_fc(hidden_states)