@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 062a1d0fab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

Fix ReplicatedLinear weight loading (#6793)

QQSong committed Jul 25, 2024 at 19:24 UTC 062a1d0fab111723ab768f94bdd48a6adc054007
1 file changed +5 -1
vllm/model_executor/layers/linear.py
+5 -1
@@ -199,12 +199,16 @@ class ReplicatedLinear(LinearBase):
199 self.input_size,
200 self.output_size,
201 self.params_dtype,
202 + weight_loader=self.weight_loader,
203 prefix=prefix)
204
205 if bias:
206 self.bias = Parameter(
207 torch.empty(self.output_size, dtype=self.params_dtype))
207 - set_weight_attrs(self.bias, {"output_dim": 0})
208 + set_weight_attrs(self.bias, {
209 + "output_dim": 0,
210 + "weight_loader": self.weight_loader,
211 + })
212 else:
213 self.register_parameter("bias", None)
214