A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
Fix ReplicatedLinear weight loading (#6793)
QQSong committed
Jul 25, 2024 at 19:24 UTC
062a1d0fab111723ab768f94bdd48a6adc054007
1 file changed
+5
-1
vllm/model_executor/layers/linear.py
+5
-1
@@ -199,12 +199,16 @@ class ReplicatedLinear(LinearBase):
199
self.input_size,
200
self.output_size,
201
self.params_dtype,
202
+ weight_loader=self.weight_loader,
203
prefix=prefix)
204
205
if bias:
206
self.bias = Parameter(
207
torch.empty(self.output_size, dtype=self.params_dtype))
207
- set_weight_attrs(self.bias, {"output_dim": 0})
208
+ set_weight_attrs(self.bias, {
209
+ "output_dim": 0,
210
+ "weight_loader": self.weight_loader,
211
+ })
212
else:
213
self.register_parameter("bias", None)
214