A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] load fc bias from config for eagle (#8790)
sohamparikh committed
Sep 25, 2024 at 02:16 UTC
3e073e66f1790f7ce339dad71514983e6e402f30
1 file changed
+10
-2
vllm/model_executor/models/eagle.py
+10
-2
@@ -44,7 +44,7 @@ class EAGLE(nn.Module):
44
self.model = model_cls(self.config.model, *args, **kwargs)
45
self.fc = nn.Linear(config.model.hidden_size * 2,
46
config.model.hidden_size,
47
- bias=False)
47
+ bias=getattr(self.config, "bias", False))
48
49
self.orig_vocab_size = config.vocab_size
50
self.truncated_vocab_size = config.truncated_vocab_size
@@ -136,10 +136,18 @@ class EAGLE(nn.Module):
136
if self.config.truncated_vocab_size < self.config.vocab_size:
137
self.token_map = nn.Parameter(loaded_weight,
138
requires_grad=False)
139
- elif name.startswith("fc."):
139
+ elif name.startswith("fc.weight"):
140
weight_loader = getattr(self.fc.weight, "weight_loader",
141
default_weight_loader)
142
weight_loader(self.fc.weight, loaded_weight)
143
+ elif name.startswith("fc.bias"):
144
+ if self.fc.bias is not None:
145
+ weight_loader = getattr(self.fc.bias, "weight_loader",
146
+ default_weight_loader)
147
+ weight_loader(self.fc.bias, loaded_weight)
148
+ else:
149
+ raise ValueError("Found bias in the loaded weights "
150
+ "but the model config doesn't have bias")
151
elif name.startswith("model.lm_head.") or name.startswith(
152
"model.model."):
153
model_weights[name.split("model.", 1)[-1]] = loaded_weight