@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3e073e66f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] load fc bias from config for eagle (#8790)

sohamparikh committed Sep 25, 2024 at 02:16 UTC 3e073e66f1790f7ce339dad71514983e6e402f30
1 file changed +10 -2
vllm/model_executor/models/eagle.py
+10 -2
@@ -44,7 +44,7 @@ class EAGLE(nn.Module):
44 self.model = model_cls(self.config.model, *args, **kwargs)
45 self.fc = nn.Linear(config.model.hidden_size * 2,
46 config.model.hidden_size,
47 - bias=False)
47 + bias=getattr(self.config, "bias", False))
48
49 self.orig_vocab_size = config.vocab_size
50 self.truncated_vocab_size = config.truncated_vocab_size
@@ -136,10 +136,18 @@ class EAGLE(nn.Module):
136 if self.config.truncated_vocab_size < self.config.vocab_size:
137 self.token_map = nn.Parameter(loaded_weight,
138 requires_grad=False)
139 - elif name.startswith("fc."):
139 + elif name.startswith("fc.weight"):
140 weight_loader = getattr(self.fc.weight, "weight_loader",
141 default_weight_loader)
142 weight_loader(self.fc.weight, loaded_weight)
143 + elif name.startswith("fc.bias"):
144 + if self.fc.bias is not None:
145 + weight_loader = getattr(self.fc.bias, "weight_loader",
146 + default_weight_loader)
147 + weight_loader(self.fc.bias, loaded_weight)
148 + else:
149 + raise ValueError("Found bias in the loaded weights "
150 + "but the model config doesn't have bias")
151 elif name.startswith("model.lm_head.") or name.startswith(
152 "model.model."):
153 model_weights[name.split("model.", 1)[-1]] = loaded_weight