A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] MLPSpeculator: Use ParallelLMHead in tie_weights=False case. (#6303)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jul 10, 2024 at 15:04 UTC
c38eba304674fdf9da4d881e46f103440e22a153
1 file changed
+1
-1
vllm/model_executor/models/mlp_speculator.py
+1
-1
@@ -110,7 +110,7 @@ class MLPSpeculator(nn.Module):
110
])
111
112
self.head = nn.ModuleList([
113
- nn.Linear(self.inner_dim, self.vocab_size, bias=False)
113
+ ParallelLMHead(self.vocab_size, self.inner_dim, bias=False)
114
for _ in range(self.max_speculative_tokens)
115
])
116
self.ln = nn.ModuleList([