A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Misc] Medusa supports custom bias (#10361)
Sky Lee committed
Nov 17, 2024 at 00:33 UTC
b98d89efd4b1a09c11c4d0cf30c9af0e93514764
1 file changed
+7
-3
vllm/model_executor/models/medusa.py
+7
-3
@@ -14,11 +14,14 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
14
15
class ResidualBlock(nn.Module):
16
17
- def __init__(self, hidden_size: int, num_layers: int) -> None:
17
+ def __init__(self, config: VllmConfig, hidden_size: int,
18
+ num_layers: int) -> None:
19
super().__init__()
20
21
self.layers = nn.ModuleList([
21
- nn.Linear(hidden_size, hidden_size, bias=False)
22
+ nn.Linear(hidden_size,
23
+ hidden_size,
24
+ bias=getattr(config, "medusa_fc_bias", False))
25
for _ in range(num_layers)
26
])
27
self.act = nn.SiLU()
@@ -49,7 +52,8 @@ class Medusa(nn.Module):
52
super().__init__()
53
self.config = config
54
self.blocks = nn.ModuleList([
52
- ResidualBlock(hidden_size=self.config.hidden_size,
55
+ ResidualBlock(config=config,
56
+ hidden_size=self.config.hidden_size,
57
num_layers=self.config.num_hidden_layers)
58
for _ in range(self.config.num_heads)
59
])