@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / b98d89efd4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Misc] Medusa supports custom bias (#10361)

Sky Lee committed Nov 17, 2024 at 00:33 UTC b98d89efd4b1a09c11c4d0cf30c9af0e93514764
1 file changed +7 -3
vllm/model_executor/models/medusa.py
+7 -3
@@ -14,11 +14,14 @@ from vllm.model_executor.sampling_metadata import SamplingMetadata
14
15 class ResidualBlock(nn.Module):
16
17 - def __init__(self, hidden_size: int, num_layers: int) -> None:
17 + def __init__(self, config: VllmConfig, hidden_size: int,
18 + num_layers: int) -> None:
19 super().__init__()
20
21 self.layers = nn.ModuleList([
21 - nn.Linear(hidden_size, hidden_size, bias=False)
22 + nn.Linear(hidden_size,
23 + hidden_size,
24 + bias=getattr(config, "medusa_fc_bias", False))
25 for _ in range(num_layers)
26 ])
27 self.act = nn.SiLU()
@@ -49,7 +52,8 @@ class Medusa(nn.Module):
52 super().__init__()
53 self.config = config
54 self.blocks = nn.ModuleList([
52 - ResidualBlock(hidden_size=self.config.hidden_size,
55 + ResidualBlock(config=config,
56 + hidden_size=self.config.hidden_size,
57 num_layers=self.config.num_hidden_layers)
58 for _ in range(self.config.num_heads)
59 ])