A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Model] Changes to MLPSpeculator to support tie_weights and input_scale (#5965)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Joshua Rosenkranz <jmrosenk@us.ibm.com>
Thomas Parnell committed
Jul 2, 2024 at 01:40 UTC
54600709b6d419fb243ce718a48ab7d40f5c3eb7
2 files changed
+81
-25
vllm/model_executor/models/mlp_speculator.py
+69
-25
@@ -13,6 +13,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
from vllm.sequence import SamplerOutput
14
from vllm.transformers_utils.configs import MLPSpeculatorConfig
15
16
+SQRT2 = 2**0.5
17
+
18
19
class MLPSpeculatorLayerNorm(nn.Module):
20
"""
@@ -26,24 +28,30 @@ class MLPSpeculatorLayerNorm(nn.Module):
28
Safety term to prevent division by zero. Make sure the chosen value
29
fits in the range of your encoding scheme
30
(i.e. fp16 requires eps >= 6e-8).
31
+ elementwise_scale_and_shift : bool
32
+ Include a learned scaling and shift term after normalization.
33
"""
34
35
def __init__(
36
self,
37
normalized_shape,
38
eps=1e-06,
39
+ elementwise_scale_and_shift=True,
40
):
41
super(MLPSpeculatorLayerNorm, self).__init__()
37
- self.weight = nn.Parameter(torch.empty(normalized_shape))
38
- self.bias = nn.Parameter(torch.empty(normalized_shape))
42
+ self.elementwise_scale_and_shift = elementwise_scale_and_shift
43
+ if self.elementwise_scale_and_shift:
44
+ self.weight = nn.Parameter(torch.empty(normalized_shape))
45
+ self.bias = nn.Parameter(torch.empty(normalized_shape))
46
self.eps = eps
47
48
def forward(self, x):
49
xf = x
50
xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
51
x = xf.type_as(x)
45
- x = self.weight * x
46
- x = x + self.bias
52
+ if self.elementwise_scale_and_shift:
53
+ x = self.weight * x
54
+ x = x + self.bias
55
return x
56
57
@@ -59,27 +67,60 @@ class MLPSpeculator(nn.Module):
67
68
self.max_speculative_tokens = config.num_lookahead_tokens
69
62
- self.emb = nn.ModuleList([
63
- VocabParallelEmbedding(config.vocab_size,
64
- self.inner_dim,
65
- org_num_embeddings=config.vocab_size)
66
- for _ in range(self.max_speculative_tokens)
67
- ])
68
-
69
- self.proj = nn.ModuleList([
70
- nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
71
- self.inner_dim,
72
- bias=False) for i in range(self.max_speculative_tokens)
73
- ])
74
-
75
- self.head = nn.ModuleList([
76
- nn.Linear(self.inner_dim, self.vocab_size, bias=False)
77
- for _ in range(self.max_speculative_tokens)
78
- ])
79
- self.ln = nn.ModuleList([
80
- MLPSpeculatorLayerNorm(self.inner_dim)
81
- for _ in range(self.max_speculative_tokens)
82
- ])
70
+ self.tie_weights = config.tie_weights
71
+ self.scale_input = config.scale_input
72
+
73
+ if self.tie_weights:
74
+ assert (
75
+ self.n_predict >
76
+ 1), "You cannot tie weights between stages when only 1 exists"
77
+ embedding = VocabParallelEmbedding(
78
+ config.vocab_size,
79
+ self.inner_dim,
80
+ org_num_embeddings=config.vocab_size)
81
+ self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens)
82
+
83
+ # the initial projection from the base model may
84
+ # have a different size, so that stays separate.
85
+ proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False)
86
+ proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False)
87
+ self.proj = nn.ModuleList([proj_first] + [proj_tied] *
88
+ (self.max_speculative_tokens - 1))
89
+
90
+ head = nn.Linear(self.inner_dim, self.vocab_size, bias=False)
91
+ self.head = nn.ModuleList([head] * self.max_speculative_tokens)
92
+
93
+ ln = MLPSpeculatorLayerNorm(self.inner_dim,
94
+ elementwise_scale_and_shift=True)
95
+ self.ln = nn.ModuleList([ln] * self.max_speculative_tokens)
96
+
97
+ else:
98
+ self.emb = nn.ModuleList([
99
+ VocabParallelEmbedding(config.vocab_size,
100
+ self.inner_dim,
101
+ org_num_embeddings=config.vocab_size)
102
+ for _ in range(self.max_speculative_tokens)
103
+ ])
104
+
105
+ self.proj = nn.ModuleList([
106
+ nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
107
+ self.inner_dim,
108
+ bias=False)
109
+ for i in range(self.max_speculative_tokens)
110
+ ])
111
+
112
+ self.head = nn.ModuleList([
113
+ nn.Linear(self.inner_dim, self.vocab_size, bias=False)
114
+ for _ in range(self.max_speculative_tokens)
115
+ ])
116
+ self.ln = nn.ModuleList([
117
+ MLPSpeculatorLayerNorm(self.inner_dim,
118
+ elementwise_scale_and_shift=True)
119
+ for _ in range(self.max_speculative_tokens)
120
+ ])
121
+ if self.scale_input:
122
+ self.ln0 = MLPSpeculatorLayerNorm(
123
+ self.emb_dim, elementwise_scale_and_shift=False)
124
125
self.state_weight = 0.5**(0.5 / config.n_predict)
126
self.emb_weight = math.sqrt(
@@ -105,6 +146,9 @@ class MLPSpeculator(nn.Module):
146
# b x 1 x d
147
previous_hidden_states = previous_hidden_states.unsqueeze(1)
148
149
+ if self.scale_input:
150
+ previous_hidden_states = self.ln0(previous_hidden_states) / SQRT2
151
+
152
# b x 1
153
last_tokens = input_ids.unsqueeze(1)
154
vllm/transformers_utils/configs/mlp_speculator.py
+12
@@ -17,6 +17,8 @@ class MLPSpeculatorConfig(PretrainedConfig):
17
n_predict: int = 3,
18
top_k_tokens_per_head: Optional[List[int]] = None,
19
n_candidates: int = 5,
20
+ tie_weights: bool = False,
21
+ scale_input: bool = False,
22
**kwargs):
23
"""
24
Initialize an MLPSpeculatorConfig
@@ -38,6 +40,14 @@ class MLPSpeculatorConfig(PretrainedConfig):
40
NOTE: This parameter is currently unused.
41
n_candidates: int
42
number of child candidates to create per sequence
43
+ tie_weights: bool
44
+ If true, use a single set of weights for every model
45
+ head/stage after the first. The initial projection
46
+ from the base model may have a different size, so that
47
+ stays separate.
48
+ scale_input: bool
49
+ if True, will scale the initial hidden states from
50
+ the base model.
51
"""
52
if top_k_tokens_per_head is None:
53
top_k_tokens_per_head = [5, 4, 3]
@@ -49,5 +59,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
59
self.top_k_tokens_per_head = top_k_tokens_per_head
60
self.n_candidates = n_candidates
61
self.num_lookahead_tokens = n_predict
62
+ self.tie_weights = tie_weights
63
+ self.scale_input = scale_input
64
65
super().__init__(**kwargs)