@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54600709b6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Model] Changes to MLPSpeculator to support tie_weights and input_scale (#5965)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com> Co-authored-by: Joshua Rosenkranz <jmrosenk@us.ibm.com>

Thomas Parnell committed Jul 2, 2024 at 01:40 UTC 54600709b6d419fb243ce718a48ab7d40f5c3eb7
2 files changed +81 -25
vllm/model_executor/models/mlp_speculator.py
+69 -25
@@ -13,6 +13,8 @@ from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 from vllm.sequence import SamplerOutput
14 from vllm.transformers_utils.configs import MLPSpeculatorConfig
15
16 +SQRT2 = 2**0.5
17 +
18
19 class MLPSpeculatorLayerNorm(nn.Module):
20 """
@@ -26,24 +28,30 @@ class MLPSpeculatorLayerNorm(nn.Module):
28 Safety term to prevent division by zero. Make sure the chosen value
29 fits in the range of your encoding scheme
30 (i.e. fp16 requires eps >= 6e-8).
31 + elementwise_scale_and_shift : bool
32 + Include a learned scaling and shift term after normalization.
33 """
34
35 def __init__(
36 self,
37 normalized_shape,
38 eps=1e-06,
39 + elementwise_scale_and_shift=True,
40 ):
41 super(MLPSpeculatorLayerNorm, self).__init__()
37 - self.weight = nn.Parameter(torch.empty(normalized_shape))
38 - self.bias = nn.Parameter(torch.empty(normalized_shape))
42 + self.elementwise_scale_and_shift = elementwise_scale_and_shift
43 + if self.elementwise_scale_and_shift:
44 + self.weight = nn.Parameter(torch.empty(normalized_shape))
45 + self.bias = nn.Parameter(torch.empty(normalized_shape))
46 self.eps = eps
47
48 def forward(self, x):
49 xf = x
50 xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
51 x = xf.type_as(x)
45 - x = self.weight * x
46 - x = x + self.bias
52 + if self.elementwise_scale_and_shift:
53 + x = self.weight * x
54 + x = x + self.bias
55 return x
56
57
@@ -59,27 +67,60 @@ class MLPSpeculator(nn.Module):
67
68 self.max_speculative_tokens = config.num_lookahead_tokens
69
62 - self.emb = nn.ModuleList([
63 - VocabParallelEmbedding(config.vocab_size,
64 - self.inner_dim,
65 - org_num_embeddings=config.vocab_size)
66 - for _ in range(self.max_speculative_tokens)
67 - ])
68 -
69 - self.proj = nn.ModuleList([
70 - nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
71 - self.inner_dim,
72 - bias=False) for i in range(self.max_speculative_tokens)
73 - ])
74 -
75 - self.head = nn.ModuleList([
76 - nn.Linear(self.inner_dim, self.vocab_size, bias=False)
77 - for _ in range(self.max_speculative_tokens)
78 - ])
79 - self.ln = nn.ModuleList([
80 - MLPSpeculatorLayerNorm(self.inner_dim)
81 - for _ in range(self.max_speculative_tokens)
82 - ])
70 + self.tie_weights = config.tie_weights
71 + self.scale_input = config.scale_input
72 +
73 + if self.tie_weights:
74 + assert (
75 + self.n_predict >
76 + 1), "You cannot tie weights between stages when only 1 exists"
77 + embedding = VocabParallelEmbedding(
78 + config.vocab_size,
79 + self.inner_dim,
80 + org_num_embeddings=config.vocab_size)
81 + self.emb = nn.ModuleList([embedding] * self.max_speculative_tokens)
82 +
83 + # the initial projection from the base model may
84 + # have a different size, so that stays separate.
85 + proj_first = nn.Linear(self.emb_dim, self.inner_dim, bias=False)
86 + proj_tied = nn.Linear(self.inner_dim, self.inner_dim, bias=False)
87 + self.proj = nn.ModuleList([proj_first] + [proj_tied] *
88 + (self.max_speculative_tokens - 1))
89 +
90 + head = nn.Linear(self.inner_dim, self.vocab_size, bias=False)
91 + self.head = nn.ModuleList([head] * self.max_speculative_tokens)
92 +
93 + ln = MLPSpeculatorLayerNorm(self.inner_dim,
94 + elementwise_scale_and_shift=True)
95 + self.ln = nn.ModuleList([ln] * self.max_speculative_tokens)
96 +
97 + else:
98 + self.emb = nn.ModuleList([
99 + VocabParallelEmbedding(config.vocab_size,
100 + self.inner_dim,
101 + org_num_embeddings=config.vocab_size)
102 + for _ in range(self.max_speculative_tokens)
103 + ])
104 +
105 + self.proj = nn.ModuleList([
106 + nn.Linear((self.emb_dim if i == 0 else self.inner_dim),
107 + self.inner_dim,
108 + bias=False)
109 + for i in range(self.max_speculative_tokens)
110 + ])
111 +
112 + self.head = nn.ModuleList([
113 + nn.Linear(self.inner_dim, self.vocab_size, bias=False)
114 + for _ in range(self.max_speculative_tokens)
115 + ])
116 + self.ln = nn.ModuleList([
117 + MLPSpeculatorLayerNorm(self.inner_dim,
118 + elementwise_scale_and_shift=True)
119 + for _ in range(self.max_speculative_tokens)
120 + ])
121 + if self.scale_input:
122 + self.ln0 = MLPSpeculatorLayerNorm(
123 + self.emb_dim, elementwise_scale_and_shift=False)
124
125 self.state_weight = 0.5**(0.5 / config.n_predict)
126 self.emb_weight = math.sqrt(
@@ -105,6 +146,9 @@ class MLPSpeculator(nn.Module):
146 # b x 1 x d
147 previous_hidden_states = previous_hidden_states.unsqueeze(1)
148
149 + if self.scale_input:
150 + previous_hidden_states = self.ln0(previous_hidden_states) / SQRT2
151 +
152 # b x 1
153 last_tokens = input_ids.unsqueeze(1)
154
vllm/transformers_utils/configs/mlp_speculator.py
+12
@@ -17,6 +17,8 @@ class MLPSpeculatorConfig(PretrainedConfig):
17 n_predict: int = 3,
18 top_k_tokens_per_head: Optional[List[int]] = None,
19 n_candidates: int = 5,
20 + tie_weights: bool = False,
21 + scale_input: bool = False,
22 **kwargs):
23 """
24 Initialize an MLPSpeculatorConfig
@@ -38,6 +40,14 @@ class MLPSpeculatorConfig(PretrainedConfig):
40 NOTE: This parameter is currently unused.
41 n_candidates: int
42 number of child candidates to create per sequence
43 + tie_weights: bool
44 + If true, use a single set of weights for every model
45 + head/stage after the first. The initial projection
46 + from the base model may have a different size, so that
47 + stays separate.
48 + scale_input: bool
49 + if True, will scale the initial hidden states from
50 + the base model.
51 """
52 if top_k_tokens_per_head is None:
53 top_k_tokens_per_head = [5, 4, 3]
@@ -49,5 +59,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
59 self.top_k_tokens_per_head = top_k_tokens_per_head
60 self.n_candidates = n_candidates
61 self.num_lookahead_tokens = n_predict
62 + self.tie_weights = tie_weights
63 + self.scale_input = scale_input
64
65 super().__init__(**kwargs)