A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[BugFix] Fix `MLPSpeculator` handling of `num_speculative_tokens` (#5876)
Nick Hill committed
Jun 27, 2024 at 10:59 UTC
691e29ecf356d2646f74c622d957ec43dbf95c3a
3 files changed
+18
-10
vllm/config.py
+7
-3
@@ -920,15 +920,19 @@ class SpeculativeConfig:
920
max_logprobs=target_model_config.max_logprobs,
921
)
922
923
- if (draft_model_config.hf_config.model_type == "mlp_speculator"
923
+ draft_hf_config = draft_model_config.hf_config
924
+ if (draft_hf_config.model_type == "mlp_speculator"
925
and target_parallel_config.world_size != 1):
926
# MLPSpeculator TP support will be added very soon
927
raise ValueError(
928
"Speculative decoding with mlp_speculator models does not "
929
"yet support distributed inferencing (TP > 1).")
930
930
- n_predict = getattr(draft_model_config.hf_config, "n_predict",
931
- None)
931
+ if (num_speculative_tokens is not None
932
+ and hasattr(draft_hf_config, "num_lookahead_tokens")):
933
+ draft_hf_config.num_lookahead_tokens = num_speculative_tokens
934
+
935
+ n_predict = getattr(draft_hf_config, "n_predict", None)
936
if n_predict is not None:
937
if num_speculative_tokens is None:
938
# Default to max value defined in draft model config.
vllm/model_executor/models/mlp_speculator.py
+8
-7
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
11
VocabParallelEmbedding)
12
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13
from vllm.sequence import SamplerOutput
14
+from vllm.transformers_utils.configs import MLPSpeculatorConfig
15
16
17
class MLPSpeculatorLayerNorm(nn.Module):
@@ -48,7 +49,7 @@ class MLPSpeculatorLayerNorm(nn.Module):
49
50
class MLPSpeculator(nn.Module):
51
51
- def __init__(self, config, **kwargs) -> None:
52
+ def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None:
53
super().__init__()
54
self.n_predict = config.n_predict
55
self.vocab_size = config.vocab_size
@@ -56,8 +57,7 @@ class MLPSpeculator(nn.Module):
57
self.inner_dim = config.inner_dim if config.inner_dim != 0 \
58
else config.emb_dim
59
59
- self.max_speculative_tokens = getattr(config, "max_speculative_tokens",
60
- self.n_predict)
60
+ self.max_speculative_tokens = config.num_lookahead_tokens
61
62
self.emb = nn.ModuleList([
63
VocabParallelEmbedding(config.vocab_size,
@@ -137,7 +137,8 @@ class MLPSpeculator(nn.Module):
137
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
138
params_dict = dict(self.named_parameters())
139
for name, loaded_weight in weights:
140
- param = params_dict[name.replace("speculator.", "")]
141
- weight_loader = getattr(param, "weight_loader",
142
- default_weight_loader)
143
- weight_loader(param, loaded_weight)
140
+ param = params_dict.get(name.replace("speculator.", ""))
141
+ if param is not None:
142
+ weight_loader = getattr(param, "weight_loader",
143
+ default_weight_loader)
144
+ weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/mlp_speculator.py
+3
@@ -35,6 +35,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
35
candidate tree.
36
For each candidate branch in the tree, head n produces topk[n]
37
additional sub-branches.
38
+ NOTE: This parameter is currently unused.
39
n_candidates: int
40
number of child candidates to create per sequence
41
"""
@@ -47,4 +48,6 @@ class MLPSpeculatorConfig(PretrainedConfig):
48
self.n_predict = n_predict
49
self.top_k_tokens_per_head = top_k_tokens_per_head
50
self.n_candidates = n_candidates
51
+ self.num_lookahead_tokens = n_predict
52
+
53
super().__init__(**kwargs)