@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 691e29ecf3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[BugFix] Fix `MLPSpeculator` handling of `num_speculative_tokens` (#5876)

Nick Hill committed Jun 27, 2024 at 10:59 UTC 691e29ecf356d2646f74c622d957ec43dbf95c3a
3 files changed +18 -10
vllm/config.py
+7 -3
@@ -920,15 +920,19 @@ class SpeculativeConfig:
920 max_logprobs=target_model_config.max_logprobs,
921 )
922
923 - if (draft_model_config.hf_config.model_type == "mlp_speculator"
923 + draft_hf_config = draft_model_config.hf_config
924 + if (draft_hf_config.model_type == "mlp_speculator"
925 and target_parallel_config.world_size != 1):
926 # MLPSpeculator TP support will be added very soon
927 raise ValueError(
928 "Speculative decoding with mlp_speculator models does not "
929 "yet support distributed inferencing (TP > 1).")
930
930 - n_predict = getattr(draft_model_config.hf_config, "n_predict",
931 - None)
931 + if (num_speculative_tokens is not None
932 + and hasattr(draft_hf_config, "num_lookahead_tokens")):
933 + draft_hf_config.num_lookahead_tokens = num_speculative_tokens
934 +
935 + n_predict = getattr(draft_hf_config, "n_predict", None)
936 if n_predict is not None:
937 if num_speculative_tokens is None:
938 # Default to max value defined in draft model config.
vllm/model_executor/models/mlp_speculator.py
+8 -7
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
11 VocabParallelEmbedding)
12 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
13 from vllm.sequence import SamplerOutput
14 +from vllm.transformers_utils.configs import MLPSpeculatorConfig
15
16
17 class MLPSpeculatorLayerNorm(nn.Module):
@@ -48,7 +49,7 @@ class MLPSpeculatorLayerNorm(nn.Module):
49
50 class MLPSpeculator(nn.Module):
51
51 - def __init__(self, config, **kwargs) -> None:
52 + def __init__(self, config: MLPSpeculatorConfig, **kwargs) -> None:
53 super().__init__()
54 self.n_predict = config.n_predict
55 self.vocab_size = config.vocab_size
@@ -56,8 +57,7 @@ class MLPSpeculator(nn.Module):
57 self.inner_dim = config.inner_dim if config.inner_dim != 0 \
58 else config.emb_dim
59
59 - self.max_speculative_tokens = getattr(config, "max_speculative_tokens",
60 - self.n_predict)
60 + self.max_speculative_tokens = config.num_lookahead_tokens
61
62 self.emb = nn.ModuleList([
63 VocabParallelEmbedding(config.vocab_size,
@@ -137,7 +137,8 @@ class MLPSpeculator(nn.Module):
137 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
138 params_dict = dict(self.named_parameters())
139 for name, loaded_weight in weights:
140 - param = params_dict[name.replace("speculator.", "")]
141 - weight_loader = getattr(param, "weight_loader",
142 - default_weight_loader)
143 - weight_loader(param, loaded_weight)
140 + param = params_dict.get(name.replace("speculator.", ""))
141 + if param is not None:
142 + weight_loader = getattr(param, "weight_loader",
143 + default_weight_loader)
144 + weight_loader(param, loaded_weight)
vllm/transformers_utils/configs/mlp_speculator.py
+3
@@ -35,6 +35,7 @@ class MLPSpeculatorConfig(PretrainedConfig):
35 candidate tree.
36 For each candidate branch in the tree, head n produces topk[n]
37 additional sub-branches.
38 + NOTE: This parameter is currently unused.
39 n_candidates: int
40 number of child candidates to create per sequence
41 """
@@ -47,4 +48,6 @@ class MLPSpeculatorConfig(PretrainedConfig):
48 self.n_predict = n_predict
49 self.top_k_tokens_per_head = top_k_tokens_per_head
50 self.n_candidates = n_candidates
51 + self.num_lookahead_tokens = n_predict
52 +
53 super().__init__(**kwargs)