A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[Bugfix] Better error message for MLPSpeculator when `num_speculative_tokens` is set too high (#5894)
Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>
Thomas Parnell committed
Jun 28, 2024 at 16:42 UTC
ec1ad0046c6cf356443ff05a63f88a8d650844df
1 file changed
+3
-3
vllm/config.py
+3
-3
@@ -956,9 +956,9 @@ class SpeculativeConfig:
956
# Verify provided value doesn't exceed the maximum
957
# supported by the draft model.
958
raise ValueError(
959
- "Expected both speculative_model and "
960
- "num_speculative_tokens to be provided, but found "
961
- f"{speculative_model=} and {num_speculative_tokens=}.")
959
+ "This speculative model supports a maximum of "
960
+ f"num_speculative_tokens={n_predict}, but "
961
+ f"{num_speculative_tokens=} was provided.")
962
963
draft_model_config.max_model_len = (
964
SpeculativeConfig._maybe_override_draft_max_model_len(