@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ec1ad0046c
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[Bugfix] Better error message for MLPSpeculator when `num_speculative_tokens` is set too high (#5894)

Signed-off-by: Thomas Parnell <tpa@zurich.ibm.com>

Thomas Parnell committed Jun 28, 2024 at 16:42 UTC ec1ad0046c6cf356443ff05a63f88a8d650844df
1 file changed +3 -3
vllm/config.py
+3 -3
@@ -956,9 +956,9 @@ class SpeculativeConfig:
956 # Verify provided value doesn't exceed the maximum
957 # supported by the draft model.
958 raise ValueError(
959 - "Expected both speculative_model and "
960 - "num_speculative_tokens to be provided, but found "
961 - f"{speculative_model=} and {num_speculative_tokens=}.")
959 + "This speculative model supports a maximum of "
960 + f"num_speculative_tokens={n_predict}, but "
961 + f"{num_speculative_tokens=} was provided.")
962
963 draft_model_config.max_model_len = (
964 SpeculativeConfig._maybe_override_draft_max_model_len(