A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[V1][Spec Decode] Use better defaults for N-gram (#15358)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Mar 23, 2025 at 10:52 UTC
bc8ed3c4bad05f5b1d37f34d5a8d324c23e84785
1 file changed
+22
-10
vllm/config.py
+22
-10
@@ -2012,18 +2012,30 @@ class SpeculativeConfig:
2012
if self.method in ("ngram", "[ngram]"):
2013
# Unified to "ngram" internally
2014
self.method = "ngram"
2015
- if self.prompt_lookup_min is None:
2016
- self.prompt_lookup_min = 1
2017
- if self.prompt_lookup_max is None or self.prompt_lookup_max < 1:
2018
- raise ValueError("prompt_lookup_max="
2019
- f"{self.prompt_lookup_max} must be > 0")
2015
+ # Set default values if not provided
2016
+ if (self.prompt_lookup_min is None
2017
+ and self.prompt_lookup_max is None):
2018
+ # TODO(woosuk): Tune these values. They are arbitrarily chosen.
2019
+ self.prompt_lookup_min = 5
2020
+ self.prompt_lookup_max = 5
2021
+ elif self.prompt_lookup_min is None:
2022
+ assert self.prompt_lookup_max is not None
2023
+ self.prompt_lookup_min = self.prompt_lookup_max
2024
+ elif self.prompt_lookup_max is None:
2025
+ assert self.prompt_lookup_min is not None
2026
+ self.prompt_lookup_max = self.prompt_lookup_min
2027
+
2028
+ # Validate values
2029
if self.prompt_lookup_min < 1:
2021
- raise ValueError("prompt_lookup_min="
2022
- f"{self.prompt_lookup_min} must be > 0")
2030
+ raise ValueError(
2031
+ f"prompt_lookup_min={self.prompt_lookup_min} must be > 0")
2032
+ if self.prompt_lookup_max < 1:
2033
+ raise ValueError(
2034
+ f"prompt_lookup_max={self.prompt_lookup_max} must be > 0")
2035
if self.prompt_lookup_min > self.prompt_lookup_max:
2024
- raise ValueError(f"prompt_lookup_min={self.prompt_lookup_min} "
2025
- "cannot be larger than prompt_lookup_max="
2026
- f"{self.prompt_lookup_max}")
2036
+ raise ValueError(
2037
+ f"prompt_lookup_min={self.prompt_lookup_min} must "
2038
+ f"be <= prompt_lookup_max={self.prompt_lookup_max}")
2039
2040
# TODO: current we still need extract vocab_size from target model
2041
# config, in future, we may try refactor it out, and set