A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 16:00 UTC
[Misc] Fix speculative config repr string (#15860)
Signed-off-by: Shangming Cai <caishangming@linux.alibaba.com>
shangmingc committed
Apr 1, 2025 at 17:26 UTC
656fd72976a86f4bc1887be228dac0ab71c56c71
1 file changed
+3
-5
vllm/config.py
+3
-5
@@ -2359,12 +2359,10 @@ class SpeculativeConfig:
2359
return self.num_speculative_tokens
2360
2361
def __repr__(self) -> str:
2362
- if self.prompt_lookup_max is not None and self.prompt_lookup_max > 0:
2363
- draft_model = "ngram"
2364
- else:
2365
- draft_model = self.draft_model_config.model
2362
+ method = self.method
2363
+ model = None if method == "ngram" else self.draft_model_config.model
2364
num_spec_tokens = self.num_speculative_tokens
2367
- return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})"
2365
+ return f"SpeculativeConfig({method=}, {model=}, {num_spec_tokens=})"
2366
2367
2368
@dataclass