A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 14:45 UTC
[V1][Spec Dec Bug Fix] Respect Spec Dec Method Specification (#16636)
Signed-off-by: Bryan Lu <yuzhelu@amazon.com>
Bryan Lu committed
Apr 16, 2025 at 19:47 UTC
2cbd4d2999e6f2ef016aa1a2e2d280b81819ab5f
1 file changed
+3
-1
vllm/config.py
+3
-1
@@ -2341,7 +2341,9 @@ class SpeculativeConfig:
2341
)
2342
2343
# Automatically detect the method
2344
- if "eagle-" in self.draft_model_config.model.lower():
2344
+ if self.method == 'eagle':
2345
+ pass
2346
+ elif "eagle-" in self.draft_model_config.model.lower():
2347
self.method = "eagle"
2348
elif self.draft_model_config.hf_config.model_type == "medusa":
2349
self.method = "medusa"