A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Bugfix] Parse SpeculativeConfig Error (#25142)
Signed-off-by: zxw <1020938856@qq.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
yyzxw committed
Sep 25, 2025 at 19:09 UTC
eaeca3cd7ff7dc5e7d28361020706a2ab7c58233
1 file changed
+12
-6
vllm/engine/arg_utils.py
+12
-6
@@ -1465,12 +1465,18 @@ class EngineArgs:
1465
return False
1466
1467
# V1 supports N-gram, Medusa, and Eagle speculative decoding.
1468
- if (self.speculative_config is not None
1469
- and self.speculative_config.get("method") == "draft_model"):
1470
- raise NotImplementedError(
1471
- "Speculative decoding with draft model is not supported yet. "
1472
- "Please consider using other speculative decoding methods "
1473
- "such as ngram, medusa, eagle, or deepseek_mtp.")
1468
+ if self.speculative_config is not None:
1469
+ # speculative_config could still be a dict at this point
1470
+ if isinstance(self.speculative_config, dict):
1471
+ method = self.speculative_config.get("method", None)
1472
+ else:
1473
+ method = self.speculative_config.method
1474
+
1475
+ if method == "draft_model":
1476
+ raise NotImplementedError(
1477
+ "Draft model speculative decoding is not supported yet. "
1478
+ "Please consider using other speculative decoding methods "
1479
+ "such as ngram, medusa, eagle, or deepseek_mtp.")
1480
1481
V1_BACKENDS = [
1482
"FLASH_ATTN_VLLM_V1",