A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
Fix passing `SpeculativeConfig` from the CLI (#22652)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Harry Mellor committed
Aug 12, 2025 at 06:13 UTC
4fbd8bb597cf392b94def04a6955f22580356d76
1 file changed
+4
vllm/engine/arg_utils.py
+4
@@ -826,6 +826,10 @@ class EngineArgs:
826
title="VllmConfig",
827
description=VllmConfig.__doc__,
828
)
829
+ # We construct SpeculativeConfig using fields from other configs in
830
+ # create_engine_config. So we set the type to a JSON string here to
831
+ # delay the Pydantic validation that comes with SpeculativeConfig.
832
+ vllm_kwargs["speculative_config"]["type"] = optional_type(json.loads)
833
vllm_group.add_argument("--speculative-config",
834
**vllm_kwargs["speculative_config"])
835
vllm_group.add_argument("--kv-transfer-config",