A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] use StoreBoolean instead of type=bool for --disable-logprobs-during-spec-decoding (#7665)
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
Travis Johnson committed
Aug 19, 2024 at 18:43 UTC
67e02fa8a405e1e1df0eb7428ad45eed20b0934b
1 file changed
+3
-1
vllm/engine/arg_utils.py
+3
-1
@@ -664,8 +664,10 @@ class EngineArgs:
664
665
parser.add_argument(
666
'--disable-logprobs-during-spec-decoding',
667
- type=bool,
667
+ action=StoreBoolean,
668
default=EngineArgs.disable_logprobs_during_spec_decoding,
669
+ nargs="?",
670
+ const="True",
671
help='If set to True, token log probabilities are not returned '
672
'during speculative decoding. If set to False, log probabilities '
673
'are returned according to the settings in SamplingParams. If '