A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 03:30 UTC
[Misc] Fix spec decode example (#20296)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
Woosuk Kwon committed
Jun 30, 2025 at 21:01 UTC
7151f92241db1bb6ef4eb0fcfed87256646d554e
1 file changed
-2
examples/offline_inference/spec_decode.py
-2
@@ -79,9 +79,7 @@ def main():
79
trust_remote_code=True,
80
tensor_parallel_size=args.tp,
81
enable_chunked_prefill=args.enable_chunked_prefill,
82
- max_num_batched_tokens=args.max_num_batched_tokens,
82
enforce_eager=args.enforce_eager,
84
- max_num_seqs=args.max_num_seqs,
83
gpu_memory_utilization=0.8,
84
speculative_config=speculative_config,
85
disable_log_stats=False,