A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
Attempt to fix GPU OOM in a spec-decoding test (#29419)
Signed-off-by: Eldar Kurtic <8884008+eldarkurtic@users.noreply.github.com>
Eldar Kurtić committed
Nov 25, 2025 at 20:23 UTC
c32a18cbe7342ac0700802b94ae98bbf928a00f0
1 file changed
+1
-1
examples/offline_inference/spec_decode.py
+1
-1
@@ -133,7 +133,7 @@ def main(args):
133
tensor_parallel_size=args.tp,
134
enable_chunked_prefill=args.enable_chunked_prefill,
135
enforce_eager=args.enforce_eager,
136
- gpu_memory_utilization=0.8,
136
+ gpu_memory_utilization=0.9,
137
speculative_config=speculative_config,
138
disable_log_stats=False,
139
max_model_len=args.max_model_len,