A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Perf] Freeze core engine proc heap after init (#24008)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Sep 4, 2025 at 07:55 UTC
e41a0fa3772c5b9bbca439ce5ed7d05803febbc1
1 file changed
+6
vllm/v1/engine/core.py
+6
@@ -1,5 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+import gc
4
import os
5
import queue
6
import signal
@@ -536,6 +537,11 @@ class EngineCoreProc(EngineCore):
537
self.step_fn = (self.step if self.batch_queue is None else
538
self.step_with_batch_queue)
539
540
+ # Mark the startup heap as static so that it's ignored by GC.
541
+ # Reduces pause times of oldest generation collections.
542
+ gc.collect()
543
+ gc.freeze()
544
+
545
@contextmanager
546
def _perform_handshakes(
547
self,