A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 22:30 UTC
[Core] Freeze garbage collector in workers after model initialization (#44363)
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Tyler Michael Smith committed
Jun 4, 2026 at 11:39 UTC
4cc78c9d5d5bb14297440d24c65423ef08e70f2a
1 file changed
+8
vllm/v1/worker/gpu_worker.py
+8
@@ -51,6 +51,7 @@ from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
51
from vllm.sequence import IntermediateTensors
52
from vllm.tasks import SupportedTask
53
from vllm.tracing import instrument
54
+from vllm.utils.gc_utils import freeze_gc_heap, maybe_attach_gc_debug_callback
55
from vllm.utils.mem_constants import GiB_bytes
56
from vllm.utils.mem_utils import MemorySnapshot, format_gib, memory_profiling
57
from vllm.utils.torch_utils import set_random_seed
@@ -722,6 +723,11 @@ class Worker(WorkerBase):
723
724
activate_triton_jit_monitor()
725
726
+ # Freeze the worker heap so the GC won't scan static objects
727
+ # (model weights, KV caches, CUDA graphs) during inference.
728
+ freeze_gc_heap()
729
+ maybe_attach_gc_debug_callback()
730
+
731
return CompilationTimes(
732
language_model=self.compilation_config.compilation_time,
733
encoder=self.compilation_config.encoder_compilation_time,
@@ -1115,6 +1121,8 @@ class Worker(WorkerBase):
1121
self._is_checkpoint_format = True
1122
1123
def shutdown(self) -> None:
1124
+ gc.unfreeze()
1125
+
1126
# has_kv_transfer_group can be None during interpreter shutdown.
1127
if ensure_kv_transfer_shutdown is not None:
1128
ensure_kv_transfer_shutdown()