A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 13, 13:15 UTC
[Perf] Move gc.freeze logic from EngineCoreProc to EngineCore for better coverage (#27896)
Signed-off-by: Jialin Ouyang <Jialin.Ouyang@gmail.com>
Jialin Ouyang committed
Nov 10, 2025 at 15:34 UTC
b30372cbd045aeac50833cd6fe6084d2edd5252c
5 files changed
+30
-14
vllm/benchmarks/serve.py
+2
-3
@@ -19,7 +19,6 @@ On the client side, run:
19
import argparse
20
import asyncio
21
import contextlib
22
-import gc
22
import importlib.util
23
import json
24
import os
@@ -49,6 +48,7 @@ from vllm.benchmarks.lib.endpoint_request_func import (
48
from vllm.benchmarks.lib.ready_checker import wait_for_endpoint
49
from vllm.benchmarks.lib.utils import convert_to_pytorch_benchmark_format, write_to_json
50
from vllm.transformers_utils.tokenizer import get_tokenizer
51
+from vllm.utils.gc_utils import freeze_gc_heap
52
53
MILLISECONDS_TO_SECONDS_CONVERSION = 1000
54
@@ -1414,8 +1414,7 @@ async def main_async(args: argparse.Namespace) -> dict[str, Any]:
1414
percentile_metrics: str = args.percentile_metrics or default_percentile_metrics
1415
1416
# Avoid GC processing "static" data - reduce pause times.
1417
- gc.collect()
1418
- gc.freeze()
1417
+ freeze_gc_heap()
1418
1419
benchmark_result = await benchmark(
1420
task_type=task_type,
vllm/distributed/parallel_state.py
+3
@@ -1483,6 +1483,9 @@ def destroy_distributed_environment():
1483
1484
1485
def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1486
+ # Ensure all objects are not freezed before cleanup
1487
+ gc.unfreeze()
1488
+
1489
destroy_model_parallel()
1490
destroy_distributed_environment()
1491
if shutdown_ray:
vllm/entrypoints/openai/api_server.py
+2
-4
@@ -1,8 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
3
import asyncio
5
-import gc
4
import hashlib
5
import importlib
6
import inspect
@@ -118,6 +116,7 @@ from vllm.reasoning import ReasoningParserManager
116
from vllm.tasks import POOLING_TASKS
117
from vllm.usage.usage_lib import UsageContext
118
from vllm.utils.argparse_utils import FlexibleArgumentParser
119
+from vllm.utils.gc_utils import freeze_gc_heap
120
from vllm.utils.network_utils import is_valid_ipv6_address
121
from vllm.utils.system_utils import decorate_logs, set_ulimit
122
from vllm.v1.engine.exceptions import EngineDeadError
@@ -153,8 +152,7 @@ async def lifespan(app: FastAPI):
152
153
# Mark the startup heap as static so that it's ignored by GC.
154
# Reduces pause times of oldest generation collections.
156
- gc.collect()
157
- gc.freeze()
155
+ freeze_gc_heap()
156
try:
157
yield
158
finally:
vllm/utils/gc_utils.py
+15
@@ -89,6 +89,21 @@ class GCDebugger:
89
)
90
91
92
+def freeze_gc_heap() -> None:
93
+ """
94
+ Freeze all objects tracked by the garbage collector. It should be invoked
95
+ after server init / warmup, to reduce GC overhead from static objects
96
+ during serving time.
97
+ """
98
+ # Ensure all static objects are pushed down to the oldest generation for
99
+ # freeze
100
+ gc.collect(0)
101
+ gc.collect(1)
102
+ gc.collect(2)
103
+ # Freeze all GC tracked objects
104
+ gc.freeze()
105
+
106
+
107
def maybe_attach_gc_debug_callback() -> None:
108
"""
109
Attached a callback for GC debug when VLLM_GC_DEBUG is enabled.
vllm/v1/engine/core.py
+8
-7
@@ -1,6 +1,5 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-import gc
3
import os
4
import queue
5
import signal
@@ -27,7 +26,10 @@ from vllm.multimodal import MULTIMODAL_REGISTRY
26
from vllm.multimodal.cache import engine_receiver_cache_from_config
27
from vllm.tasks import POOLING_TASKS, SupportedTask
28
from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
30
-from vllm.utils.gc_utils import maybe_attach_gc_debug_callback
29
+from vllm.utils.gc_utils import (
30
+ freeze_gc_heap,
31
+ maybe_attach_gc_debug_callback,
32
+)
33
from vllm.utils.hashing import get_hash_fn_by_name
34
from vllm.utils.network_utils import make_zmq_socket
35
from vllm.utils.system_utils import decorate_logs, set_process_title
@@ -197,6 +199,10 @@ class EngineCore:
199
self.step if self.batch_queue is None else self.step_with_batch_queue
200
)
201
202
+ # Mark the startup heap as static so that it's ignored by GC.
203
+ # Reduces pause times of oldest generation collections.
204
+ freeze_gc_heap()
205
+
206
def _initialize_kv_caches(
207
self, vllm_config: VllmConfig
208
) -> tuple[int, int, KVCacheConfig]:
@@ -651,11 +657,6 @@ class EngineCoreProc(EngineCore):
657
assert addresses.coordinator_input is not None
658
logger.info("Waiting for READY message from DP Coordinator...")
659
654
- # Mark the startup heap as static so that it's ignored by GC.
655
- # Reduces pause times of oldest generation collections.
656
- gc.collect()
657
- gc.freeze()
658
-
660
# If enable, attach GC debugger after static variable freeze.
661
maybe_attach_gc_debug_callback()
662