@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c485d9e25
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Core] Free CPU pinned memory on environment cleanup (#10477)

Jani Monoses committed Jan 21, 2025 at 21:56 UTC 9c485d9e252a8834ed15656838d5fbe0dc3a8f2f
1 file changed +5
vllm/distributed/parallel_state.py
+5
@@ -1183,6 +1183,11 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1183 from vllm.platforms import current_platform
1184 if not current_platform.is_cpu():
1185 torch.cuda.empty_cache()
1186 + try:
1187 + torch._C._host_emptyCache()
1188 + except AttributeError:
1189 + logger.warning(
1190 + "torch._C._host_emptyCache() only available in Pytorch >=2.5")
1191
1192
1193 def in_the_same_node_as(pg: Union[ProcessGroup, StatelessProcessGroup],