A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Core] Free CPU pinned memory on environment cleanup (#10477)
Jani Monoses committed
Jan 21, 2025 at 21:56 UTC
9c485d9e252a8834ed15656838d5fbe0dc3a8f2f
1 file changed
+5
vllm/distributed/parallel_state.py
+5
@@ -1183,6 +1183,11 @@ def cleanup_dist_env_and_memory(shutdown_ray: bool = False):
1183
from vllm.platforms import current_platform
1184
if not current_platform.is_cpu():
1185
torch.cuda.empty_cache()
1186
+ try:
1187
+ torch._C._host_emptyCache()
1188
+ except AttributeError:
1189
+ logger.warning(
1190
+ "torch._C._host_emptyCache() only available in Pytorch >=2.5")
1191
1192
1193
def in_the_same_node_as(pg: Union[ProcessGroup, StatelessProcessGroup],