@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73b3de79ea
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

explicitly del state (#784)

Xinyu Yang committed Aug 18, 2023 at 03:56 UTC 73b3de79eae5dec7459d54888083e5068890799b
1 file changed +2
vllm/model_executor/weight_utils.py
+2
@@ -76,6 +76,8 @@ def hf_model_weights_iterator(
76 state = torch.load(bin_file, map_location="cpu")
77 for name, param in state.items():
78 yield name, param
79 + del state
80 + torch.cuda.empty_cache()
81
82
83 def load_tensor_parallel_weights(