A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
explicitly del state (#784)
Xinyu Yang committed
Aug 18, 2023 at 03:56 UTC
73b3de79eae5dec7459d54888083e5068890799b
1 file changed
+2
vllm/model_executor/weight_utils.py
+2
@@ -76,6 +76,8 @@ def hf_model_weights_iterator(
76
state = torch.load(bin_file, map_location="cpu")
77
for name, param in state.items():
78
yield name, param
79
+ del state
80
+ torch.cuda.empty_cache()
81
82
83
def load_tensor_parallel_weights(