A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
[Fix] Fix memory profiling when GPU is used by multiple processes (#2863)
Woosuk Kwon committed
Feb 13, 2024 at 19:52 UTC
7e45107f51bcb38c22dd9916c61226078e8eb26d
1 file changed
+5
-1
vllm/worker/worker.py
+5
-1
@@ -84,6 +84,8 @@ class Worker:
84
torch.cuda.set_device(self.device)
85
86
_check_if_gpu_supports_dtype(self.model_config.dtype)
87
+ torch.cuda.empty_cache()
88
+ self.init_gpu_memory = torch.cuda.mem_get_info()[0]
89
else:
90
raise RuntimeError(
91
f"Not support device type: {self.device_config.device}")
@@ -126,7 +128,9 @@ class Worker:
128
# profiled peak memory.
129
torch.cuda.synchronize()
130
free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
129
- peak_memory = total_gpu_memory - free_gpu_memory
131
+ # NOTE(woosuk): Here we assume that the other processes using the same
132
+ # GPU did not change their memory usage during the profiling.
133
+ peak_memory = self.init_gpu_memory - free_gpu_memory
134
135
cache_block_size = CacheEngine.get_cache_block_size(
136
block_size, cache_dtype, self.model_config, self.parallel_config)