@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7e45107f51
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

[Fix] Fix memory profiling when GPU is used by multiple processes (#2863)

Woosuk Kwon committed Feb 13, 2024 at 19:52 UTC 7e45107f51bcb38c22dd9916c61226078e8eb26d
1 file changed +5 -1
vllm/worker/worker.py
+5 -1
@@ -84,6 +84,8 @@ class Worker:
84 torch.cuda.set_device(self.device)
85
86 _check_if_gpu_supports_dtype(self.model_config.dtype)
87 + torch.cuda.empty_cache()
88 + self.init_gpu_memory = torch.cuda.mem_get_info()[0]
89 else:
90 raise RuntimeError(
91 f"Not support device type: {self.device_config.device}")
@@ -126,7 +128,9 @@ class Worker:
128 # profiled peak memory.
129 torch.cuda.synchronize()
130 free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
129 - peak_memory = total_gpu_memory - free_gpu_memory
131 + # NOTE(woosuk): Here we assume that the other processes using the same
132 + # GPU did not change their memory usage during the profiling.
133 + peak_memory = self.init_gpu_memory - free_gpu_memory
134
135 cache_block_size = CacheEngine.get_cache_block_size(
136 block_size, cache_dtype, self.model_config, self.parallel_config)