A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Fix peak memory profiling (#2031)
Woosuk Kwon committed
Dec 12, 2023 at 22:01 UTC
30bad5c49278ec5c3836a7bf00faa1316e8827b8
2 files changed
+3
-9
vllm/utils.py
-5
@@ -40,11 +40,6 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int:
40
return int(max_shared_mem)
41
42
43
-def get_gpu_memory(gpu: int = 0) -> int:
44
- """Returns the total memory of the GPU in bytes."""
45
- return torch.cuda.get_device_properties(gpu).total_memory
46
-
47
-
43
def get_cpu_memory() -> int:
44
"""Returns the total CPU memory of the node in bytes."""
45
return psutil.virtual_memory().total
vllm/worker/worker.py
+3
-4
@@ -13,7 +13,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
13
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14
from vllm.worker.cache_engine import CacheEngine
15
from vllm.worker.model_runner import ModelRunner
16
-from vllm.utils import get_gpu_memory
16
17
18
class Worker:
@@ -81,7 +80,6 @@ class Worker:
80
# Profile the memory usage of the model and get the maximum number of
81
# cache blocks that can be allocated with the remaining free memory.
82
torch.cuda.empty_cache()
84
- torch.cuda.reset_peak_memory_stats()
83
84
# Execute a forward pass with dummy inputs to profile the memory usage
85
# of the model.
@@ -90,8 +88,9 @@ class Worker:
88
# Calculate the number of blocks that can be allocated with the
89
# profiled peak memory.
90
torch.cuda.synchronize()
93
- peak_memory = torch.cuda.max_memory_allocated()
94
- total_gpu_memory = get_gpu_memory()
91
+ free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
92
+ peak_memory = total_gpu_memory - free_gpu_memory
93
+
94
cache_block_size = CacheEngine.get_cache_block_size(
95
block_size, self.model_config, self.parallel_config)
96
num_gpu_blocks = int(