@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 30bad5c492
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Fix peak memory profiling (#2031)

Woosuk Kwon committed Dec 12, 2023 at 22:01 UTC 30bad5c49278ec5c3836a7bf00faa1316e8827b8
2 files changed +3 -9
vllm/utils.py
-5
@@ -40,11 +40,6 @@ def get_max_shared_memory_bytes(gpu: int = 0) -> int:
40 return int(max_shared_mem)
41
42
43 -def get_gpu_memory(gpu: int = 0) -> int:
44 - """Returns the total memory of the GPU in bytes."""
45 - return torch.cuda.get_device_properties(gpu).total_memory
46 -
47 -
43 def get_cpu_memory() -> int:
44 """Returns the total CPU memory of the node in bytes."""
45 return psutil.virtual_memory().total
vllm/worker/worker.py
+3 -4
@@ -13,7 +13,6 @@ from vllm.model_executor.parallel_utils.parallel_state import (
13 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
14 from vllm.worker.cache_engine import CacheEngine
15 from vllm.worker.model_runner import ModelRunner
16 -from vllm.utils import get_gpu_memory
16
17
18 class Worker:
@@ -81,7 +80,6 @@ class Worker:
80 # Profile the memory usage of the model and get the maximum number of
81 # cache blocks that can be allocated with the remaining free memory.
82 torch.cuda.empty_cache()
84 - torch.cuda.reset_peak_memory_stats()
83
84 # Execute a forward pass with dummy inputs to profile the memory usage
85 # of the model.
@@ -90,8 +88,9 @@ class Worker:
88 # Calculate the number of blocks that can be allocated with the
89 # profiled peak memory.
90 torch.cuda.synchronize()
93 - peak_memory = torch.cuda.max_memory_allocated()
94 - total_gpu_memory = get_gpu_memory()
91 + free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
92 + peak_memory = total_gpu_memory - free_gpu_memory
93 +
94 cache_block_size = CacheEngine.get_cache_block_size(
95 block_size, self.model_config, self.parallel_config)
96 num_gpu_blocks = int(