A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[ROCm] Using a more precise memory profiling (#12624)
Signed-off-by: Gregory Shtrasberg <Gregory.Shtrasberg@amd.com>
Gregory Shtrasberg committed
Feb 11, 2025 at 08:47 UTC
7539bbc6a6715dc8e5e71730e2377219b0e69e21
1 file changed
+2
-1
vllm/platforms/rocm.py
+2
-1
@@ -169,4 +169,5 @@ class RocmPlatform(Platform):
169
device: Optional[torch.types.Device] = None
170
) -> float:
171
torch.cuda.reset_peak_memory_stats(device)
172
- return torch.cuda.max_memory_allocated(device)
172
+ return torch.cuda.mem_get_info(device)[1] - torch.cuda.mem_get_info(
173
+ device)[0]