@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9ddac56311
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Platform] move current_memory_usage() into platform (#11369)

Signed-off-by: Shanshan Shen <467638484@qq.com>

Shanshan Shen committed Jan 15, 2025 at 11:38 UTC 9ddac56311b28f08e40a941296eb66fbb1be0a7a
5 files changed +31 -7
vllm/platforms/cuda.py
+7
@@ -143,6 +143,13 @@ class CudaPlatformBase(Platform):
143 if cache_config and cache_config.block_size is None:
144 cache_config.block_size = 16
145
146 + @classmethod
147 + def get_current_memory_usage(cls,
148 + device: Optional[torch.types.Device] = None
149 + ) -> float:
150 + torch.cuda.reset_peak_memory_stats(device)
151 + return torch.cuda.max_memory_allocated(device)
152 +
153 @classmethod
154 def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
155 kv_cache_dtype, block_size, use_v1) -> str:
vllm/platforms/interface.py
+9
@@ -277,6 +277,15 @@ class Platform:
277 return False
278 return True
279
280 + @classmethod
281 + def get_current_memory_usage(cls,
282 + device: Optional[torch.types.Device] = None
283 + ) -> float:
284 + """
285 + Return the memory usage in bytes.
286 + """
287 + raise NotImplementedError
288 +
289 @classmethod
290 def get_punica_wrapper(cls) -> str:
291 """
vllm/platforms/rocm.py
+7
@@ -157,3 +157,10 @@ class RocmPlatform(Platform):
157 @classmethod
158 def get_punica_wrapper(cls) -> str:
159 return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"
160 +
161 + @classmethod
162 + def get_current_memory_usage(cls,
163 + device: Optional[torch.types.Device] = None
164 + ) -> float:
165 + torch.cuda.reset_peak_memory_stats(device)
166 + return torch.cuda.max_memory_allocated(device)
vllm/platforms/xpu.py
+7
@@ -94,3 +94,10 @@ class XPUPlatform(Platform):
94 def is_pin_memory_available(cls):
95 logger.warning("Pin memory is not supported on XPU.")
96 return False
97 +
98 + @classmethod
99 + def get_current_memory_usage(cls,
100 + device: Optional[torch.types.Device] = None
101 + ) -> float:
102 + torch.xpu.reset_peak_memory_stats(device)
103 + return torch.xpu.max_memory_allocated(device)
vllm/utils.py
+1 -7
@@ -710,13 +710,7 @@ class DeviceMemoryProfiler:
710 def current_memory_usage(self) -> float:
711 # Return the memory usage in bytes.
712 from vllm.platforms import current_platform
713 - if current_platform.is_cuda_alike():
714 - torch.cuda.reset_peak_memory_stats(self.device)
715 - mem = torch.cuda.max_memory_allocated(self.device)
716 - elif current_platform.is_xpu():
717 - torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
718 - mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
719 - return mem
713 + return current_platform.get_current_memory_usage(self.device)
714
715 def __enter__(self):
716 self.initial_memory = self.current_memory_usage()