A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Platform] move current_memory_usage() into platform (#11369)
Signed-off-by: Shanshan Shen <467638484@qq.com>
Shanshan Shen committed
Jan 15, 2025 at 11:38 UTC
9ddac56311b28f08e40a941296eb66fbb1be0a7a
5 files changed
+31
-7
vllm/platforms/cuda.py
+7
@@ -143,6 +143,13 @@ class CudaPlatformBase(Platform):
143
if cache_config and cache_config.block_size is None:
144
cache_config.block_size = 16
145
146
+ @classmethod
147
+ def get_current_memory_usage(cls,
148
+ device: Optional[torch.types.Device] = None
149
+ ) -> float:
150
+ torch.cuda.reset_peak_memory_stats(device)
151
+ return torch.cuda.max_memory_allocated(device)
152
+
153
@classmethod
154
def get_attn_backend_cls(cls, selected_backend, head_size, dtype,
155
kv_cache_dtype, block_size, use_v1) -> str:
vllm/platforms/interface.py
+9
@@ -277,6 +277,15 @@ class Platform:
277
return False
278
return True
279
280
+ @classmethod
281
+ def get_current_memory_usage(cls,
282
+ device: Optional[torch.types.Device] = None
283
+ ) -> float:
284
+ """
285
+ Return the memory usage in bytes.
286
+ """
287
+ raise NotImplementedError
288
+
289
@classmethod
290
def get_punica_wrapper(cls) -> str:
291
"""
vllm/platforms/rocm.py
+7
@@ -157,3 +157,10 @@ class RocmPlatform(Platform):
157
@classmethod
158
def get_punica_wrapper(cls) -> str:
159
return "vllm.lora.punica_wrapper.punica_gpu.PunicaWrapperGPU"
160
+
161
+ @classmethod
162
+ def get_current_memory_usage(cls,
163
+ device: Optional[torch.types.Device] = None
164
+ ) -> float:
165
+ torch.cuda.reset_peak_memory_stats(device)
166
+ return torch.cuda.max_memory_allocated(device)
vllm/platforms/xpu.py
+7
@@ -94,3 +94,10 @@ class XPUPlatform(Platform):
94
def is_pin_memory_available(cls):
95
logger.warning("Pin memory is not supported on XPU.")
96
return False
97
+
98
+ @classmethod
99
+ def get_current_memory_usage(cls,
100
+ device: Optional[torch.types.Device] = None
101
+ ) -> float:
102
+ torch.xpu.reset_peak_memory_stats(device)
103
+ return torch.xpu.max_memory_allocated(device)
vllm/utils.py
+1
-7
@@ -710,13 +710,7 @@ class DeviceMemoryProfiler:
710
def current_memory_usage(self) -> float:
711
# Return the memory usage in bytes.
712
from vllm.platforms import current_platform
713
- if current_platform.is_cuda_alike():
714
- torch.cuda.reset_peak_memory_stats(self.device)
715
- mem = torch.cuda.max_memory_allocated(self.device)
716
- elif current_platform.is_xpu():
717
- torch.xpu.reset_peak_memory_stats(self.device) # type: ignore
718
- mem = torch.xpu.max_memory_allocated(self.device) # type: ignore
719
- return mem
713
+ return current_platform.get_current_memory_usage(self.device)
714
715
def __enter__(self):
716
self.initial_memory = self.current_memory_usage()