@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 889e662eae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[misc] improve memory profiling (#11809)

Signed-off-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

youkaichao committed Jan 8, 2025 at 14:36 UTC 889e662eae19fe8f30469883c6854ee4df4315a9
4 files changed +94 -8
tests/test_utils.py
+18 -1
@@ -5,6 +5,7 @@ from typing import AsyncIterator, Tuple
5
6 import pytest
7 import torch
8 +from vllm_test_utils import monitor
9
10 from vllm.utils import (FlexibleArgumentParser, StoreBoolean, deprecate_kwargs,
11 get_open_port, memory_profiling, merge_async_iterators,
@@ -289,8 +290,16 @@ def test_memory_profiling():
290
291 weights_memory_in_bytes = 128 * 1024 * 1024 * 4 # 512 MiB
292
293 + def measure_current_non_torch():
294 + free, total = torch.cuda.mem_get_info()
295 + current_used = total - free
296 + current_torch = torch.cuda.memory_reserved()
297 + current_non_torch = current_used - current_torch
298 + return current_non_torch
299 +
300 with memory_profiling(baseline_memory_in_bytes=baseline_memory_in_bytes,
293 - weights_memory_in_bytes=weights_memory_in_bytes) as result:
301 + weights_memory_in_bytes=weights_memory_in_bytes) as result, \
302 + monitor(measure_current_non_torch) as monitored_values:
303 # make a memory spike, 1 GiB
304 spike = torch.randn(256, 1024, 1024, device='cuda', dtype=torch.float32)
305 del spike
@@ -298,7 +307,15 @@ def test_memory_profiling():
307 # Add some extra non-torch memory 256 MiB (simulate NCCL)
308 handle2 = lib.cudaMalloc(256 * 1024 * 1024)
309
310 + # this is an analytic value, it is exact,
311 + # we only have 256 MiB non-torch memory increase
312 + measured_diff = monitored_values.values[-1] - monitored_values.values[0]
313 + assert measured_diff == 256 * 1024 * 1024
314 +
315 # Check that the memory usage is within 5% of the expected values
316 + # 5% tolerance is caused by PyTorch caching allocator,
317 + # we cannot control PyTorch's behavior of its internal buffers,
318 + # which causes a small error (<10 MiB in practice)
319 non_torch_ratio = result.non_torch_increase_in_bytes / (256 * 1024 * 1024) # noqa
320 torch_peak_ratio = result.torch_peak_increase_in_bytes / (1024 * 1024 * 1024) # noqa
321 assert abs(non_torch_ratio - 1) <= 0.05
tests/vllm_test_utils/vllm_test_utils/__init__.py
+2 -1
@@ -4,5 +4,6 @@ It does not import any vLLM modules.
4 """
5
6 from .blame import BlameResult, blame
7 +from .monitor import MonitoredValues, monitor
8
8 -__all__ = ["blame", "BlameResult"]
9 +__all__ = ["blame", "BlameResult", "monitor", "MonitoredValues"]
tests/vllm_test_utils/vllm_test_utils/monitor.py new
+68
@@ -0,0 +1,68 @@
1 +import contextlib
2 +import dataclasses
3 +import sys
4 +import traceback
5 +from typing import Callable, Generator, Generic, TypeVar
6 +
7 +_T = TypeVar("_T")
8 +
9 +
10 +@dataclasses.dataclass
11 +class MonitoredValues(Generic[_T]):
12 + values: list[_T] = dataclasses.field(default_factory=list)
13 + trace_stacks: list[str] = dataclasses.field(default_factory=list)
14 +
15 +
16 +@contextlib.contextmanager
17 +def monitor(
18 + measure_func: Callable[[],
19 + _T]) -> Generator[MonitoredValues[_T], None, None]:
20 + """
21 + Trace the function calls to continuously monitor the change of
22 + a value.
23 +
24 + Usage:
25 +
26 + ```python
27 +
28 + def measure_func():
29 + ... # measure the current value
30 + return current_value
31 +
32 + with monitor(measure_func) as monitored_values:
33 + # do something
34 +
35 + monitored_values.values # all changes of the values
36 + monitored_values.trace_stacks # trace stacks of every change
37 + ```
38 + """
39 + monitored_values = MonitoredValues[_T]()
40 +
41 + def _trace_calls(frame, event, arg=None):
42 + nonlocal monitored_values
43 + if event in ['line']:
44 + # triggered by every line of Python code.
45 + # only Python functions will trigger it,
46 + # c/cpp functions will not trigger it.
47 + try:
48 + # Temporarily disable the trace function
49 + sys.settrace(None)
50 + # do a measurement
51 + current_value = measure_func()
52 + if len(monitored_values.values
53 + ) == 0 or current_value != monitored_values.values[-1]:
54 + monitored_values.values.append(current_value)
55 + monitored_values.trace_stacks.append("".join(
56 + traceback.format_stack()))
57 + # Re-enable the trace function
58 + sys.settrace(_trace_calls)
59 + except NameError:
60 + # modules are deleted during shutdown
61 + pass
62 + return _trace_calls
63 +
64 + try:
65 + sys.settrace(_trace_calls)
66 + yield monitored_values
67 + finally:
68 + sys.settrace(None)
vllm/utils.py
+6 -6
@@ -1742,10 +1742,10 @@ class MemorySnapshot:
1742 timestamp: float = 0.0
1743
1744 def measure(self):
1745 - self.torch_peak_in_bytes = torch.cuda.memory_stats(
1746 - )["allocated_bytes.all.peak"]
1747 - self.torch_memory_in_bytes = torch.cuda.memory_stats(
1748 - )["allocated_bytes.all.current"]
1745 + self.torch_peak_in_bytes = torch.cuda.max_memory_reserved()
1746 + # torch.cuda.memory_reserved() is how many bytes
1747 + # PyTorch gets from cuda (by calling cudaMalloc, etc.)
1748 + self.torch_memory_in_bytes = torch.cuda.memory_reserved()
1749 self.timestamp = time.time()
1750
1751 def __sub__(self, other: "MemorySnapshot") -> "MemorySnapshot":
@@ -1822,10 +1822,10 @@ def memory_profiling(
1822
1823 The memory used for loading weights (a.) is directly given from the argument `weights_memory_in_bytes`.
1824
1825 - The increase of ``torch.cuda.memory_stats()["allocated_bytes.all.peak"]` after profiling gives (b.).
1825 + The increase of `torch.cuda.memory_stats()["allocated_bytes.all.peak"]` after profiling gives (b.).
1826
1827 (c.) is tricky. We measure the total memory used in this GPU (`torch.cuda.mem_get_info()[1] - torch.cuda.mem_get_info()[0]`),
1828 - subtract the baseline memory, the memory used by the model weights, and diff of `torch.cuda.memory_stats()["allocated_bytes.all.current"]`.
1828 + subtract the baseline memory, the memory used by the model weights, and diff of `torch.cuda.memory_reserved()`.
1829 """ # noqa
1830 torch.cuda.reset_peak_memory_stats()
1831