@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / da02cb4b27
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[core] further polish memory profiling (#12126)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Jan 18, 2025 at 12:25 UTC da02cb4b274ab8bcebb8b8e677ff4b43440bc499
3 files changed +85 -67
tests/test_utils.py
+12 -14
@@ -9,10 +9,10 @@ import torch
9 from vllm_test_utils import monitor
10
11 from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
12 -from vllm.utils import (FlexibleArgumentParser, PlaceholderModule,
13 - StoreBoolean, bind_kv_cache, deprecate_kwargs,
14 - get_open_port, memory_profiling, merge_async_iterators,
15 - supports_kw)
12 +from vllm.utils import (FlexibleArgumentParser, MemorySnapshot,
13 + PlaceholderModule, StoreBoolean, bind_kv_cache,
14 + deprecate_kwargs, get_open_port, memory_profiling,
15 + merge_async_iterators, supports_kw)
16
17 from .utils import error_on_warning, fork_new_process_for_each_test
18
@@ -284,14 +284,13 @@ def test_memory_profiling():
284 # 512 MiB allocation outside of this instance
285 handle1 = lib.cudaMalloc(512 * 1024 * 1024)
286
287 - baseline_memory_in_bytes = \
288 - torch.cuda.mem_get_info()[1] - torch.cuda.mem_get_info()[0]
287 + baseline_snapshot = MemorySnapshot()
288
289 # load weights
290
291 weights = torch.randn(128, 1024, 1024, device='cuda', dtype=torch.float32)
292
294 - weights_memory_in_bytes = 128 * 1024 * 1024 * 4 # 512 MiB
293 + weights_memory = 128 * 1024 * 1024 * 4 # 512 MiB
294
295 def measure_current_non_torch():
296 free, total = torch.cuda.mem_get_info()
@@ -300,8 +299,8 @@ def test_memory_profiling():
299 current_non_torch = current_used - current_torch
300 return current_non_torch
301
303 - with memory_profiling(baseline_memory_in_bytes=baseline_memory_in_bytes,
304 - weights_memory_in_bytes=weights_memory_in_bytes) as result, \
302 + with memory_profiling(baseline_snapshot=baseline_snapshot,
303 + weights_memory=weights_memory) as result, \
304 monitor(measure_current_non_torch) as monitored_values:
305 # make a memory spike, 1 GiB
306 spike = torch.randn(256, 1024, 1024, device='cuda', dtype=torch.float32)
@@ -316,13 +315,12 @@ def test_memory_profiling():
315 assert measured_diff == 256 * 1024 * 1024
316
317 # Check that the memory usage is within 5% of the expected values
319 - # 5% tolerance is caused by PyTorch caching allocator,
320 - # we cannot control PyTorch's behavior of its internal buffers,
318 + # 5% tolerance is caused by cuda runtime.
319 + # we cannot control cuda runtime in the granularity of bytes,
320 # which causes a small error (<10 MiB in practice)
322 - non_torch_ratio = result.non_torch_increase_in_bytes / (256 * 1024 * 1024) # noqa
323 - torch_peak_ratio = result.torch_peak_increase_in_bytes / (1024 * 1024 * 1024) # noqa
321 + non_torch_ratio = result.non_torch_increase / (256 * 1024 * 1024) # noqa
322 assert abs(non_torch_ratio - 1) <= 0.05
325 - assert abs(torch_peak_ratio - 1) <= 0.05
323 + assert result.torch_peak_increase == 1024 * 1024 * 1024
324 del weights
325 lib.cudaFree(handle1)
326 lib.cudaFree(handle2)
vllm/utils.py
+56 -39
@@ -1923,36 +1923,57 @@ def kill_process_tree(pid: int):
1923 @dataclass
1924 class MemorySnapshot:
1925 """Memory snapshot."""
1926 - torch_peak_in_bytes: int = 0
1927 - torch_memory_in_bytes: int = 0
1926 + torch_peak: int = 0
1927 + cuda_memory: int = 0
1928 + torch_memory: int = 0
1929 + non_torch_memory: int = 0
1930 timestamp: float = 0.0
1931 + auto_measure: bool = True
1932 +
1933 + def __post_init__(self):
1934 + if self.auto_measure:
1935 + self.measure()
1936
1937 def measure(self):
1931 - self.torch_peak_in_bytes = torch.cuda.max_memory_reserved()
1938 + # we measure the torch peak memory usage via allocated_bytes,
1939 + # rather than `torch.cuda.memory_reserved()` .
1940 + # After `torch.cuda.reset_peak_memory_stats()`,
1941 + # `torch.cuda.memory_reserved()` will keep growing, and only shrink
1942 + # when we call `torch.cuda.empty_cache()` or OOM happens.
1943 + self.torch_peak = torch.cuda.memory_stats().get(
1944 + "allocated_bytes.all.peak", 0)
1945 +
1946 + self.cuda_memory = torch.cuda.mem_get_info(
1947 + )[1] - torch.cuda.mem_get_info()[0]
1948 +
1949 # torch.cuda.memory_reserved() is how many bytes
1950 # PyTorch gets from cuda (by calling cudaMalloc, etc.)
1934 - self.torch_memory_in_bytes = torch.cuda.memory_reserved()
1951 + # this is used to measure the non-torch memory usage
1952 + self.torch_memory = torch.cuda.memory_reserved()
1953 +
1954 + self.non_torch_memory = self.cuda_memory - self.torch_memory
1955 self.timestamp = time.time()
1956
1957 def __sub__(self, other: "MemorySnapshot") -> "MemorySnapshot":
1938 - """support a - b"""
1958 return MemorySnapshot(
1940 - torch_peak_in_bytes=self.torch_peak_in_bytes -
1941 - other.torch_peak_in_bytes,
1942 - torch_memory_in_bytes=self.torch_memory_in_bytes -
1943 - other.torch_memory_in_bytes,
1944 - timestamp=self.timestamp - other.timestamp)
1959 + torch_peak=self.torch_peak - other.torch_peak,
1960 + cuda_memory=self.cuda_memory - other.cuda_memory,
1961 + torch_memory=self.torch_memory - other.torch_memory,
1962 + non_torch_memory=self.non_torch_memory - other.non_torch_memory,
1963 + timestamp=self.timestamp - other.timestamp,
1964 + auto_measure=False,
1965 + )
1966
1967
1968 @dataclass
1969 class MemoryProfilingResult:
1949 - """Memory profiling result.
1950 - """ # noqa
1951 - baseline_memory_in_bytes: int = 0
1952 - non_kv_cache_memory_in_bytes: int = 0
1953 - torch_peak_increase_in_bytes: int = 0
1954 - non_torch_increase_in_bytes: int = 0
1955 - weights_memory_in_bytes: float = 0
1970 + """Memory profiling result. All numbers are in bytes.
1971 + """
1972 + non_kv_cache_memory: int = 0
1973 + torch_peak_increase: int = 0
1974 + non_torch_increase: int = 0
1975 + weights_memory: float = 0
1976 + before_create: MemorySnapshot = field(default_factory=MemorySnapshot)
1977 before_profile: MemorySnapshot = field(default_factory=MemorySnapshot)
1978 after_profile: MemorySnapshot = field(default_factory=MemorySnapshot)
1979 profile_time: float = 0.0
@@ -1960,18 +1981,14 @@ class MemoryProfilingResult:
1981
1982 @contextlib.contextmanager
1983 def memory_profiling(
1963 - baseline_memory_in_bytes: int, weights_memory_in_bytes: int
1964 -) -> Generator[MemoryProfilingResult, None, None]:
1984 + baseline_snapshot: MemorySnapshot,
1985 + weights_memory: int) -> Generator[MemoryProfilingResult, None, None]:
1986 """Memory profiling context manager.
1966 - baseline_memory_in_bytes: memory used by all the components other than
1967 - the current vLLM instance. It contains: memory used by other processes, memory
1968 - used by another vLLM instance in the same process, etc. It is usually measured
1969 - before the current vLLM instance initialize the device. And we assume it is
1970 - constant during the profiling of the current vLLM instance.
1971 - weights_memory_in_bytes: memory used by PyTorch when loading the model weights.
1987 + baseline_snapshot: the memory snapshot before the current vLLM instance.
1988 + weights_memory: memory used by PyTorch when loading the model weights.
1989 Note that, before loading the model weights, we also initialize the device
1990 and distributed environment, which may consume some memory. This part is not
1974 - included in the weights_memory_in_bytes because PyTorch does not control it.
1991 + included in the weights_memory because PyTorch does not control it.
1992
1993 The memory in one GPU can be classified into 3 categories:
1994 1. memory used by anything other than the current vLLM instance.
@@ -2006,20 +2023,21 @@ def memory_profiling(
2023 b. 2 GiB reserved for the peak activation tensors (category 2)
2024 c. 1 GiB used by non-torch components (category 3)
2025
2009 - The memory used for loading weights (a.) is directly given from the argument `weights_memory_in_bytes`.
2026 + The memory used for loading weights (a.) is directly given from the argument `weights_memory`.
2027
2011 - The increase of `torch.cuda.memory_stats()["allocated_bytes.all.peak"]` after profiling gives (b.).
2028 + The increase of `torch.cuda.memory_stats()["allocated_bytes.all.peak"]` during profiling gives (b.).
2029
2013 - (c.) is tricky. We measure the total memory used in this GPU (`torch.cuda.mem_get_info()[1] - torch.cuda.mem_get_info()[0]`),
2014 - subtract the baseline memory, the memory used by the model weights, and diff of `torch.cuda.memory_reserved()`.
2030 + The increase of `non_torch_memory` from creating the current vLLM instance until after profiling to get (c.).
2031 """ # noqa
2032 + gc.collect()
2033 + torch.cuda.empty_cache()
2034 torch.cuda.reset_peak_memory_stats()
2035
2036 result = MemoryProfilingResult()
2037
2020 - result.baseline_memory_in_bytes = baseline_memory_in_bytes
2038 + result.before_create = baseline_snapshot
2039 # the part of memory used for holding the model weights
2022 - result.weights_memory_in_bytes = weights_memory_in_bytes
2040 + result.weights_memory = weights_memory
2041
2042 result.before_profile.measure()
2043
@@ -2030,13 +2048,12 @@ def memory_profiling(
2048
2049 result.after_profile.measure()
2050
2033 - diff = result.after_profile - result.before_profile
2034 - result.torch_peak_increase_in_bytes = diff.torch_peak_in_bytes
2035 - current_cuda_memory_bytes = torch.cuda.mem_get_info(
2036 - )[1] - torch.cuda.mem_get_info()[0]
2037 - result.non_torch_increase_in_bytes = current_cuda_memory_bytes - baseline_memory_in_bytes - weights_memory_in_bytes - diff.torch_memory_in_bytes # noqa
2038 - result.profile_time = diff.timestamp
2039 - result.non_kv_cache_memory_in_bytes = result.non_torch_increase_in_bytes + result.torch_peak_increase_in_bytes + result.weights_memory_in_bytes # noqa
2051 + diff_profile = result.after_profile - result.before_profile
2052 + diff_from_create = result.after_profile - result.before_create
2053 + result.torch_peak_increase = diff_profile.torch_peak
2054 + result.non_torch_increase = diff_from_create.non_torch_memory
2055 + result.profile_time = diff_profile.timestamp
2056 + result.non_kv_cache_memory = result.non_torch_increase + result.torch_peak_increase + result.weights_memory # noqa
2057
2058
2059 # Adapted from: https://github.com/sgl-project/sglang/blob/v0.4.1/python/sglang/srt/utils.py#L630 # noqa: E501
vllm/worker/worker.py
+17 -14
@@ -21,7 +21,8 @@ from vllm.platforms import current_platform
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23 SequenceGroupMetadata, SequenceGroupMetadataDelta)
24 -from vllm.utils import GiB_bytes, bind_kv_cache, memory_profiling
24 +from vllm.utils import (GiB_bytes, MemorySnapshot, bind_kv_cache,
25 + memory_profiling)
26 from vllm.worker.cache_engine import CacheEngine
27 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
28 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -137,7 +138,8 @@ class Worker(LocalOrDistributedWorkerBase):
138 _check_if_gpu_supports_dtype(self.model_config.dtype)
139 gc.collect()
140 torch.cuda.empty_cache()
140 - self.init_gpu_memory = torch.cuda.mem_get_info()[0]
141 + torch.cuda.reset_peak_memory_stats()
142 + self.baseline_snapshot = MemorySnapshot()
143 else:
144 raise RuntimeError(
145 f"Not support device type: {self.device_config.device}")
@@ -192,10 +194,9 @@ class Worker(LocalOrDistributedWorkerBase):
194
195 # Execute a forward pass with dummy inputs to profile the memory usage
196 # of the model.
195 - with memory_profiling(baseline_memory_in_bytes=total_gpu_memory -
196 - self.init_gpu_memory,
197 - weights_memory_in_bytes=self.model_runner.
198 - model_memory_usage) as result:
197 + with memory_profiling(
198 + self.baseline_snapshot,
199 + weights_memory=self.model_runner.model_memory_usage) as result:
200 self.model_runner.profile_run()
201
202 self._assert_memory_footprint_increased_during_profiling()
@@ -203,7 +204,7 @@ class Worker(LocalOrDistributedWorkerBase):
204 memory_for_current_instance = total_gpu_memory * \
205 self.cache_config.gpu_memory_utilization
206 available_kv_cache_memory = (memory_for_current_instance -
206 - result.non_kv_cache_memory_in_bytes)
207 + result.non_kv_cache_memory)
208
209 # Calculate the number of blocks that can be allocated with the
210 # profiled peak memory.
@@ -226,11 +227,11 @@ class Worker(LocalOrDistributedWorkerBase):
227 f"({self.cache_config.gpu_memory_utilization:.2f})"
228 f" = {(memory_for_current_instance / GiB_bytes):.2f}GiB\n"
229 "model weights take "
229 - f"{(result.weights_memory_in_bytes / GiB_bytes):.2f}GiB;"
230 + f"{(result.weights_memory / GiB_bytes):.2f}GiB;"
231 " non_torch_memory takes "
231 - f"{(result.non_torch_increase_in_bytes / GiB_bytes):.2f}GiB;"
232 + f"{(result.non_torch_increase / GiB_bytes):.2f}GiB;"
233 " PyTorch activation peak memory takes "
233 - f"{(result.torch_peak_increase_in_bytes / GiB_bytes):.2f}GiB;"
234 + f"{(result.torch_peak_increase / GiB_bytes):.2f}GiB;"
235 " the rest of the memory reserved for KV Cache is "
236 f"{(available_kv_cache_memory / GiB_bytes):.2f}GiB.")
237
@@ -246,11 +247,13 @@ class Worker(LocalOrDistributedWorkerBase):
247 def _assert_memory_footprint_increased_during_profiling(self):
248 # NOTE(woosuk): Here we assume that the other processes using the same
249 # GPU did not change their memory usage during the profiling.
249 - free_gpu_memory, _ = torch.cuda.mem_get_info()
250 - assert self.init_gpu_memory - free_gpu_memory > 0, (
250 + free_gpu_memory, total = torch.cuda.mem_get_info()
251 + cuda_memory = total - free_gpu_memory
252 + assert self.baseline_snapshot.cuda_memory < cuda_memory, (
253 "Error in memory profiling. "
252 - f"Initial free memory {self.init_gpu_memory}, current free memory"
253 - f" {free_gpu_memory}. This happens when the GPU memory was "
254 + f"Initial used memory {self.baseline_snapshot.cuda_memory}, "
255 + f"currently used memory {cuda_memory}. "
256 + f"This happens when the GPU memory was "
257 "not properly cleaned up before initializing the vLLM instance.")
258
259 def initialize_cache(self, num_gpu_blocks: int,