@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 551603feff
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[core] overhaul memory profiling and fix backward compatibility (#10511)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 16, 2024 at 13:32 UTC 551603feffd9b4ba98ccdd34e02e403e04db88c1
8 files changed +236 -60
tests/entrypoints/llm/test_gpu_utilization.py new
+25
@@ -0,0 +1,25 @@
1 +from vllm import LLM, SamplingParams
2 +
3 +
4 +def test_gpu_memory_utilization():
5 + prompts = [
6 + "Hello, my name is",
7 + "The president of the United States is",
8 + "The capital of France is",
9 + "The future of AI is",
10 + ]
11 + sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
12 +
13 + # makes sure gpu_memory_utilization is per-instance limit,
14 + # not a global limit
15 + llms = [
16 + LLM(model="facebook/opt-125m",
17 + gpu_memory_utilization=0.3,
18 + enforce_eager=True) for i in range(3)
19 + ]
20 + for llm in llms:
21 + outputs = llm.generate(prompts, sampling_params)
22 + for output in outputs:
23 + prompt = output.prompt
24 + generated_text = output.outputs[0].text
25 + print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
tests/entrypoints/llm/test_lazy_outlines.py
+1 -1
@@ -36,7 +36,7 @@ def run_lmfe(sample_regex):
36 llm = LLM(model="facebook/opt-125m",
37 enforce_eager=True,
38 guided_decoding_backend="lm-format-enforcer",
39 - gpu_memory_utilization=0.6)
39 + gpu_memory_utilization=0.3)
40 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
41 outputs = llm.generate(
42 prompts=[
tests/test_utils.py
+42 -2
@@ -5,11 +5,13 @@ from functools import partial
5 from typing import AsyncIterator, Tuple
6
7 import pytest
8 +import torch
9
10 from vllm.utils import (FlexibleArgumentParser, StoreBoolean, deprecate_kwargs,
10 - get_open_port, merge_async_iterators, supports_kw)
11 + get_open_port, memory_profiling, merge_async_iterators,
12 + supports_kw)
13
12 -from .utils import error_on_warning
14 +from .utils import error_on_warning, fork_new_process_for_each_test
15
16
17 @pytest.mark.asyncio
@@ -270,3 +272,41 @@ def test_supports_kw(callable,kw_name,requires_kw_only,
272 requires_kw_only=requires_kw_only,
273 allow_var_kwargs=allow_var_kwargs
274 ) == is_supported
275 +
276 +
277 +@fork_new_process_for_each_test
278 +def test_memory_profiling():
279 + # Fake out some model loading + inference memory usage to test profiling
280 + # Memory used by other processes will show up as cuda usage outside of torch
281 + from vllm.distributed.device_communicators.cuda_wrapper import (
282 + CudaRTLibrary)
283 + lib = CudaRTLibrary()
284 + # 512 MiB allocation outside of this instance
285 + handle1 = lib.cudaMalloc(512 * 1024 * 1024)
286 +
287 + baseline_memory_in_bytes = \
288 + torch.cuda.mem_get_info()[1] - torch.cuda.mem_get_info()[0]
289 +
290 + # load weights
291 +
292 + weights = torch.randn(128, 1024, 1024, device='cuda', dtype=torch.float32)
293 +
294 + weights_memory_in_bytes = 128 * 1024 * 1024 * 4 # 512 MiB
295 +
296 + with memory_profiling(baseline_memory_in_bytes=baseline_memory_in_bytes,
297 + weights_memory_in_bytes=weights_memory_in_bytes) as result:
298 + # make a memory spike, 1 GiB
299 + spike = torch.randn(256, 1024, 1024, device='cuda', dtype=torch.float32)
300 + del spike
301 +
302 + # Add some extra non-torch memory 256 MiB (simulate NCCL)
303 + handle2 = lib.cudaMalloc(256 * 1024 * 1024)
304 +
305 + # Check that the memory usage is within 5% of the expected values
306 + non_torch_ratio = result.non_torch_increase_in_bytes / (256 * 1024 * 1024) # noqa
307 + torch_peak_ratio = result.torch_peak_increase_in_bytes / (1024 * 1024 * 1024) # noqa
308 + assert abs(non_torch_ratio - 1) <= 0.05
309 + assert abs(torch_peak_ratio - 1) <= 0.05
310 + del weights
311 + lib.cudaFree(handle1)
312 + lib.cudaFree(handle2)
tests/worker/test_profile.py
+9 -9
@@ -31,10 +31,6 @@ def test_gpu_memory_profiling():
31 is_driver_worker=True,
32 )
33
34 - # Load the model so we can profile it
35 - worker.init_device()
36 - worker.load_model()
37 -
34 # Set 10GiB as the total gpu ram to be device-agnostic
35 def mock_mem_info():
36 current_usage = torch.cuda.memory_stats(
@@ -46,20 +42,24 @@ def test_gpu_memory_profiling():
42
43 from unittest.mock import patch
44 with patch("torch.cuda.mem_get_info", side_effect=mock_mem_info):
45 + # Load the model so we can profile it
46 + worker.init_device()
47 + worker.load_model()
48 gpu_blocks, _ = worker.determine_num_available_blocks()
49
51 - # Peak vram usage by torch should be 0.7077 GiB
50 + # Peak vram usage by torch should be 0.47 GiB
51 + # Model weights take 0.25 GiB
52 # No memory should be allocated outside of torch
53 # 9.0 GiB should be the utilization target
54 - # 8.2923 GiB should be available for the KV cache
54 + # 8.28 GiB should be available for the KV cache
55 block_size = CacheEngine.get_cache_block_size(
56 engine_config.cache_config, engine_config.model_config,
57 engine_config.parallel_config)
58
59 - expected_blocks = (8.2923 * 1024**3) // block_size
59 + expected_blocks = (8.28 * 1024**3) // block_size
60
61 # Check within a small tolerance for portability
62 # Hardware, kernel, or dependency changes could all affect memory
63 # utilization.
64 - # A 10 block tolerance here should be about 6MB of wiggle room.
65 - assert abs(gpu_blocks - expected_blocks) < 10
64 + # A 100 block tolerance here should be about 60MB of wiggle room.
65 + assert abs(gpu_blocks - expected_blocks) < 100
vllm/engine/arg_utils.py
+6 -5
@@ -487,11 +487,12 @@ class EngineArgs:
487 help='The fraction of GPU memory to be used for the model '
488 'executor, which can range from 0 to 1. For example, a value of '
489 '0.5 would imply 50%% GPU memory utilization. If unspecified, '
490 - 'will use the default value of 0.9. This is a global gpu memory '
491 - 'utilization limit, for example if 50%% of the gpu memory is '
492 - 'already used before vLLM starts and --gpu-memory-utilization is '
493 - 'set to 0.9, then only 40%% of the gpu memory will be allocated '
494 - 'to the model executor.')
490 + 'will use the default value of 0.9. This is a per-instance '
491 + 'limit, and only applies to the current vLLM instance.'
492 + 'It does not matter if you have another vLLM instance running '
493 + 'on the same GPU. For example, if you have two vLLM instances '
494 + 'running on the same GPU, you can set the GPU memory utilization '
495 + 'to 0.5 for each instance.')
496 parser.add_argument(
497 '--num-gpu-blocks-override',
498 type=int,
vllm/utils.py
+123 -2
@@ -23,10 +23,12 @@ import weakref
23 from asyncio import FIRST_COMPLETED, AbstractEventLoop, Future, Task
24 from collections import UserDict, defaultdict
25 from collections.abc import Iterable, Mapping
26 +from dataclasses import dataclass, field
27 from functools import lru_cache, partial, wraps
28 from typing import (TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable,
28 - Dict, Generic, Hashable, List, Literal, Optional,
29 - OrderedDict, Set, Tuple, Type, TypeVar, Union, overload)
29 + Dict, Generator, Generic, Hashable, List, Literal,
30 + Optional, OrderedDict, Set, Tuple, Type, TypeVar, Union,
31 + overload)
32 from uuid import uuid4
33
34 import numpy as np
@@ -1664,3 +1666,122 @@ def kill_process_tree(pid: int):
1666 # Finally kill the parent
1667 with contextlib.suppress(ProcessLookupError):
1668 os.kill(pid, signal.SIGKILL)
1669 +
1670 +
1671 +@dataclass
1672 +class MemorySnapshot:
1673 + """Memory snapshot."""
1674 + torch_peak_in_bytes: int = 0
1675 + torch_memory_in_bytes: int = 0
1676 + timestamp: float = 0.0
1677 +
1678 + def measure(self):
1679 + self.torch_peak_in_bytes = torch.cuda.memory_stats(
1680 + )["allocated_bytes.all.peak"]
1681 + self.torch_memory_in_bytes = torch.cuda.memory_stats(
1682 + )["allocated_bytes.all.current"]
1683 + self.timestamp = time.time()
1684 +
1685 + def __sub__(self, other: "MemorySnapshot") -> "MemorySnapshot":
1686 + """support a - b"""
1687 + return MemorySnapshot(
1688 + torch_peak_in_bytes=self.torch_peak_in_bytes -
1689 + other.torch_peak_in_bytes,
1690 + torch_memory_in_bytes=self.torch_memory_in_bytes -
1691 + other.torch_memory_in_bytes,
1692 + timestamp=self.timestamp - other.timestamp)
1693 +
1694 +
1695 +@dataclass
1696 +class MemoryProfilingResult:
1697 + """Memory profiling result.
1698 + """ # noqa
1699 + baseline_memory_in_bytes: int = 0
1700 + non_kv_cache_memory_in_bytes: int = 0
1701 + torch_peak_increase_in_bytes: int = 0
1702 + non_torch_increase_in_bytes: int = 0
1703 + weights_memory_in_bytes: float = 0
1704 + before_profile: MemorySnapshot = field(default_factory=MemorySnapshot)
1705 + after_profile: MemorySnapshot = field(default_factory=MemorySnapshot)
1706 + profile_time: float = 0.0
1707 +
1708 +
1709 +@contextlib.contextmanager
1710 +def memory_profiling(
1711 + baseline_memory_in_bytes: int, weights_memory_in_bytes: int
1712 +) -> Generator[MemoryProfilingResult, None, None]:
1713 + """Memory profiling context manager.
1714 + baseline_memory_in_bytes: memory used by all the components other than
1715 + the current vLLM instance. It contains: memory used by other processes, memory
1716 + used by another vLLM instance in the same process, etc. It is usually measured
1717 + before the current vLLM instance initialize the device. And we assume it is
1718 + constant during the profiling of the current vLLM instance.
1719 + weights_memory_in_bytes: memory used by PyTorch when loading the model weights.
1720 + Note that, before loading the model weights, we also initialize the device
1721 + and distributed environment, which may consume some memory. This part is not
1722 + included in the weights_memory_in_bytes because PyTorch does not control it.
1723 +
1724 + The memory in one GPU can be classified into 3 categories:
1725 + 1. memory used by anything other than the current vLLM instance.
1726 + 2. memory used by torch in the current vLLM instance.
1727 + 3. memory used in the current vLLM instance, but not by torch.
1728 +
1729 + A quantitive example:
1730 +
1731 + Before creating the current vLLM instance:
1732 + category 1: 1 GiB
1733 + category 2: 0 GiB
1734 + category 3: 0 GiB
1735 +
1736 + After creating the current vLLM instance and loading the model,
1737 + (i.e. before profiling):
1738 + category 1: 1 GiB
1739 + category 2: 2 GiB (model weights take 2 GiB)
1740 + category 3: 0.5 GiB (memory used by NCCL)
1741 +
1742 + During profiling (peak):
1743 + category 1: 1 GiB
1744 + category 2: 4 GiB (peak activation tensors take 2 GiB)
1745 + category 3: 1 GiB (memory used by NCCL + buffers for some attention backends)
1746 +
1747 + After profiling:
1748 + category 1: 1 GiB
1749 + category 2: 3 GiB (after garbage-collecting activation tensors)
1750 + category 3: 1 GiB (memory used by NCCL + buffers for some attention backends)
1751 +
1752 + In this case, non-kv cache takes 5 GiB in total, including:
1753 + a. 2 GiB used by the model weights (category 2)
1754 + b. 2 GiB reserved for the peak activation tensors (category 2)
1755 + c. 1 GiB used by non-torch components (category 3)
1756 +
1757 + The memory used for loading weights (a.) is directly given from the argument `weights_memory_in_bytes`.
1758 +
1759 + The increase of ``torch.cuda.memory_stats()["allocated_bytes.all.peak"]` after profiling gives (b.).
1760 +
1761 + (c.) is tricky. We measure the total memory used in this GPU (`torch.cuda.mem_get_info()[1] - torch.cuda.mem_get_info()[0]`),
1762 + subtract the baseline memory, the memory used by the model weights, and diff of `torch.cuda.memory_stats()["allocated_bytes.all.current"]`.
1763 + """ # noqa
1764 + torch.cuda.reset_peak_memory_stats()
1765 +
1766 + result = MemoryProfilingResult()
1767 +
1768 + result.baseline_memory_in_bytes = baseline_memory_in_bytes
1769 + # the part of memory used for holding the model weights
1770 + result.weights_memory_in_bytes = weights_memory_in_bytes
1771 +
1772 + result.before_profile.measure()
1773 +
1774 + yield result
1775 +
1776 + gc.collect()
1777 + torch.cuda.empty_cache()
1778 +
1779 + result.after_profile.measure()
1780 +
1781 + diff = result.after_profile - result.before_profile
1782 + result.torch_peak_increase_in_bytes = diff.torch_peak_in_bytes
1783 + current_cuda_memory_bytes = torch.cuda.mem_get_info(
1784 + )[1] - torch.cuda.mem_get_info()[0]
1785 + result.non_torch_increase_in_bytes = current_cuda_memory_bytes - baseline_memory_in_bytes - weights_memory_in_bytes - diff.torch_memory_in_bytes # noqa
1786 + result.profile_time = diff.timestamp
1787 + result.non_kv_cache_memory_in_bytes = result.non_torch_increase_in_bytes + result.torch_peak_increase_in_bytes + result.weights_memory_in_bytes # noqa
vllm/worker/multi_step_model_runner.py
+2 -1
@@ -645,7 +645,8 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
645 return model_input
646
647 def load_model(self) -> None:
648 - return self._base_model_runner.load_model()
648 + self._base_model_runner.load_model()
649 + self.model_memory_usage = self._base_model_runner.model_memory_usage
650
651 def save_sharded_state(
652 self,
vllm/worker/worker.py
+28 -40
@@ -1,7 +1,6 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -import time
4 from typing import Dict, List, Optional, Set, Tuple, Type, Union
5
6 import torch
@@ -22,6 +21,7 @@ from vllm.platforms import current_platform
21 from vllm.prompt_adapter.request import PromptAdapterRequest
22 from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
23 SequenceGroupMetadata, SequenceGroupMetadataDelta)
24 +from vllm.utils import GiB_bytes, memory_profiling
25 from vllm.worker.cache_engine import CacheEngine
26 from vllm.worker.enc_dec_model_runner import EncoderDecoderModelRunner
27 from vllm.worker.model_runner import GPUModelRunnerBase, ModelRunner
@@ -192,33 +192,22 @@ class Worker(LocalOrDistributedWorkerBase):
192 torch.cuda.reset_peak_memory_stats()
193
194 free_memory_pre_profile, total_gpu_memory = torch.cuda.mem_get_info()
195 - start_time = time.time()
195
196 # Execute a forward pass with dummy inputs to profile the memory usage
197 # of the model.
199 - self.model_runner.profile_run()
200 - torch.cuda.synchronize()
198 + with memory_profiling(baseline_memory_in_bytes=total_gpu_memory -
199 + self.init_gpu_memory,
200 + weights_memory_in_bytes=self.model_runner.
201 + model_memory_usage) as result:
202 + self.model_runner.profile_run()
203 + torch.cuda.synchronize()
204
205 self._assert_memory_footprint_increased_during_profiling()
206
204 - # Get the peak memory allocation recorded by torch
205 - peak_memory = torch.cuda.memory_stats()["allocated_bytes.all.peak"]
206 -
207 - # Check for any memory left around that may have been allocated on the
208 - # gpu outside of `torch`. NCCL operations, for example, can use a few
209 - # GB during a forward pass
210 - torch.cuda.empty_cache()
211 - torch_allocated_bytes = torch.cuda.memory_stats(
212 - )["allocated_bytes.all.current"]
213 - total_allocated_bytes = torch.cuda.mem_get_info(
214 - )[1] - torch.cuda.mem_get_info()[0]
215 - non_torch_allocations = total_allocated_bytes - torch_allocated_bytes
216 - if non_torch_allocations > 0:
217 - peak_memory += non_torch_allocations
218 -
219 - available_kv_cache_memory = (
220 - total_gpu_memory * self.cache_config.gpu_memory_utilization -
221 - peak_memory)
207 + memory_for_current_instance = total_gpu_memory * \
208 + self.cache_config.gpu_memory_utilization
209 + available_kv_cache_memory = (memory_for_current_instance -
210 + result.non_kv_cache_memory_in_bytes)
211
212 # Calculate the number of blocks that can be allocated with the
213 # profiled peak memory.
@@ -233,24 +222,23 @@ class Worker(LocalOrDistributedWorkerBase):
222 num_gpu_blocks = max(num_gpu_blocks, 0)
223 num_cpu_blocks = max(num_cpu_blocks, 0)
224
236 - end_time = time.time()
237 - logger.info(
238 - "Memory profiling results: "
239 - "duration=%.2f seconds, "
240 - "total_gpu_memory=%.2fGiB, "
241 - "initial_memory_usage=%.2fGiB, "
242 - "peak_torch_memory=%.2fGiB, "
243 - "memory_usage_post_profile=%.2fGiB, "
244 - "non_torch_memory=%.2fGiB, "
245 - "kv_cache_size=%.2fGiB, "
246 - "gpu_memory_utilization=%.2f.", end_time - start_time,
247 - total_gpu_memory / (1024**3),
248 - (total_gpu_memory - free_memory_pre_profile) / (1024**3),
249 - (peak_memory - non_torch_allocations) / (1024**3),
250 - total_allocated_bytes / (1024**3),
251 - non_torch_allocations / (1024**3),
252 - available_kv_cache_memory / (1024**3),
253 - self.cache_config.gpu_memory_utilization)
225 + msg = (f"Memory profiling takes {result.profile_time:.2f} seconds\n"
226 + "the current vLLM instance can use "
227 + "total_gpu_memory "
228 + f"({(total_gpu_memory / GiB_bytes):.2f}GiB)"
229 + " x gpu_memory_utilization "
230 + f"({self.cache_config.gpu_memory_utilization:.2f})"
231 + f" = {(memory_for_current_instance / GiB_bytes):.2f}GiB\n"
232 + "model weights take "
233 + f"{(result.weights_memory_in_bytes / GiB_bytes):.2f}GiB;"
234 + " non_torch_memory takes "
235 + f"{(result.non_torch_increase_in_bytes / GiB_bytes):.2f}GiB;"
236 + " PyTorch activation peak memory takes "
237 + f"{(result.torch_peak_increase_in_bytes / GiB_bytes):.2f}GiB;"
238 + " the rest of the memory reserved for KV Cache is "
239 + f"{(available_kv_cache_memory / GiB_bytes):.2f}GiB.")
240 +
241 + logger.info(msg)
242
243 # Final cleanup
244 if self.model_runner.lora_manager: