@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / de4008e2ab
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Bugfix][Core] Use torch.cuda.memory_stats() to profile peak memory usage (#9352)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>

Joe Runde committed Oct 17, 2024 at 21:47 UTC de4008e2abc50b8a5d72d7ba553037f03cf97caa
4 files changed +122 -17
tests/entrypoints/llm/test_lazy_outlines.py
+3 -1
@@ -26,10 +26,12 @@ def test_lazy_outlines(sample_regex):
26 # make sure outlines is not imported
27 assert 'outlines' not in sys.modules
28
29 + # The second LLM needs to request a higher gpu_memory_utilization because
30 + # the first LLM has already allocated a full 30% of the gpu memory.
31 llm = LLM(model="facebook/opt-125m",
32 enforce_eager=True,
33 guided_decoding_backend="lm-format-enforcer",
32 - gpu_memory_utilization=0.3)
34 + gpu_memory_utilization=0.6)
35 sampling_params = SamplingParams(temperature=0.8, top_p=0.95)
36 outputs = llm.generate(
37 prompts=[
tests/entrypoints/offline_mode/test_offline_mode.py
+1 -1
@@ -44,7 +44,7 @@ def test_offline_mode(llm: LLM, monkeypatch):
44 LLM(model=MODEL_NAME,
45 max_num_batched_tokens=4096,
46 tensor_parallel_size=1,
47 - gpu_memory_utilization=0.10,
47 + gpu_memory_utilization=0.20,
48 enforce_eager=True)
49 finally:
50 # Reset the environment after the test
tests/worker/test_profile.py new
+69
@@ -0,0 +1,69 @@
1 +import torch
2 +
3 +from vllm.engine.arg_utils import EngineArgs
4 +from vllm.utils import get_distributed_init_method, get_ip, get_open_port
5 +from vllm.worker.cache_engine import CacheEngine
6 +from vllm.worker.worker import Worker
7 +
8 +
9 +def test_gpu_memory_profiling():
10 + # Tests the gpu profiling that happens in order to determine the number of
11 + # KV cache blocks that we can allocate on the GPU.
12 + # This test mocks the maximum available gpu memory so that it can run on
13 + # any gpu setup.
14 +
15 + # Set up engine args to build a worker.
16 + engine_args = EngineArgs(model="facebook/opt-125m",
17 + dtype="half",
18 + load_format="dummy")
19 + engine_config = engine_args.create_engine_config()
20 + engine_config.cache_config.num_gpu_blocks = 1000
21 + engine_config.cache_config.num_cpu_blocks = 1000
22 +
23 + # Create the worker.
24 + distributed_init_method = get_distributed_init_method(
25 + get_ip(), get_open_port())
26 + worker = Worker(
27 + model_config=engine_config.model_config,
28 + parallel_config=engine_config.parallel_config,
29 + scheduler_config=engine_config.scheduler_config,
30 + device_config=engine_config.device_config,
31 + cache_config=engine_config.cache_config,
32 + load_config=engine_config.load_config,
33 + local_rank=0,
34 + rank=0,
35 + distributed_init_method=distributed_init_method,
36 + is_driver_worker=True,
37 + )
38 +
39 + # Load the model so we can profile it
40 + worker.init_device()
41 + worker.load_model()
42 +
43 + # Set 10GiB as the total gpu ram to be device-agnostic
44 + def mock_mem_info():
45 + current_usage = torch.cuda.memory_stats(
46 + )["allocated_bytes.all.current"]
47 + mock_total_bytes = 10 * 1024**3
48 + free = mock_total_bytes - current_usage
49 +
50 + return (free, mock_total_bytes)
51 +
52 + from unittest.mock import patch
53 + with patch("torch.cuda.mem_get_info", side_effect=mock_mem_info):
54 + gpu_blocks, _ = worker.determine_num_available_blocks()
55 +
56 + # Peak vram usage by torch should be 0.7077 GiB
57 + # Non-torch allocations should be 0.0079 GiB
58 + # 9.0 GiB should be the utilization target
59 + # 8.2843 GiB should be available for the KV cache
60 + block_size = CacheEngine.get_cache_block_size(
61 + engine_config.cache_config, engine_config.model_config,
62 + engine_config.parallel_config)
63 +
64 + expected_blocks = (8.2843 * 1024**3) // block_size
65 +
66 + # Check within a small tolerance for portability
67 + # Hardware, kernel, or dependency changes could all affect memory
68 + # utilization
69 + assert abs(gpu_blocks - expected_blocks) < 5
vllm/worker/worker.py
+49 -15
@@ -217,42 +217,76 @@ class Worker(LocalOrDistributedWorkerBase):
217 # Profile the memory usage of the model and get the maximum number of
218 # cache blocks that can be allocated with the remaining free memory.
219 torch.cuda.empty_cache()
220 + torch.cuda.reset_peak_memory_stats()
221 +
222 + free_memory_pre_profile, total_gpu_memory = torch.cuda.mem_get_info()
223
224 # Execute a forward pass with dummy inputs to profile the memory usage
225 # of the model.
226 self.model_runner.profile_run()
227 + torch.cuda.synchronize()
228 +
229 + self._assert_memory_footprint_increased_during_profiling()
230 +
231 + # Get the peak memory allocation recorded by torch
232 + peak_memory = torch.cuda.memory_stats()["allocated_bytes.all.peak"]
233 +
234 + # Check for any memory left around that may have been allocated on the
235 + # gpu outside of `torch`. NCCL operations, for example, can use a few
236 + # GB during a forward pass
237 + torch.cuda.empty_cache()
238 + # After emptying the torch cache, any other increase in gpu ram should
239 + # be from non-torch allocations.
240 + non_torch_allocations = free_memory_pre_profile - \
241 + torch.cuda.mem_get_info()[0]
242 + if non_torch_allocations > 0:
243 + peak_memory += non_torch_allocations
244 +
245 + available_kv_cache_memory = (
246 + total_gpu_memory * self.cache_config.gpu_memory_utilization -
247 + peak_memory)
248
249 # Calculate the number of blocks that can be allocated with the
250 # profiled peak memory.
227 - torch.cuda.synchronize()
228 - free_gpu_memory, total_gpu_memory = torch.cuda.mem_get_info()
229 - # NOTE(woosuk): Here we assume that the other processes using the same
230 - # GPU did not change their memory usage during the profiling.
231 - peak_memory = self.init_gpu_memory - free_gpu_memory
232 - assert peak_memory > 0, (
233 - "Error in memory profiling. "
234 - f"Initial free memory {self.init_gpu_memory}, current free memory"
235 - f" {free_gpu_memory}. This happens when the GPU memory was "
236 - "not properly cleaned up before initializing the vLLM instance.")
237 -
251 cache_block_size = self.get_cache_block_size_bytes()
252 if cache_block_size == 0:
253 num_gpu_blocks = 0
254 num_cpu_blocks = 0
255 else:
243 - num_gpu_blocks = int(
244 - (total_gpu_memory * self.cache_config.gpu_memory_utilization -
245 - peak_memory) // cache_block_size)
256 + num_gpu_blocks = int(available_kv_cache_memory // cache_block_size)
257 num_cpu_blocks = int(self.cache_config.swap_space_bytes //
258 cache_block_size)
259 num_gpu_blocks = max(num_gpu_blocks, 0)
260 num_cpu_blocks = max(num_cpu_blocks, 0)
261 +
262 + logger.info(
263 + "Memory profiling results: total_gpu_memory=%.2fGiB"
264 + " initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
265 + " non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
266 + " gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
267 + (total_gpu_memory - free_memory_pre_profile) / (1024**3),
268 + (peak_memory - non_torch_allocations) / (1024**3),
269 + non_torch_allocations / (1024**3),
270 + available_kv_cache_memory / (1024**3),
271 + self.cache_config.gpu_memory_utilization)
272 +
273 + # Final cleanup
274 if self.model_runner.lora_manager:
275 self.model_runner.remove_all_loras()
276 gc.collect()
253 - torch.cuda.empty_cache()
277 +
278 return num_gpu_blocks, num_cpu_blocks
279
280 + def _assert_memory_footprint_increased_during_profiling(self):
281 + # NOTE(woosuk): Here we assume that the other processes using the same
282 + # GPU did not change their memory usage during the profiling.
283 + free_gpu_memory, _ = torch.cuda.mem_get_info()
284 + assert self.init_gpu_memory - free_gpu_memory > 0, (
285 + "Error in memory profiling. "
286 + f"Initial free memory {self.init_gpu_memory}, current free memory"
287 + f" {free_gpu_memory}. This happens when the GPU memory was "
288 + "not properly cleaned up before initializing the vLLM instance.")
289 +
290 def initialize_cache(self, num_gpu_blocks: int,
291 num_cpu_blocks: int) -> None:
292 """Allocate GPU and CPU KV cache with the specified number of blocks.