A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
:bug: fix torch memory profiling (#9516)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
Joe Runde committed
Oct 18, 2024 at 20:25 UTC
380e18639f315a696bd5dcc93a24f250573b95a9
3 files changed
+14
-11
tests/quantization/test_bitsandbytes.py
+1
-2
@@ -107,8 +107,7 @@ def validate_generated_texts(hf_runner,
107
quantization='bitsandbytes',
108
load_format='bitsandbytes',
109
tensor_parallel_size=vllm_tp_size,
110
- enforce_eager=False,
111
- gpu_memory_utilization=0.8) as llm:
110
+ enforce_eager=False) as llm:
111
vllm_outputs = llm.generate_greedy(prompts, 8)
112
vllm_logs = log_generated_texts(prompts, vllm_outputs, "VllmRunner")
113
tests/worker/test_profile.py
+6
-5
@@ -54,16 +54,17 @@ def test_gpu_memory_profiling():
54
gpu_blocks, _ = worker.determine_num_available_blocks()
55
56
# Peak vram usage by torch should be 0.7077 GiB
57
- # Non-torch allocations should be 0.0079 GiB
57
+ # No memory should be allocated outside of torch
58
# 9.0 GiB should be the utilization target
59
- # 8.2843 GiB should be available for the KV cache
59
+ # 8.2923 GiB should be available for the KV cache
60
block_size = CacheEngine.get_cache_block_size(
61
engine_config.cache_config, engine_config.model_config,
62
engine_config.parallel_config)
63
64
- expected_blocks = (8.2843 * 1024**3) // block_size
64
+ expected_blocks = (8.2923 * 1024**3) // block_size
65
66
# Check within a small tolerance for portability
67
# Hardware, kernel, or dependency changes could all affect memory
68
- # utilization
69
- assert abs(gpu_blocks - expected_blocks) < 5
68
+ # utilization.
69
+ # A 10 block tolerance here should be about 6MB of wiggle room.
70
+ assert abs(gpu_blocks - expected_blocks) < 10
vllm/worker/worker.py
+7
-4
@@ -232,10 +232,11 @@ class Worker(LocalOrDistributedWorkerBase):
232
# gpu outside of `torch`. NCCL operations, for example, can use a few
233
# GB during a forward pass
234
torch.cuda.empty_cache()
235
- # After emptying the torch cache, any other increase in gpu ram should
236
- # be from non-torch allocations.
237
- non_torch_allocations = free_memory_pre_profile - \
238
- torch.cuda.mem_get_info()[0]
235
+ torch_allocated_bytes = torch.cuda.memory_stats(
236
+ )["allocated_bytes.all.current"]
237
+ total_allocated_bytes = torch.cuda.mem_get_info(
238
+ )[1] - torch.cuda.mem_get_info()[0]
239
+ non_torch_allocations = total_allocated_bytes - torch_allocated_bytes
240
if non_torch_allocations > 0:
241
peak_memory += non_torch_allocations
242
@@ -259,10 +260,12 @@ class Worker(LocalOrDistributedWorkerBase):
260
logger.info(
261
"Memory profiling results: total_gpu_memory=%.2fGiB"
262
" initial_memory_usage=%.2fGiB peak_torch_memory=%.2fGiB"
263
+ " memory_usage_post_profile=%.2fGib"
264
" non_torch_memory=%.2fGiB kv_cache_size=%.2fGiB"
265
" gpu_memory_utilization=%.2f", total_gpu_memory / (1024**3),
266
(total_gpu_memory - free_memory_pre_profile) / (1024**3),
267
(peak_memory - non_torch_allocations) / (1024**3),
268
+ total_allocated_bytes / (1024**3),
269
non_torch_allocations / (1024**3),
270
available_kv_cache_memory / (1024**3),
271
self.cache_config.gpu_memory_utilization)