A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Measure model memory usage (#3120)
Michael Goin committed
Mar 7, 2024 at 11:42 UTC
385da2dae2b90e5273da8dfce881727bd9c574a1
2 files changed
+37
-6
vllm/utils.py
+25
@@ -3,6 +3,7 @@ import os
3
import socket
4
import subprocess
5
import uuid
6
+import gc
7
from platform import uname
8
from typing import List, Tuple, Union
9
from packaging.version import parse, Version
@@ -309,3 +310,27 @@ def create_kv_caches_with_random(
310
f"Does not support value cache of type {cache_dtype}")
311
value_caches.append(value_cache)
312
return key_caches, value_caches
313
+
314
+
315
+class measure_cuda_memory:
316
+
317
+ def __init__(self, device=None):
318
+ self.device = device
319
+
320
+ def current_memory_usage(self) -> float:
321
+ # Return the memory usage in bytes.
322
+ torch.cuda.reset_peak_memory_stats(self.device)
323
+ mem = torch.cuda.max_memory_allocated(self.device)
324
+ return mem
325
+
326
+ def __enter__(self):
327
+ self.initial_memory = self.current_memory_usage()
328
+ # This allows us to call methods of the context manager if needed
329
+ return self
330
+
331
+ def __exit__(self, exc_type, exc_val, exc_tb):
332
+ self.final_memory = self.current_memory_usage()
333
+ self.consumed_memory = self.final_memory - self.initial_memory
334
+
335
+ # Force garbage collection
336
+ gc.collect()
vllm/worker/model_runner.py
+12
-6
@@ -21,7 +21,7 @@ from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
21
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
22
from vllm.lora.layers import LoRAMapping
23
from vllm.lora.request import LoRARequest
24
-from vllm.utils import in_wsl
24
+from vllm.utils import in_wsl, measure_cuda_memory
25
26
logger = init_logger(__name__)
27
@@ -85,11 +85,17 @@ class ModelRunner:
85
self.model_config.enforce_eager = True
86
87
def load_model(self) -> None:
88
- self.model = get_model(self.model_config,
89
- self.device_config,
90
- lora_config=self.lora_config,
91
- parallel_config=self.parallel_config,
92
- scheduler_config=self.scheduler_config)
88
+ with measure_cuda_memory() as m:
89
+ self.model = get_model(self.model_config,
90
+ self.device_config,
91
+ lora_config=self.lora_config,
92
+ parallel_config=self.parallel_config,
93
+ scheduler_config=self.scheduler_config)
94
+
95
+ self.model_memory_usage = m.consumed_memory
96
+ logger.info(
97
+ f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98
+ )
99
100
vocab_size = self.model.config.vocab_size
101