@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 385da2dae2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Measure model memory usage (#3120)

Michael Goin committed Mar 7, 2024 at 11:42 UTC 385da2dae2b90e5273da8dfce881727bd9c574a1
2 files changed +37 -6
vllm/utils.py
+25
@@ -3,6 +3,7 @@ import os
3 import socket
4 import subprocess
5 import uuid
6 +import gc
7 from platform import uname
8 from typing import List, Tuple, Union
9 from packaging.version import parse, Version
@@ -309,3 +310,27 @@ def create_kv_caches_with_random(
310 f"Does not support value cache of type {cache_dtype}")
311 value_caches.append(value_cache)
312 return key_caches, value_caches
313 +
314 +
315 +class measure_cuda_memory:
316 +
317 + def __init__(self, device=None):
318 + self.device = device
319 +
320 + def current_memory_usage(self) -> float:
321 + # Return the memory usage in bytes.
322 + torch.cuda.reset_peak_memory_stats(self.device)
323 + mem = torch.cuda.max_memory_allocated(self.device)
324 + return mem
325 +
326 + def __enter__(self):
327 + self.initial_memory = self.current_memory_usage()
328 + # This allows us to call methods of the context manager if needed
329 + return self
330 +
331 + def __exit__(self, exc_type, exc_val, exc_tb):
332 + self.final_memory = self.current_memory_usage()
333 + self.consumed_memory = self.final_memory - self.initial_memory
334 +
335 + # Force garbage collection
336 + gc.collect()
vllm/worker/model_runner.py
+12 -6
@@ -21,7 +21,7 @@ from vllm.sequence import SamplerOutput, SequenceData, SequenceGroupMetadata
21 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
22 from vllm.lora.layers import LoRAMapping
23 from vllm.lora.request import LoRARequest
24 -from vllm.utils import in_wsl
24 +from vllm.utils import in_wsl, measure_cuda_memory
25
26 logger = init_logger(__name__)
27
@@ -85,11 +85,17 @@ class ModelRunner:
85 self.model_config.enforce_eager = True
86
87 def load_model(self) -> None:
88 - self.model = get_model(self.model_config,
89 - self.device_config,
90 - lora_config=self.lora_config,
91 - parallel_config=self.parallel_config,
92 - scheduler_config=self.scheduler_config)
88 + with measure_cuda_memory() as m:
89 + self.model = get_model(self.model_config,
90 + self.device_config,
91 + lora_config=self.lora_config,
92 + parallel_config=self.parallel_config,
93 + scheduler_config=self.scheduler_config)
94 +
95 + self.model_memory_usage = m.consumed_memory
96 + logger.info(
97 + f"Loading model weights took {self.model_memory_usage / float(2**30):.4f} GB"
98 + )
99
100 vocab_size = self.model.config.vocab_size
101