A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 22:15 UTC
[Misc] Capture and log the time of loading weights (#13666)
Jun Duan committed
Feb 22, 2025 at 01:06 UTC
68d535ef442384478797aa64738ec8e96b6b43d3
2 files changed
+10
-5
vllm/v1/worker/gpu_model_runner.py
+5
-3
@@ -1048,6 +1048,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1048
def load_model(self) -> None:
1049
logger.info("Starting to load model %s...", self.model_config.model)
1050
with DeviceMemoryProfiler() as m: # noqa: SIM117
1051
+ time_before_load = time.perf_counter()
1052
self.model = get_model(vllm_config=self.vllm_config)
1053
if self.lora_config:
1054
self.model = self.load_lora_model(self.model,
@@ -1055,10 +1056,11 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1056
self.scheduler_config,
1057
self.lora_config,
1058
self.device)
1058
-
1059
+ time_after_load = time.perf_counter()
1060
self.model_memory_usage = m.consumed_memory
1060
- logger.info("Loading model weights took %.4f GB",
1061
- self.model_memory_usage / float(2**30))
1061
+ logger.info("Loading model weights took %.4f GB and %.6f seconds",
1062
+ self.model_memory_usage / float(2**30),
1063
+ time_after_load - time_before_load)
1064
1065
def _get_prompt_logprobs_dict(
1066
self,
vllm/worker/model_runner.py
+5
-2
@@ -1109,11 +1109,14 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1109
def load_model(self) -> None:
1110
logger.info("Starting to load model %s...", self.model_config.model)
1111
with DeviceMemoryProfiler(self.device) as m:
1112
+ time_before_load = time.perf_counter()
1113
self.model = get_model(vllm_config=self.vllm_config)
1114
+ time_after_load = time.perf_counter()
1115
1116
self.model_memory_usage = m.consumed_memory
1115
- logger.info("Loading model weights took %.4f GB",
1116
- self.model_memory_usage / float(2**30))
1117
+ logger.info("Loading model weights took %.4f GB and %.6f seconds",
1118
+ self.model_memory_usage / float(2**30),
1119
+ time_after_load - time_before_load)
1120
1121
if self.lora_config:
1122
assert supports_lora(