@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7c2749a4fd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[misc] add start loading models for users information (#6670)

youkaichao committed Jul 22, 2024 at 20:08 UTC 7c2749a4fdd92150940cc80c1027a1e173099a24
1 file changed +1
vllm/worker/model_runner.py
+1
@@ -677,6 +677,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
677 int(self.cache_config.cpu_offload_gb * 1024**3))
678
679 def load_model(self) -> None:
680 + logger.info("Starting to load model %s...", self.model_config.model)
681 with CudaMemoryProfiler() as m:
682 self.model = get_model(model_config=self.model_config,
683 device_config=self.device_config,