@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a8b150c595
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Init model on GPU to reduce CPU memory footprint (#1796)

ljss committed Nov 28, 2023 at 03:18 UTC a8b150c5950a353d458b47ce319585698ef41e3f
1 file changed +2 -3
vllm/model_executor/model_loader.py
+2 -3
@@ -87,9 +87,9 @@ def get_model(model_config: ModelConfig) -> nn.Module:
87 with _set_default_torch_dtype(model_config.dtype):
88 # Create a model instance.
89 # The weights will be initialized as empty tensors.
90 - model = model_class(model_config.hf_config, linear_method)
90 + with torch.device("cuda"):
91 + model = model_class(model_config.hf_config, linear_method)
92 if model_config.load_format == "dummy":
92 - model = model.cuda()
93 # NOTE(woosuk): For accurate performance evaluation, we assign
94 # random values to the weights.
95 initialize_dummy_weights(model)
@@ -97,5 +97,4 @@ def get_model(model_config: ModelConfig) -> nn.Module:
97 # Load the weights from the cached or downloaded files.
98 model.load_weights(model_config.model, model_config.download_dir,
99 model_config.load_format, model_config.revision)
100 - model = model.cuda()
100 return model.eval()