A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Init model on GPU to reduce CPU memory footprint (#1796)
ljss committed
Nov 28, 2023 at 03:18 UTC
a8b150c5950a353d458b47ce319585698ef41e3f
1 file changed
+2
-3
vllm/model_executor/model_loader.py
+2
-3
@@ -87,9 +87,9 @@ def get_model(model_config: ModelConfig) -> nn.Module:
87
with _set_default_torch_dtype(model_config.dtype):
88
# Create a model instance.
89
# The weights will be initialized as empty tensors.
90
- model = model_class(model_config.hf_config, linear_method)
90
+ with torch.device("cuda"):
91
+ model = model_class(model_config.hf_config, linear_method)
92
if model_config.load_format == "dummy":
92
- model = model.cuda()
93
# NOTE(woosuk): For accurate performance evaluation, we assign
94
# random values to the weights.
95
initialize_dummy_weights(model)
@@ -97,5 +97,4 @@ def get_model(model_config: ModelConfig) -> nn.Module:
97
# Load the weights from the cached or downloaded files.
98
model.load_weights(model_config.model, model_config.download_dir,
99
model_config.load_format, model_config.revision)
100
- model = model.cuda()
100
return model.eval()