A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 08:00 UTC
Set torch default dtype in a context manager (#971)
Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>
Antoni Baum committed
Sep 6, 2023 at 23:39 UTC
005ba458b52fe2cf9837201d05644eadcdf10ca0
1 file changed
+24
-15
vllm/model_executor/model_loader.py
+24
-15
@@ -1,4 +1,5 @@
1
"""Utilities for selecting and loading models."""
2
+import contextlib
3
from typing import Type
4
5
import torch
@@ -30,6 +31,15 @@ _MODEL_REGISTRY = {
31
}
32
33
34
+@contextlib.contextmanager
35
+def _set_default_torch_dtype(dtype: torch.dtype):
36
+ """Sets the default torch dtype to the given dtype."""
37
+ old_dtype = torch.get_default_dtype()
38
+ torch.set_default_dtype(dtype)
39
+ yield
40
+ torch.set_default_dtype(old_dtype)
41
+
42
+
43
def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
44
architectures = getattr(config, "architectures", [])
45
for arch in architectures:
@@ -42,19 +52,18 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
52
53
def get_model(model_config: ModelConfig) -> nn.Module:
54
model_class = _get_model_architecture(model_config.hf_config)
45
- torch.set_default_dtype(model_config.dtype)
46
-
47
- # Create a model instance.
48
- # The weights will be initialized as empty tensors.
49
- model = model_class(model_config.hf_config)
50
- if model_config.use_dummy_weights:
51
- model = model.cuda()
52
- # NOTE(woosuk): For accurate performance evaluation, we assign
53
- # random values to the weights.
54
- initialize_dummy_weights(model)
55
- else:
56
- # Load the weights from the cached or downloaded files.
57
- model.load_weights(model_config.model, model_config.download_dir,
58
- model_config.use_np_weights)
59
- model = model.cuda()
55
+ with _set_default_torch_dtype(model_config.dtype):
56
+ # Create a model instance.
57
+ # The weights will be initialized as empty tensors.
58
+ model = model_class(model_config.hf_config)
59
+ if model_config.use_dummy_weights:
60
+ model = model.cuda()
61
+ # NOTE(woosuk): For accurate performance evaluation, we assign
62
+ # random values to the weights.
63
+ initialize_dummy_weights(model)
64
+ else:
65
+ # Load the weights from the cached or downloaded files.
66
+ model.load_weights(model_config.model, model_config.download_dir,
67
+ model_config.use_np_weights)
68
+ model = model.cuda()
69
return model.eval()