@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 005ba458b5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 08:00 UTC

Set torch default dtype in a context manager (#971)

Signed-off-by: Antoni Baum <antoni.baum@protonmail.com>

Antoni Baum committed Sep 6, 2023 at 23:39 UTC 005ba458b52fe2cf9837201d05644eadcdf10ca0
1 file changed +24 -15
vllm/model_executor/model_loader.py
+24 -15
@@ -1,4 +1,5 @@
1 """Utilities for selecting and loading models."""
2 +import contextlib
3 from typing import Type
4
5 import torch
@@ -30,6 +31,15 @@ _MODEL_REGISTRY = {
31 }
32
33
34 +@contextlib.contextmanager
35 +def _set_default_torch_dtype(dtype: torch.dtype):
36 + """Sets the default torch dtype to the given dtype."""
37 + old_dtype = torch.get_default_dtype()
38 + torch.set_default_dtype(dtype)
39 + yield
40 + torch.set_default_dtype(old_dtype)
41 +
42 +
43 def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
44 architectures = getattr(config, "architectures", [])
45 for arch in architectures:
@@ -42,19 +52,18 @@ def _get_model_architecture(config: PretrainedConfig) -> Type[nn.Module]:
52
53 def get_model(model_config: ModelConfig) -> nn.Module:
54 model_class = _get_model_architecture(model_config.hf_config)
45 - torch.set_default_dtype(model_config.dtype)
46 -
47 - # Create a model instance.
48 - # The weights will be initialized as empty tensors.
49 - model = model_class(model_config.hf_config)
50 - if model_config.use_dummy_weights:
51 - model = model.cuda()
52 - # NOTE(woosuk): For accurate performance evaluation, we assign
53 - # random values to the weights.
54 - initialize_dummy_weights(model)
55 - else:
56 - # Load the weights from the cached or downloaded files.
57 - model.load_weights(model_config.model, model_config.download_dir,
58 - model_config.use_np_weights)
59 - model = model.cuda()
55 + with _set_default_torch_dtype(model_config.dtype):
56 + # Create a model instance.
57 + # The weights will be initialized as empty tensors.
58 + model = model_class(model_config.hf_config)
59 + if model_config.use_dummy_weights:
60 + model = model.cuda()
61 + # NOTE(woosuk): For accurate performance evaluation, we assign
62 + # random values to the weights.
63 + initialize_dummy_weights(model)
64 + else:
65 + # Load the weights from the cached or downloaded files.
66 + model.load_weights(model_config.model, model_config.download_dir,
67 + model_config.use_np_weights)
68 + model = model.cuda()
69 return model.eval()