@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c84e924287
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 09:15 UTC

[Minor] Fix a dtype bug (#79)

Woosuk Kwon committed May 6, 2023 at 02:12 UTC c84e924287fbaf923994865806e7ebc93b4070e6
1 file changed +5 -1
cacheflow/models/model_utils.py
+5 -1
@@ -37,7 +37,11 @@ _MEMORY_ANALYZERS = {
37
38
39 def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
40 - config_dtype: torch.dtype = getattr(config, 'torch_dtype', torch.float32)
40 + # NOTE: getattr(config, 'torch_dtype', torch.float32) is not correct
41 + # because config.torch_dtype can be None.
42 + config_dtype = getattr(config, 'torch_dtype', None)
43 + if config_dtype is None:
44 + config_dtype = torch.float32
45 if dtype == 'default':
46 if config_dtype == torch.float32:
47 # Following the common practice, we use float16 for float32 models.