A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 09:15 UTC
[Minor] Fix a dtype bug (#79)
Woosuk Kwon committed
May 6, 2023 at 02:12 UTC
c84e924287fbaf923994865806e7ebc93b4070e6
1 file changed
+5
-1
cacheflow/models/model_utils.py
+5
-1
@@ -37,7 +37,11 @@ _MEMORY_ANALYZERS = {
37
38
39
def _get_dtype(config: PretrainedConfig, dtype: str) -> torch.dtype:
40
- config_dtype: torch.dtype = getattr(config, 'torch_dtype', torch.float32)
40
+ # NOTE: getattr(config, 'torch_dtype', torch.float32) is not correct
41
+ # because config.torch_dtype can be None.
42
+ config_dtype = getattr(config, 'torch_dtype', None)
43
+ if config_dtype is None:
44
+ config_dtype = torch.float32
45
if dtype == 'default':
46
if config_dtype == torch.float32:
47
# Following the common practice, we use float16 for float32 models.