A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Bugfix] Add logs for all model dtype casting (#4717)
Michael Goin committed
May 9, 2024 at 14:36 UTC
be0c5180ac0832a0b285d0845d458798bb3f0f4f
1 file changed
+3
vllm/config.py
+3
@@ -1063,6 +1063,7 @@ def _get_and_verify_dtype(
1063
if config_dtype == torch.float32:
1064
# Following the common practice, we use float16 for float32
1065
# models.
1066
+ logger.info("Casting torch.float32 to torch.float16.")
1067
torch_dtype = torch.float16
1068
else:
1069
torch_dtype = config_dtype
@@ -1087,9 +1088,11 @@ def _get_and_verify_dtype(
1088
if torch_dtype != config_dtype:
1089
if torch_dtype == torch.float32:
1090
# Upcasting to float32 is allowed.
1091
+ logger.info("Upcasting %s to %s.", config_dtype, torch_dtype)
1092
pass
1093
elif config_dtype == torch.float32:
1094
# Downcasting from float32 to float16 or bfloat16 is allowed.
1095
+ logger.info("Downcasting %s to %s.", config_dtype, torch_dtype)
1096
pass
1097
else:
1098
# Casting between float16 and bfloat16 is allowed with a warning.