@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / be0c5180ac
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Bugfix] Add logs for all model dtype casting (#4717)

Michael Goin committed May 9, 2024 at 14:36 UTC be0c5180ac0832a0b285d0845d458798bb3f0f4f
1 file changed +3
vllm/config.py
+3
@@ -1063,6 +1063,7 @@ def _get_and_verify_dtype(
1063 if config_dtype == torch.float32:
1064 # Following the common practice, we use float16 for float32
1065 # models.
1066 + logger.info("Casting torch.float32 to torch.float16.")
1067 torch_dtype = torch.float16
1068 else:
1069 torch_dtype = config_dtype
@@ -1087,9 +1088,11 @@ def _get_and_verify_dtype(
1088 if torch_dtype != config_dtype:
1089 if torch_dtype == torch.float32:
1090 # Upcasting to float32 is allowed.
1091 + logger.info("Upcasting %s to %s.", config_dtype, torch_dtype)
1092 pass
1093 elif config_dtype == torch.float32:
1094 # Downcasting from float32 to float16 or bfloat16 is allowed.
1095 + logger.info("Downcasting %s to %s.", config_dtype, torch_dtype)
1096 pass
1097 else:
1098 # Casting between float16 and bfloat16 is allowed with a warning.