A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix] Prioritize dtype in root config before checking text config (#17629)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
May 4, 2025 at 20:43 UTC
2858830c39da0ae153bc1328dbba7680f5fbebe1
1 file changed
+4
-2
vllm/config.py
+4
-2
@@ -2954,10 +2954,12 @@ def _get_and_verify_dtype(
2954
) -> torch.dtype:
2955
# NOTE: getattr(config, "torch_dtype", torch.float32) is not correct
2956
# because config.torch_dtype can be None.
2957
- config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
2957
+ config_dtype = getattr(config, "torch_dtype", None)
2958
2959
- # Fallback for multi-modal models if the root config
2959
+ # Fallbacks for multi-modal models if the root config
2960
# does not define torch_dtype
2961
+ if config_dtype is None:
2962
+ config_dtype = getattr(config.get_text_config(), "torch_dtype", None)
2963
if config_dtype is None and hasattr(config, "vision_config"):
2964
config_dtype = getattr(config.vision_config, "torch_dtype", None)
2965