A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] Fix default weight loading for scalars (#7534)
Michael Goin committed
Aug 15, 2024 at 16:10 UTC
21313e09e3f9448817016290da20d0db1adf3664
1 file changed
+11
-5
vllm/model_executor/model_loader/weight_utils.py
+11
-5
@@ -516,11 +516,17 @@ def default_weight_loader(param: torch.Tensor,
516
loaded_weight: torch.Tensor) -> None:
517
"""Default weight loader."""
518
try:
519
- assert param.size() == loaded_weight.size(), (
520
- f"Attempted to load weight ({loaded_weight.size()}) "
521
- f"into parameter ({param.size()})")
522
-
523
- param.data.copy_(loaded_weight)
519
+ if param.numel() == 1 and loaded_weight.numel() == 1:
520
+ # Sometimes scalar values aren't considered tensors with shapes
521
+ # so if both param and loaded_weight are a scalar,
522
+ # "broadcast" instead of copy
523
+ param.data.fill_(loaded_weight.item())
524
+ else:
525
+ assert param.size() == loaded_weight.size(), (
526
+ f"Attempted to load weight ({loaded_weight.size()}) "
527
+ f"into parameter ({param.size()})")
528
+
529
+ param.data.copy_(loaded_weight)
530
except Exception:
531
# NOTE: This exception is added for the purpose of setting breakpoint to
532
# debug weight loading issues.