@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 21313e09e3
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] Fix default weight loading for scalars (#7534)

Michael Goin committed Aug 15, 2024 at 16:10 UTC 21313e09e3f9448817016290da20d0db1adf3664
1 file changed +11 -5
vllm/model_executor/model_loader/weight_utils.py
+11 -5
@@ -516,11 +516,17 @@ def default_weight_loader(param: torch.Tensor,
516 loaded_weight: torch.Tensor) -> None:
517 """Default weight loader."""
518 try:
519 - assert param.size() == loaded_weight.size(), (
520 - f"Attempted to load weight ({loaded_weight.size()}) "
521 - f"into parameter ({param.size()})")
522 -
523 - param.data.copy_(loaded_weight)
519 + if param.numel() == 1 and loaded_weight.numel() == 1:
520 + # Sometimes scalar values aren't considered tensors with shapes
521 + # so if both param and loaded_weight are a scalar,
522 + # "broadcast" instead of copy
523 + param.data.fill_(loaded_weight.item())
524 + else:
525 + assert param.size() == loaded_weight.size(), (
526 + f"Attempted to load weight ({loaded_weight.size()}) "
527 + f"into parameter ({param.size()})")
528 +
529 + param.data.copy_(loaded_weight)
530 except Exception:
531 # NOTE: This exception is added for the purpose of setting breakpoint to
532 # debug weight loading issues.