@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 72fc97a0f1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Bugfix] Fix torch dynamo fixes caused by `replace_parameters` (#8748)

Lucas Wilkinson committed Sep 24, 2024 at 14:33 UTC 72fc97a0f100b92f1ff6c6a16e27d12f1c7569aa
1 file changed +8 -4
vllm/model_executor/layers/quantization/utils/layer_utils.py
+8 -4
@@ -21,13 +21,17 @@ def replace_parameter(mod: torch.nn.Module, name: str,
21 new: Union[torch.Tensor, torch.nn.Parameter]) -> None:
22
23 old = getattr(mod, name)
24 - if old.dtype == new.dtype and \
24 + if type(old) is type(new) and old.dtype == new.dtype and \
25 old.untyped_storage().nbytes() == new.untyped_storage().nbytes():
26 # If we can just update in-place to avoid re-registering
27 # can be faster if the underlying storage is the same
28 update_tensor_inplace(old, new)
29 else:
30 - # Fallback re-register parameter
30 + # Fallback re-register parameter, convert to Parameter if necessary
31 + # this not only ensures we don't register a tensor as a parameter, but
32 + # also ensures that all parameter subclasses get re-registered as
33 + # parameters for `torch.compile` compatibility
34 if not isinstance(new, torch.nn.Parameter):
32 - new = torch.nn.Parameter(new)
33 - mod.register_parameter(name, torch.nn.Parameter(new))
35 + new = torch.nn.Parameter(new, requires_grad=False)
36 + mod.register_parameter(name,
37 + torch.nn.Parameter(new, requires_grad=False))