A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Bugfix] Fix torch dynamo fixes caused by `replace_parameters` (#8748)
Lucas Wilkinson committed
Sep 24, 2024 at 14:33 UTC
72fc97a0f100b92f1ff6c6a16e27d12f1c7569aa
1 file changed
+8
-4
vllm/model_executor/layers/quantization/utils/layer_utils.py
+8
-4
@@ -21,13 +21,17 @@ def replace_parameter(mod: torch.nn.Module, name: str,
21
new: Union[torch.Tensor, torch.nn.Parameter]) -> None:
22
23
old = getattr(mod, name)
24
- if old.dtype == new.dtype and \
24
+ if type(old) is type(new) and old.dtype == new.dtype and \
25
old.untyped_storage().nbytes() == new.untyped_storage().nbytes():
26
# If we can just update in-place to avoid re-registering
27
# can be faster if the underlying storage is the same
28
update_tensor_inplace(old, new)
29
else:
30
- # Fallback re-register parameter
30
+ # Fallback re-register parameter, convert to Parameter if necessary
31
+ # this not only ensures we don't register a tensor as a parameter, but
32
+ # also ensures that all parameter subclasses get re-registered as
33
+ # parameters for `torch.compile` compatibility
34
if not isinstance(new, torch.nn.Parameter):
32
- new = torch.nn.Parameter(new)
33
- mod.register_parameter(name, torch.nn.Parameter(new))
35
+ new = torch.nn.Parameter(new, requires_grad=False)
36
+ mod.register_parameter(name,
37
+ torch.nn.Parameter(new, requires_grad=False))