A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 12, 13:30 UTC
Revert back to torch.equal over torch.allclose from #28819 (#29086)
Signed-off-by: Eldar Kurtic <8884008+eldarkurtic@users.noreply.github.com>
Eldar Kurtić committed
Nov 25, 2025 at 15:23 UTC
0231ce836a27ad0ed722e4b7162821baeeb19fdf
1 file changed
+7
-4
vllm/v1/spec_decode/eagle.py
+7
-4
@@ -1055,11 +1055,11 @@ class EagleProposer:
1055
elif (
1056
isinstance(target_embed_tokens.weight, torch.Tensor)
1057
and isinstance(self.model.model.embed_tokens.weight, torch.Tensor)
1058
- and torch.allclose(
1058
+ # TODO: Offload to CPU for comparison to avoid extra GPU memory
1059
+ # usage in CI testing environments with limited GPU memory
1060
+ and torch.equal(
1061
target_embed_tokens.weight.cpu(),
1062
self.model.model.embed_tokens.weight.cpu(),
1061
- rtol=1e-5,
1062
- atol=1e-7,
1063
)
1064
):
1065
share_embeddings = True
@@ -1105,8 +1105,11 @@ class EagleProposer:
1105
hasattr(target_language_model, "lm_head")
1106
and isinstance(target_language_model.lm_head.weight, torch.Tensor)
1107
and isinstance(self.model.lm_head.weight, torch.Tensor)
1108
+ # TODO: Offload to CPU for comparison to avoid extra GPU memory
1109
+ # usage in CI testing environments with limited GPU memory
1110
and torch.equal(
1109
- target_language_model.lm_head.weight, self.model.lm_head.weight
1111
+ target_language_model.lm_head.weight.cpu(),
1112
+ self.model.lm_head.weight.cpu(),
1113
)
1114
):
1115
share_lm_head = True