A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 11:00 UTC
fix(test): recompute Jina ColBERT rotary inv_freq cleared by transformers v5 weight loader (#39176)
Signed-off-by: Ilya Boytsov <ilyaboytsov1805@gmail.com>
Ilya Boytsov committed
Apr 7, 2026 at 16:40 UTC
6e1100889e6a675d17ad82815acf8f02f1cc419e
1 file changed
+8
tests/models/language/pooling/test_colbert.py
+8
@@ -109,6 +109,14 @@ def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
109
**extra,
110
).to(device)
111
model.eval()
112
+
113
+ # Transformers 5.0 weight materialization can clear non-persistent
114
+ # buffers (e.g. rotary inv_freq) that were registered with
115
+ # persistent=False. Re-compute them so the model produces valid output.
116
+ for mod in model.modules():
117
+ if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
118
+ mod.inv_freq = mod._compute_inv_freq(device=device)
119
+
120
return model
121
122