[Bugfix][multi_modal] Fix pos_ids being unitialized for minicpmv2.6 in hf runner (#51432)
Signed-off-by: Dino Music <Dino.Music@amd.com>
music-dino committed
Aug 8, 2026 at 06:09 UTC
a828536fb53aedc76216eb0c0289d7417e3851f7
1 file changed
+18
tests/models/multimodal/generation/vlm_utils/model_utils.py
+18
@@ -922,6 +922,22 @@ def _internvl_generate(
922
return outputs
923
924
925
+def _restore_resampler_pos_cache(hf_model: HfRunner) -> None:
926
+ """Recompute the resampler's 2D sin/cos position cache after loading.
927
+
928
+ `from_pretrained` materializes non-persistent buffers as uninitialized
929
+ memory, leaving `pos_embed` as zeros or NaN instead of the values computed
930
+ in `__init__`.
931
+ """
932
+ restored = 0
933
+ for module in hf_model.model.modules():
934
+ if not hasattr(module, "_set_2d_pos_cache"):
935
+ continue
936
+ module._set_2d_pos_cache(module.max_size, module.pos_embed.device)
937
+ restored += 1
938
+ assert restored, "no resampler pos cache found to restore"
939
+
940
+
941
def minicpmv_25_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
942
orig_generate = hf_model.model.generate
943
@@ -965,6 +981,8 @@ def minicpmo_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
981
982
983
def minicpmv_26_patch_hf_runner(hf_model: HfRunner) -> HfRunner:
984
+ _restore_resampler_pos_cache(hf_model)
985
+
986
orig_generate = hf_model.model.generate
987
988
def _generate(self, *args, image_sizes=None, **kwargs):