[Bugfix] Fix OpenPangu multimodal embedding merge (#55941)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Taneem Ibrahim committed
Sep 8, 2026 at 18:53 UTC
5af4cc33ec0837ded8a18b368ec58a9fd4e4d5cd
2 files changed
+68
-10
tests/models/multimodal/test_openpangu_vl.py
new
+55
@@ -0,0 +1,55 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+from types import SimpleNamespace
5
+
6
+import pytest
7
+import torch
8
+from torch import nn
9
+
10
+from vllm.model_executor.models.openpangu_vl import (
11
+ OpenPanguVLForConditionalGeneration,
12
+)
13
+
14
+
15
+class _LanguageModel(nn.Module):
16
+ def __init__(self, hidden_size: int) -> None:
17
+ super().__init__()
18
+ self.hidden_size = hidden_size
19
+
20
+ def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
21
+ return input_ids.float().unsqueeze(-1).expand(-1, self.hidden_size).clone()
22
+
23
+
24
+class _OpenPanguVL(OpenPanguVLForConditionalGeneration):
25
+ def __init__(self, hidden_size: int) -> None:
26
+ nn.Module.__init__(self)
27
+ self.config = SimpleNamespace(image_token_id=41, video_token_id=42)
28
+ self.language_model = _LanguageModel(hidden_size)
29
+
30
+ def get_language_model(self) -> _LanguageModel:
31
+ return self.language_model
32
+
33
+
34
+@pytest.mark.parametrize("device_type", ["cpu", "cuda"])
35
+def test_get_input_embeddings_merges_image_and_video_embeddings(
36
+ device_type: str,
37
+) -> None:
38
+ if device_type == "cuda" and not torch.cuda.is_available():
39
+ pytest.skip("CUDA is not available")
40
+
41
+ device = torch.device(device_type)
42
+ hidden_size = 4
43
+ model = _OpenPanguVL(hidden_size).to(device)
44
+ input_ids = torch.tensor([7, 41, 41, 8, 42, 9], device=device)
45
+ multimodal_embeddings = (
46
+ torch.full((2, hidden_size), -1.0, device=device),
47
+ torch.full((1, hidden_size), -2.0, device=device),
48
+ )
49
+
50
+ actual = model.get_input_embeddings(input_ids, multimodal_embeddings)
51
+
52
+ expected = model.language_model.embed_input_ids(input_ids)
53
+ expected[1:3] = -1.0
54
+ expected[4] = -2.0
55
+ torch.testing.assert_close(actual, expected)
vllm/model_executor/models/openpangu_vl.py
+13
-10
@@ -1039,17 +1039,20 @@ class OpenPanguVLForConditionalGeneration(
1039
def get_input_embeddings(
1040
self,
1041
input_ids: torch.Tensor,
1042
- multimodal_embeddings=None,
1042
+ multimodal_embeddings: MultiModalEmbeddings | None = None,
1043
) -> torch.Tensor:
1044
- inputs_embeds = self.language_model.embed_input_ids(input_ids)
1045
- if multimodal_embeddings is not None:
1046
- inputs_embeds = self.embed_input_ids(
1047
- input_ids,
1048
- inputs_embeds,
1049
- multimodal_embeddings,
1050
- [self.config.image_token_id, self.config.video_token_id],
1051
- )
1052
- return inputs_embeds
1044
+ if multimodal_embeddings is None:
1045
+ return self.language_model.embed_input_ids(input_ids)
1046
+
1047
+ mm_token_ids = input_ids.new_tensor(
1048
+ [self.config.image_token_id, self.config.video_token_id]
1049
+ )
1050
+ is_multimodal = torch.isin(input_ids, mm_token_ids)
1051
+ return self.embed_input_ids(
1052
+ input_ids,
1053
+ multimodal_embeddings,
1054
+ is_multimodal=is_multimodal,
1055
+ )
1056
1057
def _process_image_input(self, image_input) -> tuple[torch.Tensor, ...]:
1058
grid_thw = image_input["image_grid_thw"]