A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 22:45 UTC
[Model][Ernie4.5-VL] Support video metadata for timestamp rendering (#31274)
Signed-off-by: dengsonghe <dengsonghe@baidu.com> Co-authored-by: dengsonghe <dengsonghe@baidu.com>
SongHe committed
Dec 25, 2025 at 22:07 UTC
2d6001f491bc29223dd3f8150688daeaded275ee
2 files changed
+82
-5
tests/models/multimodal/processing/test_common.py
+2
-1
@@ -104,7 +104,8 @@ _IGNORE_MM_KEYS = {
104
}
105
106
MM_DATA_PATCHES = {
107
- # GLM4.1V and Qwen3-VL requires video metadata to be included in the input
107
+ # Ernie4.5-VL, GLM4.1V and Qwen3-VL requires video metadata
108
+ "ernie4_5_moe_vl": qwen3_vl_patch_mm_data,
109
"glm4v": glm4_1v_patch_mm_data,
110
"glm4v_moe": glm4_1v_patch_mm_data,
111
"qwen3_vl": qwen3_vl_patch_mm_data,
vllm/model_executor/models/ernie45_vl.py
+80
-4
@@ -21,7 +21,7 @@
21
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
# See the License for the specific language governing permissions and
23
# limitations under the License.
24
-"""Inference-only Erine VL model compatible with HuggingFace weights."""
24
+"""Inference-only Ernie VL model compatible with HuggingFace weights."""
25
26
import itertools
27
import math
@@ -41,7 +41,7 @@ from vllm.attention.layers.mm_encoder_attention import (
41
MMEncoderAttention,
42
)
43
from vllm.config import MultiModalConfig, VllmConfig
44
-from vllm.config.multimodal import BaseDummyOptions
44
+from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
45
from vllm.distributed import parallel_state
46
from vllm.distributed import utils as dist_utils
47
from vllm.logger import init_logger
@@ -64,7 +64,7 @@ from vllm.multimodal.inputs import (
64
MultiModalFieldConfig,
65
MultiModalKwargsItems,
66
)
67
-from vllm.multimodal.parse import ImageSize, MultiModalDataItems
67
+from vllm.multimodal.parse import ImageSize, MultiModalDataItems, MultiModalDataParser
68
from vllm.multimodal.processing import (
69
BaseMultiModalProcessor,
70
BaseProcessingInfo,
@@ -952,6 +952,11 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
952
953
954
class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessingInfo]):
955
+ def _get_data_parser(self) -> MultiModalDataParser:
956
+ return MultiModalDataParser(
957
+ video_needs_metadata=True,
958
+ )
959
+
960
def _pixel_values_norm(
961
self,
962
pixel_values: torch.Tensor,
@@ -1010,8 +1015,25 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin
1015
mm_data["images"] = []
1016
if "videos" not in mm_data:
1017
mm_data["videos"] = []
1018
+
1019
+ # Check if HF processor supports video metadata
1020
+ hf_processor = self.info.get_hf_processor(**mm_kwargs)
1021
+ supports_video_metadata = getattr(
1022
+ hf_processor, "supports_video_metadata", False
1023
+ )
1024
+
1025
+ if mm_data["videos"] and not supports_video_metadata:
1026
+ # Old HF processor, unwrap tuple to pure frames
1027
+ logger.warning_once(
1028
+ "HF processor doesn't support video metadata. "
1029
+ "Timestamps will NOT be rendered. Please upgrade the model."
1030
+ )
1031
+ mm_data["videos"] = [
1032
+ v[0] if isinstance(v, tuple) else v for v in mm_data["videos"]
1033
+ ]
1034
+
1035
processor_output = self.info.ctx.call_hf_processor(
1014
- self.info.get_hf_processor(**mm_kwargs),
1036
+ hf_processor,
1037
dict(text=[prompt], images=mm_data["images"], videos=mm_data["videos"]),
1038
dict(**mm_kwargs, **tok_kwargs),
1039
)
@@ -1163,6 +1185,60 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
1185
),
1186
}
1187
1188
+ def _get_dummy_videos(
1189
+ self,
1190
+ *,
1191
+ width: int,
1192
+ height: int,
1193
+ num_frames: int,
1194
+ num_videos: int,
1195
+ overrides: VideoDummyOptions | None = None,
1196
+ ):
1197
+ if overrides:
1198
+ if overrides.num_frames:
1199
+ if overrides.num_frames > num_frames:
1200
+ logger.warning(
1201
+ "video.num_frames override (%d) exceeds model's "
1202
+ "maximum number of frames (%d), will be ignored",
1203
+ overrides.num_frames,
1204
+ num_frames,
1205
+ )
1206
+ num_frames = min(num_frames, overrides.num_frames)
1207
+ if overrides.width:
1208
+ if overrides.width > width:
1209
+ logger.warning(
1210
+ "video.width override (%d) exceeds model's "
1211
+ "maximum width (%d), will be ignored",
1212
+ overrides.width,
1213
+ width,
1214
+ )
1215
+ width = min(width, overrides.width)
1216
+ if overrides.height:
1217
+ if overrides.height > height:
1218
+ logger.warning(
1219
+ "video.height override (%d) exceeds model's "
1220
+ "maximum height (%d), will be ignored",
1221
+ overrides.height,
1222
+ height,
1223
+ )
1224
+ height = min(height, overrides.height)
1225
+ num_frames = max(num_frames, 2) # ernie4.5-vl requires at least 2 frames
1226
+
1227
+ video = np.full((num_frames, width, height, 3), 255, dtype=np.uint8)
1228
+ video_items = []
1229
+ for i in range(num_videos):
1230
+ video_metadata = {
1231
+ "fps": 2.0,
1232
+ "duration": num_frames / 2.0,
1233
+ "total_num_frames": num_frames,
1234
+ "frames_indices": [i for i in range(num_frames)],
1235
+ "video_backend": "opencv",
1236
+ "do_sample_frames": False,
1237
+ }
1238
+ video_item = (video.copy(), video_metadata)
1239
+ video_items.append(video_item)
1240
+ return video_items
1241
+
1242
1243
@MULTIMODAL_REGISTRY.register_processor(
1244
Ernie4_5VLMultiModalProcessor,