| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # coding=utf-8 |
| 4 | # Copyright 2025 The OpenBMB Team. All rights reserved. |
| 5 | # |
| 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | # you may not use this file except in compliance with the License. |
| 8 | # You may obtain a copy of the License at |
| 9 | # |
| 10 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | # |
| 12 | # Unless required by applicable law or agreed to in writing, software |
| 13 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | # See the License for the specific language governing permissions and |
| 16 | # limitations under the License. |
| 17 | """Processor class for MiniCPMO.""" |
| 18 | |
| 19 | import math |
| 20 | from typing import TypeAlias |
| 21 | |
| 22 | import numpy as np |
| 23 | import regex |
| 24 | import torch |
| 25 | from transformers.image_processing_utils import BatchFeature |
| 26 | from transformers.image_utils import ImageInput |
| 27 | from transformers.processing_utils import ProcessorMixin |
| 28 | from transformers.tokenization_utils_base import PreTokenizedInput, TextInput |
| 29 | from transformers.utils import TensorType |
| 30 | |
| 31 | MiniCPMOBatchFeature: TypeAlias = BatchFeature |
| 32 | |
| 33 | |
| 34 | class MiniCPMOProcessor(ProcessorMixin): |
| 35 | r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image |
| 36 | processor and a MiniCPMV tokenizer into a single processor. |
| 37 | |
| 38 | [`MiniCPMVProcessor`] offers all the functionalities of |
| 39 | [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the |
| 40 | [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] |
| 41 | for more information. |
| 42 | |
| 43 | Args: |
| 44 | image_processor ([`MiniCPMVImageProcessor`], *optional*): |
| 45 | The image processor is a required input. |
| 46 | tokenizer ([`LlamaTokenizerWrapper`], *optional*): |
| 47 | The tokenizer is a required input. |
| 48 | |
| 49 | """ |
| 50 | |
| 51 | attributes = ["image_processor", "feature_extractor", "tokenizer"] |
| 52 | feature_extractor_class = "WhisperFeatureExtractor" |
| 53 | image_processor_class = "AutoImageProcessor" |
| 54 | tokenizer_class = "AutoTokenizer" |
| 55 | |
| 56 | def __init__( |
| 57 | self, |
| 58 | image_processor=None, |
| 59 | feature_extractor=None, |
| 60 | tokenizer=None, |
| 61 | pool_step=2, |
| 62 | ): |
| 63 | super().__init__(image_processor, feature_extractor, tokenizer) |
| 64 | # Mirror the MiniCPMVProcessor guard: newer (transformers v5.7+) |
| 65 | # MiniCPM image processors may drop the legacy `version` attribute, |
| 66 | # so fall back to None instead of hard-crashing. `version` only |
| 67 | # special-cases the 2.5 tokenization path; other values take the |
| 68 | # default branch. |
| 69 | self.version = getattr(image_processor, "version", None) |
| 70 | self.pool_step = pool_step |
| 71 | |
| 72 | def __call__( |
| 73 | self, |
| 74 | text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput], |
| 75 | images: ImageInput = None, |
| 76 | audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]] = None, |
| 77 | audio_parts: list | None = None, |
| 78 | max_length: int | None = None, |
| 79 | do_pad: bool | None = True, |
| 80 | max_slice_nums: int | None = None, |
| 81 | use_image_id: bool = True, |
| 82 | chunk_input: bool = False, |
| 83 | return_tensors: str | TensorType | None = TensorType.PYTORCH, |
| 84 | sampling_rate: int | None = 16000, |
| 85 | **kwargs, |
| 86 | ) -> MiniCPMOBatchFeature: |
| 87 | if images is not None: |
| 88 | image_inputs = self.image_processor( |
| 89 | images, |
| 90 | do_pad=do_pad, |
| 91 | max_slice_nums=max_slice_nums, |
| 92 | return_tensors=return_tensors, |
| 93 | ) |
| 94 | else: |
| 95 | image_inputs = None |
| 96 | |
| 97 | if audios is not None: |
| 98 | audio_features, audio_feature_lens, audio_phs = self.audio_feature_extract( |
| 99 | audios, audio_parts, chunk_input, sampling_rate |
| 100 | ) |
| 101 | else: |
| 102 | audio_features, audio_feature_lens, audio_phs = [], [], [] |
| 103 | |
| 104 | model_inputs = self._convert_omni_to_inputs( |
| 105 | image_inputs, |
| 106 | audio_phs, |
| 107 | text, |
| 108 | max_slice_nums=max_slice_nums, |
| 109 | use_image_id=use_image_id, |
| 110 | max_length=max_length, |
| 111 | **kwargs, |
| 112 | ) |
| 113 | |
| 114 | model_inputs["audio_features"] = audio_features |
| 115 | model_inputs["audio_feature_lens"] = audio_feature_lens |
| 116 | |
| 117 | return MiniCPMOBatchFeature(data={**model_inputs}) |
| 118 | |
| 119 | def get_audio_placeholder(self, audio_lens, chunk_input, chunk_length): |
| 120 | pool_step = self.pool_step |
| 121 | feature_lens = math.ceil(audio_lens / self.feature_extractor.hop_length) |
| 122 | |
| 123 | feature_lens = (feature_lens - 1) // 2 + 1 |
| 124 | output_lens = (feature_lens - pool_step) // pool_step + 1 |
| 125 | |
| 126 | audio_start = getattr(self.tokenizer, "audio_start", "<audio>") |
| 127 | audio_end = getattr(self.tokenizer, "audio_end", "</audio>") |
| 128 | |
| 129 | if chunk_input: |
| 130 | fbank_feat_in_chunk = int(chunk_length * 100) |
| 131 | cnn_feat_in_chunk = (fbank_feat_in_chunk - 1) // 2 + 1 |
| 132 | audio_embeds_in_chunk = (cnn_feat_in_chunk - pool_step) // pool_step + 1 |
| 133 | num_audio_chunks = ( |
| 134 | output_lens + audio_embeds_in_chunk - 1 |
| 135 | ) // audio_embeds_in_chunk |
| 136 | |
| 137 | place_holders = "" |
| 138 | total_unk_len = 0 |
| 139 | for _ in range(num_audio_chunks): |
| 140 | unk_len = min(audio_embeds_in_chunk, output_lens - total_unk_len) |
| 141 | place_holders += audio_start + "<unk>" * unk_len + audio_end |
| 142 | total_unk_len += unk_len |
| 143 | audio_placeholder = place_holders |
| 144 | else: |
| 145 | audio_placeholder = audio_start + "<unk>" * output_lens + audio_end |
| 146 | |
| 147 | return audio_placeholder |
| 148 | |
| 149 | def audio_feature_extract( |
| 150 | self, |
| 151 | audios: np.ndarray | list[np.ndarray] | list[list[np.ndarray]], |
| 152 | audio_parts: list | None = None, |
| 153 | chunk_input: bool | None = False, |
| 154 | sampling_rate: int | None = None, |
| 155 | chunk_length: int | None = 1, |
| 156 | **kwargs, |
| 157 | ): |
| 158 | if isinstance(audios, np.ndarray): |
| 159 | audios_list = [[audios]] |
| 160 | elif isinstance(audios[0], np.ndarray): |
| 161 | audios_list = [audios] |
| 162 | else: |
| 163 | audios_list = audios |
| 164 | |
| 165 | if audio_parts is not None: |
| 166 | assert len(audio_parts) == len(audios_list) |
| 167 | for parts, audios in zip(audio_parts, audios_list): |
| 168 | assert len(parts) == len(audios) |
| 169 | |
| 170 | audio_feature_lens_list = [] |
| 171 | audio_ph_list = [] |
| 172 | |
| 173 | audio_features_all = [] |
| 174 | |
| 175 | # audio placeholder not dependent on audio_parts |
| 176 | for audios in audios_list: |
| 177 | if audios: |
| 178 | audio_ph_list.append( |
| 179 | [ |
| 180 | self.get_audio_placeholder(len(a), chunk_input, chunk_length) |
| 181 | for a in audios |
| 182 | ] |
| 183 | ) |
| 184 | else: |
| 185 | audio_ph_list.append([]) |
| 186 | |
| 187 | for idx, audios in enumerate(audios_list): |
| 188 | if audio_parts is not None: |
| 189 | # same audio part merge |
| 190 | audio_part = audio_parts[idx] |
| 191 | merge_audio = [] |
| 192 | cur_audio = [] |
| 193 | for aid, (part, audio) in enumerate(zip(audio_part, audios)): |
| 194 | if aid == 0 or audio_part[aid] == audio_part[aid - 1]: |
| 195 | cur_audio.append(audio) |
| 196 | else: |
| 197 | merge_audio.append(np.hstack(cur_audio)) |
| 198 | cur_audio = [audio] |
| 199 | if cur_audio: |
| 200 | merge_audio.append(np.hstack(cur_audio)) |
| 201 | |
| 202 | else: |
| 203 | merge_audio = audios |
| 204 | |
| 205 | audio_feature_lens = [] |
| 206 | |
| 207 | # If the audio exceeds 30 seconds, split it into chunks every 30 seconds. |
| 208 | final_merge_audio = [] |
| 209 | max_audio_inp_len = 30 * (sampling_rate or 16000) |
| 210 | for audio in merge_audio: |
| 211 | if len(audio) <= max_audio_inp_len: |
| 212 | final_merge_audio.append(audio) |
| 213 | else: |
| 214 | for i in range(math.ceil(len(audio) / max_audio_inp_len)): |
| 215 | final_merge_audio.append( |
| 216 | audio[i * max_audio_inp_len : (i + 1) * max_audio_inp_len] |
| 217 | ) |
| 218 | |
| 219 | if audios: |
| 220 | audio_inputs = self.feature_extractor( |
| 221 | final_merge_audio, |
| 222 | sampling_rate=sampling_rate, |
| 223 | return_attention_mask=True, |
| 224 | padding="max_length", |
| 225 | return_tensors="pt", |
| 226 | **kwargs, |
| 227 | ) |
| 228 | audio_feature = audio_inputs["input_features"] |
| 229 | actual_lens = audio_inputs["attention_mask"].sum(dim=1) |
| 230 | |
| 231 | for feat, lens in zip(audio_feature, actual_lens): |
| 232 | audio_features_all.append(feat[:, :lens]) |
| 233 | audio_feature_lens.append(lens) |
| 234 | |
| 235 | audio_feature_lens = torch.hstack(audio_feature_lens) |
| 236 | audio_feature_lens_list.append(audio_feature_lens) |
| 237 | else: |
| 238 | audio_feature_lens_list.append([]) |
| 239 | |
| 240 | if audio_features_all: |
| 241 | audio_features = [i.permute(1, 0) for i in audio_features_all] |
| 242 | audio_features = torch.nn.utils.rnn.pad_sequence( |
| 243 | audio_features, batch_first=True, padding_value=0.0 |
| 244 | ).permute(0, 2, 1) |
| 245 | else: |
| 246 | audio_features = [] |
| 247 | |
| 248 | return audio_features, audio_feature_lens_list, audio_ph_list |
| 249 | |
| 250 | # Copied from transformers.models.clip.processing_clip.CLIPProcessor.batch_decode |
| 251 | # with CLIP->Llama |
| 252 | def batch_decode(self, *args, **kwargs): |
| 253 | """This method forwards all its arguments to LlamaTokenizerFast's |
| 254 | [`~PythonBackend.batch_decode`]. Please refer to the |
| 255 | docstring of this method for more information. |
| 256 | """ |
| 257 | output_ids = args[0] |
| 258 | result_text = [] |
| 259 | for result in output_ids: |
| 260 | result = result[result != 0] |
| 261 | if len(result) > 0 and result[0] == self.tokenizer.bos_id: |
| 262 | result = result[1:] |
| 263 | if len(result) > 0 and result[-1] == self.tokenizer.eos_id: |
| 264 | result = result[:-1] |
| 265 | result_text.append( |
| 266 | self.tokenizer.decode(result, *args[1:], **kwargs).strip() |
| 267 | ) |
| 268 | return result_text |
| 269 | |
| 270 | # Copied from transformers.models.clip.processing_clip.CLIPProcessor.decode |
| 271 | # with CLIP->Llama |
| 272 | def decode(self, *args, **kwargs): |
| 273 | """This method forwards all its arguments to LlamaTokenizerFast's |
| 274 | [`~PythonBackend.decode`]. Please refer to the docstring |
| 275 | of this method for more information. |
| 276 | """ |
| 277 | result = args[0] |
| 278 | result = result[result != 0] |
| 279 | if len(result) > 0 and result[0] == self.tokenizer.bos_id: |
| 280 | result = result[1:] |
| 281 | if len(result) > 0 and ( |
| 282 | result[-1] == self.tokenizer.eos_id |
| 283 | or ( |
| 284 | hasattr(self.tokenizer, "eot_id") |
| 285 | and result[-1] == self.tokenizer.eot_id |
| 286 | ) |
| 287 | ): |
| 288 | result = result[:-1] |
| 289 | return self.tokenizer.decode(result, *args[1:], **kwargs).strip() |
| 290 | |
| 291 | def _convert(self, input_str, max_inp_length: int | None = None, **kwargs): |
| 292 | input_ids = self.tokenizer.encode(input_str, **kwargs) |
| 293 | if max_inp_length is not None: |
| 294 | input_ids = input_ids[:max_inp_length] |
| 295 | input_ids = torch.tensor(input_ids, dtype=torch.int32) |
| 296 | |
| 297 | ## image bound |
| 298 | start_cond = (input_ids == self.tokenizer.im_start_id) | ( |
| 299 | input_ids == self.tokenizer.slice_start_id |
| 300 | ) |
| 301 | end_cond = (input_ids == self.tokenizer.im_end_id) | ( |
| 302 | input_ids == self.tokenizer.slice_end_id |
| 303 | ) |
| 304 | |
| 305 | image_start_idx = torch.where(start_cond)[0] |
| 306 | image_start_idx += 1 |
| 307 | image_end_idx = torch.where(end_cond)[0] |
| 308 | |
| 309 | assert len(image_start_idx) == len(image_end_idx), ( |
| 310 | f"The number of image start tokens ({len(image_start_idx)}) " |
| 311 | f"and end tokens ({len(image_end_idx)}) must match." |
| 312 | ) |
| 313 | |
| 314 | image_bounds = torch.hstack( |
| 315 | [ |
| 316 | image_start_idx.unsqueeze(-1), |
| 317 | image_end_idx.unsqueeze(-1), |
| 318 | ] |
| 319 | ) |
| 320 | |
| 321 | ## audio bound |
| 322 | audio_start_idx = torch.where(input_ids == self.tokenizer.audio_start_id)[0] |
| 323 | audio_end_idx = torch.where(input_ids == self.tokenizer.audio_end_id)[0] |
| 324 | assert len(audio_start_idx) == len(audio_end_idx) |
| 325 | audio_bounds = torch.hstack( |
| 326 | [(audio_start_idx + 1).unsqueeze(-1), audio_end_idx.unsqueeze(-1)] |
| 327 | ) |
| 328 | |
| 329 | spk_start_idx = torch.where(input_ids == self.tokenizer.spk_start_id)[0] |
| 330 | spk_end_idx = torch.where(input_ids == self.tokenizer.spk_end_id)[0] |
| 331 | assert len(spk_start_idx) == len(spk_end_idx) |
| 332 | spk_bounds = torch.hstack( |
| 333 | [(spk_start_idx + 1).unsqueeze(-1), spk_end_idx.unsqueeze(-1)] |
| 334 | ) |
| 335 | |
| 336 | return input_ids, image_bounds, audio_bounds, spk_bounds |
| 337 | |
| 338 | def _convert_omni_to_inputs( |
| 339 | self, |
| 340 | images, |
| 341 | audio_phs, |
| 342 | texts: str | list[str], |
| 343 | truncation=None, |
| 344 | max_length=None, |
| 345 | max_slice_nums=None, |
| 346 | use_image_id=None, |
| 347 | return_tensors=None, |
| 348 | **kwargs, |
| 349 | ): |
| 350 | if images is None and audio_phs is None: |
| 351 | model_inputs = self.tokenizer( |
| 352 | texts, |
| 353 | return_tensors=return_tensors, |
| 354 | truncation=truncation, |
| 355 | max_length=max_length, |
| 356 | **kwargs, |
| 357 | ) |
| 358 | return MiniCPMOBatchFeature(data={**model_inputs}) |
| 359 | |
| 360 | image_tag = "(<image>./</image>)" |
| 361 | image_pattern = r"\(<image>./</image>\)" |
| 362 | audio_tag = "(<audio>./</audio>)" |
| 363 | audio_pattern = r"\(<audio>./</audio>\)" |
| 364 | split_pattern = rf"({image_pattern}|{audio_pattern})" |
| 365 | |
| 366 | if isinstance(texts, str): |
| 367 | texts = [texts] |
| 368 | |
| 369 | bs = len(texts) |
| 370 | if images is not None: |
| 371 | images, image_sizes, tgt_sizes = ( |
| 372 | images["pixel_values"], |
| 373 | images["image_sizes"], |
| 374 | images["tgt_sizes"], |
| 375 | ) |
| 376 | else: |
| 377 | images, image_sizes, tgt_sizes = [[]] * bs, [[]] * bs, [[]] * bs |
| 378 | |
| 379 | input_ids_list = [] |
| 380 | image_bounds_list = [] |
| 381 | audio_bounds_list = [] |
| 382 | spk_bounds_list = [] |
| 383 | |
| 384 | for index, text in enumerate(texts): |
| 385 | text_chunks = regex.split(split_pattern, text) |
| 386 | |
| 387 | image_tags = regex.findall(image_pattern, text) |
| 388 | audio_tags = regex.findall(audio_pattern, text) |
| 389 | |
| 390 | if image_tags: |
| 391 | assert images is not None |
| 392 | assert len(image_tags) == len(image_sizes[index]) |
| 393 | if audio_tags: |
| 394 | assert audio_phs is not None |
| 395 | assert len(audio_tags) == len(audio_phs[index]) |
| 396 | |
| 397 | image_id = 0 |
| 398 | audio_id = 0 |
| 399 | for i, chunk in enumerate(text_chunks): |
| 400 | if chunk == image_tag: |
| 401 | image_placeholder = ( |
| 402 | self.image_processor.get_slice_image_placeholder( |
| 403 | image_sizes[index][image_id], |
| 404 | image_id, |
| 405 | max_slice_nums, |
| 406 | use_image_id, |
| 407 | ) |
| 408 | ) |
| 409 | image_id += 1 |
| 410 | text_chunks[i] = image_placeholder |
| 411 | elif chunk == audio_tag: |
| 412 | audio_placeholder = audio_phs[index][audio_id] |
| 413 | audio_id += 1 |
| 414 | text_chunks[i] = audio_placeholder |
| 415 | |
| 416 | final_text = "".join(text_chunks) |
| 417 | input_ids, image_bounds, audio_bounds, spk_bounds = self._convert( |
| 418 | final_text, max_length, **kwargs |
| 419 | ) |
| 420 | |
| 421 | input_ids_list.append(input_ids) |
| 422 | image_bounds_list.append(image_bounds) |
| 423 | audio_bounds_list.append(audio_bounds) |
| 424 | spk_bounds_list.append(spk_bounds) |
| 425 | |
| 426 | padded_input_ids, padding_lengths = self.pad( |
| 427 | input_ids_list, padding_side="left" |
| 428 | ) |
| 429 | attention_mask = torch.ones_like(padded_input_ids, dtype=torch.bool) |
| 430 | for i, length in enumerate(padding_lengths): |
| 431 | image_bounds_list[i] = image_bounds_list[i] + length |
| 432 | audio_bounds_list[i] = audio_bounds_list[i] + length |
| 433 | spk_bounds_list[i] = spk_bounds_list[i] + length |
| 434 | attention_mask[i, :length] = False |
| 435 | |
| 436 | data = { |
| 437 | "input_ids": padded_input_ids, |
| 438 | "attention_mask": attention_mask, |
| 439 | "pixel_values": images, |
| 440 | "image_sizes": image_sizes, |
| 441 | "image_bound": image_bounds_list, |
| 442 | "tgt_sizes": tgt_sizes, |
| 443 | "audio_bounds": audio_bounds_list, |
| 444 | "spk_bounds": spk_bounds_list, |
| 445 | } |
| 446 | |
| 447 | return data |
| 448 | |
| 449 | @property |
| 450 | # Copied from |
| 451 | # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names |
| 452 | def model_input_names(self): |
| 453 | tokenizer_input_names = self.tokenizer.model_input_names |
| 454 | image_processor_input_names = self.image_processor.model_input_names |
| 455 | feature_extractor_input_names = self.feature_extractor.model_input_names |
| 456 | return list( |
| 457 | dict.fromkeys( |
| 458 | tokenizer_input_names |
| 459 | + image_processor_input_names |
| 460 | + feature_extractor_input_names |
| 461 | ) |
| 462 | ) |
| 463 | |
| 464 | def pad( |
| 465 | self, |
| 466 | inputs, |
| 467 | max_length=None, |
| 468 | padding_value=0, |
| 469 | padding_side="left", |
| 470 | ): |
| 471 | if not inputs: |
| 472 | return torch.empty(0), [] |
| 473 | |
| 474 | items = [] |
| 475 | if isinstance(inputs[0], list): |
| 476 | assert isinstance(inputs[0][0], torch.Tensor) |
| 477 | for it in inputs: |
| 478 | for tr in it: |
| 479 | items.append(tr) |
| 480 | else: |
| 481 | assert isinstance(inputs[0], torch.Tensor) |
| 482 | items = inputs |
| 483 | |
| 484 | batch_size = len(items) |
| 485 | shape = items[0].shape |
| 486 | dim = len(shape) |
| 487 | assert dim <= 2 |
| 488 | if max_length is None: |
| 489 | max_length = 0 |
| 490 | max_length = max(max_length, max(item.shape[-1] for item in items)) |
| 491 | min_length = min(item.shape[-1] for item in items) |
| 492 | dtype = items[0].dtype |
| 493 | |
| 494 | if dim == 0: |
| 495 | return torch.stack([item for item in items], dim=0), [0] |
| 496 | elif dim == 1: |
| 497 | if max_length == min_length: |
| 498 | return ( |
| 499 | torch.stack([item for item in items], dim=0), |
| 500 | [0] * batch_size, |
| 501 | ) |
| 502 | tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value |
| 503 | else: |
| 504 | tensor = ( |
| 505 | torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) |
| 506 | + padding_value |
| 507 | ) |
| 508 | |
| 509 | padding_length = [] |
| 510 | for i, item in enumerate(items): |
| 511 | if dim == 1: |
| 512 | if padding_side == "left": |
| 513 | tensor[i, -len(item) :] = item.clone() |
| 514 | else: |
| 515 | tensor[i, : len(item)] = item.clone() |
| 516 | elif dim == 2: |
| 517 | if padding_side == "left": |
| 518 | tensor[i, -len(item) :, :] = item.clone() |
| 519 | else: |
| 520 | tensor[i, : len(item), :] = item.clone() |
| 521 | padding_length.append(tensor.shape[-1] - len(item)) |
| 522 | |
| 523 | return tensor, padding_length |