| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # Copyright 2024 The HuggingFace Inc. team. |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
| 16 | """Processor class for MiniCPMV.""" |
| 17 | |
| 18 | from typing import TypeAlias |
| 19 | |
| 20 | import regex |
| 21 | import torch |
| 22 | from transformers.image_processing_utils import BatchFeature |
| 23 | from transformers.image_utils import ImageInput |
| 24 | from transformers.processing_utils import ProcessorMixin |
| 25 | from transformers.tokenization_utils_base import ( |
| 26 | PaddingStrategy, |
| 27 | PreTokenizedInput, |
| 28 | TextInput, |
| 29 | TruncationStrategy, |
| 30 | ) |
| 31 | from transformers.utils import TensorType |
| 32 | |
| 33 | MiniCPMVBatchFeature: TypeAlias = BatchFeature |
| 34 | |
| 35 | |
| 36 | class MiniCPMVProcessor(ProcessorMixin): |
| 37 | r"""Constructs a MiniCPMV processor which wraps a MiniCPMV image |
| 38 | processor and a MiniCPMV tokenizer into a single processor. |
| 39 | |
| 40 | [`MiniCPMVProcessor`] offers all the functionalities of |
| 41 | [`MiniCPMVImageProcessor`] and [`LlamaTokenizerWrapper`]. See the |
| 42 | [`~MiniCPMVProcessor.__call__`] and [`~MiniCPMVProcessor.decode`] |
| 43 | for more information. |
| 44 | |
| 45 | Args: |
| 46 | image_processor ([`MiniCPMVImageProcessor`], *optional*): |
| 47 | The image processor is a required input. |
| 48 | tokenizer ([`LlamaTokenizerWrapper`], *optional*): |
| 49 | The tokenizer is a required input. |
| 50 | |
| 51 | """ |
| 52 | |
| 53 | attributes = ["image_processor", "tokenizer"] |
| 54 | image_processor_class = "AutoImageProcessor" |
| 55 | tokenizer_class = "AutoTokenizer" |
| 56 | |
| 57 | def __init__(self, image_processor=None, tokenizer=None, version=None): |
| 58 | super().__init__(image_processor, tokenizer) |
| 59 | self.version = version |
| 60 | |
| 61 | def __call__( |
| 62 | self, |
| 63 | text: TextInput | PreTokenizedInput | list[TextInput] | list[PreTokenizedInput], |
| 64 | images: ImageInput = None, |
| 65 | padding: bool | str | PaddingStrategy = False, |
| 66 | truncation: bool | str | TruncationStrategy = None, |
| 67 | max_length: int | None = None, |
| 68 | do_pad: bool | None = True, |
| 69 | return_tensors: str | TensorType | None = TensorType.PYTORCH, |
| 70 | ) -> MiniCPMVBatchFeature: |
| 71 | """Run the vendored MiniCPMV processor on a (text, images) pair. |
| 72 | |
| 73 | Batched inputs are supported following the upstream MiniCPM-V |
| 74 | processor flow. ``images`` is forwarded to the underlying image |
| 75 | processor and ``text`` is tokenized with image placeholders |
| 76 | replaced by the appropriate slice tokens. Returns a |
| 77 | ``MiniCPMVBatchFeature`` with at minimum ``input_ids`` and (when |
| 78 | images are provided) ``pixel_values``, ``image_sizes``, |
| 79 | ``image_bound`` and ``tgt_sizes``. |
| 80 | """ |
| 81 | if images is not None: |
| 82 | image_inputs = self.image_processor( |
| 83 | images, do_pad=do_pad, return_tensors=return_tensors |
| 84 | ) |
| 85 | else: |
| 86 | image_inputs = {} |
| 87 | return self._convert_images_texts_to_inputs( |
| 88 | image_inputs, text, max_length=max_length |
| 89 | ) |
| 90 | |
| 91 | # Copied from transformers.models.clip.processing_clip.CLIPProcessor |
| 92 | # .batch_decode with CLIP->Llama |
| 93 | def batch_decode(self, *args, **kwargs): |
| 94 | """This method forwards all its arguments to LlamaTokenizerFast's |
| 95 | [`~PythonBackend.batch_decode`]. Please refer to the |
| 96 | docstring of this method for more information. |
| 97 | """ |
| 98 | output_ids = args[0] |
| 99 | result_text = [] |
| 100 | |
| 101 | bos_id = getattr( |
| 102 | self.tokenizer, |
| 103 | "bos_token_id", |
| 104 | getattr(self.tokenizer, "bos_id", 1), |
| 105 | ) |
| 106 | eos_id = getattr( |
| 107 | self.tokenizer, |
| 108 | "eos_token_id", |
| 109 | getattr(self.tokenizer, "eos_id", 2), |
| 110 | ) |
| 111 | |
| 112 | for result in output_ids: |
| 113 | result = result[result != 0] |
| 114 | if len(result) > 0 and result[0] == bos_id: |
| 115 | result = result[1:] |
| 116 | if len(result) > 0 and result[-1] == eos_id: |
| 117 | result = result[:-1] |
| 118 | result_text.append( |
| 119 | self.tokenizer.decode(result, *args[1:], **kwargs).strip() |
| 120 | ) |
| 121 | return result_text |
| 122 | |
| 123 | # Copied from transformers.models.clip.processing_clip.CLIPProcessor |
| 124 | # .decode with CLIP->Llama |
| 125 | def decode(self, *args, **kwargs): |
| 126 | """This method forwards all its arguments to LlamaTokenizerFast's |
| 127 | [`~PythonBackend.decode`]. Please refer to the docstring |
| 128 | of this method for more information. |
| 129 | """ |
| 130 | result = args[0] |
| 131 | result = result[result != 0] |
| 132 | |
| 133 | bos_id = getattr( |
| 134 | self.tokenizer, |
| 135 | "bos_token_id", |
| 136 | getattr(self.tokenizer, "bos_id", 1), |
| 137 | ) |
| 138 | eos_id = getattr( |
| 139 | self.tokenizer, |
| 140 | "eos_token_id", |
| 141 | getattr(self.tokenizer, "eos_id", 2), |
| 142 | ) |
| 143 | eot_id = getattr(self.tokenizer, "eot_id", None) |
| 144 | |
| 145 | if len(result) > 0 and result[0] == bos_id: |
| 146 | result = result[1:] |
| 147 | if len(result) > 0 and ( |
| 148 | result[-1] == eos_id or (eot_id is not None and result[-1] == eot_id) |
| 149 | ): |
| 150 | result = result[:-1] |
| 151 | return self.tokenizer.decode(result, *args[1:], **kwargs).strip() |
| 152 | |
| 153 | def _convert(self, input_str, max_inp_length: int | None = None): |
| 154 | add_bos = getattr(self.tokenizer, "add_bos_token", False) |
| 155 | if self.version == (2, 5) or add_bos: |
| 156 | input_ids = self.tokenizer.encode(input_str) |
| 157 | else: |
| 158 | bos_id = getattr( |
| 159 | self.tokenizer, |
| 160 | "bos_token_id", |
| 161 | getattr(self.tokenizer, "bos_id", 1), |
| 162 | ) |
| 163 | input_ids = [bos_id] + self.tokenizer.encode(input_str) |
| 164 | |
| 165 | if max_inp_length is not None: |
| 166 | input_ids = input_ids[:max_inp_length] |
| 167 | input_ids = torch.tensor(input_ids, dtype=torch.int32) |
| 168 | |
| 169 | im_start_id = getattr( |
| 170 | self.tokenizer, |
| 171 | "im_start_id", |
| 172 | self.tokenizer.convert_tokens_to_ids("<im_start>"), |
| 173 | ) |
| 174 | im_end_id = getattr( |
| 175 | self.tokenizer, |
| 176 | "im_end_id", |
| 177 | self.tokenizer.convert_tokens_to_ids("<im_end>"), |
| 178 | ) |
| 179 | |
| 180 | image_start_tokens = torch.where(input_ids == im_start_id)[0] |
| 181 | image_start_tokens += 1 |
| 182 | image_end_tokens = torch.where(input_ids == im_end_id)[0] |
| 183 | assert len(image_start_tokens) == len(image_end_tokens), ( |
| 184 | f"The number of image start tokens ({len(image_start_tokens)}) " |
| 185 | f"and end tokens ({len(image_end_tokens)}) must match." |
| 186 | ) |
| 187 | image_bounds = torch.hstack( |
| 188 | [ |
| 189 | image_start_tokens.unsqueeze(-1), |
| 190 | image_end_tokens.unsqueeze(-1), |
| 191 | ] |
| 192 | ) |
| 193 | return input_ids, image_bounds |
| 194 | |
| 195 | def _convert_images_texts_to_inputs( |
| 196 | self, |
| 197 | images, |
| 198 | texts, |
| 199 | do_pad=False, |
| 200 | truncation=None, |
| 201 | max_length=None, |
| 202 | return_tensors=None, |
| 203 | ): |
| 204 | if not len(images): |
| 205 | model_inputs = self.tokenizer( |
| 206 | texts, |
| 207 | return_tensors=return_tensors, |
| 208 | padding=do_pad, |
| 209 | truncation=truncation, |
| 210 | max_length=max_length, |
| 211 | ) |
| 212 | return MiniCPMVBatchFeature(data={**model_inputs}) |
| 213 | |
| 214 | pattern = "(<image>./</image>)" |
| 215 | images_val = images["pixel_values"] |
| 216 | image_sizes = images["image_sizes"] |
| 217 | tgt_sizes = images["tgt_sizes"] |
| 218 | |
| 219 | if isinstance(texts, str): |
| 220 | texts = [texts] |
| 221 | |
| 222 | input_ids_list = [] |
| 223 | image_bounds_list = [] |
| 224 | |
| 225 | for index, text in enumerate(texts): |
| 226 | image_tags = regex.findall(pattern, text) |
| 227 | assert len(image_tags) == len(image_sizes[index]) |
| 228 | text_chunks = text.split(pattern) |
| 229 | final_text = "" |
| 230 | for i in range(len(image_tags)): |
| 231 | placeholder = self.image_processor.get_slice_image_placeholder( |
| 232 | image_sizes[index][i] |
| 233 | ) |
| 234 | final_text = final_text + text_chunks[i] + placeholder |
| 235 | final_text += text_chunks[-1] |
| 236 | input_ids, image_bounds = self._convert(final_text, max_length) |
| 237 | input_ids_list.append(input_ids) |
| 238 | image_bounds_list.append(image_bounds) |
| 239 | |
| 240 | padded_input_ids, padding_lengths = self.pad( |
| 241 | input_ids_list, |
| 242 | padding_side="left", |
| 243 | ) |
| 244 | for i, length in enumerate(padding_lengths): |
| 245 | image_bounds_list[i] = image_bounds_list[i] + length |
| 246 | |
| 247 | return MiniCPMVBatchFeature( |
| 248 | data={ |
| 249 | "input_ids": padded_input_ids, |
| 250 | "attention_mask": padded_input_ids.ne(0), |
| 251 | "pixel_values": images_val, |
| 252 | "image_sizes": image_sizes, |
| 253 | "image_bound": image_bounds_list, |
| 254 | "tgt_sizes": tgt_sizes, |
| 255 | } |
| 256 | ) |
| 257 | |
| 258 | @property |
| 259 | # Copied from |
| 260 | # transformers.models.clip.processing_clip.CLIPProcessor.model_input_names |
| 261 | def model_input_names(self): |
| 262 | tokenizer_input_names = self.tokenizer.model_input_names |
| 263 | image_processor_input_names = self.image_processor.model_input_names |
| 264 | return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) |
| 265 | |
| 266 | # Copied from openbmb/MiniCPM-V-4_5 processing_minicpmv.py. |
| 267 | def pad(self, inputs, max_length=None, padding_value=0, padding_side="left"): |
| 268 | if not inputs: |
| 269 | return torch.empty(0), [] |
| 270 | |
| 271 | items = [] |
| 272 | if isinstance(inputs[0], list): |
| 273 | assert isinstance(inputs[0][0], torch.Tensor) |
| 274 | for it in inputs: |
| 275 | for tr in it: |
| 276 | items.append(tr) |
| 277 | else: |
| 278 | assert isinstance(inputs[0], torch.Tensor) |
| 279 | items = inputs |
| 280 | |
| 281 | batch_size = len(items) |
| 282 | shape = items[0].shape |
| 283 | dim = len(shape) |
| 284 | assert dim <= 2 |
| 285 | if max_length is None: |
| 286 | max_length = 0 |
| 287 | max_length = max(max_length, max(item.shape[-1] for item in items)) |
| 288 | min_length = min(item.shape[-1] for item in items) |
| 289 | dtype = items[0].dtype |
| 290 | |
| 291 | if dim == 0: |
| 292 | return torch.stack([item for item in items], dim=0), [0] |
| 293 | elif dim == 1: |
| 294 | if max_length == min_length: |
| 295 | return torch.stack([item for item in items], dim=0), [0] * batch_size |
| 296 | tensor = torch.zeros((batch_size, max_length), dtype=dtype) + padding_value |
| 297 | else: |
| 298 | tensor = ( |
| 299 | torch.zeros((batch_size, max_length, shape[-1]), dtype=dtype) |
| 300 | + padding_value |
| 301 | ) |
| 302 | |
| 303 | padding_lengths = [] |
| 304 | for i, item in enumerate(items): |
| 305 | if dim == 1: |
| 306 | if padding_side == "left": |
| 307 | tensor[i, -len(item) :] = item.clone() |
| 308 | else: |
| 309 | tensor[i, : len(item)] = item.clone() |
| 310 | elif dim == 2: |
| 311 | if padding_side == "left": |
| 312 | tensor[i, -len(item) :, :] = item.clone() |
| 313 | else: |
| 314 | tensor[i, : len(item), :] = item.clone() |
| 315 | padding_lengths.append(tensor.shape[-1] - len(item)) |
| 316 | |
| 317 | return tensor, padding_lengths |