| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | import math |
| 4 | from functools import cached_property |
| 5 | |
| 6 | import numpy as np |
| 7 | import PIL |
| 8 | import torch |
| 9 | from transformers import BatchFeature |
| 10 | from transformers.image_utils import ImageInput |
| 11 | from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack |
| 12 | from transformers.tokenization_utils_base import PreTokenizedInput, TextInput |
| 13 | |
| 14 | __all__ = ["Ovis2_5Processor"] |
| 15 | IMAGE_TOKEN = "<image>" |
| 16 | VIDEO_TOKEN = "<video>" |
| 17 | MIN_PIXELS = 448 * 448 |
| 18 | MAX_PIXELS = 1792 * 1792 |
| 19 | |
| 20 | |
| 21 | class Ovis2_5ProcessorKwargs(ProcessingKwargs, total=False): # type: ignore[call-arg] |
| 22 | _defaults = { |
| 23 | "text_kwargs": { |
| 24 | "padding": False, |
| 25 | }, |
| 26 | "images_kwargs": { |
| 27 | "do_convert_rgb": True, |
| 28 | }, |
| 29 | "videos_kwargs": { |
| 30 | "do_convert_rgb": True, |
| 31 | }, |
| 32 | } |
| 33 | |
| 34 | |
| 35 | class Ovis2_5Processor(ProcessorMixin): |
| 36 | r"""Constructs an Ovis processor which wraps an Ovis image processor |
| 37 | and a Qwen2 tokenizer into a single processor. |
| 38 | [`OvisProcessor`] offers all the functionalities of |
| 39 | [`Qwen2VLImageProcessor`] and [`Qwen2TokenizerFast`]. |
| 40 | See the [`~OvisProcessor.__call__`] and [`~OvisProcessor.decode`] |
| 41 | for more information. |
| 42 | |
| 43 | Args: |
| 44 | image_processor ([`Qwen2VLImageProcessor`], *optional*): |
| 45 | The image processor is a required input. |
| 46 | tokenizer ([`Qwen2TokenizerFast`], *optional*): |
| 47 | The tokenizer is a required input. |
| 48 | chat_template (`str`, *optional*): A Jinja template which will |
| 49 | be used to convert lists of messages in a chat into |
| 50 | a tokenizable string. |
| 51 | |
| 52 | """ |
| 53 | |
| 54 | attributes = ["image_processor", "tokenizer"] |
| 55 | valid_kwargs = ["chat_template", "image_pad_token"] |
| 56 | |
| 57 | image_processor_class = "AutoImageProcessor" |
| 58 | tokenizer_class = "AutoTokenizer" |
| 59 | |
| 60 | def __init__( |
| 61 | self, |
| 62 | image_processor=None, |
| 63 | tokenizer=None, |
| 64 | chat_template=None, |
| 65 | image_pad_token=None, |
| 66 | patch_size=16, |
| 67 | hidden_stride=2, |
| 68 | temporal_patch_size=1, |
| 69 | **kwargs, |
| 70 | ): |
| 71 | self.image_token = IMAGE_TOKEN |
| 72 | self.video_token = VIDEO_TOKEN |
| 73 | self.image_pad_token = "<|image_pad|>" |
| 74 | |
| 75 | self.patch_size = patch_size |
| 76 | self.hidden_stride = hidden_stride |
| 77 | self.temporal_patch_size = temporal_patch_size |
| 78 | super().__init__(image_processor, tokenizer, chat_template=chat_template) |
| 79 | |
| 80 | @cached_property |
| 81 | def extra_special_tokens(self): |
| 82 | required_tokens = { |
| 83 | "image_token": "<image>", |
| 84 | "video_token": "<video>", |
| 85 | "visual_atom": "<ovis_visual_atom>", |
| 86 | "image_start": "<ovis_image_start>", |
| 87 | "image_end": "<ovis_image_end>", |
| 88 | "video_start": "<ovis_video_start>", |
| 89 | "video_end": "<ovis_video_end>", |
| 90 | "image_pad": "<|image_pad|>", |
| 91 | } |
| 92 | |
| 93 | # The checkpoint defines both `additional_special_tokens` and |
| 94 | # `extra_special_tokens`, with the latter empty. Transformers ignores |
| 95 | # the former because the latter is explicitly empty, so the tokens are |
| 96 | # missing from the vocab. Re-add them to restore the expected ids. |
| 97 | self.tokenizer.add_tokens(list(required_tokens.values()), special_tokens=True) |
| 98 | |
| 99 | # in TokenizerPool tokenizer.get_added_vocab() gets values from an instance |
| 100 | # that was updated with tokenizer.add_tokens(), so it is favourable over |
| 101 | # self.tokenizer.convert_tokens_to_ids(), which refers to a copy before |
| 102 | # mutation by tokenizer.add_tokens() |
| 103 | added_vocab = self.tokenizer.get_added_vocab() |
| 104 | |
| 105 | return { |
| 106 | key: added_vocab[token_name] for key, token_name in required_tokens.items() |
| 107 | } |
| 108 | |
| 109 | def __call__( |
| 110 | self, |
| 111 | images: ImageInput = None, |
| 112 | videos: np.ndarray | list[ImageInput] = None, |
| 113 | text: TextInput |
| 114 | | PreTokenizedInput |
| 115 | | list[TextInput] |
| 116 | | list[PreTokenizedInput] = None, |
| 117 | **kwargs: Unpack[Ovis2_5ProcessorKwargs], |
| 118 | ) -> BatchFeature: |
| 119 | """Main method to prepare for the model one or several sequences(s) |
| 120 | and image(s). This method forwards the `text`and `kwargs` arguments |
| 121 | to Qwen2TokenizerFast's [`~Qwen2TokenizerFast.__call__`] if `text` |
| 122 | is not `None` to encode the text. To prepare the vision inputs, |
| 123 | this method forwards the `vision_infos` and `kwargs` arguments to |
| 124 | Qwen2VLImageProcessor's [`~Qwen2VLImageProcessor.__call__`] |
| 125 | if `vision_infos` is not `None`. |
| 126 | |
| 127 | Args: |
| 128 | images (`PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, or a list): |
| 129 | The image or batch of images to be prepared. Each image can |
| 130 | be a PIL image, NumPy array or PyTorch tensor. Both |
| 131 | channels-first and channels-last formats are supported. |
| 132 | videos (`np.ndarray`, `torch.Tensor`, or a list of either): |
| 133 | The video or batch of videos to be prepared. Each video can |
| 134 | be a 4D NumPy array or PyTorch tensor, or a nested list of |
| 135 | 3D frames. Both channels-first and channels-last formats are |
| 136 | supported. |
| 137 | text (`str`, `list[str]`, `list[list[str]]`): |
| 138 | The sequence or batch of sequences to be encoded. Each |
| 139 | sequence can be a string or a list of strings (pretokenized |
| 140 | string). If the sequences are provided as list of strings |
| 141 | (pretokenized), you must set `is_split_into_words=True` (to |
| 142 | lift the ambiguity with a batch of sequences). |
| 143 | return_tensors (`str` or [`~utils.TensorType`], *optional*): |
| 144 | If set, will return tensors of a particular framework. |
| 145 | Acceptable values are: |
| 146 | - `'tf'`: Return TensorFlow `tf.constant` objects. |
| 147 | - `'pt'`: Return PyTorch `torch.Tensor` objects. |
| 148 | - `'np'`: Return NumPy `np.ndarray` objects. |
| 149 | - `'jax'`: Return JAX `jnp.ndarray` objects. |
| 150 | |
| 151 | Returns: |
| 152 | [`BatchFeature`]: A [`BatchFeature`] with the following fields: |
| 153 | - **input_ids** -- list of token ids to be fed to a model. |
| 154 | Returned when `text` is not `None`. |
| 155 | - **attention_mask** -- list of indices specifying which tokens |
| 156 | should be attended to by the model (when |
| 157 | `return_attention_mask=True` or if *"attention_mask"* |
| 158 | is in `self.model_input_names` and if `text` is not `None`). |
| 159 | - **pixel_values** -- Pixel values to be fed to a model. |
| 160 | Returned when `images` is not `None`. |
| 161 | - **pixel_values_videos** -- Pixel values of videos to be fed to |
| 162 | a model. Returned when `videos` is not `None`. |
| 163 | - **image_grid_thw** -- list of image 3D grid in LLM. Returned |
| 164 | when `images` is not `None`. |
| 165 | - **video_grid_thw** -- list of video 3D grid in LLM. Returned |
| 166 | when `videos` is not `None`. |
| 167 | - **second_per_grid_ts** -- list of video seconds per time grid. |
| 168 | Returned when `videos` is not `None`. |
| 169 | |
| 170 | """ |
| 171 | output_kwargs = self._merge_kwargs( |
| 172 | Ovis2_5ProcessorKwargs, |
| 173 | tokenizer_init_kwargs=self.tokenizer.init_kwargs, |
| 174 | **kwargs, |
| 175 | ) |
| 176 | # Process all images first |
| 177 | visual_features = {} |
| 178 | output = BatchFeature() |
| 179 | if images is not None: |
| 180 | processed_images = [] |
| 181 | image_placeholders_list = [] |
| 182 | grids = [] |
| 183 | # Process each image |
| 184 | for image in images if isinstance(images, list) else [images]: |
| 185 | pixel_values, image_placeholders, grid = self.preprocess_multidata( |
| 186 | images=image, |
| 187 | **output_kwargs["images_kwargs"], |
| 188 | ) |
| 189 | processed_images.append(pixel_values) |
| 190 | image_placeholders_list.append(image_placeholders) |
| 191 | grids.append(grid) |
| 192 | |
| 193 | # assign all processed images |
| 194 | if processed_images: |
| 195 | visual_features["image_placeholders"] = image_placeholders_list |
| 196 | output["pixel_values"] = processed_images |
| 197 | output["grids"] = grids |
| 198 | |
| 199 | if videos is not None: |
| 200 | processed_videos = [] |
| 201 | videos_placeholders_list = [] |
| 202 | grids = [] |
| 203 | # Process each video |
| 204 | for video in videos if isinstance(videos, list) else [videos]: |
| 205 | pixel_values, video_placeholders, grid = self.preprocess_multidata( |
| 206 | video=video, |
| 207 | **output_kwargs["videos_kwargs"], |
| 208 | ) |
| 209 | processed_videos.append(pixel_values) |
| 210 | videos_placeholders_list.append(video_placeholders) |
| 211 | grids.append(grid) |
| 212 | # assign all processed videos |
| 213 | if processed_videos: |
| 214 | visual_features["video_placeholders"] = videos_placeholders_list |
| 215 | output["video_pixel_values"] = processed_videos |
| 216 | output["video_grids"] = grids |
| 217 | |
| 218 | # Process text input |
| 219 | if text is not None: |
| 220 | if not isinstance(text, list): |
| 221 | text = [text] |
| 222 | tokenized_batched_text = self._tokenize_with_visual_symbol(text) |
| 223 | image_token_id = self.get_token_value("image_token") |
| 224 | video_token_id = self.get_token_value("video_token") |
| 225 | replaced_ids_list = [] |
| 226 | image_idx = 0 |
| 227 | video_idx = 0 |
| 228 | for ids_tensor in tokenized_batched_text: |
| 229 | has_image_tokens = ( |
| 230 | image_token_id in ids_tensor |
| 231 | and "image_placeholders" in visual_features |
| 232 | and image_idx < len(visual_features["image_placeholders"]) |
| 233 | ) |
| 234 | has_video_tokens = ( |
| 235 | video_token_id in ids_tensor |
| 236 | and "video_placeholders" in visual_features |
| 237 | and video_idx < len(visual_features["video_placeholders"]) |
| 238 | ) |
| 239 | if has_image_tokens or has_video_tokens: |
| 240 | # Convert to list for easier manipulation |
| 241 | ids_list = ids_tensor.tolist() |
| 242 | new_ids = [] |
| 243 | |
| 244 | # Replace placeholders |
| 245 | for token_id in ids_list: |
| 246 | if token_id == image_token_id: |
| 247 | new_ids.extend( |
| 248 | visual_features["image_placeholders"][image_idx] |
| 249 | ) |
| 250 | image_idx += 1 |
| 251 | elif token_id == video_token_id: |
| 252 | new_ids.extend( |
| 253 | visual_features["video_placeholders"][video_idx] |
| 254 | ) |
| 255 | video_idx += 1 |
| 256 | else: |
| 257 | new_ids.append(token_id) |
| 258 | # Convert back to tensor |
| 259 | ids_tensor = torch.tensor(new_ids, dtype=torch.long) |
| 260 | replaced_ids_list.append(ids_tensor) |
| 261 | if replaced_ids_list: |
| 262 | replaced_and_tokenized_ids = torch.stack(replaced_ids_list) |
| 263 | else: |
| 264 | replaced_and_tokenized_ids = torch.tensor([], dtype=torch.long) |
| 265 | output["input_ids"] = replaced_and_tokenized_ids |
| 266 | |
| 267 | return output |
| 268 | # If only images were provided |
| 269 | return BatchFeature(data=visual_features) |
| 270 | |
| 271 | def _tokenize_with_visual_symbol(self, text_list: list[str]) -> torch.LongTensor: |
| 272 | batch_token_ids = [] |
| 273 | for text in text_list: |
| 274 | token_ids = [] |
| 275 | video_token_id = self.get_token_value("video_token") |
| 276 | image_token_id = self.get_token_value("image_token") |
| 277 | video_split_texts = text.split(self.video_token) |
| 278 | |
| 279 | for j, video_segment in enumerate(video_split_texts): |
| 280 | image_split_texts = video_segment.split(self.image_token) |
| 281 | text_chunks = [ |
| 282 | self.tokenizer(chunk, add_special_tokens=False).input_ids |
| 283 | for chunk in image_split_texts |
| 284 | ] |
| 285 | segment_tokens = [] |
| 286 | for i, chunk in enumerate(text_chunks): |
| 287 | segment_tokens.extend(chunk) |
| 288 | if i < len(text_chunks) - 1: |
| 289 | segment_tokens.append(image_token_id) |
| 290 | token_ids.extend(segment_tokens) |
| 291 | if j < len(video_split_texts) - 1: |
| 292 | token_ids.append(video_token_id) |
| 293 | |
| 294 | batch_token_ids.append(token_ids) |
| 295 | return torch.tensor(batch_token_ids, dtype=torch.long) |
| 296 | |
| 297 | # Copied from qwen2_vl |
| 298 | def smart_resize( |
| 299 | self, |
| 300 | height: int, |
| 301 | width: int, |
| 302 | factor: int = 28, |
| 303 | min_pixels: int = MIN_PIXELS, |
| 304 | max_pixels: int = MAX_PIXELS, |
| 305 | ): |
| 306 | """Rescales the image so that the following conditions are met: |
| 307 | 1. Both dimensions (height and width) are divisible by 'factor'. |
| 308 | 2. The total number of pixels is within the range |
| 309 | ['min_pixels', 'max_pixels']. |
| 310 | 3. The aspect ratio of the image is maintained as closely as possible. |
| 311 | """ |
| 312 | if height < factor or width < factor: |
| 313 | print( |
| 314 | f"height:{height} or width:{width} must be larger than factor:{factor}" |
| 315 | ) |
| 316 | if height < width: |
| 317 | width = round(factor / height * width) |
| 318 | height = factor |
| 319 | else: |
| 320 | height = round(factor / width * height) |
| 321 | width = factor |
| 322 | |
| 323 | elif max(height, width) / min(height, width) > 200: |
| 324 | print( |
| 325 | f"absolute aspect ratio must be smaller than 200, " |
| 326 | f"got {max(height, width) / min(height, width)}" |
| 327 | ) |
| 328 | if height > width: |
| 329 | height = 200 * width |
| 330 | else: |
| 331 | width = 200 * height |
| 332 | |
| 333 | h_bar = round(height / factor) * factor |
| 334 | w_bar = round(width / factor) * factor |
| 335 | if h_bar * w_bar > max_pixels: |
| 336 | beta = math.sqrt((height * width) / max_pixels) |
| 337 | h_bar = math.floor(height / beta / factor) * factor |
| 338 | w_bar = math.floor(width / beta / factor) * factor |
| 339 | elif h_bar * w_bar < min_pixels: |
| 340 | beta = math.sqrt(min_pixels / (height * width)) |
| 341 | h_bar = math.ceil(height * beta / factor) * factor |
| 342 | w_bar = math.ceil(width * beta / factor) * factor |
| 343 | return h_bar, w_bar |
| 344 | |
| 345 | def get_token_value(self, tok): |
| 346 | return self.extra_special_tokens[tok] |
| 347 | |
| 348 | def construct_visual_indicators(self, grid, is_video: bool = False): |
| 349 | if is_video: |
| 350 | start_token = self.get_token_value("video_start") |
| 351 | end_token = self.get_token_value("video_end") |
| 352 | else: |
| 353 | start_token = self.get_token_value("image_start") |
| 354 | end_token = self.get_token_value("image_end") |
| 355 | |
| 356 | image_placeholders = [start_token, self.get_token_value("visual_atom")] |
| 357 | if grid[0] * grid[1] > 1: |
| 358 | for r in range(grid[0]): |
| 359 | for c in range(grid[1]): |
| 360 | image_placeholders.append(self.get_token_value("visual_atom")) |
| 361 | |
| 362 | image_placeholders.append(end_token) |
| 363 | return image_placeholders |
| 364 | |
| 365 | def construct_visual_placeholders(self, grid, is_video: bool = False): |
| 366 | visual_placeholders = self.construct_visual_indicators((1, 1), is_video) |
| 367 | |
| 368 | image_atom_token_id = self.get_token_value("visual_atom") |
| 369 | # Extract the padding token ID from tokenizer |
| 370 | image_padding_token_id = self.get_token_value("image_pad") |
| 371 | |
| 372 | num_image_atoms = grid[0] * grid[1] * grid[2] |
| 373 | num_image_atoms //= self.hidden_stride**2 |
| 374 | num_image_atoms //= self.temporal_patch_size |
| 375 | |
| 376 | # Create a new list with padding tokens inserted |
| 377 | padded_placeholder_tokens = [] |
| 378 | for token in visual_placeholders: |
| 379 | if token == image_atom_token_id: |
| 380 | padded_placeholder_tokens.extend( |
| 381 | [image_padding_token_id] * num_image_atoms |
| 382 | ) |
| 383 | else: |
| 384 | padded_placeholder_tokens.append(image_padding_token_id) |
| 385 | return padded_placeholder_tokens |
| 386 | |
| 387 | def preprocess_multidata( |
| 388 | self, |
| 389 | images: PIL.Image.Image | list[PIL.Image.Image] | None = None, |
| 390 | video: list[PIL.Image.Image] | np.ndarray | None = None, |
| 391 | do_convert_rgb: bool | None = True, |
| 392 | min_pixels: int = MIN_PIXELS, |
| 393 | max_pixels: int = MAX_PIXELS, |
| 394 | return_tensors: str | None = "pt", |
| 395 | ): |
| 396 | is_video = False |
| 397 | if images is not None: |
| 398 | if not isinstance(images, list): |
| 399 | images = [images] |
| 400 | elif video is not None: |
| 401 | is_video = True |
| 402 | # type of video in dummy_mm_data is np.ndarray |
| 403 | if isinstance(video, np.ndarray): |
| 404 | images = [] |
| 405 | for i in range(video.shape[0]): |
| 406 | image = PIL.Image.fromarray(video[i].astype(np.uint8)) |
| 407 | images.append(image) |
| 408 | elif isinstance(video, list): |
| 409 | images = video |
| 410 | else: |
| 411 | raise ValueError("Either images or video should be provided.") |
| 412 | assert images is not None |
| 413 | min_pixels = min( |
| 414 | max_pixels if max_pixels is not None else MAX_PIXELS, |
| 415 | min_pixels if min_pixels is not None else MIN_PIXELS, |
| 416 | ) |
| 417 | images = [ |
| 418 | image.convert("RGB") if do_convert_rgb and image.mode != "RGB" else image |
| 419 | for image in images |
| 420 | ] |
| 421 | |
| 422 | width, height = images[0].size |
| 423 | resized_height, resized_width = height, width |
| 424 | processed_images = [] |
| 425 | for image in images: |
| 426 | resized_height, resized_width = self.smart_resize( |
| 427 | height, |
| 428 | width, |
| 429 | factor=self.patch_size * self.hidden_stride, |
| 430 | min_pixels=min_pixels, |
| 431 | max_pixels=max_pixels, |
| 432 | ) |
| 433 | new_size = dict(height=resized_height, width=resized_width) |
| 434 | image_pt = self.image_processor.preprocess(image, size=new_size)[ |
| 435 | "pixel_values" |
| 436 | ][0] |
| 437 | |
| 438 | processed_images.append(image_pt) |
| 439 | |
| 440 | patches = np.array(processed_images) |
| 441 | if patches.shape[0] % self.temporal_patch_size != 0: |
| 442 | num_to_pad = self.temporal_patch_size - ( |
| 443 | patches.shape[0] % self.temporal_patch_size |
| 444 | ) |
| 445 | repeats = np.repeat(patches[-1][np.newaxis], num_to_pad, axis=0) |
| 446 | patches = np.concatenate([patches, repeats], axis=0) |
| 447 | channel = patches.shape[1] |
| 448 | grid_t = patches.shape[0] // self.temporal_patch_size |
| 449 | grid_h = resized_height // self.patch_size |
| 450 | grid_w = resized_width // self.patch_size |
| 451 | |
| 452 | patches = patches.reshape( |
| 453 | grid_t, |
| 454 | self.temporal_patch_size, |
| 455 | channel, |
| 456 | grid_h // self.hidden_stride, |
| 457 | self.hidden_stride, |
| 458 | self.patch_size, |
| 459 | grid_w // self.hidden_stride, |
| 460 | self.hidden_stride, |
| 461 | self.patch_size, |
| 462 | ) |
| 463 | patches = patches.transpose(0, 3, 6, 4, 7, 2, 1, 5, 8) |
| 464 | flatten_patches = patches.reshape( |
| 465 | grid_t * grid_h * grid_w, |
| 466 | channel * self.temporal_patch_size * self.patch_size * self.patch_size, |
| 467 | ) |
| 468 | |
| 469 | visual_placeholders = self.construct_visual_placeholders( |
| 470 | [grid_t, grid_h, grid_w], is_video |
| 471 | ) |
| 472 | return ( |
| 473 | torch.tensor(flatten_patches), |
| 474 | visual_placeholders, |
| 475 | torch.tensor([[grid_t, grid_h, grid_w]]), |
| 476 | ) |