| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | from fractions import Fraction |
| 5 | |
| 6 | from transformers import LlavaNextProcessor |
| 7 | from transformers.image_processing_utils import select_best_resolution |
| 8 | |
| 9 | |
| 10 | class Granite4VisionProcessor(LlavaNextProcessor): |
| 11 | """Processor for Granite 4 Vision. |
| 12 | |
| 13 | Extends LlavaNextProcessor to account for the Window Q-Former |
| 14 | downsampling when computing the number of image features. |
| 15 | |
| 16 | This processor is needed because the granite4_vision processor type |
| 17 | is not yet in the transformers version pinned by vLLM. |
| 18 | """ |
| 19 | |
| 20 | model_type = "granite4_vision" |
| 21 | |
| 22 | def __init__( |
| 23 | self, |
| 24 | image_processor=None, |
| 25 | tokenizer=None, |
| 26 | patch_size=None, |
| 27 | vision_feature_select_strategy=None, |
| 28 | chat_template=None, |
| 29 | image_token="<image>", |
| 30 | num_additional_image_tokens=0, |
| 31 | downsample_rate=None, |
| 32 | **kwargs, |
| 33 | ): |
| 34 | super().__init__( |
| 35 | image_processor=image_processor, |
| 36 | tokenizer=tokenizer, |
| 37 | patch_size=patch_size, |
| 38 | vision_feature_select_strategy=vision_feature_select_strategy, |
| 39 | chat_template=chat_template, |
| 40 | image_token=image_token, |
| 41 | num_additional_image_tokens=num_additional_image_tokens, |
| 42 | ) |
| 43 | self.downsample_rate = downsample_rate |
| 44 | |
| 45 | def _get_number_of_features( |
| 46 | self, |
| 47 | orig_height: int, |
| 48 | orig_width: int, |
| 49 | height: int, |
| 50 | width: int, |
| 51 | ) -> int: |
| 52 | image_grid_pinpoints = self.image_processor.image_grid_pinpoints |
| 53 | |
| 54 | height_best_resolution, width_best_resolution = select_best_resolution( |
| 55 | [orig_height, orig_width], image_grid_pinpoints |
| 56 | ) |
| 57 | scale_height = height_best_resolution // height |
| 58 | scale_width = width_best_resolution // width |
| 59 | |
| 60 | patches_height = height // self.patch_size |
| 61 | patches_width = width // self.patch_size |
| 62 | if self.downsample_rate is not None: |
| 63 | ds_rate = Fraction(self.downsample_rate) |
| 64 | patches_height = int(patches_height * ds_rate) |
| 65 | patches_width = int(patches_width * ds_rate) |
| 66 | |
| 67 | unpadded_features, newline_features = self._get_unpadded_features( |
| 68 | orig_height, |
| 69 | orig_width, |
| 70 | patches_height, |
| 71 | patches_width, |
| 72 | scale_height, |
| 73 | scale_width, |
| 74 | ) |
| 75 | base_features = ( |
| 76 | patches_height * patches_width + self.num_additional_image_tokens |
| 77 | ) |
| 78 | return unpadded_features + newline_features + base_features |