| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | # adapted from https://github.com/deepseek-ai/DeepSeek-OCR/blob/main/DeepSeek-OCR-master/DeepSeek-OCR-vllm/process/image_process.py |
| 4 | # and https://github.com/deepseek-ai/DeepSeek-OCR-2/blob/main/DeepSeek-OCR2-master/DeepSeek-OCR2-vllm/process/image_process.py |
| 5 | import math |
| 6 | from typing import Literal |
| 7 | |
| 8 | import torch |
| 9 | import torchvision.transforms as T |
| 10 | from PIL import Image, ImageOps |
| 11 | from transformers import BatchFeature, LlamaTokenizerFast |
| 12 | from transformers.processing_utils import ProcessorMixin |
| 13 | |
| 14 | # TODO(Isotr0py): change modes for variants |
| 15 | # see: https://github.com/deepseek-ai/DeepSeek-OCR/blob/8cf003d38821fa1b19c73da3bd1b0dc262ea8136/DeepSeek-OCR-master/DeepSeek-OCR-vllm/config.py#L1-L6 |
| 16 | # Tiny: base_size = 512, image_size = 512, crop_mode = False |
| 17 | # Small: base_size = 640, image_size = 640, crop_mode = False |
| 18 | # Base: base_size = 1024, image_size = 1024, crop_mode = False |
| 19 | # Large: base_size = 1280, image_size = 1280, crop_mode = False |
| 20 | # Gundam: base_size = 1024, image_size = 640, crop_mode = True |
| 21 | BASE_SIZE = 1024 |
| 22 | IMAGE_SIZE = 640 |
| 23 | CROP_MODE = True |
| 24 | |
| 25 | # TODO(Isotr0py): Expose as mm_kwargs |
| 26 | MIN_CROPS = 2 |
| 27 | MAX_CROPS = 6 # max:9; If your GPU memory is small, it is recommended to set it to 6. |
| 28 | |
| 29 | |
| 30 | def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size): |
| 31 | best_ratio_diff = float("inf") |
| 32 | best_ratio = (1, 1) |
| 33 | area = width * height |
| 34 | for ratio in target_ratios: |
| 35 | target_aspect_ratio = ratio[0] / ratio[1] |
| 36 | ratio_diff = abs(aspect_ratio - target_aspect_ratio) |
| 37 | if ratio_diff < best_ratio_diff: |
| 38 | best_ratio_diff = ratio_diff |
| 39 | best_ratio = ratio |
| 40 | elif ratio_diff == best_ratio_diff: |
| 41 | if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]: |
| 42 | best_ratio = ratio |
| 43 | return best_ratio |
| 44 | |
| 45 | |
| 46 | def calculate_aspect_ratios( |
| 47 | min_num: int = MIN_CROPS, max_num: int = MAX_CROPS |
| 48 | ) -> list[tuple[int, int]]: |
| 49 | target_ratios: set[tuple[int, int]] = set( |
| 50 | (i, j) |
| 51 | for n in range(min_num, max_num + 1) |
| 52 | for i in range(1, n + 1) |
| 53 | for j in range(1, n + 1) |
| 54 | if i * j <= max_num and i * j >= min_num |
| 55 | ) |
| 56 | sorted_target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1]) |
| 57 | return sorted_target_ratios |
| 58 | |
| 59 | |
| 60 | def count_tiles( |
| 61 | orig_width, |
| 62 | orig_height, |
| 63 | min_num=MIN_CROPS, |
| 64 | max_num=MAX_CROPS, |
| 65 | image_size=640, |
| 66 | use_thumbnail=False, |
| 67 | ): |
| 68 | aspect_ratio = orig_width / orig_height |
| 69 | |
| 70 | # calculate the existing image aspect ratio |
| 71 | target_ratios = calculate_aspect_ratios(min_num, max_num) |
| 72 | |
| 73 | # find the closest aspect ratio to the target |
| 74 | target_aspect_ratio = find_closest_aspect_ratio( |
| 75 | aspect_ratio, target_ratios, orig_width, orig_height, image_size |
| 76 | ) |
| 77 | |
| 78 | return target_aspect_ratio |
| 79 | |
| 80 | |
| 81 | def dynamic_preprocess( |
| 82 | image, min_num=MIN_CROPS, max_num=MAX_CROPS, image_size=640, use_thumbnail=False |
| 83 | ): |
| 84 | orig_width, orig_height = image.size |
| 85 | aspect_ratio = orig_width / orig_height |
| 86 | |
| 87 | # calculate the existing image aspect ratio |
| 88 | target_ratios = calculate_aspect_ratios(min_num, max_num) |
| 89 | |
| 90 | # find the closest aspect ratio to the target |
| 91 | target_aspect_ratio = find_closest_aspect_ratio( |
| 92 | aspect_ratio, target_ratios, orig_width, orig_height, image_size |
| 93 | ) |
| 94 | |
| 95 | # calculate the target width and height |
| 96 | target_width = image_size * target_aspect_ratio[0] |
| 97 | target_height = image_size * target_aspect_ratio[1] |
| 98 | blocks = target_aspect_ratio[0] * target_aspect_ratio[1] |
| 99 | |
| 100 | # resize the image |
| 101 | resized_img = image.resize((target_width, target_height)) |
| 102 | processed_images = [] |
| 103 | for i in range(blocks): |
| 104 | box = ( |
| 105 | (i % (target_width // image_size)) * image_size, |
| 106 | (i // (target_width // image_size)) * image_size, |
| 107 | ((i % (target_width // image_size)) + 1) * image_size, |
| 108 | ((i // (target_width // image_size)) + 1) * image_size, |
| 109 | ) |
| 110 | # split the image |
| 111 | split_img = resized_img.crop(box) |
| 112 | processed_images.append(split_img) |
| 113 | assert len(processed_images) == blocks |
| 114 | if use_thumbnail and len(processed_images) != 1: |
| 115 | thumbnail_img = image.resize((image_size, image_size)) |
| 116 | processed_images.append(thumbnail_img) |
| 117 | return processed_images, target_aspect_ratio |
| 118 | |
| 119 | |
| 120 | class ImageTransform: |
| 121 | def __init__( |
| 122 | self, |
| 123 | mean: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 124 | std: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 125 | normalize: bool = True, |
| 126 | ): |
| 127 | self.mean = mean |
| 128 | self.std = std |
| 129 | self.normalize = normalize |
| 130 | |
| 131 | transform_pipelines = [T.ToTensor()] |
| 132 | |
| 133 | if normalize: |
| 134 | transform_pipelines.append(T.Normalize(mean, std)) |
| 135 | |
| 136 | self.transform = T.Compose(transform_pipelines) |
| 137 | |
| 138 | def __call__(self, pil_img: Image.Image): |
| 139 | x = self.transform(pil_img) |
| 140 | return x |
| 141 | |
| 142 | |
| 143 | class DeepseekOCRProcessor(ProcessorMixin): |
| 144 | tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") |
| 145 | attributes = ["tokenizer"] |
| 146 | |
| 147 | def __init__( |
| 148 | self, |
| 149 | tokenizer: LlamaTokenizerFast, |
| 150 | patch_size: int = 16, |
| 151 | downsample_ratio: int = 4, |
| 152 | image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 153 | image_std: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 154 | normalize: bool = True, |
| 155 | image_token: str = "<image>", |
| 156 | pad_token: str = "<｜▁pad▁｜>", |
| 157 | add_special_token: bool = False, |
| 158 | sft_format: str = "deepseek", |
| 159 | mask_prompt: bool = True, |
| 160 | ignore_id: int = -100, |
| 161 | image_size: int = IMAGE_SIZE, |
| 162 | base_size: int = BASE_SIZE, |
| 163 | strategy: Literal["v1", "v2"] = "v1", |
| 164 | max_crops: int = MAX_CROPS, |
| 165 | **kwargs, |
| 166 | ): |
| 167 | self.image_size = image_size |
| 168 | self.base_size = base_size |
| 169 | self.max_crops = max_crops |
| 170 | |
| 171 | # image token calculation strategy for |
| 172 | # Deepseek-OCR and Deepseek-OCR-2 |
| 173 | self.strategy = strategy |
| 174 | assert strategy in ["v1", "v2"], "Only 'v1' and 'v2' strategies are supported." |
| 175 | |
| 176 | self.patch_size = 16 |
| 177 | self.image_mean = image_mean |
| 178 | self.image_std = image_std |
| 179 | self.normalize = normalize |
| 180 | self.downsample_ratio = 4 |
| 181 | |
| 182 | self.image_transform = ImageTransform( |
| 183 | mean=image_mean, std=image_std, normalize=normalize |
| 184 | ) |
| 185 | |
| 186 | self.tokenizer = tokenizer |
| 187 | self.tokenizer.padding_side = "left" # must set this，padding side with make a difference in batch inference # noqa: E501 |
| 188 | |
| 189 | # add the pad_token as special token to use 'tokenizer.pad_token' |
| 190 | # and 'tokenizer.pad_token_id' |
| 191 | if self.tokenizer.pad_token is None: |
| 192 | self.tokenizer.add_special_tokens({"pad_token": pad_token}) |
| 193 | |
| 194 | # add image token |
| 195 | self.image_token_id = self.tokenizer.vocab.get(image_token) |
| 196 | self.image_token = image_token |
| 197 | self.pad_token = pad_token |
| 198 | self.add_special_token = add_special_token |
| 199 | self.sft_format = sft_format |
| 200 | self.mask_prompt = mask_prompt |
| 201 | self.ignore_id = ignore_id |
| 202 | |
| 203 | super().__init__( |
| 204 | tokenizer, |
| 205 | **kwargs, |
| 206 | ) |
| 207 | |
| 208 | @property |
| 209 | def bos_id(self): |
| 210 | return self.tokenizer.bos_token_id |
| 211 | |
| 212 | @property |
| 213 | def eos_id(self): |
| 214 | return self.tokenizer.eos_token_id |
| 215 | |
| 216 | @property |
| 217 | def pad_id(self): |
| 218 | return self.tokenizer.pad_token_id |
| 219 | |
| 220 | def encode(self, text: str, bos: bool = True, eos: bool = False): |
| 221 | t = self.tokenizer.encode(text, add_special_tokens=False) |
| 222 | if bos: |
| 223 | t = [self.bos_id] + t |
| 224 | if eos: |
| 225 | t = t + [self.eos_id] |
| 226 | return t |
| 227 | |
| 228 | def decode(self, t: list[int], **kwargs) -> str: |
| 229 | return self.tokenizer.decode(t, **kwargs) |
| 230 | |
| 231 | def process_one( |
| 232 | self, |
| 233 | prompt: str, |
| 234 | images: list[Image.Image], |
| 235 | crop_mode: bool = CROP_MODE, |
| 236 | ): |
| 237 | """Args: |
| 238 | prompt (str): the formatted prompt; |
| 239 | images (List[ImageType]): the list of images; |
| 240 | crop_mode (bool): if True, then crop the image; |
| 241 | |
| 242 | Returns: |
| 243 | outputs (BaseProcessorOutput): the output of the processor, |
| 244 | - input_ids (torch.LongTensor): [N + image tokens] |
| 245 | - target_ids (torch.LongTensor): [N + image tokens] |
| 246 | - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] |
| 247 | - image_id (int): the id of the image token |
| 248 | - num_image_tokens (List[int]): the number of image tokens |
| 249 | |
| 250 | """ |
| 251 | if prompt is None or images is None: |
| 252 | raise ValueError("prompt and images must be used at the same time.") |
| 253 | |
| 254 | sft_format = prompt |
| 255 | |
| 256 | ( |
| 257 | input_ids, |
| 258 | pixel_values, |
| 259 | images_crop, |
| 260 | images_seq_mask, |
| 261 | images_spatial_crop, |
| 262 | num_image_tokens, |
| 263 | _, |
| 264 | ) = self.tokenize_with_images( |
| 265 | conversation=sft_format, |
| 266 | images=images, |
| 267 | bos=True, |
| 268 | eos=True, |
| 269 | cropping=crop_mode, |
| 270 | ) |
| 271 | |
| 272 | prepare = BatchFeature( |
| 273 | data=dict( |
| 274 | input_ids=input_ids, |
| 275 | pixel_values=pixel_values, |
| 276 | images_crop=images_crop, |
| 277 | images_seq_mask=images_seq_mask, |
| 278 | images_spatial_crop=images_spatial_crop, |
| 279 | num_image_tokens=num_image_tokens, |
| 280 | ), |
| 281 | tensor_type="pt", |
| 282 | ) |
| 283 | return prepare |
| 284 | |
| 285 | def __call__( |
| 286 | self, |
| 287 | *, |
| 288 | prompt: str, |
| 289 | images: list[Image.Image], |
| 290 | crop_mode: bool = CROP_MODE, |
| 291 | **kwargs, |
| 292 | ): |
| 293 | prepare = self.process_one( |
| 294 | prompt=prompt, |
| 295 | images=images, |
| 296 | crop_mode=crop_mode, |
| 297 | ) |
| 298 | |
| 299 | return prepare |
| 300 | |
| 301 | def tokenize_with_images( |
| 302 | self, |
| 303 | conversation: str, |
| 304 | images: list[Image.Image], |
| 305 | bos: bool = True, |
| 306 | eos: bool = True, |
| 307 | cropping: bool = True, |
| 308 | ): |
| 309 | """Tokenize text with <image> tags.""" |
| 310 | num_image_tags = conversation.count(self.image_token) |
| 311 | if num_image_tags != len(images): |
| 312 | raise ValueError( |
| 313 | f"Number of {self.image_token!r} tokens in prompt " |
| 314 | f"({num_image_tags}) does not match number of images " |
| 315 | f"({len(images)})." |
| 316 | ) |
| 317 | text_splits = conversation.split(self.image_token) |
| 318 | images_list, images_crop_list, images_seq_mask, images_spatial_crop = ( |
| 319 | [], |
| 320 | [], |
| 321 | [], |
| 322 | [], |
| 323 | ) |
| 324 | image_shapes = [] |
| 325 | num_image_tokens = [] |
| 326 | tokenized_str = [] |
| 327 | for text_sep, image in zip(text_splits, images): |
| 328 | tokenized_sep = self.encode(text_sep, bos=False, eos=False) |
| 329 | tokenized_str += tokenized_sep |
| 330 | images_seq_mask += [False] * len(tokenized_sep) |
| 331 | |
| 332 | image_shapes.append(image.size) |
| 333 | |
| 334 | images_crop_raw = [] |
| 335 | if image.size[0] <= self.image_size and image.size[1] <= self.image_size: |
| 336 | crop_ratio = [1, 1] |
| 337 | elif cropping: |
| 338 | images_crop_raw, crop_ratio = dynamic_preprocess( |
| 339 | image, image_size=self.image_size, max_num=self.max_crops |
| 340 | ) |
| 341 | else: |
| 342 | crop_ratio = [1, 1] |
| 343 | |
| 344 | if not cropping: |
| 345 | image = image.resize((self.image_size, self.image_size)) |
| 346 | |
| 347 | global_view = ImageOps.pad( |
| 348 | image, |
| 349 | (self.base_size, self.base_size), |
| 350 | color=tuple(int(x * 255) for x in self.image_transform.mean), |
| 351 | ) |
| 352 | images_list.append(self.image_transform(global_view)) |
| 353 | |
| 354 | num_width_tiles, num_height_tiles = crop_ratio |
| 355 | images_spatial_crop.append([num_width_tiles, num_height_tiles]) |
| 356 | |
| 357 | if num_width_tiles > 1 or num_height_tiles > 1: |
| 358 | for cropped_image in images_crop_raw: |
| 359 | images_crop_list.append(self.image_transform(cropped_image)) |
| 360 | |
| 361 | num_queries = math.ceil( |
| 362 | (self.image_size // self.patch_size) / self.downsample_ratio |
| 363 | ) |
| 364 | num_queries_base = math.ceil( |
| 365 | (self.base_size // self.patch_size) / self.downsample_ratio |
| 366 | ) |
| 367 | |
| 368 | num_tokens_base = ( |
| 369 | (num_queries_base * (num_queries_base + 1)) |
| 370 | if self.strategy == "v1" |
| 371 | else num_queries_base * num_queries_base |
| 372 | ) |
| 373 | tokenized_image = [self.image_token_id] * num_tokens_base |
| 374 | |
| 375 | tokenized_image += [self.image_token_id] |
| 376 | if num_width_tiles > 1 or num_height_tiles > 1: |
| 377 | num_tokens_per_row = ( |
| 378 | num_queries * num_width_tiles + 1 |
| 379 | if self.strategy == "v1" |
| 380 | else num_queries * num_width_tiles |
| 381 | ) |
| 382 | local_row = [self.image_token_id] * num_tokens_per_row |
| 383 | tokenized_image += local_row * (num_queries * num_height_tiles) |
| 384 | tokenized_str += tokenized_image |
| 385 | images_seq_mask += [True] * len(tokenized_image) |
| 386 | num_image_tokens.append(len(tokenized_image)) |
| 387 | |
| 388 | """process the last text split""" |
| 389 | tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) |
| 390 | tokenized_str += tokenized_sep |
| 391 | images_seq_mask += [False] * len(tokenized_sep) |
| 392 | |
| 393 | """add the bos and eos tokens""" |
| 394 | if bos: |
| 395 | tokenized_str = [self.bos_id] + tokenized_str |
| 396 | images_seq_mask = [False] + images_seq_mask |
| 397 | if eos: |
| 398 | tokenized_str = tokenized_str + [self.eos_id] |
| 399 | images_seq_mask = images_seq_mask + [False] |
| 400 | |
| 401 | assert len(tokenized_str) == len(images_seq_mask), ( |
| 402 | f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} " |
| 403 | f"is not equal to images_seq_mask's length {len(images_seq_mask)}." |
| 404 | ) |
| 405 | |
| 406 | masked_tokenized_str = [] |
| 407 | for token_index in tokenized_str: |
| 408 | if token_index != self.image_token_id: |
| 409 | masked_tokenized_str.append(token_index) |
| 410 | else: |
| 411 | masked_tokenized_str.append(self.ignore_id) |
| 412 | |
| 413 | assert ( |
| 414 | len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) |
| 415 | ), ( |
| 416 | f"tokenized_str's length {len(tokenized_str)}, " |
| 417 | f"input_ids' length {len(masked_tokenized_str)}, " |
| 418 | f"images_seq_mask's length {len(images_seq_mask)}, are not equal." |
| 419 | ) |
| 420 | |
| 421 | input_ids = torch.LongTensor(tokenized_str) |
| 422 | target_ids = torch.LongTensor(masked_tokenized_str) |
| 423 | images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) |
| 424 | |
| 425 | # set input_ids < 0 | input_ids == self.image_token_id as ignore_id |
| 426 | target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( |
| 427 | self.ignore_id |
| 428 | ) |
| 429 | input_ids[input_ids < 0] = self.pad_id |
| 430 | |
| 431 | # Remove the ending eos token |
| 432 | assert input_ids[-1] == self.eos_id |
| 433 | input_ids = input_ids[:-1] |
| 434 | target_ids = target_ids[:-1] |
| 435 | images_seq_mask = images_seq_mask[:-1] |
| 436 | |
| 437 | if len(images_list) == 0: |
| 438 | pixel_values = torch.zeros((0, 3, self.base_size, self.base_size)) |
| 439 | images_spatial_crop = torch.zeros((0, 2), dtype=torch.long) |
| 440 | images_crop = torch.zeros((0, 3, self.image_size, self.image_size)) |
| 441 | else: |
| 442 | pixel_values = torch.stack(images_list, dim=0) |
| 443 | images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) |
| 444 | if images_crop_list: |
| 445 | images_crop = torch.stack(images_crop_list, dim=0) |
| 446 | else: |
| 447 | images_crop = torch.zeros((0, 3, self.image_size, self.image_size)) |
| 448 | |
| 449 | input_ids = input_ids.unsqueeze(0) |
| 450 | |
| 451 | return ( |
| 452 | input_ids, |
| 453 | pixel_values, |
| 454 | images_crop, |
| 455 | images_seq_mask, |
| 456 | images_spatial_crop, |
| 457 | num_image_tokens, |
| 458 | image_shapes, |
| 459 | ) |