| 1 | # SPDX-License-Identifier: Apache-2.0 |
| 2 | # SPDX-FileCopyrightText: Copyright contributors to the vLLM project |
| 3 | |
| 4 | # ruff: noqa: E501 |
| 5 | # coding=utf-8 |
| 6 | # adapted from https://github.com/deepseek-ai/DeepSeek-VL2/blob/ff23960c5cf9e6874b44be38af930cfb0ccbb620/deepseek_vl2/models/processing_deepseek_vl_v2.py |
| 7 | # Copyright (c) 2023-2024 DeepSeek. |
| 8 | # |
| 9 | # Permission is hereby granted, free of charge, to any person obtaining a copy of |
| 10 | # this software and associated documentation files (the "Software"), to deal in |
| 11 | # the Software without restriction, including without limitation the rights to |
| 12 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of |
| 13 | # the Software, and to permit persons to whom the Software is furnished to do so, |
| 14 | # subject to the following conditions: |
| 15 | # |
| 16 | # The above copyright notice and this permission notice shall be included in all |
| 17 | # copies or substantial portions of the Software. |
| 18 | # |
| 19 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 20 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS |
| 21 | # FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR |
| 22 | # COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER |
| 23 | # IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN |
| 24 | # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 25 | |
| 26 | import math |
| 27 | from typing import Any |
| 28 | |
| 29 | import torch |
| 30 | import torchvision.transforms as T |
| 31 | from PIL import Image, ImageOps |
| 32 | from transformers import BatchFeature, LlamaTokenizerFast |
| 33 | from transformers.processing_utils import ProcessorMixin |
| 34 | |
| 35 | |
| 36 | class ImageTransform: |
| 37 | def __init__( |
| 38 | self, |
| 39 | mean: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 40 | std: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 41 | normalize: bool = True, |
| 42 | ): |
| 43 | self.mean = mean |
| 44 | self.std = std |
| 45 | self.normalize = normalize |
| 46 | |
| 47 | transform_pipelines = [T.ToTensor()] |
| 48 | |
| 49 | if normalize: |
| 50 | transform_pipelines.append(T.Normalize(mean, std)) |
| 51 | |
| 52 | self.transform = T.Compose(transform_pipelines) |
| 53 | |
| 54 | def __call__(self, pil_img: Image.Image): |
| 55 | x = self.transform(pil_img) |
| 56 | return x |
| 57 | |
| 58 | |
| 59 | class DeepseekVLV2Processor(ProcessorMixin): |
| 60 | tokenizer_class = ("LlamaTokenizer", "LlamaTokenizerFast") |
| 61 | attributes = ["tokenizer"] |
| 62 | |
| 63 | def __init__( |
| 64 | self, |
| 65 | tokenizer: LlamaTokenizerFast, |
| 66 | candidate_resolutions: tuple[tuple[int, int]], |
| 67 | patch_size: int, |
| 68 | downsample_ratio: int, |
| 69 | image_mean: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 70 | image_std: tuple[float, float, float] = (0.5, 0.5, 0.5), |
| 71 | normalize: bool = True, |
| 72 | image_token: str = "<image>", |
| 73 | pad_token: str = "<｜▁pad▁｜>", |
| 74 | add_special_token: bool = False, |
| 75 | sft_format: str = "deepseek", |
| 76 | mask_prompt: bool = True, |
| 77 | ignore_id: int = -100, |
| 78 | **kwargs, |
| 79 | ): |
| 80 | self.candidate_resolutions = candidate_resolutions |
| 81 | self.image_size = candidate_resolutions[0][0] |
| 82 | self.patch_size = patch_size |
| 83 | self.image_mean = image_mean |
| 84 | self.image_std = image_std |
| 85 | self.normalize = normalize |
| 86 | self.downsample_ratio = downsample_ratio |
| 87 | |
| 88 | self.image_transform = ImageTransform( |
| 89 | mean=image_mean, std=image_std, normalize=normalize |
| 90 | ) |
| 91 | self.tokenizer = tokenizer |
| 92 | self.tokenizer.padding_side = "left" # must set this，padding side with make a difference in batch inference |
| 93 | |
| 94 | # add the pad_token as special token to use 'tokenizer.pad_token' and 'tokenizer.pad_token_id' |
| 95 | if tokenizer.pad_token is None: |
| 96 | self.tokenizer.add_special_tokens({"pad_token": pad_token}) |
| 97 | |
| 98 | # add image token |
| 99 | image_token_id = self.tokenizer.vocab.get(image_token) |
| 100 | if image_token_id is None: |
| 101 | special_tokens = [image_token] |
| 102 | special_tokens_dict = {"additional_special_tokens": special_tokens} |
| 103 | self.tokenizer.add_special_tokens(special_tokens_dict) |
| 104 | self.image_token_id = self.tokenizer.vocab.get(image_token) |
| 105 | |
| 106 | # add five special tokens for grounding-related tasks |
| 107 | # <|ref|>, <|/ref|>, <|det|>, <|/det|>, <|grounding|> |
| 108 | special_tokens = ["<|ref|>", "<|/ref|>", "<|det|>", "<|/det|>", "<|grounding|>"] |
| 109 | special_tokens_dict = {"additional_special_tokens": special_tokens} |
| 110 | self.tokenizer.add_special_tokens(special_tokens_dict) |
| 111 | |
| 112 | # add special tokens for SFT data |
| 113 | special_tokens = ["<|User|>", "<|Assistant|>"] |
| 114 | special_tokens_dict = {"additional_special_tokens": special_tokens} |
| 115 | self.tokenizer.add_special_tokens(special_tokens_dict) |
| 116 | |
| 117 | self.image_token = image_token |
| 118 | self.pad_token = pad_token |
| 119 | self.add_special_token = add_special_token |
| 120 | self.sft_format = sft_format |
| 121 | self.mask_prompt = mask_prompt |
| 122 | self.ignore_id = ignore_id |
| 123 | |
| 124 | super().__init__( |
| 125 | tokenizer, |
| 126 | **kwargs, |
| 127 | ) |
| 128 | |
| 129 | def select_best_resolution(self, image_size): |
| 130 | # used for cropping |
| 131 | original_width, original_height = image_size |
| 132 | best_fit = None |
| 133 | max_effective_resolution = 0 |
| 134 | min_wasted_resolution = float("inf") |
| 135 | |
| 136 | for width, height in self.candidate_resolutions: |
| 137 | scale = min(width / original_width, height / original_height) |
| 138 | downscaled_width, downscaled_height = ( |
| 139 | int(original_width * scale), |
| 140 | int(original_height * scale), |
| 141 | ) |
| 142 | effective_resolution = min( |
| 143 | downscaled_width * downscaled_height, original_width * original_height |
| 144 | ) |
| 145 | wasted_resolution = (width * height) - effective_resolution |
| 146 | |
| 147 | if effective_resolution > max_effective_resolution or ( |
| 148 | effective_resolution == max_effective_resolution |
| 149 | and wasted_resolution < min_wasted_resolution |
| 150 | ): |
| 151 | max_effective_resolution = effective_resolution |
| 152 | min_wasted_resolution = wasted_resolution |
| 153 | best_fit = (width, height) |
| 154 | |
| 155 | return best_fit |
| 156 | |
| 157 | @property |
| 158 | def bos_id(self): |
| 159 | return self.tokenizer.bos_token_id |
| 160 | |
| 161 | @property |
| 162 | def eos_id(self): |
| 163 | return self.tokenizer.eos_token_id |
| 164 | |
| 165 | @property |
| 166 | def pad_id(self): |
| 167 | return self.tokenizer.pad_token_id |
| 168 | |
| 169 | def encode(self, text: str, bos: bool = True, eos: bool = False): |
| 170 | t = self.tokenizer.encode(text, add_special_tokens=False) |
| 171 | |
| 172 | if bos: |
| 173 | t = [self.bos_id] + t |
| 174 | if eos: |
| 175 | t = t + [self.eos_id] |
| 176 | |
| 177 | return t |
| 178 | |
| 179 | def decode(self, t: list[int], **kwargs) -> str: |
| 180 | return self.tokenizer.decode(t, **kwargs) |
| 181 | |
| 182 | def process_one( |
| 183 | self, |
| 184 | prompt: str, |
| 185 | images: list[Image.Image], |
| 186 | inference_mode: bool = True, |
| 187 | **kwargs: Any, |
| 188 | ): |
| 189 | """Args: |
| 190 | prompt (str): the formatted prompt; |
| 191 | images (list[ImageType]): the list of images; |
| 192 | inference_mode (bool): if True, then remove the last eos token; |
| 193 | **kwargs: Additional keyword arguments. |
| 194 | |
| 195 | Returns: |
| 196 | outputs (BaseProcessorOutput): the output of the processor, |
| 197 | - input_ids (torch.LongTensor): [N + image tokens] |
| 198 | - target_ids (torch.LongTensor): [N + image tokens] |
| 199 | - pixel_values (torch.FloatTensor): [n_patches, 3, H, W] |
| 200 | - image_id (int): the id of the image token |
| 201 | - num_image_tokens (list[int]): the number of image tokens |
| 202 | |
| 203 | """ |
| 204 | if prompt is None or images is None: |
| 205 | raise ValueError("prompt and images must be used at the same time.") |
| 206 | |
| 207 | sft_format = prompt |
| 208 | ( |
| 209 | tokenized_str, |
| 210 | images_list, |
| 211 | images_seq_mask, |
| 212 | images_spatial_crop, |
| 213 | num_image_tokens, |
| 214 | ) = self.tokenize_with_images( |
| 215 | sft_format, images, bos=True, eos=True, cropping=len(images) <= 2 |
| 216 | ) |
| 217 | masked_tokenized_str = [] |
| 218 | for token_index in tokenized_str: |
| 219 | if token_index != self.image_token_id: |
| 220 | masked_tokenized_str.append(token_index) |
| 221 | else: |
| 222 | masked_tokenized_str.append(self.ignore_id) |
| 223 | |
| 224 | assert ( |
| 225 | len(tokenized_str) == len(images_seq_mask) == len(masked_tokenized_str) |
| 226 | ), ( |
| 227 | f"tokenized_str's length {len(tokenized_str)}, input_ids' length {len(masked_tokenized_str)}, " |
| 228 | f"imags_seq_mask's length {len(images_seq_mask)}, are not equal" |
| 229 | ) |
| 230 | |
| 231 | input_ids = torch.LongTensor(tokenized_str) |
| 232 | target_ids = torch.LongTensor(masked_tokenized_str) |
| 233 | images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool) |
| 234 | |
| 235 | # set input_ids < 0 | input_ids == self.image_token_id as ignore_id |
| 236 | target_ids[(input_ids < 0) | (input_ids == self.image_token_id)] = ( |
| 237 | self.ignore_id |
| 238 | ) |
| 239 | input_ids[input_ids < 0] = self.pad_id |
| 240 | |
| 241 | if inference_mode: |
| 242 | # Remove the ending eos token |
| 243 | assert input_ids[-1] == self.eos_id |
| 244 | input_ids = input_ids[:-1] |
| 245 | target_ids = target_ids[:-1] |
| 246 | images_seq_mask = images_seq_mask[:-1] |
| 247 | |
| 248 | if len(images_list) == 0: |
| 249 | pixel_values = torch.zeros((1, 3, self.image_size, self.image_size)) |
| 250 | images_spatial_crop = torch.zeros((1, 2), dtype=torch.long) |
| 251 | else: |
| 252 | pixel_values = torch.stack(images_list, dim=0) |
| 253 | images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long) |
| 254 | |
| 255 | input_ids = input_ids.unsqueeze(0) |
| 256 | |
| 257 | prepare = BatchFeature( |
| 258 | data=dict( |
| 259 | input_ids=input_ids, |
| 260 | pixel_values=pixel_values, |
| 261 | images_seq_mask=images_seq_mask, |
| 262 | images_spatial_crop=images_spatial_crop, |
| 263 | num_image_tokens=num_image_tokens, |
| 264 | ), |
| 265 | tensor_type="pt", |
| 266 | ) |
| 267 | return prepare |
| 268 | |
| 269 | def __call__( |
| 270 | self, |
| 271 | *, |
| 272 | text: str, |
| 273 | images: list[Image.Image], |
| 274 | inference_mode: bool = True, |
| 275 | **kwargs: Any, |
| 276 | ): |
| 277 | """Args: |
| 278 | text (str): the formatted prompt; |
| 279 | images (list[ImageType]): the list of images; |
| 280 | inference_mode (bool): if True, then remove the last eos token; |
| 281 | **kwargs: |
| 282 | |
| 283 | Returns: |
| 284 | outputs (BaseProcessorOutput): the output of the processor, |
| 285 | - input_ids (torch.LongTensor): [N + image tokens] |
| 286 | - images (torch.FloatTensor): [n_images, 3, H, W] |
| 287 | - image_id (int): the id of the image token |
| 288 | - num_image_tokens (list[int]): the number of image tokens |
| 289 | |
| 290 | """ |
| 291 | prepare = self.process_one( |
| 292 | prompt=text, |
| 293 | images=images, |
| 294 | inference_mode=inference_mode, |
| 295 | ) |
| 296 | |
| 297 | return prepare |
| 298 | |
| 299 | def tokenize_with_images( |
| 300 | self, |
| 301 | conversation: str, |
| 302 | images: list[Image.Image], |
| 303 | bos: bool = True, |
| 304 | eos: bool = True, |
| 305 | cropping: bool = True, |
| 306 | ): |
| 307 | """Tokenize text with <image> tags.""" |
| 308 | num_image_tags = conversation.count(self.image_token) |
| 309 | if num_image_tags != len(images): |
| 310 | raise ValueError( |
| 311 | f"Number of {self.image_token!r} tokens in prompt " |
| 312 | f"({num_image_tags}) does not match number of images " |
| 313 | f"({len(images)})." |
| 314 | ) |
| 315 | text_splits = conversation.split(self.image_token) |
| 316 | images_list, images_seq_mask, images_spatial_crop = [], [], [] |
| 317 | num_image_tokens = [] |
| 318 | tokenized_str = [] |
| 319 | for text_sep, image in zip(text_splits, images): |
| 320 | """encode text_sep""" |
| 321 | tokenized_sep = self.encode(text_sep, bos=False, eos=False) |
| 322 | tokenized_str += tokenized_sep |
| 323 | images_seq_mask += [False] * len(tokenized_sep) |
| 324 | |
| 325 | """select best resolution for anyres""" |
| 326 | if cropping: |
| 327 | best_width, best_height = self.select_best_resolution(image.size) |
| 328 | else: |
| 329 | best_width, best_height = self.image_size, self.image_size |
| 330 | |
| 331 | """process the global view""" |
| 332 | global_view = ImageOps.pad( |
| 333 | image, |
| 334 | (self.image_size, self.image_size), |
| 335 | color=tuple(int(x * 255) for x in self.image_transform.mean), |
| 336 | ) |
| 337 | images_list.append(self.image_transform(global_view)) |
| 338 | |
| 339 | """process the local views""" |
| 340 | local_view = ImageOps.pad( |
| 341 | image, |
| 342 | (best_width, best_height), |
| 343 | color=tuple(int(x * 255) for x in self.image_transform.mean), |
| 344 | ) |
| 345 | for i in range(0, best_height, self.image_size): |
| 346 | for j in range(0, best_width, self.image_size): |
| 347 | images_list.append( |
| 348 | self.image_transform( |
| 349 | local_view.crop( |
| 350 | (j, i, j + self.image_size, i + self.image_size) |
| 351 | ) |
| 352 | ) |
| 353 | ) |
| 354 | |
| 355 | """record height / width crop num""" |
| 356 | num_width_tiles, num_height_tiles = ( |
| 357 | best_width // self.image_size, |
| 358 | best_height // self.image_size, |
| 359 | ) |
| 360 | images_spatial_crop.append([num_width_tiles, num_height_tiles]) |
| 361 | |
| 362 | """add image tokens""" |
| 363 | h = w = math.ceil( |
| 364 | (self.image_size // self.patch_size) / self.downsample_ratio |
| 365 | ) |
| 366 | # global views tokens h * (w + 1), 1 is for line separator |
| 367 | tokenized_image = [self.image_token_id] * h * (w + 1) |
| 368 | # add a separator between global and local views |
| 369 | tokenized_image += [self.image_token_id] |
| 370 | # local views tokens, (num_height_tiles * h) * (num_width_tiles * w + 1) |
| 371 | tokenized_image += ( |
| 372 | [self.image_token_id] |
| 373 | * (num_height_tiles * h) |
| 374 | * (num_width_tiles * w + 1) |
| 375 | ) |
| 376 | |
| 377 | tokenized_str += tokenized_image |
| 378 | images_seq_mask += [True] * len(tokenized_image) |
| 379 | num_image_tokens.append(len(tokenized_image)) |
| 380 | |
| 381 | """process the last text split""" |
| 382 | tokenized_sep = self.encode(text_splits[-1], bos=False, eos=False) |
| 383 | tokenized_str += tokenized_sep |
| 384 | images_seq_mask += [False] * len(tokenized_sep) |
| 385 | |
| 386 | """add the bos and eos tokens""" |
| 387 | if bos: |
| 388 | tokenized_str = [self.bos_id] + tokenized_str |
| 389 | images_seq_mask = [False] + images_seq_mask |
| 390 | if eos: |
| 391 | tokenized_str = tokenized_str + [self.eos_id] |
| 392 | images_seq_mask = images_seq_mask + [False] |
| 393 | |
| 394 | assert len(tokenized_str) == len(images_seq_mask), ( |
| 395 | f"tokenize_with_images func: tokenized_str's length {len(tokenized_str)} is not equal to imags_seq_mask's length {len(images_seq_mask)}" |
| 396 | ) |
| 397 | |
| 398 | return ( |
| 399 | tokenized_str, |
| 400 | images_list, |
| 401 | images_seq_mask, |
| 402 | images_spatial_crop, |
| 403 | num_image_tokens, |
| 404 | ) |